swf-monitor/scripts/batch-log-learn.py

0001 #!/usr/bin/env python3
0002 """batch-log-learn.py — mine the batch-record corpus before it ages out.
0003
0004 The prod-ops agent's doer for the learning pass (docs/ERROR_ATTRIBUTION.md,
0005 Retention and learning). It reads every captured condor event log and
0006 writes the knowledge base beside them: the catalogue of event codes that
0007 actually occur with counts and an example each, the taxonomy of recurring
0008 reasons with the jobs and days they appear on, and the lines that are
0009 boilerplate because they are in most logs. It then rescues one raw log per
0010 pattern, and every log matching no pattern, under ``keep``, so retention
0011 can delete a date directory whole without deciding anything about content.
0012
0013 The parser improves from what the corpus contains rather than from an
0014 author's expectation of it, and every entry in the knowledge base keeps the
0015 raw evidence behind it.
0016
0017 Django-bootstrap standalone script — also usable by hand::
0018
0019     cd /data/wenauseic/github/swf-monitor/src
0020     ../../swf-testbed/.venv/bin/python ../scripts/batch-log-learn.py [--no-rescue]
0021
0022 The last stdout line is a JSON summary; the readable report is above it.
0023 """
0024 import argparse
0025 import json
0026 import os
0027 import sys
0028
0029 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
0030 sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src'))
0031 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swf_monitor_project.settings')
0032
0033 import django  # noqa: E402
0034 django.setup()
0035
0036 from monitor_app import batch_records  # noqa: E402
0037
0038
0039 def main():
0040     parser = argparse.ArgumentParser()
0041     parser.add_argument('--no-rescue', action='store_true',
0042                         help='build the knowledge base, rescue nothing')
0043     parser.add_argument('--root', default=None, help='override the store root')
0044     parser.add_argument('--quiet', action='store_true')
0045     args = parser.parse_args()
0046
0047     knowledge = batch_records.learn(root=args.root, rescue=not args.no_rescue)
0048
0049     if not args.quiet:
0050         print(f"logs read: {knowledge['logs']}")
0051         print(f"event codes: {len(knowledge['codes'])}, "
0052               f"unnamed by the parser: {', '.join(knowledge['unnamed_codes']) or 'none'}")
0053         for code in knowledge['codes'][:12]:
0054             name = code['name'] or f"event {code['code']} (unnamed)"
0055             print(f"  {code['code']}  {name:<24} {code['count']:5d} in {code['logs']} logs")
0056         print(f"reason patterns: {len(knowledge['patterns'])}")
0057         for pattern in knowledge['patterns'][:12]:
0058             print(f"  {pattern['count']:5d}  {pattern['event']:<18} "
0059                   f"{pattern['example'][:100]}")
0060         print(f"boilerplate lines: {len(knowledge['boilerplate'])}")
0061         if 'rescued' in knowledge:
0062             print(f"rescued: {knowledge['rescued']}")
0063
0064     print(json.dumps({
0065         'logs': knowledge['logs'],
0066         'codes': len(knowledge['codes']),
0067         'unnamed_codes': knowledge['unnamed_codes'],
0068         'patterns': len(knowledge['patterns']),
0069         'boilerplate': len(knowledge['boilerplate']),
0070         'rescued': knowledge.get('rescued', {}),
0071     }))
0072     return 0
0073
0074
0075 if __name__ == '__main__':
0076     sys.exit(main())