File indexing completed on 2026-08-12 09:36:13
0001
0002 """AppLog retention sweep for the swf_applog table.
0003
0004 Deletes log rows older than the retention window, in bounded batches so
0005 the table is never long-locked. Retention is per level: the default
0006 keeps every level 30 days; --error-days widens the window for ERROR and
0007 CRITICAL rows when a longer post-mortem trail is wanted.
0008
0009 Standalone, no Django. Database credentials come from the same DB_* env
0010 vars the monitor settings read (source the deployment .env). Intended
0011 for cron; every run logs row counts before and after. See
0012 docs/CACHED_PRODUCTS.md for the page-serving half of the log-volume
0013 story.
0014 """
0015 import argparse
0016 import logging
0017 import os
0018 import sys
0019 import time
0020
0021 import psycopg
0022
0023 logging.basicConfig(
0024 level=logging.INFO,
0025 format="%(asctime)s %(levelname)s applog-retention %(message)s")
0026 log = logging.getLogger(__name__)
0027
0028 BATCH_ROWS = 50000
0029
0030
0031 def connect():
0032 return psycopg.connect(
0033 dbname=os.environ.get("DB_NAME", "swfdb"),
0034 user=os.environ.get("DB_USER", "admin"),
0035 password=os.environ["DB_PASSWORD"],
0036 host=os.environ.get("DB_HOST", "localhost"),
0037 port=os.environ.get("DB_PORT", "5432"),
0038 )
0039
0040
0041 def batched_delete(conn, where_sql, params, dry_run):
0042 """Delete matching rows in id-batches; returns rows deleted."""
0043 total = 0
0044 while True:
0045 with conn.cursor() as cur:
0046 if dry_run:
0047 cur.execute(
0048 f"SELECT count(*) FROM swf_applog WHERE {where_sql}",
0049 params)
0050 count = cur.fetchone()[0]
0051 log.info("dry run: %d rows match: %s", count, where_sql)
0052 return count
0053 cur.execute(
0054 f"DELETE FROM swf_applog WHERE id IN ("
0055 f"SELECT id FROM swf_applog WHERE {where_sql} "
0056 f"LIMIT {BATCH_ROWS})", params)
0057 deleted = cur.rowcount
0058 conn.commit()
0059 total += deleted
0060 if deleted:
0061 log.info("deleted batch of %d (total %d)", deleted, total)
0062 if deleted < BATCH_ROWS:
0063 return total
0064 time.sleep(0.5)
0065
0066
0067 def main():
0068 parser = argparse.ArgumentParser(description=__doc__)
0069 parser.add_argument("--days", type=int, default=30,
0070 help="retention window in days (default 30)")
0071 parser.add_argument("--error-days", type=int, default=None,
0072 help="retention for ERROR/CRITICAL rows "
0073 "(default: same as --days)")
0074 parser.add_argument("--dry-run", action="store_true",
0075 help="count matching rows, delete nothing")
0076 args = parser.parse_args()
0077 error_days = (args.error_days if args.error_days is not None
0078 else args.days)
0079
0080 try:
0081 conn = connect()
0082 except Exception as e:
0083 log.error("database connection failed: %s", e)
0084 return 1
0085 try:
0086 with conn.cursor() as cur:
0087 cur.execute("SELECT count(*) FROM swf_applog")
0088 before = cur.fetchone()[0]
0089 log.info("rows before: %d (retention %dd, errors %dd%s)",
0090 before, args.days, error_days,
0091 ", dry run" if args.dry_run else "")
0092 deleted = batched_delete(
0093 conn,
0094 "timestamp < now() - make_interval(days => %s) "
0095 "AND levelname NOT IN ('ERROR', 'CRITICAL')",
0096 (args.days,), args.dry_run)
0097 deleted += batched_delete(
0098 conn,
0099 "timestamp < now() - make_interval(days => %s) "
0100 "AND levelname IN ('ERROR', 'CRITICAL')",
0101 (error_days,), args.dry_run)
0102 if not args.dry_run:
0103 with conn.cursor() as cur:
0104 cur.execute("SELECT count(*) FROM swf_applog")
0105 after = cur.fetchone()[0]
0106 log.info("done: removed %d rows, %d remain", deleted, after)
0107 except Exception as e:
0108 log.error("retention sweep failed: %s", e)
0109 return 1
0110 finally:
0111 conn.close()
0112 return 0
0113
0114
0115 if __name__ == "__main__":
0116 sys.exit(main())
