diff --git a/.last-branch b/.last-branch index b7b7c98..1598c06 100644 --- a/.last-branch +++ b/.last-branch @@ -1 +1 @@ -v20260106-18-runchecks-popup-objects-fallback +v20260106-19-missed-run-detection-threshold diff --git a/containers/backupchecks/src/backend/app/main/routes_daily_jobs.py b/containers/backupchecks/src/backend/app/main/routes_daily_jobs.py index e1b325b..890bc29 100644 --- a/containers/backupchecks/src/backend/app/main/routes_daily_jobs.py +++ b/containers/backupchecks/src/backend/app/main/routes_daily_jobs.py @@ -1,5 +1,5 @@ from .routes_shared import * # noqa: F401,F403 -from .routes_shared import _format_datetime, _get_or_create_settings, _apply_overrides_to_run, _infer_schedule_map_from_runs +from .routes_shared import _format_datetime, _get_or_create_settings, _apply_overrides_to_run, _infer_schedule_map_from_runs, _infer_monthly_schedule_from_runs # Grace window for today's Expected/Missed transition. # A job is only marked Missed after the latest expected time plus this grace. @@ -85,6 +85,23 @@ def daily_jobs(): for job in jobs: schedule_map = _infer_schedule_map_from_runs(job.id) expected_times = schedule_map.get(weekday_idx) or [] + + # If no weekly schedule is inferred (e.g. monthly jobs), try monthly inference. + if not expected_times: + monthly = _infer_monthly_schedule_from_runs(job.id) + if monthly: + dom = int(monthly.get("day_of_month") or 0) + mtimes = monthly.get("times") or [] + # For months shorter than dom, treat the last day of month as the scheduled day. + try: + import calendar as _calendar + last_dom = _calendar.monthrange(target_date.year, target_date.month)[1] + except Exception: + last_dom = target_date.day + scheduled_dom = dom if (dom and dom <= last_dom) else last_dom + if target_date.day == scheduled_dom: + expected_times = list(mtimes) + if not expected_times: continue diff --git a/containers/backupchecks/src/backend/app/main/routes_run_checks.py b/containers/backupchecks/src/backend/app/main/routes_run_checks.py index 27760f8..12908f3 100644 --- a/containers/backupchecks/src/backend/app/main/routes_run_checks.py +++ b/containers/backupchecks/src/backend/app/main/routes_run_checks.py @@ -1,5 +1,7 @@ from __future__ import annotations +import calendar + from datetime import date, datetime, time, timedelta, timezone from flask import jsonify, render_template, request @@ -13,6 +15,7 @@ from .routes_shared import ( _get_ui_timezone_name, _get_or_create_settings, _infer_schedule_map_from_runs, + _infer_monthly_schedule_from_runs, _to_amsterdam_date, main_bp, roles_required, @@ -84,7 +87,13 @@ def _ensure_missed_runs_for_job(job: Job, start_from: date, end_inclusive: date) """ tz = _get_ui_timezone() schedule_map = _infer_schedule_map_from_runs(job.id) or {} - if not schedule_map: + has_weekly_times = any((schedule_map.get(i) or []) for i in range(7)) + + monthly = None + if not has_weekly_times: + monthly = _infer_monthly_schedule_from_runs(job.id) + + if (not has_weekly_times) and (not monthly): return 0 today_local = _to_amsterdam_date(datetime.utcnow()) or datetime.utcnow().date() @@ -120,6 +129,8 @@ def _ensure_missed_runs_for_job(job: Job, start_from: date, end_inclusive: date) inserted = 0 d = start_from while d <= end_inclusive: + if not has_weekly_times: + break weekday = d.weekday() times = schedule_map.get(weekday) or [] if not times: @@ -177,6 +188,82 @@ def _ensure_missed_runs_for_job(job: Job, start_from: date, end_inclusive: date) d = d + timedelta(days=1) + + # Monthly expected slots (fallback when no stable weekly schedule is detected) + if (not has_weekly_times) and monthly: + try: + dom = int(monthly.get("day_of_month") or 0) + except Exception: + dom = 0 + times = monthly.get("times") or [] + + if dom > 0 and times: + # Iterate months in the window [start_from, end_inclusive] + cur = date(start_from.year, start_from.month, 1) + end_marker = date(end_inclusive.year, end_inclusive.month, 1) + + while cur <= end_marker: + try: + last_dom = calendar.monthrange(cur.year, cur.month)[1] + except Exception: + last_dom = 28 + scheduled_dom = dom if dom <= last_dom else last_dom + scheduled_date = date(cur.year, cur.month, scheduled_dom) + + if scheduled_date >= start_from and scheduled_date <= end_inclusive: + for hhmm in times: + hm = _parse_hhmm(hhmm) + if not hm: + continue + hh, mm = hm + + local_dt = datetime.combine(scheduled_date, time(hour=hh, minute=mm)) + if tz: + local_dt = local_dt.replace(tzinfo=tz) + + # Only generate missed runs for past slots. + if local_dt > now_local_dt: + continue + + slot_utc_naive = _utc_naive_from_local(local_dt) + + window_start = slot_utc_naive - MISSED_GRACE_WINDOW + window_end = slot_utc_naive + MISSED_GRACE_WINDOW + + exists = ( + db.session.query(JobRun.id) + .filter( + JobRun.job_id == job.id, + JobRun.run_at.isnot(None), + or_( + and_(JobRun.missed.is_(False), JobRun.mail_message_id.isnot(None)), + and_(JobRun.missed.is_(True), JobRun.mail_message_id.is_(None)), + ), + JobRun.run_at >= window_start, + JobRun.run_at <= window_end, + ) + .first() + ) + if exists: + continue + + miss = JobRun( + job_id=job.id, + run_at=slot_utc_naive, + status="Missed", + missed=True, + remark=None, + mail_message_id=None, + ) + db.session.add(miss) + inserted += 1 + + # Next month + if cur.month == 12: + cur = date(cur.year + 1, 1, 1) + else: + cur = date(cur.year, cur.month + 1, 1) + if inserted: db.session.commit() return inserted diff --git a/containers/backupchecks/src/backend/app/main/routes_shared.py b/containers/backupchecks/src/backend/app/main/routes_shared.py index e73ee9e..7a8430c 100644 --- a/containers/backupchecks/src/backend/app/main/routes_shared.py +++ b/containers/backupchecks/src/backend/app/main/routes_shared.py @@ -6,6 +6,7 @@ import json import re import html as _html import math +import calendar import datetime as datetime_module from functools import wraps @@ -612,7 +613,13 @@ def _infer_schedule_map_from_runs(job_id: int): """Infer weekly schedule blocks (15-min) from historical runs. Returns dict weekday->sorted list of 'HH:MM' strings in configured UI local time. + + Notes: + - Only considers real runs that came from mail reports (mail_message_id is not NULL). + - Synthetic missed rows never influence schedule inference. + - To reduce noise, a weekday/time bucket must occur at least MIN_OCCURRENCES times. """ + MIN_OCCURRENCES = 3 schedule = {i: [] for i in range(7)} # 0=Mon .. 6=Sun # Certain job types are informational and should never participate in schedule @@ -632,6 +639,7 @@ def _infer_schedule_map_from_runs(job_id: int): return schedule except Exception: pass + try: # Only infer schedules from real runs that came from mail reports. # Synthetic "Missed" rows must never influence schedule inference. @@ -653,13 +661,13 @@ def _infer_schedule_map_from_runs(job_id: int): if not runs: return schedule - # Convert run_at to UI local time and bucket into 15-minute blocks + # Convert run_at to UI local time and bucket into 15-minute blocks. try: tz = _get_ui_timezone() except Exception: tz = None - seen = {i: set() for i in range(7)} + counts = {i: {} for i in range(7)} # weekday -> { "HH:MM": count } for r in runs: if not r.run_at: continue @@ -678,14 +686,139 @@ def _infer_schedule_map_from_runs(job_id: int): minute_bucket = (dt.minute // 15) * 15 hh = dt.hour tstr = f"{hh:02d}:{minute_bucket:02d}" - seen[wd].add(tstr) + counts[wd][tstr] = int(counts[wd].get(tstr, 0)) + 1 for wd in range(7): - schedule[wd] = sorted(seen[wd]) + # Keep only buckets that occur frequently enough. + keep = [t for t, c in counts[wd].items() if int(c) >= MIN_OCCURRENCES] + schedule[wd] = sorted(keep) return schedule +def _infer_monthly_schedule_from_runs(job_id: int): + """Infer a monthly schedule from historical runs. + + Returns: + dict with keys: + - day_of_month (int) + - times (list[str] of 'HH:MM' 15-min buckets) + or None if not enough evidence. + + Rules: + - Uses only real mail-based runs (mail_message_id is not NULL) and excludes synthetic missed rows. + - Requires at least MIN_OCCURRENCES occurrences for the inferred day-of-month. + - Uses a simple cadence heuristic: typical gaps between runs must be >= 20 days to qualify as monthly. + """ + MIN_OCCURRENCES = 3 + + try: + # Same "real run" rule as weekly inference. + runs = ( + JobRun.query + .filter( + JobRun.job_id == job_id, + JobRun.run_at.isnot(None), + JobRun.missed.is_(False), + JobRun.mail_message_id.isnot(None), + ) + .order_by(JobRun.run_at.asc()) + .limit(500) + .all() + ) + except Exception: + runs = [] + + if len(runs) < MIN_OCCURRENCES: + return None + + try: + tz = _get_ui_timezone() + except Exception: + tz = None + + # Convert and keep local datetimes. + local_dts = [] + for r in runs: + if not r.run_at: + continue + dt = r.run_at + if tz is not None: + try: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime_module.timezone.utc).astimezone(tz) + else: + dt = dt.astimezone(tz) + except Exception: + pass + local_dts.append(dt) + + if len(local_dts) < MIN_OCCURRENCES: + return None + + # Cadence heuristic: monthly jobs shouldn't look weekly. + local_dts_sorted = sorted(local_dts) + gaps = [] + for i in range(1, len(local_dts_sorted)): + try: + delta_days = (local_dts_sorted[i] - local_dts_sorted[i - 1]).total_seconds() / 86400.0 + if delta_days > 0: + gaps.append(delta_days) + except Exception: + continue + + if gaps: + gaps_sorted = sorted(gaps) + median_gap = gaps_sorted[len(gaps_sorted) // 2] + # If it looks like a weekly/daily cadence, do not classify as monthly. + if median_gap < 20.0: + return None + + # Count day-of-month occurrences and time buckets on that day. + dom_counts = {} + time_counts_by_dom = {} # dom -> { "HH:MM": count } + for dt in local_dts: + dom = int(dt.day) + dom_counts[dom] = int(dom_counts.get(dom, 0)) + 1 + + minute_bucket = (dt.minute // 15) * 15 + tstr = f"{int(dt.hour):02d}:{int(minute_bucket):02d}" + if dom not in time_counts_by_dom: + time_counts_by_dom[dom] = {} + time_counts_by_dom[dom][tstr] = int(time_counts_by_dom[dom].get(tstr, 0)) + 1 + + # Pick the most common day-of-month with enough occurrences. + best_dom = None + best_dom_count = 0 + for dom, c in dom_counts.items(): + if int(c) >= MIN_OCCURRENCES and int(c) > best_dom_count: + best_dom = int(dom) + best_dom_count = int(c) + + if best_dom is None: + return None + + # Times on that day must also be stable. Keep frequent buckets; otherwise fall back to the top bucket. + time_counts = time_counts_by_dom.get(best_dom) or {} + keep_times = [t for t, c in time_counts.items() if int(c) >= MIN_OCCURRENCES] + if not keep_times: + # Fallback: choose the single most common time bucket for that day. + best_t = None + best_c = 0 + for t, c in time_counts.items(): + if int(c) > best_c: + best_t = t + best_c = int(c) + if best_t: + keep_times = [best_t] + + keep_times = sorted(set(keep_times)) + if not keep_times: + return None + + return {"day_of_month": int(best_dom), "times": keep_times} + + def _schedule_map_to_desc(schedule_map): weekday_names = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"] any_times = any(schedule_map.get(i) for i in range(7)) diff --git a/docs/changelog.md b/docs/changelog.md index 11f4493..5cb1d60 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -139,6 +139,15 @@ Removed an incorrectly indented redirect statement so the module loads correctly - Added legacy fallback to load objects via JobRun.objects for older data during/after upgrades. - Added mail-based fallback to load objects via MailObject when no run-linked objects exist yet. - Updated imports in routes_run_checks to include JobObject and MailObject used by the fallback logic. + +--- + +## v20260106-19-missed-run-detection-threshold +- Improved weekly schedule inference by requiring a time bucket to occur at least 3 times before it is considered “expected” (reduces outlier noise and false missed runs). +- Added monthly schedule inference based on real mail-based runs (>= 3 occurrences) with a cadence check to avoid classifying weekly jobs as monthly. +- Updated missed run generation (Run Checks) to use monthly inference when no stable weekly schedule exists, so monthly jobs are marked missed on the correct expected day instead of a week later. +- Updated Daily Jobs to show expected entries for inferred monthly jobs on the scheduled day-of-month (with last-day-of-month fallback for short months). + ================================================================================================================================================ ## v0.1.16