From 4dff0303a388197ae95beb20e960267bccee253e Mon Sep 17 00:00:00 2001 From: Ivo Oskamp Date: Fri, 20 Mar 2026 12:57:55 +0100 Subject: [PATCH] Auto-commit local changes before build (2026-03-20 12:57:55) --- .../src/backend/app/cove_importer.py | 109 +++++++++++++++++- .../src/backend/app/main/routes_cove.py | 82 ++++++++++++- .../src/backend/app/main/routes_jobs.py | 1 + .../src/backend/app/main/routes_run_checks.py | 30 ++++- .../src/templates/main/job_detail.html | 61 ++++++++-- .../src/templates/main/run_checks.html | 46 ++++++-- docs/changelog-claude.md | 19 +++ docs/technical-notes-codex.md | 11 ++ 8 files changed, 336 insertions(+), 23 deletions(-) diff --git a/containers/backupchecks/src/backend/app/cove_importer.py b/containers/backupchecks/src/backend/app/cove_importer.py index 8b339e7..fc70659 100644 --- a/containers/backupchecks/src/backend/app/cove_importer.py +++ b/containers/backupchecks/src/backend/app/cove_importer.py @@ -13,7 +13,7 @@ from __future__ import annotations import logging import re -from datetime import datetime, timezone +from datetime import date, datetime, timedelta, timezone from typing import Any import requests @@ -297,6 +297,101 @@ def _fmt_utc(dt: datetime | None) -> str: return dt.strftime("%Y-%m-%d %H:%M UTC") +def _backfill_colorbar_runs( + cove_acc, + job, + colorbar: str, + last_run_at: datetime, +) -> int: + """Create historical JobRun records from the 28-day colorbar string (D09F08). + + The colorbar encodes backup status for each of the last 28 days. This is + called after an account is first linked so that historical data is visible + immediately instead of only accumulating day-by-day going forward. + + Supported formats: + - Continuous string: "55525555..." (each char = 1 day, oldest first) + - Separated: "5 5 2 5 5..." or "5,5,2,5,5..." + + Status code 0 means no backup ran that day and is skipped. + + Returns the number of runs created. + """ + from .models import JobRun # local import to avoid circular deps + + if not colorbar or not last_run_at: + return 0 + + # Normalise to a list of raw code strings + stripped = colorbar.strip() + if " " in stripped or "," in stripped: + raw_codes = re.split(r"[,\s]+", stripped) + else: + raw_codes = list(stripped) + + if not raw_codes: + return 0 + + # Oldest day is first (index 0), newest day is last (index -1). + # The newest entry corresponds to the real run already created; skip it. + newest_date = last_run_at.date() + ref_time = last_run_at.time() # use same time-of-day as the real run + + created = 0 + for i, code_raw in enumerate(raw_codes): + try: + code = int(str(code_raw).strip()) + except (ValueError, TypeError): + continue + + if code == 0: + continue # no backup that day + + # How many days before the newest date is this position? + days_ago = len(raw_codes) - 1 - i + + if days_ago == 0: + # Most-recent day — real run already exists with precise timestamp + continue + + run_date = newest_date - timedelta(days=days_ago) + run_dt = datetime( + run_date.year, run_date.month, run_date.day, + ref_time.hour, ref_time.minute, ref_time.second, + ) + + date_str = run_date.isoformat() + external_id = f"cove-colorbar-{cove_acc.account_id}-{date_str}" + + existing = JobRun.query.filter_by( + job_id=job.id, external_id=external_id + ).first() + if existing: + continue + + status = _map_status(code) + run = JobRun( + job_id=job.id, + mail_message_id=None, + run_at=run_dt, + status=status, + remark=( + f"Cove historical run (28-day colorbar) | " + f"date: {date_str} | " + f"status: {_status_label(code)} ({code})" + ), + missed=False, + override_applied=False, + source_type="cove_api", + external_id=external_id, + ) + db.session.add(run) + db.session.flush() + created += 1 + + return created + + def run_cove_import(settings, include_reasons: bool = False): """Fetch Cove account statistics and update the staging table + JobRuns. @@ -519,6 +614,18 @@ def _process_account(account: dict) -> str: if job.customer_id: _persist_datasource_objects(flat, job.customer_id, job.id, run.id, last_run_at) + # Backfill historical runs from the 28-day colorbar when this is the first + # real run for this job (i.e. the account was just linked). Deduplication + # via external_id makes this safe to call on every import. + if colorbar_28d: + backfilled = _backfill_colorbar_runs(cove_acc, job, colorbar_28d, last_run_at) + if backfilled: + logger.info( + "Cove backfill: created %d historical runs for account %s", + backfilled, + account_id, + ) + db.session.commit() return "created" diff --git a/containers/backupchecks/src/backend/app/main/routes_cove.py b/containers/backupchecks/src/backend/app/main/routes_cove.py index 382b89a..3e347e0 100644 --- a/containers/backupchecks/src/backend/app/main/routes_cove.py +++ b/containers/backupchecks/src/backend/app/main/routes_cove.py @@ -9,7 +9,7 @@ import re from .routes_shared import * # noqa: F401,F403 from .routes_shared import _log_admin_event -from ..cove_importer import CoveImportError, run_cove_import +from ..cove_importer import CoveImportError, run_cove_import, DATASOURCE_LABELS as _COVE_DS_LABELS from ..models import CoveAccount, Customer, Job, JobRun, SystemSettings @@ -311,3 +311,83 @@ def cove_account_unlink(cove_account_db_id: int): ) flash("Cove account unlinked.", "success") return redirect(url_for("main.cove_accounts")) + + +@main_bp.route("/cove/run//detail") +@login_required +@roles_required("admin", "operator", "viewer") +def cove_run_detail(run_id: int): + """Return structured Cove run details as JSON for the job detail popup.""" + from ..database import db + from sqlalchemy import text as _sql_text + + run = JobRun.query.get_or_404(run_id) + if getattr(run, "source_type", None) != "cove_api": + return jsonify({"status": "error", "message": "Not a Cove run"}), 400 + + cove_acc = CoveAccount.query.filter_by(job_id=run.job_id).first() + + cove_summary = None + if cove_acc: + raw_ds = (cove_acc.datasource_types or "").strip().upper() + ds_codes = re.findall(r"D\d{1,2}", raw_ds) if raw_ds else [] + ds_labels: list[str] = [] + for code in ds_codes: + lbl = _COVE_DS_LABELS.get(code, code) + if lbl not in ds_labels: + ds_labels.append(lbl) + cove_summary = { + "account_name": cove_acc.account_name or "", + "computer_name": cove_acc.computer_name or "", + "customer_name": cove_acc.customer_name or "", + "datasources": ", ".join(ds_labels) if ds_labels else (raw_ds or ""), + "last_run_at": cove_acc.last_run_at.strftime("%Y-%m-%d %H:%M") if cove_acc.last_run_at else "", + "status": run.status or "", + } + + # Per-run datasource objects from run_object_links + obj_rows = db.session.execute( + _sql_text(""" + SELECT co.object_name AS name, rol.status, rol.error_message + FROM run_object_links rol + JOIN customer_objects co ON co.id = rol.customer_object_id + WHERE rol.run_id = :run_id + ORDER BY co.object_name ASC + """), + {"run_id": run_id}, + ).mappings().all() + objects = [ + { + "name": r["name"] or "", + "type": "", + "status": r["status"] or "", + "error_message": r["error_message"] or "", + } + for r in obj_rows + ] + + account_label = "" + if cove_acc: + account_label = cove_acc.account_name or cove_acc.computer_name or f"account {cove_acc.account_id}" + + return jsonify({ + "status": "ok", + "meta": { + "subject": account_label, + "from_address": "", + "backup_software": "Cove Data Protection", + "backup_type": "", + "job_name": "", + "overall_status": run.status or "", + "overall_message": run.remark or "", + "customer_name": cove_acc.customer_name if cove_acc else "", + "received_at": run.run_at.strftime("%Y-%m-%d %H:%M") if run.run_at else "", + "parsed_at": "", + "has_eml": False, + }, + "cove_summary": cove_summary, + "cloud_connect_summary": None, + "objects": objects, + "body_html": "", + "mail": None, + }) diff --git a/containers/backupchecks/src/backend/app/main/routes_jobs.py b/containers/backupchecks/src/backend/app/main/routes_jobs.py index c1e6df4..baea853 100644 --- a/containers/backupchecks/src/backend/app/main/routes_jobs.py +++ b/containers/backupchecks/src/backend/app/main/routes_jobs.py @@ -512,6 +512,7 @@ def job_detail(job_id: int): "ticket_codes": ticket_codes, "remark_items": remark_items, "mail_message_id": r.mail_message_id, + "source_type": getattr(r, "source_type", "") or "", "reviewed_by": (r.reviewed_by.username if getattr(r, "reviewed_by", None) else ""), "reviewed_at": _format_datetime(r.reviewed_at) if r.reviewed_at else "", } diff --git a/containers/backupchecks/src/backend/app/main/routes_run_checks.py b/containers/backupchecks/src/backend/app/main/routes_run_checks.py index 7227e5e..32e908f 100644 --- a/containers/backupchecks/src/backend/app/main/routes_run_checks.py +++ b/containers/backupchecks/src/backend/app/main/routes_run_checks.py @@ -1,6 +1,7 @@ from __future__ import annotations import calendar +import re import threading from datetime import date, datetime, time, timedelta, timezone @@ -1482,10 +1483,36 @@ def run_checks_details(): has_eml = False body_html = "" cloud_connect_summary = None + cove_summary = None + + # For Cove API runs, suppress the mail section entirely and show + # structured Cove account details instead. + if getattr(run, "source_type", None) == "cove_api": + from ..models import CoveAccount + from ..cove_importer import DATASOURCE_LABELS as _COVE_DS_LABELS + _cove_acc = CoveAccount.query.filter_by(job_id=job.id).first() + if _cove_acc: + # Translate raw datasource code string (e.g. "D01D19") to labels + _raw_ds = (_cove_acc.datasource_types or "").strip().upper() + _ds_codes = re.findall(r"D\d{1,2}", _raw_ds) if _raw_ds else [] + _ds_labels: list[str] = [] + for _code in _ds_codes: + _lbl = _COVE_DS_LABELS.get(_code, _code) + if _lbl not in _ds_labels: + _ds_labels.append(_lbl) + cove_summary = { + "account_name": _cove_acc.account_name or "", + "computer_name": _cove_acc.computer_name or "", + "customer_name": _cove_acc.customer_name or "", + "datasources": ", ".join(_ds_labels) if _ds_labels else (_raw_ds or ""), + "last_run_at": _format_datetime(_cove_acc.last_run_at) if _cove_acc.last_run_at else "", + "status": run.status or "", + } + # No mail_meta / body_html for Cove runs — Cove has no email # For Cloud Connect runs, suppress the raw report email (it contains all # tenants) and replace it with a structured summary from the staging account. - if getattr(run, "source_type", None) == "cloud_connect": + elif getattr(run, "source_type", None) == "cloud_connect": from ..models import CloudConnectAccount _cc_acc = CloudConnectAccount.query.filter_by(job_id=job.id).first() if _cc_acc: @@ -1644,6 +1671,7 @@ def run_checks_details(): "mail": mail_meta, "body_html": body_html, "cloud_connect_summary": cloud_connect_summary, + "cove_summary": cove_summary, "objects": objects_payload, "autotask_ticket_id": getattr(run, "autotask_ticket_id", None), "autotask_ticket_number": getattr(run, "autotask_ticket_number", None) or "", diff --git a/containers/backupchecks/src/templates/main/job_detail.html b/containers/backupchecks/src/templates/main/job_detail.html index c4035fa..4aa39fc 100644 --- a/containers/backupchecks/src/templates/main/job_detail.html +++ b/containers/backupchecks/src/templates/main/job_detail.html @@ -108,7 +108,7 @@ {% if history_rows %} {% for r in history_rows %} - + {{ r.run_day }} {{ r.run_at|local_datetime }} {% set _s = (r.status or "")|lower %} @@ -273,6 +273,19 @@ + + +
Mail
@@ -688,13 +701,19 @@ function renderObjects(objects) { rows.forEach(function (row) { row.addEventListener("click", function () { - var messageId = row.getAttribute("data-message-id"); - var runId = row.getAttribute("data-run-id"); - if (!messageId) return; + var messageId = row.getAttribute("data-message-id"); + var runId = row.getAttribute("data-run-id"); + var sourceType = row.getAttribute("data-source-type") || ""; + if (!messageId && !runId) return; currentRunId = runId ? parseInt(runId, 10) : null; - var detailUrl = "{{ url_for('main.inbox_message_detail', message_id=0) }}".replace("0", messageId); - if (runId) detailUrl += "?run_id=" + encodeURIComponent(runId); + var detailUrl; + if (sourceType === "cove_api" && runId && !messageId) { + detailUrl = "{{ url_for('main.cove_run_detail', run_id=0) }}".replace("0", runId); + } else { + detailUrl = "{{ url_for('main.inbox_message_detail', message_id=0) }}".replace("0", messageId); + if (runId) detailUrl += "?run_id=" + encodeURIComponent(runId); + } fetch(detailUrl) .then(function (resp) { if (!resp.ok) throw new Error("Failed to load message details"); @@ -739,7 +758,11 @@ function renderObjects(objects) { var overallEl = document.getElementById("run_msg_overall"); if (overallEl) { // For CC runs use the run's own status; for regular runs use the mail's overall_status. - var overallStatus = (data.cloud_connect_summary ? (data.cloud_connect_summary.status || "") : (meta.overall_status || "")); + var overallStatus = ( + data.cove_summary ? (data.cove_summary.status || "") : + data.cloud_connect_summary ? (data.cloud_connect_summary.status || "") : + (meta.overall_status || "") + ); var d = statusDotClass(overallStatus); overallEl.innerHTML = (d ? ('') : '') + escapeHtml(overallStatus); } @@ -759,13 +782,27 @@ function renderObjects(objects) { } } - var ccPanel = document.getElementById("jdm_cc_summary_panel"); + var ccPanel = document.getElementById("jdm_cc_summary_panel"); + var covePanel = document.getElementById("jdm_cove_summary_panel"); var mailHeading = document.getElementById("jdm_mail_heading"); var mailToggle = document.getElementById("jdm_mail_toggle"); var mailBody = document.getElementById("jdm_mail_iframe_body"); var bodyFrame = document.getElementById("run_msg_body_container_iframe"); - if (data.cloud_connect_summary) { + if (data.cove_summary) { + var cs = data.cove_summary; + document.getElementById("jdm_cove_account").textContent = cs.account_name || "—"; + document.getElementById("jdm_cove_computer").textContent = cs.computer_name || "—"; + document.getElementById("jdm_cove_customer").textContent = cs.customer_name || "—"; + document.getElementById("jdm_cove_datasources").textContent = cs.datasources || "—"; + document.getElementById("jdm_cove_last_run").textContent = cs.last_run_at || "—"; + document.getElementById("jdm_cove_status").textContent = cs.status || "—"; + if (covePanel) covePanel.style.display = ""; + if (ccPanel) ccPanel.style.display = "none"; + if (mailHeading) mailHeading.style.display = "none"; + if (mailToggle) mailToggle.style.display = "none"; + if (mailBody) mailBody.style.display = "none"; + } else if (data.cloud_connect_summary) { var s = data.cloud_connect_summary; document.getElementById("jdm_cc_user").textContent = s.user || ""; document.getElementById("jdm_cc_section").textContent = s.section || ""; @@ -774,14 +811,16 @@ function renderObjects(objects) { document.getElementById("jdm_cc_free").textContent = s.free_space || "—"; document.getElementById("jdm_cc_last_active").textContent = s.last_active || "—"; document.getElementById("jdm_cc_status").textContent = s.status || "—"; + if (covePanel) covePanel.style.display = "none"; if (ccPanel) ccPanel.style.display = ""; - if (mailHeading) mailHeading.textContent = "Source report email"; + if (mailHeading) { mailHeading.style.display = ""; mailHeading.textContent = "Source report email"; } if (mailToggle) { mailToggle.style.display = ""; mailToggle.textContent = "show"; } if (mailBody) mailBody.style.display = "none"; if (bodyFrame) bodyFrame.srcdoc = wrapMailHtml(data.body_html || ""); } else { + if (covePanel) covePanel.style.display = "none"; if (ccPanel) ccPanel.style.display = "none"; - if (mailHeading) mailHeading.textContent = "Mail"; + if (mailHeading) { mailHeading.style.display = ""; mailHeading.textContent = "Mail"; } if (mailToggle) mailToggle.style.display = "none"; if (mailBody) mailBody.style.display = ""; if (bodyFrame) bodyFrame.srcdoc = wrapMailHtml(data.body_html || ""); diff --git a/containers/backupchecks/src/templates/main/run_checks.html b/containers/backupchecks/src/templates/main/run_checks.html index 1b44d11..3876893 100644 --- a/containers/backupchecks/src/templates/main/run_checks.html +++ b/containers/backupchecks/src/templates/main/run_checks.html @@ -331,6 +331,19 @@
+ + +
Mail
@@ -1447,12 +1460,26 @@ table.addEventListener('change', function (e) { } var ccPanel = document.getElementById('rcm_cc_summary_panel'); + var covePanel = document.getElementById('rcm_cove_summary_panel'); var mailHeading = document.getElementById('rcm_mail_heading'); var mailToggle = document.getElementById('rcm_mail_toggle'); var mailBody = document.getElementById('rcm_mail_iframe_body'); var bodyFrame = document.getElementById('rcm_body_iframe'); - if (run.cloud_connect_summary) { + if (run.cove_summary) { + var cs = run.cove_summary; + document.getElementById('rcm_cove_account').textContent = cs.account_name || '—'; + document.getElementById('rcm_cove_computer').textContent = cs.computer_name || '—'; + document.getElementById('rcm_cove_customer').textContent = cs.customer_name || '—'; + document.getElementById('rcm_cove_datasources').textContent = cs.datasources || '—'; + document.getElementById('rcm_cove_last_run').textContent = cs.last_run_at || '—'; + document.getElementById('rcm_cove_status').textContent = cs.status || '—'; + if (covePanel) covePanel.style.display = ''; + if (ccPanel) ccPanel.style.display = 'none'; + if (mailHeading) mailHeading.style.display = 'none'; + if (mailToggle) mailToggle.style.display = 'none'; + if (mailBody) mailBody.style.display = 'none'; + } else if (run.cloud_connect_summary) { var s = run.cloud_connect_summary; document.getElementById('rcc_user').textContent = s.user || ''; document.getElementById('rcc_section').textContent = s.section || ''; @@ -1461,17 +1488,18 @@ table.addEventListener('change', function (e) { document.getElementById('rcc_free').textContent = s.free_space || '—'; document.getElementById('rcc_last_active').textContent = s.last_active || '—'; document.getElementById('rcc_status').textContent = s.status || '—'; - if (ccPanel) ccPanel.style.display = ''; - if (mailHeading) mailHeading.textContent = 'Source report email'; - if (mailToggle) mailToggle.style.display = ''; - if (mailToggle) mailToggle.textContent = 'show'; + if (covePanel) covePanel.style.display = 'none'; + if (ccPanel) ccPanel.style.display = ''; + if (mailHeading) { mailHeading.style.display = ''; mailHeading.textContent = 'Source report email'; } + if (mailToggle) { mailToggle.style.display = ''; mailToggle.textContent = 'show'; } if (mailBody) mailBody.style.display = 'none'; if (bodyFrame) bodyFrame.srcdoc = wrapMailHtml(run.body_html || ''); } else { - if (ccPanel) ccPanel.style.display = 'none'; - if (mailHeading) mailHeading.textContent = 'Mail'; - if (mailToggle) mailToggle.style.display = 'none'; - if (mailBody) mailBody.style.display = ''; + if (covePanel) covePanel.style.display = 'none'; + if (ccPanel) ccPanel.style.display = 'none'; + if (mailHeading) { mailHeading.style.display = ''; mailHeading.textContent = 'Mail'; } + if (mailToggle) mailToggle.style.display = 'none'; + if (mailBody) mailBody.style.display = ''; if (bodyFrame) { bodyFrame.srcdoc = wrapMailHtml(run.body_html || (run.missed ? '
No email for missed run.
' : '')); } diff --git a/docs/changelog-claude.md b/docs/changelog-claude.md index 6ad3dc7..6364f1f 100644 --- a/docs/changelog-claude.md +++ b/docs/changelog-claude.md @@ -2,6 +2,25 @@ This file documents all changes made to this project via Claude Code. +## [2026-03-20] (3) + +### Added +- Cove importer: historical run backfill from 28-day colorbar (`D09F08`): + - When a new run is created for a linked job, `_backfill_colorbar_runs()` reconstructs up to 27 days of history from the colorbar field + - Each non-zero colorbar position creates a `JobRun` with `source_type="cove_api"` and `external_id="cove-colorbar-{account_id}-{date}"` + - Uses same time-of-day as the real run for approximate `run_at` timestamps + - Fully idempotent: `external_id` deduplication prevents duplicates on subsequent imports + - Resolves the issue where only the most-recent session was visible after first linking an account + +- Cove run details popup in job detail page: + - Cove run rows in job detail history table are now clickable (even without a mail message) + - New endpoint `GET /cove/run//detail` returns structured Cove account info and per-datasource objects + - Popup shows: account name, computer, customer (Cove), readable datasource labels, last run, status + - Mail section is hidden entirely for Cove runs (no email involved) + - `routes_jobs.py`: `source_type` added to `history_rows` dict so JS can detect Cove runs + - `job_detail.html`: rows with `source_type=cove_api` get `data-source-type` attribute and are made clickable; JS routes to `/cove/run//detail` instead of `inbox_message_detail` + - Run Checks popup (`routes_run_checks.py`): `cove_summary` added to run payload for `source_type=cove_api` runs with same structured details; mail section hidden for Cove runs + ## [2026-03-20] (2) ### Changed diff --git a/docs/technical-notes-codex.md b/docs/technical-notes-codex.md index 1e86d8d..c1c7e36 100644 --- a/docs/technical-notes-codex.md +++ b/docs/technical-notes-codex.md @@ -241,6 +241,17 @@ Deduplication is enforced per linked job: - check `JobRun.query.filter_by(job_id=job.id, external_id=external_id).first()` - this prevents cross-job collisions when accounts are relinked. +Historical (colorbar) runs use `external_id = f"cove-colorbar-{account_id}-{date_str}"` (e.g. `cove-colorbar-4378343-2026-03-15`). + +### Historical Backfill (28-day colorbar) +When a new run is created for a job, `_backfill_colorbar_runs()` is called to reconstruct up to 27 additional days of history from the `D09F08` colorbar field. + +- Each character in the colorbar = one day's status (oldest first, position 0 = 27 days ago, last position = today) +- Status 0 = no backup that day → skipped +- `run_at` = same time-of-day as the real run, but on the historical date +- Idempotent: `external_id` deduplication prevents duplicates on subsequent imports +- Only creates runs for days where a backup actually ran (non-zero status code) + ### Run Enrichment - Cove-created `JobRun.remark` contains account/computer/customer and last status/timestamp summary. - Per-datasource run object records include: