From 461c46b0ff5febda5ccbbdbad809e2559fdcaa14 Mon Sep 17 00:00:00 2001
From: Ivo Oskamp <ivooskamp@oskamp.info>
Date: Fri, 20 Mar 2026 09:56:22 +0100
Subject: [PATCH] Auto-commit local changes before build (2026-03-20 09:56:22)

---
 .../src/backend/app/cloud_connect_importer.py | 19 ++++---
 .../backend/app/main/routes_cloud_connect.py  | 51 +++++++++++++++++--
 .../src/backend/app/main/routes_run_checks.py | 31 ++++++++++-
 .../src/templates/main/run_checks.html        | 37 ++++++++++++--
 docs/changelog-claude.md                      |  7 +++
 docs/technical-notes-codex.md                 | 15 +++---
 6 files changed, 139 insertions(+), 21 deletions(-)

diff --git a/containers/backupchecks/src/backend/app/cloud_connect_importer.py b/containers/backupchecks/src/backend/app/cloud_connect_importer.py
index 2eeed0e..415ccda 100644
--- a/containers/backupchecks/src/backend/app/cloud_connect_importer.py
+++ b/containers/backupchecks/src/backend/app/cloud_connect_importer.py
@@ -26,7 +26,7 @@ from datetime import datetime, timedelta
 from typing import Optional
 
 from .database import db
-from .models import CloudConnectAccount, Customer, Job, JobRun
+from .models import CloudConnectAccount, Customer, Job, JobRun, MailMessage
 
 logger = logging.getLogger(__name__)
 
@@ -214,6 +214,13 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
         return {"total": 0, "linked": 0, "unlinked": 0, "created": 0, "skipped": 0}
 
     now = datetime.utcnow()
+
+    # Use the mail's received_at as the report date so that re-importing
+    # historical emails creates runs on the correct calendar day, not today.
+    _mail_msg = MailMessage.query.get(mail_message_id)
+    report_dt   = (_mail_msg.received_at if _mail_msg and _mail_msg.received_at else now)
+    report_date = report_dt.date().isoformat()
+
     counters = {"total": len(rows), "linked": 0, "unlinked": 0, "created": 0, "skipped": 0}
 
     for row in rows:
@@ -248,21 +255,19 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
             counters["unlinked"] += 1
             continue
 
-        # Account is linked — create a JobRun if not already present for today.
+        # Account is linked — create a JobRun if not already present for this report date.
         job = Job.query.get(acc.job_id)
         if not job:
             counters["skipped"] += 1
             continue
 
-        # Deduplicate: one run per job per calendar day (report is daily).
-        run_date = now.date().isoformat()
-        external_id = f"vcc-{user}-{section}-{run_date}".lower().replace(" ", "_")
+        # Deduplicate: one run per job per report date.
+        external_id = f"vcc-{user}-{section}-{report_date}".lower().replace(" ", "_")
 
         existing = JobRun.query.filter_by(job_id=job.id, external_id=external_id).first()
         if existing:
             # Update status in case re-import happens same day with different result.
             existing.status = row["status"]
-            existing.run_at = now
             db.session.add(existing)
             counters["skipped"] += 1
             counters["linked"] += 1
@@ -273,7 +278,7 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
         run = JobRun(
             job_id=job.id,
             mail_message_id=mail_message_id,
-            run_at=now,
+            run_at=report_dt,
             status=row["status"],
             remark=error_message or None,
             missed=False,
diff --git a/containers/backupchecks/src/backend/app/main/routes_cloud_connect.py b/containers/backupchecks/src/backend/app/main/routes_cloud_connect.py
index 0beaeb4..f4bb517 100644
--- a/containers/backupchecks/src/backend/app/main/routes_cloud_connect.py
+++ b/containers/backupchecks/src/backend/app/main/routes_cloud_connect.py
@@ -103,6 +103,37 @@ def cloud_connect_scan_inbox():
     return redirect(url_for("main.cloud_connect_accounts"))
 
 
+def _import_historical_runs(acc) -> int:
+    """Re-process all stored Cloud Connect report emails for a newly linked account.
+
+    Returns the total number of new JobRun records created.
+    """
+    mails = (
+        MailMessage.query
+        .filter(
+            db.func.lower(MailMessage.backup_type) == "cloud connect report",
+            MailMessage.location != "deleted",
+            MailMessage.html_body.isnot(None),
+        )
+        .order_by(MailMessage.received_at.asc())
+        .all()
+    )
+    runs_created = 0
+    for mail in mails:
+        try:
+            result = upsert_cloud_connect_report(
+                mail_message_id=mail.id,
+                html_body=mail.html_body or "",
+            )
+            runs_created += result.get("created", 0)
+        except Exception as exc:
+            _log_admin_event(
+                event_type="cloud_connect_historical_import_error",
+                message=f"Failed to re-process mail {mail.id}: {exc}",
+            )
+    return runs_created
+
+
 @main_bp.route("/cloud-connect/accounts/<int:cc_account_db_id>/link", methods=["POST"])
 @login_required
 @roles_required("admin", "operator")
@@ -133,12 +164,18 @@ def cloud_connect_account_link(cc_account_db_id: int):
         acc.job_id = job.id
         db.session.commit()
 
+        runs_created = _import_historical_runs(acc)
+
         _log_admin_event(
             event_type="cloud_connect_account_linked",
-            message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to new job '{job_name}'",
+            message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to new job '{job_name}' ({runs_created} historical run(s) created)",
             details=f"customer={customer.name}, job_name={job_name}",
         )
-        flash(f"Job '{job_name}' created and linked to '{acc.user}' ({acc.section}).", "success")
+        flash(
+            f"Job '{job_name}' created and linked to '{acc.user}' ({acc.section}). "
+            f"{runs_created} historical run(s) imported.",
+            "success",
+        )
 
     elif action == "link":
         job_id = request.form.get("job_id", type=int)
@@ -150,12 +187,18 @@ def cloud_connect_account_link(cc_account_db_id: int):
         acc.job_id = job.id
         db.session.commit()
 
+        runs_created = _import_historical_runs(acc)
+
         _log_admin_event(
             event_type="cloud_connect_account_linked",
-            message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to existing job '{job.job_name}'",
+            message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to existing job '{job.job_name}' ({runs_created} historical run(s) created)",
             details=f"job_id={job.id}, job_name={job.job_name}",
         )
-        flash(f"Linked '{acc.user}' ({acc.section}) to job '{job.job_name}'.", "success")
+        flash(
+            f"Linked '{acc.user}' ({acc.section}) to job '{job.job_name}'. "
+            f"{runs_created} historical run(s) imported.",
+            "success",
+        )
 
     else:
         flash("Unknown action.", "danger")
diff --git a/containers/backupchecks/src/backend/app/main/routes_run_checks.py b/containers/backupchecks/src/backend/app/main/routes_run_checks.py
index e94b4a9..e66a07a 100644
--- a/containers/backupchecks/src/backend/app/main/routes_run_checks.py
+++ b/containers/backupchecks/src/backend/app/main/routes_run_checks.py
@@ -1475,12 +1475,40 @@ def run_checks_details():
         mail_meta = None
         has_eml = False
         body_html = ""
-        if msg:
+        cloud_connect_summary = None
+
+        # For Cloud Connect runs, suppress the raw report email (it contains all
+        # tenants) and replace it with a structured summary from the staging account.
+        if getattr(run, "source_type", None) == "cloud_connect":
+            from ..models import CloudConnectAccount
+            _cc_acc = CloudConnectAccount.query.filter_by(job_id=job.id).first()
+            if _cc_acc:
+                cloud_connect_summary = {
+                    "user":        _cc_acc.user or "",
+                    "section":     _cc_acc.section or "",
+                    "repo_name":   _cc_acc.repo_name or "",
+                    "repo_type":   _cc_acc.repo_type or "",
+                    "used_space":  _cc_acc.used_space or "",
+                    "total_quota": _cc_acc.total_quota or "",
+                    "free_space":  _cc_acc.free_space or "",
+                    "last_active": _cc_acc.last_active_raw or "",
+                    "status":      _cc_acc.last_status or "",
+                }
+            # Keep mail meta and EML link for audit trail; skip body HTML.
+            if msg:
+                mail_meta = {
+                    "from_address": msg.from_address or "",
+                    "subject":      msg.subject or "",
+                    "received_at":  _format_datetime(msg.received_at),
+                }
+                has_eml = bool(getattr(msg, "eml_stored_at", None))
+        elif msg:
             mail_meta = {
                 "from_address": msg.from_address or "",
                 "subject": msg.subject or "",
                 "received_at": _format_datetime(msg.received_at),
             }
+        if msg and cloud_connect_summary is None:
             def _is_blank_text(s):
                 return s is None or (isinstance(s, str) and s.strip() == "")
 
@@ -1608,6 +1636,7 @@ def run_checks_details():
                 "has_eml": bool(has_eml),
                 "mail": mail_meta,
                 "body_html": body_html,
+                "cloud_connect_summary": cloud_connect_summary,
                 "objects": objects_payload,
                 "autotask_ticket_id": getattr(run, "autotask_ticket_id", None),
                 "autotask_ticket_number": getattr(run, "autotask_ticket_number", None) or "",
diff --git a/containers/backupchecks/src/templates/main/run_checks.html b/containers/backupchecks/src/templates/main/run_checks.html
index 2cfc0bc..11020b1 100644
--- a/containers/backupchecks/src/templates/main/run_checks.html
+++ b/containers/backupchecks/src/templates/main/run_checks.html
@@ -318,7 +318,20 @@
                 </dd>
               </dl>
 
-              <div class="mb-3 rcm-mail-panel">
+              <div class="mb-3 rcm-mail-panel" id="rcm_cc_summary_panel" style="display:none;">
+                <h6>Cloud Connect</h6>
+                <dl class="row mb-0 dl-compact" id="rcm_cc_summary_dl">
+                  <dt class="col-4">User</dt>      <dd class="col-8" id="rcc_user"></dd>
+                  <dt class="col-4">Section</dt>   <dd class="col-8" id="rcc_section"></dd>
+                  <dt class="col-4">Repository</dt><dd class="col-8" id="rcc_repo"></dd>
+                  <dt class="col-4">Used / Quota</dt><dd class="col-8" id="rcc_used"></dd>
+                  <dt class="col-4">Free</dt>      <dd class="col-8" id="rcc_free"></dd>
+                  <dt class="col-4">Last active</dt><dd class="col-8" id="rcc_last_active"></dd>
+                  <dt class="col-4">Status</dt>    <dd class="col-8" id="rcc_status"></dd>
+                </dl>
+              </div>
+
+              <div class="mb-3 rcm-mail-panel" id="rcm_mail_iframe_panel">
                 <h6>Mail</h6>
                 <iframe
                   id="rcm_body_iframe"
@@ -1421,9 +1434,27 @@ table.addEventListener('change', function (e) {
         document.getElementById('rcm_received').textContent = '';
       }
 
+      var ccPanel = document.getElementById('rcm_cc_summary_panel');
+      var iframePanel = document.getElementById('rcm_mail_iframe_panel');
       var bodyFrame = document.getElementById('rcm_body_iframe');
-      if (bodyFrame) {
-        bodyFrame.srcdoc = wrapMailHtml(run.body_html || (run.missed ? '<div class="text-muted">No email for missed run.</div>' : ''));
+
+      if (run.cloud_connect_summary) {
+        var s = run.cloud_connect_summary;
+        document.getElementById('rcc_user').textContent        = s.user || '';
+        document.getElementById('rcc_section').textContent     = s.section || '';
+        document.getElementById('rcc_repo').textContent        = s.repo_name + (s.repo_type ? ' (' + s.repo_type + ')' : '');
+        document.getElementById('rcc_used').textContent        = (s.used_space || '—') + ' / ' + (s.total_quota || '—');
+        document.getElementById('rcc_free').textContent        = s.free_space || '—';
+        document.getElementById('rcc_last_active').textContent = s.last_active || '—';
+        document.getElementById('rcc_status').textContent      = s.status || '—';
+        if (ccPanel)     ccPanel.style.display     = '';
+        if (iframePanel) iframePanel.style.display = 'none';
+      } else {
+        if (ccPanel)     ccPanel.style.display     = 'none';
+        if (iframePanel) iframePanel.style.display = '';
+        if (bodyFrame) {
+          bodyFrame.srcdoc = wrapMailHtml(run.body_html || (run.missed ? '<div class="text-muted">No email for missed run.</div>' : ''));
+        }
       }
 
       var emlBtn = document.getElementById('rcm_eml_btn');
diff --git a/docs/changelog-claude.md b/docs/changelog-claude.md
index a0aae2f..379ef08 100644
--- a/docs/changelog-claude.md
+++ b/docs/changelog-claude.md
@@ -4,6 +4,13 @@ This file documents all changes made to this project via Claude Code.
 
 ## [2026-03-20]
 
+### Fixed
+- "Delete all jobs" in Settings → Maintenance no longer times out on large datasets:
+  - Replaced ORM-based deletion (loaded all jobs/runs into Python memory, deleted object by object) with direct SQL `DELETE FROM` statements in FK order — handles 650K+ rows in seconds
+  - Added `job_run_review_events` to the FK cleanup sequence (was causing a FK violation)
+  - Added `cove_accounts` and `cloud_connect_accounts` unlinking before job deletion
+  - Gunicorn worker timeout raised from default 30 s to 120 s (`Dockerfile`)
+
 ### Changed
 - Cloud Connect Accounts page: replaced per-row "Link / Create Job" button and inline modals with clickable rows and a single shared modal (mirrors Inbox UX):
   - Clicking an unmatched row opens a modal pre-filled with the account's user and section
diff --git a/docs/technical-notes-codex.md b/docs/technical-notes-codex.md
index ae5da3a..e1bae7b 100644
--- a/docs/technical-notes-codex.md
+++ b/docs/technical-notes-codex.md
@@ -85,12 +85,15 @@ File: `containers/backupchecks/src/backend/app/models.py`
   - `FeedbackItem`, `FeedbackVote`, `FeedbackReply`, `FeedbackAttachment`
 
 ### Foreign Key Relationships & Deletion Order
-Critical deletion order to avoid constraint violations:
-1. Clean auxiliary tables (ticket_job_runs, remark_job_runs, scopes, overrides)
-2. Unlink mails from jobs (UPDATE mail_messages SET job_id = NULL)
-3. Delete mail_objects
-4. Delete jobs (cascades to job_runs)
-5. Delete mails
+Critical deletion order to avoid constraint violations (used in "Delete all jobs" maintenance route):
+1. Unlink staging accounts: `UPDATE cove_accounts SET job_id = NULL`, `UPDATE cloud_connect_accounts SET job_id = NULL`
+2. Unlink mails: `UPDATE mail_messages SET job_id = NULL, location = 'inbox'`
+3. Delete FK tables referencing `job_runs`: `remark_job_runs`, `ticket_job_runs`, `run_object_links`, `job_run_review_events`
+4. Delete FK tables referencing `jobs`: `job_object_links`, `ticket_scopes`, `remark_scopes`, `overrides`
+5. `DELETE FROM job_runs`
+6. `DELETE FROM jobs`
+
+Note: always use direct SQL (`DELETE FROM`) for bulk deletions — ORM-level deletes load all objects into Python memory and time out on large datasets.
 
 ### Key Model Fields
 **MailMessage model:**