Auto-commit local changes before build (2026-03-20 09:56:22)

containers/backupchecks/src/backend/app/cloud_connect_importer.py

@@ -26,7 +26,7 @@ from datetime import datetime, timedelta
 from typing import Optional
 
 from .database import db
-from .models import CloudConnectAccount, Customer, Job, JobRun
+from .models import CloudConnectAccount, Customer, Job, JobRun, MailMessage
 
 logger = logging.getLogger(__name__)
 
@@ -214,6 +214,13 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
         return {"total": 0, "linked": 0, "unlinked": 0, "created": 0, "skipped": 0}
 
     now = datetime.utcnow()
+
+    # Use the mail's received_at as the report date so that re-importing
+    # historical emails creates runs on the correct calendar day, not today.
+    _mail_msg = MailMessage.query.get(mail_message_id)
+    report_dt   = (_mail_msg.received_at if _mail_msg and _mail_msg.received_at else now)
+    report_date = report_dt.date().isoformat()
+
     counters = {"total": len(rows), "linked": 0, "unlinked": 0, "created": 0, "skipped": 0}
 
     for row in rows:
@@ -248,21 +255,19 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
             counters["unlinked"] += 1
             continue
 
-        # Account is linked — create a JobRun if not already present for today.
+        # Account is linked — create a JobRun if not already present for this report date.
         job = Job.query.get(acc.job_id)
         if not job:
             counters["skipped"] += 1
             continue
 
-        # Deduplicate: one run per job per calendar day (report is daily).
+        # Deduplicate: one run per job per report date.
-        run_date = now.date().isoformat()
+        external_id = f"vcc-{user}-{section}-{report_date}".lower().replace(" ", "_")
-        external_id = f"vcc-{user}-{section}-{run_date}".lower().replace(" ", "_")
 
         existing = JobRun.query.filter_by(job_id=job.id, external_id=external_id).first()
         if existing:
             # Update status in case re-import happens same day with different result.
             existing.status = row["status"]
-            existing.run_at = now
             db.session.add(existing)
             counters["skipped"] += 1
             counters["linked"] += 1
@@ -273,7 +278,7 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
         run = JobRun(
             job_id=job.id,
             mail_message_id=mail_message_id,
-            run_at=now,
+            run_at=report_dt,
             status=row["status"],
             remark=error_message or None,
             missed=False,

containers/backupchecks/src/backend/app/main/routes_cloud_connect.py

@@ -103,6 +103,37 @@ def cloud_connect_scan_inbox():
     return redirect(url_for("main.cloud_connect_accounts"))
 
 
+def _import_historical_runs(acc) -> int:
+    """Re-process all stored Cloud Connect report emails for a newly linked account.
+
+    Returns the total number of new JobRun records created.
+    """
+    mails = (
+        MailMessage.query
+        .filter(
+            db.func.lower(MailMessage.backup_type) == "cloud connect report",
+            MailMessage.location != "deleted",
+            MailMessage.html_body.isnot(None),
+        )
+        .order_by(MailMessage.received_at.asc())
+        .all()
+    )
+    runs_created = 0
+    for mail in mails:
+        try:
+            result = upsert_cloud_connect_report(
+                mail_message_id=mail.id,
+                html_body=mail.html_body or "",
+            )
+            runs_created += result.get("created", 0)
+        except Exception as exc:
+            _log_admin_event(
+                event_type="cloud_connect_historical_import_error",
+                message=f"Failed to re-process mail {mail.id}: {exc}",
+            )
+    return runs_created
+
+
 @main_bp.route("/cloud-connect/accounts/<int:cc_account_db_id>/link", methods=["POST"])
 @login_required
 @roles_required("admin", "operator")
@@ -133,12 +164,18 @@ def cloud_connect_account_link(cc_account_db_id: int):
         acc.job_id = job.id
         db.session.commit()
 
+        runs_created = _import_historical_runs(acc)
+
         _log_admin_event(
             event_type="cloud_connect_account_linked",
-            message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to new job '{job_name}'",
+            message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to new job '{job_name}' ({runs_created} historical run(s) created)",
             details=f"customer={customer.name}, job_name={job_name}",
         )
-        flash(f"Job '{job_name}' created and linked to '{acc.user}' ({acc.section}).", "success")
+        flash(
+            f"Job '{job_name}' created and linked to '{acc.user}' ({acc.section}). "
+            f"{runs_created} historical run(s) imported.",
+            "success",
+        )
 
     elif action == "link":
         job_id = request.form.get("job_id", type=int)
@@ -150,12 +187,18 @@ def cloud_connect_account_link(cc_account_db_id: int):
         acc.job_id = job.id
         db.session.commit()
 
+        runs_created = _import_historical_runs(acc)
+
         _log_admin_event(
             event_type="cloud_connect_account_linked",
-            message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to existing job '{job.job_name}'",
+            message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to existing job '{job.job_name}' ({runs_created} historical run(s) created)",
             details=f"job_id={job.id}, job_name={job.job_name}",
         )
-        flash(f"Linked '{acc.user}' ({acc.section}) to job '{job.job_name}'.", "success")
+        flash(
+            f"Linked '{acc.user}' ({acc.section}) to job '{job.job_name}'. "
+            f"{runs_created} historical run(s) imported.",
+            "success",
+        )
 
     else:
         flash("Unknown action.", "danger")

containers/backupchecks/src/backend/app/main/routes_run_checks.py

@@ -1475,12 +1475,40 @@ def run_checks_details():
         mail_meta = None
         has_eml = False
         body_html = ""
-        if msg:
+        cloud_connect_summary = None
+
+        # For Cloud Connect runs, suppress the raw report email (it contains all
+        # tenants) and replace it with a structured summary from the staging account.
+        if getattr(run, "source_type", None) == "cloud_connect":
+            from ..models import CloudConnectAccount
+            _cc_acc = CloudConnectAccount.query.filter_by(job_id=job.id).first()
+            if _cc_acc:
+                cloud_connect_summary = {
+                    "user":        _cc_acc.user or "",
+                    "section":     _cc_acc.section or "",
+                    "repo_name":   _cc_acc.repo_name or "",
+                    "repo_type":   _cc_acc.repo_type or "",
+                    "used_space":  _cc_acc.used_space or "",
+                    "total_quota": _cc_acc.total_quota or "",
+                    "free_space":  _cc_acc.free_space or "",
+                    "last_active": _cc_acc.last_active_raw or "",
+                    "status":      _cc_acc.last_status or "",
+                }
+            # Keep mail meta and EML link for audit trail; skip body HTML.
+            if msg:
+                mail_meta = {
+                    "from_address": msg.from_address or "",
+                    "subject":      msg.subject or "",
+                    "received_at":  _format_datetime(msg.received_at),
+                }
+                has_eml = bool(getattr(msg, "eml_stored_at", None))
+        elif msg:
             mail_meta = {
                 "from_address": msg.from_address or "",
                 "subject": msg.subject or "",
                 "received_at": _format_datetime(msg.received_at),
             }
+        if msg and cloud_connect_summary is None:
             def _is_blank_text(s):
                 return s is None or (isinstance(s, str) and s.strip() == "")
 
@@ -1608,6 +1636,7 @@ def run_checks_details():
                 "has_eml": bool(has_eml),
                 "mail": mail_meta,
                 "body_html": body_html,
+                "cloud_connect_summary": cloud_connect_summary,
                 "objects": objects_payload,
                 "autotask_ticket_id": getattr(run, "autotask_ticket_id", None),
                 "autotask_ticket_number": getattr(run, "autotask_ticket_number", None) or "",

containers/backupchecks/src/templates/main/run_checks.html

@@ -318,7 +318,20 @@
                 </dd>
               </dl>
 
-              <div class="mb-3 rcm-mail-panel">
+              <div class="mb-3 rcm-mail-panel" id="rcm_cc_summary_panel" style="display:none;">
+                <h6>Cloud Connect</h6>
+                <dl class="row mb-0 dl-compact" id="rcm_cc_summary_dl">
+                  <dt class="col-4">User</dt>      <dd class="col-8" id="rcc_user"></dd>
+                  <dt class="col-4">Section</dt>   <dd class="col-8" id="rcc_section"></dd>
+                  <dt class="col-4">Repository</dt><dd class="col-8" id="rcc_repo"></dd>
+                  <dt class="col-4">Used / Quota</dt><dd class="col-8" id="rcc_used"></dd>
+                  <dt class="col-4">Free</dt>      <dd class="col-8" id="rcc_free"></dd>
+                  <dt class="col-4">Last active</dt><dd class="col-8" id="rcc_last_active"></dd>
+                  <dt class="col-4">Status</dt>    <dd class="col-8" id="rcc_status"></dd>
+                </dl>
+              </div>
+
+              <div class="mb-3 rcm-mail-panel" id="rcm_mail_iframe_panel">
                 <h6>Mail</h6>
                 <iframe
                   id="rcm_body_iframe"
@@ -1421,9 +1434,27 @@ table.addEventListener('change', function (e) {
         document.getElementById('rcm_received').textContent = '';
       }
 
+      var ccPanel = document.getElementById('rcm_cc_summary_panel');
+      var iframePanel = document.getElementById('rcm_mail_iframe_panel');
       var bodyFrame = document.getElementById('rcm_body_iframe');
-      if (bodyFrame) {
+
-        bodyFrame.srcdoc = wrapMailHtml(run.body_html || (run.missed ? '<div class="text-muted">No email for missed run.</div>' : ''));
+      if (run.cloud_connect_summary) {
+        var s = run.cloud_connect_summary;
+        document.getElementById('rcc_user').textContent        = s.user || '';
+        document.getElementById('rcc_section').textContent     = s.section || '';
+        document.getElementById('rcc_repo').textContent        = s.repo_name + (s.repo_type ? ' (' + s.repo_type + ')' : '');
+        document.getElementById('rcc_used').textContent        = (s.used_space || '—') + ' / ' + (s.total_quota || '—');
+        document.getElementById('rcc_free').textContent        = s.free_space || '—';
+        document.getElementById('rcc_last_active').textContent = s.last_active || '—';
+        document.getElementById('rcc_status').textContent      = s.status || '—';
+        if (ccPanel)     ccPanel.style.display     = '';
+        if (iframePanel) iframePanel.style.display = 'none';
+      } else {
+        if (ccPanel)     ccPanel.style.display     = 'none';
+        if (iframePanel) iframePanel.style.display = '';
+        if (bodyFrame) {
+          bodyFrame.srcdoc = wrapMailHtml(run.body_html || (run.missed ? '<div class="text-muted">No email for missed run.</div>' : ''));
+        }
       }
 
       var emlBtn = document.getElementById('rcm_eml_btn');

docs/changelog-claude.md

@@ -4,6 +4,13 @@ This file documents all changes made to this project via Claude Code.
 
 ## [2026-03-20]
 
+### Fixed
+- "Delete all jobs" in Settings → Maintenance no longer times out on large datasets:
+  - Replaced ORM-based deletion (loaded all jobs/runs into Python memory, deleted object by object) with direct SQL `DELETE FROM` statements in FK order — handles 650K+ rows in seconds
+  - Added `job_run_review_events` to the FK cleanup sequence (was causing a FK violation)
+  - Added `cove_accounts` and `cloud_connect_accounts` unlinking before job deletion
+  - Gunicorn worker timeout raised from default 30 s to 120 s (`Dockerfile`)
+
 ### Changed
 - Cloud Connect Accounts page: replaced per-row "Link / Create Job" button and inline modals with clickable rows and a single shared modal (mirrors Inbox UX):
   - Clicking an unmatched row opens a modal pre-filled with the account's user and section

docs/technical-notes-codex.md

@@ -85,12 +85,15 @@ File: `containers/backupchecks/src/backend/app/models.py`
   - `FeedbackItem`, `FeedbackVote`, `FeedbackReply`, `FeedbackAttachment`
 
 ### Foreign Key Relationships & Deletion Order
-Critical deletion order to avoid constraint violations:
+Critical deletion order to avoid constraint violations (used in "Delete all jobs" maintenance route):
-1. Clean auxiliary tables (ticket_job_runs, remark_job_runs, scopes, overrides)
+1. Unlink staging accounts: `UPDATE cove_accounts SET job_id = NULL`, `UPDATE cloud_connect_accounts SET job_id = NULL`
-2. Unlink mails from jobs (UPDATE mail_messages SET job_id = NULL)
+2. Unlink mails: `UPDATE mail_messages SET job_id = NULL, location = 'inbox'`
-3. Delete mail_objects
+3. Delete FK tables referencing `job_runs`: `remark_job_runs`, `ticket_job_runs`, `run_object_links`, `job_run_review_events`
-4. Delete jobs (cascades to job_runs)
+4. Delete FK tables referencing `jobs`: `job_object_links`, `ticket_scopes`, `remark_scopes`, `overrides`
-5. Delete mails
+5. `DELETE FROM job_runs`
+6. `DELETE FROM jobs`
+
+Note: always use direct SQL (`DELETE FROM`) for bulk deletions — ORM-level deletes load all objects into Python memory and time out on large datasets.
 
 ### Key Model Fields
 **MailMessage model:**