ivooskamp/backupchecks
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Auto-commit local changes before build (2026-03-20 09:56:22)

Browse Source
This commit is contained in:
Ivo Oskamp 2026-03-20 09:56:22 +01:00
parent 5da3137adc
commit 461c46b0ff
6 changed files with 139 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
containers/backupchecks/src/backend/app/cloud_connect_importer.py
View File

@ -26,7 +26,7 @@ from datetime import datetime, timedelta
from typing import Optional
from .database import db
from .models import CloudConnectAccount, Customer, Job, JobRun
from .models import CloudConnectAccount, Customer, Job, JobRun, MailMessage
logger = logging.getLogger(__name__)
@ -214,6 +214,13 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
return {"total": 0, "linked": 0, "unlinked": 0, "created": 0, "skipped": 0}
now = datetime.utcnow()
# Use the mail's received_at as the report date so that re-importing
# historical emails creates runs on the correct calendar day, not today.
_mail_msg = MailMessage.query.get(mail_message_id)
report_dt = (_mail_msg.received_at if _mail_msg and _mail_msg.received_at else now)
report_date = report_dt.date().isoformat()
counters = {"total": len(rows), "linked": 0, "unlinked": 0, "created": 0, "skipped": 0}
for row in rows:
@ -248,21 +255,19 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
counters["unlinked"] += 1
continue
# Account is linked — create a JobRun if not already present for today.
# Account is linked — create a JobRun if not already present for this report date.
job = Job.query.get(acc.job_id)
if not job:
counters["skipped"] += 1
continue
# Deduplicate: one run per job per calendar day (report is daily).
run_date = now.date().isoformat()
external_id = f"vcc-{user}-{section}-{run_date}".lower().replace(" ", "_")
# Deduplicate: one run per job per report date.
external_id = f"vcc-{user}-{section}-{report_date}".lower().replace(" ", "_")
existing = JobRun.query.filter_by(job_id=job.id, external_id=external_id).first()
if existing:
# Update status in case re-import happens same day with different result.
existing.status = row["status"]
existing.run_at = now
db.session.add(existing)
counters["skipped"] += 1
counters["linked"] += 1
@ -273,7 +278,7 @@ def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
run = JobRun(
job_id=job.id,
mail_message_id=mail_message_id,
run_at=now,
run_at=report_dt,
status=row["status"],
remark=error_message or None,
missed=False,

51
containers/backupchecks/src/backend/app/main/routes_cloud_connect.py
View File

@ -103,6 +103,37 @@ def cloud_connect_scan_inbox():
return redirect(url_for("main.cloud_connect_accounts"))
def _import_historical_runs(acc) -> int:
"""Re-process all stored Cloud Connect report emails for a newly linked account.
Returns the total number of new JobRun records created.
"""
mails = (
MailMessage.query
.filter(
db.func.lower(MailMessage.backup_type) == "cloud connect report",
MailMessage.location != "deleted",
MailMessage.html_body.isnot(None),
)
.order_by(MailMessage.received_at.asc())
.all()
)
runs_created = 0
for mail in mails:
try:
result = upsert_cloud_connect_report(
mail_message_id=mail.id,
html_body=mail.html_body or "",
)
runs_created += result.get("created", 0)
except Exception as exc:
_log_admin_event(
event_type="cloud_connect_historical_import_error",
message=f"Failed to re-process mail {mail.id}: {exc}",
)
return runs_created
@main_bp.route("/cloud-connect/accounts/<int:cc_account_db_id>/link", methods=["POST"])
@login_required
@roles_required("admin", "operator")
@ -133,12 +164,18 @@ def cloud_connect_account_link(cc_account_db_id: int):
acc.job_id = job.id
db.session.commit()
runs_created = _import_historical_runs(acc)
_log_admin_event(
event_type="cloud_connect_account_linked",
message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to new job '{job_name}'",
message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to new job '{job_name}' ({runs_created} historical run(s) created)",
details=f"customer={customer.name}, job_name={job_name}",
)
flash(f"Job '{job_name}' created and linked to '{acc.user}' ({acc.section}).", "success")
flash(
f"Job '{job_name}' created and linked to '{acc.user}' ({acc.section}). "
f"{runs_created} historical run(s) imported.",
"success",
)
elif action == "link":
job_id = request.form.get("job_id", type=int)
@ -150,12 +187,18 @@ def cloud_connect_account_link(cc_account_db_id: int):
acc.job_id = job.id
db.session.commit()
runs_created = _import_historical_runs(acc)
_log_admin_event(
event_type="cloud_connect_account_linked",
message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to existing job '{job.job_name}'",
message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to existing job '{job.job_name}' ({runs_created} historical run(s) created)",
details=f"job_id={job.id}, job_name={job.job_name}",
)
flash(f"Linked '{acc.user}' ({acc.section}) to job '{job.job_name}'.", "success")
flash(
f"Linked '{acc.user}' ({acc.section}) to job '{job.job_name}'. "
f"{runs_created} historical run(s) imported.",
"success",
)
else:
flash("Unknown action.", "danger")

29
containers/backupchecks/src/backend/app/main/routes_run_checks.py
View File

@ -1475,12 +1475,40 @@ def run_checks_details():
mail_meta = None
has_eml = False
body_html = ""
cloud_connect_summary = None
# For Cloud Connect runs, suppress the raw report email (it contains all
# tenants) and replace it with a structured summary from the staging account.
if getattr(run, "source_type", None) == "cloud_connect":
from ..models import CloudConnectAccount
_cc_acc = CloudConnectAccount.query.filter_by(job_id=job.id).first()
if _cc_acc:
cloud_connect_summary = {
"user": _cc_acc.user or "",
"section": _cc_acc.section or "",
"repo_name": _cc_acc.repo_name or "",
"repo_type": _cc_acc.repo_type or "",
"used_space": _cc_acc.used_space or "",
"total_quota": _cc_acc.total_quota or "",
"free_space": _cc_acc.free_space or "",
"last_active": _cc_acc.last_active_raw or "",
"status": _cc_acc.last_status or "",
}
# Keep mail meta and EML link for audit trail; skip body HTML.
if msg:
mail_meta = {
"from_address": msg.from_address or "",
"subject": msg.subject or "",
"received_at": _format_datetime(msg.received_at),
}
has_eml = bool(getattr(msg, "eml_stored_at", None))
elif msg:
mail_meta = {
"from_address": msg.from_address or "",
"subject": msg.subject or "",
"received_at": _format_datetime(msg.received_at),
}
if msg and cloud_connect_summary is None:
def _is_blank_text(s):
return s is None or (isinstance(s, str) and s.strip() == "")
@ -1608,6 +1636,7 @@ def run_checks_details():
"has_eml": bool(has_eml),
"mail": mail_meta,
"body_html": body_html,
"cloud_connect_summary": cloud_connect_summary,
"objects": objects_payload,
"autotask_ticket_id": getattr(run, "autotask_ticket_id", None),
"autotask_ticket_number": getattr(run, "autotask_ticket_number", None) or "",

33
containers/backupchecks/src/templates/main/run_checks.html
View File

@ -318,7 +318,20 @@
</dd>
</dl>
<div class="mb-3 rcm-mail-panel">
<div class="mb-3 rcm-mail-panel" id="rcm_cc_summary_panel" style="display:none;">
<h6>Cloud Connect</h6>
<dl class="row mb-0 dl-compact" id="rcm_cc_summary_dl">
<dt class="col-4">User</dt> <dd class="col-8" id="rcc_user"></dd>
<dt class="col-4">Section</dt> <dd class="col-8" id="rcc_section"></dd>
<dt class="col-4">Repository</dt><dd class="col-8" id="rcc_repo"></dd>
<dt class="col-4">Used / Quota</dt><dd class="col-8" id="rcc_used"></dd>
<dt class="col-4">Free</dt> <dd class="col-8" id="rcc_free"></dd>
<dt class="col-4">Last active</dt><dd class="col-8" id="rcc_last_active"></dd>
<dt class="col-4">Status</dt> <dd class="col-8" id="rcc_status"></dd>
</dl>
</div>
<div class="mb-3 rcm-mail-panel" id="rcm_mail_iframe_panel">
<h6>Mail</h6>
<iframe
id="rcm_body_iframe"
@ -1421,10 +1434,28 @@ table.addEventListener('change', function (e) {
document.getElementById('rcm_received').textContent = '';
}
var ccPanel = document.getElementById('rcm_cc_summary_panel');
var iframePanel = document.getElementById('rcm_mail_iframe_panel');
var bodyFrame = document.getElementById('rcm_body_iframe');
if (run.cloud_connect_summary) {
var s = run.cloud_connect_summary;
document.getElementById('rcc_user').textContent = s.user || '';
document.getElementById('rcc_section').textContent = s.section || '';
document.getElementById('rcc_repo').textContent = s.repo_name + (s.repo_type ? ' (' + s.repo_type + ')' : '');
document.getElementById('rcc_used').textContent = (s.used_space || '—') + ' / ' + (s.total_quota || '—');
document.getElementById('rcc_free').textContent = s.free_space || '—';
document.getElementById('rcc_last_active').textContent = s.last_active || '—';
document.getElementById('rcc_status').textContent = s.status || '—';
if (ccPanel) ccPanel.style.display = '';
if (iframePanel) iframePanel.style.display = 'none';
} else {
if (ccPanel) ccPanel.style.display = 'none';
if (iframePanel) iframePanel.style.display = '';
if (bodyFrame) {
bodyFrame.srcdoc = wrapMailHtml(run.body_html || (run.missed ? '<div class="text-muted">No email for missed run.</div>' : ''));
}
}
var emlBtn = document.getElementById('rcm_eml_btn');
if (emlBtn) {

7
docs/changelog-claude.md
View File

@ -4,6 +4,13 @@ This file documents all changes made to this project via Claude Code.
## [2026-03-20]
### Fixed
- "Delete all jobs" in Settings → Maintenance no longer times out on large datasets:
- Replaced ORM-based deletion (loaded all jobs/runs into Python memory, deleted object by object) with direct SQL `DELETE FROM` statements in FK order — handles 650K+ rows in seconds
- Added `job_run_review_events` to the FK cleanup sequence (was causing a FK violation)
- Added `cove_accounts` and `cloud_connect_accounts` unlinking before job deletion
- Gunicorn worker timeout raised from default 30 s to 120 s (`Dockerfile`)
### Changed
- Cloud Connect Accounts page: replaced per-row "Link / Create Job" button and inline modals with clickable rows and a single shared modal (mirrors Inbox UX):
- Clicking an unmatched row opens a modal pre-filled with the account's user and section

15
docs/technical-notes-codex.md
View File

@ -85,12 +85,15 @@ File: `containers/backupchecks/src/backend/app/models.py`
- `FeedbackItem`, `FeedbackVote`, `FeedbackReply`, `FeedbackAttachment`
### Foreign Key Relationships & Deletion Order
Critical deletion order to avoid constraint violations:
1. Clean auxiliary tables (ticket_job_runs, remark_job_runs, scopes, overrides)
2. Unlink mails from jobs (UPDATE mail_messages SET job_id = NULL)
3. Delete mail_objects
4. Delete jobs (cascades to job_runs)
5. Delete mails
Critical deletion order to avoid constraint violations (used in "Delete all jobs" maintenance route):
1. Unlink staging accounts: `UPDATE cove_accounts SET job_id = NULL`, `UPDATE cloud_connect_accounts SET job_id = NULL`
2. Unlink mails: `UPDATE mail_messages SET job_id = NULL, location = 'inbox'`
3. Delete FK tables referencing `job_runs`: `remark_job_runs`, `ticket_job_runs`, `run_object_links`, `job_run_review_events`
4. Delete FK tables referencing `jobs`: `job_object_links`, `ticket_scopes`, `remark_scopes`, `overrides`
5. `DELETE FROM job_runs`
6. `DELETE FROM jobs`
Note: always use direct SQL (`DELETE FROM`) for bulk deletions — ORM-level deletes load all objects into Python memory and time out on large datasets.
### Key Model Fields
**MailMessage model:**