From 282601269c342ff1e738209e527ea08c022ca255 Mon Sep 17 00:00:00 2001 From: Ivo Oskamp Date: Mon, 23 Mar 2026 13:46:42 +0100 Subject: [PATCH] Release prep v0.2.3 fixes and changelog updates --- .../backupchecks/src/backend/app/changelog.py | 47 ++++ .../src/backend/app/cove_importer.py | 132 +++++++++-- .../src/backend/app/main/routes_cove.py | 2 + .../src/backend/app/main/routes_run_checks.py | 207 +++++++++++++++--- .../src/backend/app/parsers/synology.py | 2 +- .../backupchecks/src/static/css/layout.css | 37 ++++ .../src/templates/main/customers.html | 2 +- .../src/templates/main/job_detail.html | 17 +- .../src/templates/main/run_checks.html | 15 ++ docs/changelog-claude.md | 28 +++ docs/changelog.md | 20 ++ docs/technical-notes-codex.md | 16 +- 12 files changed, 473 insertions(+), 52 deletions(-) diff --git a/containers/backupchecks/src/backend/app/changelog.py b/containers/backupchecks/src/backend/app/changelog.py index 5d8a63a..a68b035 100644 --- a/containers/backupchecks/src/backend/app/changelog.py +++ b/containers/backupchecks/src/backend/app/changelog.py @@ -3,6 +3,53 @@ Changelog data structure for Backupchecks """ CHANGELOG = [ + { + "version": "v0.2.3", + "date": "2026-03-23", + "summary": "Update release that improves Autotask existing-ticket linking transparency, Cove object detail propagation, and Customers page link styling consistency.", + "sections": [ + { + "title": "Added", + "type": "feature", + "changes": [ + "Autotask Link existing ticket now also posts a ticket note when an additional Backupchecks alert/run is linked, including customer, job, run context and a Backupchecks deep-link" + ] + }, + { + "title": "Changed", + "type": "improvement", + "changes": [ + "Link existing Autotask API response now includes note_posted and note_warning fields for operator visibility", + "Customers page customer-name links to filtered Jobs now use sidebar-matching text and hover styling instead of default blue link styling" + ] + }, + { + "title": "Fixed", + "type": "bugfix", + "changes": [ + "Autotask object retrieval for ticket composition now reads objects from run_object_links/customer_objects first (same source as Run Checks UI), then falls back to legacy job_objects/mail_objects, preventing missing object details", + "Cove Accounts and Cove-run object visibility improved by consistently using persisted run object links as primary object source, aligning operator view and ticket details", + "Autotask ticket affected-objects list now includes only problem objects (failed/error/warning/missed) and excludes completed/success objects" + ] + } + ] + }, + { + "version": "v0.2.2", + "date": "2026-03-23", + "summary": "Hotfix release that corrects Synology Active Backup for Business parsing for completion mails using has been wording, preventing wrong job-name extraction.", + "sections": [ + { + "title": "Fixed", + "type": "bugfix", + "changes": [ + "Synology Active Backup for Business parser now recognizes completion mails using has been completed wording", + "Prevents fallback to the generic Synology Active Backup parser that could incorrectly take the bracketed subject prefix as job name", + "ABB mails like backup task dc001 on DS220p has been completed now keep the expected identity: backup_software=Synology, backup_type=Active Backup for Business, job_name=dc001" + ] + } + ] + }, { "version": "v0.2.1", "date": "2026-03-20", diff --git a/containers/backupchecks/src/backend/app/cove_importer.py b/containers/backupchecks/src/backend/app/cove_importer.py index fc70659..2f21d2e 100644 --- a/containers/backupchecks/src/backend/app/cove_importer.py +++ b/containers/backupchecks/src/backend/app/cove_importer.py @@ -45,6 +45,13 @@ COVE_COLUMNS = [ "D23F00", "D23F15", # M365 Teams ] +# Optional datasource-specific columns for datasources that may be active in I78 +# but are not always available in every tenant/API scope. +COVE_OPTIONAL_COLUMNS = [ + "D2F00", "D2F15", # System State + "D6F00", "D6F15", # Network Shares +] + # Mapping from Cove status code to Backupchecks status string STATUS_MAP: dict[int, str] = { 1: "Warning", # In process @@ -75,14 +82,21 @@ STATUS_LABELS: dict[int, str] = { 12: "Restarted", } -# Datasource label mapping (column prefix → human-readable label) +# Datasource label mapping (Cove datasource code → human-readable label). +# Keep both canonical and zero-padded aliases so UI summary and importer stay in sync. DATASOURCE_LABELS: dict[str, str] = { "D1": "Files & Folders", + "D01": "Files & Folders", + "D2": "System State", + "D02": "System State", + "D6": "Network Shares", + "D06": "Network Shares", "D10": "VssMsSql", "D11": "VssSharePoint", "D19": "M365 Exchange", "D20": "M365 OneDrive", "D5": "M365 SharePoint", + "D05": "M365 SharePoint", "D23": "M365 Teams", } @@ -147,6 +161,7 @@ def _cove_enumerate( partner_id: int, start: int, count: int, + columns: list[str] | None = None, ) -> list[dict]: """Call EnumerateAccountStatistics and return a list of account dicts. @@ -162,7 +177,7 @@ def _cove_enumerate( "PartnerId": partner_id, "StartRecordNumber": start, "RecordsCount": count, - "Columns": COVE_COLUMNS, + "Columns": columns or COVE_COLUMNS, } }, } @@ -201,6 +216,12 @@ def _cove_enumerate( return [] +def _is_cove_security_column_error(exc: Exception) -> bool: + """Return True when Enumerate failed due to restricted columns/security scope.""" + msg = str(exc or "").lower() + return ("security reasons" in msg) or ("13501" in msg) + + def _flatten_settings(account: dict) -> dict: """Convert the Settings array in an account dict to a flat key→value dict. @@ -216,6 +237,42 @@ def _flatten_settings(account: dict) -> dict: return flat +def _normalize_ds_code(code: str) -> str: + """Normalize datasource codes like D01 -> D1.""" + m = re.fullmatch(r"D(\d{1,2})", (code or "").strip().upper()) + if not m: + return (code or "").strip().upper() + return f"D{int(m.group(1))}" + + +def _parse_active_datasource_codes(raw: Any) -> list[str]: + """Extract unique active datasource codes from I78 (e.g. D01D02D10).""" + text = str(raw or "").strip().upper() + if not text: + return [] + seen: set[str] = set() + out: list[str] = [] + for code in re.findall(r"D\d{1,2}", text): + norm = _normalize_ds_code(code) + if not norm or norm in seen: + continue + seen.add(norm) + out.append(norm) + return out + + +def _label_for_ds_code(code: str) -> str: + """Resolve human label for a datasource code (supports canonical + padded aliases).""" + norm = _normalize_ds_code(code) + if norm in DATASOURCE_LABELS: + return DATASOURCE_LABELS[norm] + m = re.fullmatch(r"D(\d{1,2})", norm) + if m: + padded = f"D{int(m.group(1)):02d}" + return DATASOURCE_LABELS.get(padded, norm) + return DATASOURCE_LABELS.get(code, code) + + def _map_status(code: Any) -> str: """Map a Cove status code (int) to a Backupchecks status string.""" if code is None: @@ -436,11 +493,26 @@ def run_cove_import(settings, include_reasons: bool = False): page_size = 250 start = 0 + base_columns = list(COVE_COLUMNS) + extended_columns = base_columns + list(COVE_OPTIONAL_COLUMNS) + use_optional_columns = True + while True: + columns = extended_columns if use_optional_columns else base_columns try: - accounts = _cove_enumerate(url, visa, partner_id, start, page_size) - except CoveImportError: - raise + accounts = _cove_enumerate(url, visa, partner_id, start, page_size, columns=columns) + except CoveImportError as exc: + # Some tenants block specific datasource columns; fall back safely. + if use_optional_columns and _is_cove_security_column_error(exc): + logger.warning( + "Cove import: optional datasource columns blocked by API scope; " + "falling back to base columns. Error: %s", + exc, + ) + use_optional_columns = False + accounts = _cove_enumerate(url, visa, partner_id, start, page_size, columns=base_columns) + else: + raise except Exception as exc: raise CoveImportError(f"Unexpected error fetching accounts at offset {start}: {exc}") from exc @@ -584,6 +656,9 @@ def _process_account(account: dict) -> str: # Cove session was previously stored under another job. existing = JobRun.query.filter_by(job_id=job.id, external_id=external_id).first() if existing: + # Keep objects in sync even when the run itself is a duplicate session. + if job.customer_id: + _persist_datasource_objects(flat, job.customer_id, job.id, existing.id, last_run_at) db.session.commit() return "skip_duplicate" @@ -638,18 +713,49 @@ def _persist_datasource_objects( observed_at: datetime, ) -> None: """Create run_object_links for each active datasource found in the account stats.""" - for ds_prefix, ds_label in DATASOURCE_LABELS.items(): - status_key = f"{ds_prefix}F00" + # Use I78 as source-of-truth for active datasources so object count matches Cove UI. + ds_codes = _parse_active_datasource_codes(flat.get("I78")) + + # Fallback: when I78 is missing, derive from present DxxF00 keys to avoid empty objects. + if not ds_codes: + seen: set[str] = set() + for key in flat.keys(): + m = re.fullmatch(r"(D\d{1,2})F00", str(key or "").upper()) + if not m: + continue + norm = _normalize_ds_code(m.group(1)) + if norm and norm not in seen: + seen.add(norm) + ds_codes.append(norm) + + overall_status_code = flat.get("D09F00") + overall_last_ts = _ts_to_dt(flat.get("D09F15")) + + for ds_code in ds_codes: + ds_label = _label_for_ds_code(ds_code) + + status_key = f"{ds_code}F00" + ts_key = f"{ds_code}F15" status_code = flat.get(status_key) + + uses_overall_fallback = status_code is None if status_code is None: - continue + status_code = overall_status_code status = _map_status(status_code) - ds_last_ts = _ts_to_dt(flat.get(f"{ds_prefix}F15")) - status_msg = ( - f"Cove datasource status: {_status_label(status_code)} " - f"({status_code}); last session: {_fmt_utc(ds_last_ts)}" - ) + ds_last_ts = _ts_to_dt(flat.get(ts_key)) or overall_last_ts + + if uses_overall_fallback: + status_msg = ( + f"Cove datasource status: {_status_label(status_code)} " + f"({status_code}); datasource-specific status not returned by API columns; " + f"using overall account status; last session: {_fmt_utc(ds_last_ts)}" + ) + else: + status_msg = ( + f"Cove datasource status: {_status_label(status_code)} " + f"({status_code}); last session: {_fmt_utc(ds_last_ts)}" + ) # Use the same SQLAlchemy session/transaction as JobRun creation. # A separate engine connection cannot reliably see the uncommitted run row. diff --git a/containers/backupchecks/src/backend/app/main/routes_cove.py b/containers/backupchecks/src/backend/app/main/routes_cove.py index 3e347e0..4ccf433 100644 --- a/containers/backupchecks/src/backend/app/main/routes_cove.py +++ b/containers/backupchecks/src/backend/app/main/routes_cove.py @@ -19,6 +19,8 @@ _COVE_DATASOURCE_LABELS = { "D1": "Files & Folders", "D02": "System State", "D2": "System State", + "D06": "Network Shares", + "D6": "Network Shares", "D10": "VssMsSql", "D11": "VssSharePoint", "D19": "M365 Exchange", diff --git a/containers/backupchecks/src/backend/app/main/routes_run_checks.py b/containers/backupchecks/src/backend/app/main/routes_run_checks.py index 32e908f..315446f 100644 --- a/containers/backupchecks/src/backend/app/main/routes_run_checks.py +++ b/containers/backupchecks/src/backend/app/main/routes_run_checks.py @@ -645,36 +645,110 @@ def _compose_autotask_ticket_description( lines.append("Summary:") lines.append(overall_message) lines.append("") - lines.append("Multiple objects reported messages. See Backupchecks for full details.") - else: - # Fallback to object-level messages with a hard limit. + + # Always include object-level details so technicians can immediately see + # which objects failed/warned from the ticket itself. + def _object_priority(obj: dict[str, str]) -> int: + combined = f"{obj.get('status', '')} {obj.get('error_message', '')}".strip().lower() + if any(x in combined for x in ("failed", "error", "missed")): + return 0 + if "warning" in combined: + return 1 + return 2 + + def _is_problem_object(status: str, err: str) -> bool: + s = (status or "").strip().lower() + e = (err or "").strip().lower() + combined = f"{s} {e}".strip() + + if any(x in combined for x in ("failed", "error", "warning", "missed")): + return True + + if s in ("success", "succeeded", "completed", "ok"): + return False + + if "completed" in combined: + return False + + # Keep uncommon non-success statuses visible. + return bool(s or e) + + candidates: list[dict[str, str]] = [] + for o in objects_payload or []: + name = (o.get("name") or "").strip() + err = (o.get("error_message") or "").strip() + st = (o.get("status") or "").strip() + if not name: + continue + if not _is_problem_object(st, err): + continue + candidates.append( + { + "name": name, + "status": st, + "error_message": err, + } + ) + + if candidates: + sorted_candidates = sorted( + candidates, + key=lambda x: (_object_priority(x), x["name"].lower()), + ) + lines.append("Affected objects:") limit = 10 shown = 0 - total = 0 - for o in objects_payload or []: - name = (o.get("name") or "").strip() - err = (o.get("error_message") or "").strip() - st = (o.get("status") or "").strip() - if not name: - continue - if not err and not st: - continue - total += 1 + for o in sorted_candidates: if shown >= limit: - continue - msg = err or st - lines.append(f"- {name}: {msg}") + break + msg = o["error_message"] or o["status"] + lines.append(f"- {o['name']}: {msg}") shown += 1 - - if total == 0: - lines.append("No detailed object messages available. See Backupchecks for full details.") - elif total > shown: - lines.append(f"And {int(total - shown)} additional objects reported similar messages.") + if len(sorted_candidates) > shown: + lines.append(f"And {int(len(sorted_candidates) - shown)} additional objects reported messages.") + else: + lines.append("No detailed object messages available. See Backupchecks for full details.") lines.append("") lines.append(f"Backupchecks details: {job_link}") return "\n".join(lines).strip() + "\n" + +def _compose_autotask_link_existing_note( + *, + settings, + job: Job, + run: JobRun, + ticket_number: str, + linked_run_count: int, +) -> str: + tz_name = _get_ui_timezone_name() or "Europe/Amsterdam" + run_at_str = _format_datetime(run.run_at) if run.run_at else "-" + actor = (getattr(current_user, "email", None) or getattr(current_user, "username", None) or "operator") + + base_url = (getattr(settings, "autotask_base_url", None) or "").strip() + job_rel = url_for("main.job_detail", job_id=job.id) + job_link = urljoin(base_url.rstrip("/") + "/", job_rel.lstrip("/")) + if run.id: + job_link = f"{job_link}?run_id={int(run.id)}" + + lines: list[str] = [] + lines.append("[Backupchecks] Additional alert linked to this ticket.") + lines.append(f"Customer: {job.customer.name if job.customer else ''}") + lines.append(f"Job: {job.job_name or ''}") + lines.append(f"Backup: {job.backup_software or ''} / {job.backup_type or ''}") + lines.append(f"Run ID: {int(run.id) if run.id else 0}") + lines.append(f"Run at ({tz_name}): {run_at_str}") + lines.append(f"Run status: {run.status or ''}") + lines.append(f"Linked active runs for this job: {int(linked_run_count or 0)}") + if ticket_number: + lines.append(f"Ticket: {ticket_number}") + lines.append(f"Linked by: {actor}") + lines.append("") + lines.append(f"Backupchecks details: {job_link}") + return "\n".join(lines).strip() + "\n" + + # Grace window for matching real runs to an expected schedule slot. # A run within +/- 1 hour of the inferred schedule time counts as fulfilling the slot. MISSED_GRACE_WINDOW = timedelta(hours=1) @@ -1790,21 +1864,58 @@ def api_run_checks_create_autotask_ticket(): overall_message = (getattr(msg, "overall_message", None) or "") if msg else "" objects_payload: list[dict[str, str]] = [] - try: - objs = run.objects.order_by(JobObject.object_name.asc()).all() - except Exception: - objs = list(run.objects or []) - for o in objs or []: - objects_payload.append( - { - "name": getattr(o, "object_name", "") or "", - "type": getattr(o, "object_type", "") or "", - "status": getattr(o, "status", "") or "", - "error_message": getattr(o, "error_message", "") or "", - } - ) - if (not objects_payload) and msg: + # Preferred source: run_object_links/customer_objects, also used by Run Checks UI + # for Cove/cloud sourced runs. + try: + rows = ( + db.session.execute( + text( + """ + SELECT + co.object_name AS name, + rol.status AS status, + rol.error_message AS error_message + FROM run_object_links rol + JOIN customer_objects co ON co.id = rol.customer_object_id + WHERE rol.run_id = :run_id + ORDER BY co.object_name ASC + """ + ), + {"run_id": run.id}, + ) + .mappings() + .all() + ) + for rr in rows: + objects_payload.append( + { + "name": rr.get("name") or "", + "type": "", + "status": rr.get("status") or "", + "error_message": rr.get("error_message") or "", + } + ) + except Exception: + pass + + # Fallback for legacy mail parser sourced runs. + if not objects_payload: + try: + objs = run.objects.order_by(JobObject.object_name.asc()).all() + except Exception: + objs = list(run.objects or []) + for o in objs or []: + objects_payload.append( + { + "name": getattr(o, "object_name", "") or "", + "type": getattr(o, "object_type", "") or "", + "status": getattr(o, "status", "") or "", + "error_message": getattr(o, "error_message", "") or "", + } + ) + + if (not objects_payload) and msg and getattr(run, "source_type", None) != "cloud_connect": try: mos = MailObject.query.filter_by(mail_message_id=msg.id).order_by(MailObject.object_name.asc()).all() except Exception: @@ -2193,12 +2304,38 @@ def api_run_checks_autotask_link_existing_ticket(): db.session.rollback() return jsonify({"status": "error", "message": "Failed to persist Autotask ticket link."}), 500 + note_posted = False + note_warning = "" + try: + settings = _get_or_create_settings() + note_body = _compose_autotask_link_existing_note( + settings=settings, + job=job, + run=run, + ticket_number=tnum, + linked_run_count=len(run_ids), + ) + client.create_ticket_note( + { + "ticketID": int(ticket_id), + "title": "Backupchecks", + "description": note_body, + "publish": 1, + } + ) + note_posted = True + except Exception as exc: + note_posted = False + note_warning = f"Autotask ticket was linked, but posting the link-update note failed: {exc}" + return jsonify( { "status": "ok", "ticket_id": int(ticket_id), "ticket_number": tnum, "internal_ticket_id": int(getattr(internal_ticket, "id", 0) or 0) if internal_ticket else 0, + "note_posted": bool(note_posted), + "note_warning": note_warning, } ) diff --git a/containers/backupchecks/src/backend/app/parsers/synology.py b/containers/backupchecks/src/backend/app/parsers/synology.py index 192a642..73d9403 100644 --- a/containers/backupchecks/src/backend/app/parsers/synology.py +++ b/containers/backupchecks/src/backend/app/parsers/synology.py @@ -265,7 +265,7 @@ _ABB_SUBJECT_RE = re.compile(r"\bactive\s+backup\s+for\s+business\b", re.I) # "backup task vSphere-Task-1 on KANTOOR-NEW was skipped" _ABB_COMPLETED_RE = re.compile( r"\b(?:virtuele\s+machine\s+)?(?:de\s+)?back-?up\s*(?:taak|job)\s+(?:van\s+deze\s+taak\s+)?(?P.+?)\s+op\s+(?P[A-Za-z0-9._-]+)\s+is\s+(?Pvoltooid|gedeeltelijk\s+voltooid|genegeerd)\b" - r"|\b(?:virtual\s+machine\s+)?(?:the\s+)?back-?up\s+(?:task|job)\s+(?P.+?)\s+on\s+(?P[A-Za-z0-9._-]+)\s+(?:is\s+|was\s+)?(?Pcompleted|finished|has\s+completed|partially\s+completed|skipped|ignored)\b", + r"|\b(?:virtual\s+machine\s+)?(?:the\s+)?back-?up\s+(?:task|job)\s+(?P.+?)\s+on\s+(?P[A-Za-z0-9._-]+)\s+(?:(?:is|was|has\s+been|has)\s+)?(?Pcompleted|finished|partially\s+completed|skipped|ignored)\b", re.I, ) diff --git a/containers/backupchecks/src/static/css/layout.css b/containers/backupchecks/src/static/css/layout.css index 19e289b..a31ab2e 100644 --- a/containers/backupchecks/src/static/css/layout.css +++ b/containers/backupchecks/src/static/css/layout.css @@ -173,6 +173,24 @@ body.bc-body { color: var(--bc-sidebar-active-text); } +.bc-sidebar-link-inline { + display: inline-flex; + align-items: center; + padding: 2px 6px; + margin: -2px -6px; + border-radius: 6px; + text-decoration: none; + color: var(--bc-sidebar-text); + font-weight: 500; + transition: background var(--bc-transition), color var(--bc-transition); +} + +.bc-sidebar-link-inline:hover { + background: rgba(255,255,255,0.06); + color: var(--bc-sidebar-text-hover); +} + + .bc-nav-icon { display: flex; align-items: center; flex-shrink: 0; } .bc-nav-label-text { flex: 1; overflow: hidden; text-overflow: ellipsis; } @@ -412,6 +430,25 @@ body.bc-body { white-space: nowrap; } +/* Keep long object/datasource values fully visible in operational modals. */ +#rcm_objects_table th, +#rcm_objects_table td, +#dj_objects_table th, +#dj_objects_table td, +#run_msg_objects_container table th, +#run_msg_objects_container table td { + white-space: normal; + overflow-wrap: anywhere; + word-break: break-word; +} + +#rcm_cove_datasources, +#jdm_cove_datasources { + white-space: normal; + overflow-wrap: anywhere; + word-break: break-word; +} + /* Cards */ .card { border-radius: var(--bc-radius); diff --git a/containers/backupchecks/src/templates/main/customers.html b/containers/backupchecks/src/templates/main/customers.html index 4a78bf7..25e3b36 100644 --- a/containers/backupchecks/src/templates/main/customers.html +++ b/containers/backupchecks/src/templates/main/customers.html @@ -50,7 +50,7 @@ {% for c in customers %} - + {{ c.name }} diff --git a/containers/backupchecks/src/templates/main/job_detail.html b/containers/backupchecks/src/templates/main/job_detail.html index 2053417..dac2e01 100644 --- a/containers/backupchecks/src/templates/main/job_detail.html +++ b/containers/backupchecks/src/templates/main/job_detail.html @@ -183,6 +183,14 @@ @media (min-width: 1400px) { .modal-xxl { max-width: 1400px; } } #run_msg_body_container_iframe { height: 55vh; } #run_msg_objects_container { max-height: 25vh; overflow: auto; } + + #jobRunMessageModal.is-cove #jdm_mail_iframe_panel { + display: none !important; + } + + #jobRunMessageModal.is-cove #run_msg_objects_container { + max-height: 55vh; + } @@ -286,7 +294,7 @@ -
+
Mail