From 934a495867e6f371fd3eeee445aed8efad275646 Mon Sep 17 00:00:00 2001 From: Ivo Oskamp Date: Tue, 13 Jan 2026 15:07:59 +0100 Subject: [PATCH] Auto-commit local changes before build (2026-01-13 15:07:59) --- .last-branch | 2 +- .../src/backend/app/auto_importer_service.py | 4 +- .../src/backend/app/mail_importer.py | 91 +++++- .../src/backend/app/main/routes_inbox.py | 260 +++++++++++++++--- .../src/backend/app/main/routes_settings.py | 4 +- .../src/backend/app/main/routes_shared.py | 2 +- .../src/backend/app/object_persistence.py | 63 ++++- .../src/backend/app/parsers/veeam.py | 88 ++++++ docs/changelog.md | 11 + 9 files changed, 466 insertions(+), 59 deletions(-) diff --git a/.last-branch b/.last-branch index 8e70fb3..96a874e 100644 --- a/.last-branch +++ b/.last-branch @@ -1 +1 @@ -v20260113-07-job-delete-fix +v20260113-08-vspc-object-linking diff --git a/containers/backupchecks/src/backend/app/auto_importer_service.py b/containers/backupchecks/src/backend/app/auto_importer_service.py index 2132172..7e2a274 100644 --- a/containers/backupchecks/src/backend/app/auto_importer_service.py +++ b/containers/backupchecks/src/backend/app/auto_importer_service.py @@ -7,7 +7,7 @@ from datetime import datetime from .admin_logging import log_admin_event from .mail_importer import MailImportError, run_auto_import from .models import SystemSettings -from .object_persistence import persist_objects_for_approved_run +from .object_persistence import persist_objects_for_auto_run _AUTO_IMPORTER_THREAD_NAME = "auto_importer" @@ -80,7 +80,7 @@ def start_auto_importer(app) -> None: persisted_errors = 0 for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs: try: - persisted_objects += persist_objects_for_approved_run( + persisted_objects += persist_objects_for_auto_run( int(customer_id), int(job_id), int(run_id), int(mail_message_id) ) except Exception as exc: diff --git a/containers/backupchecks/src/backend/app/mail_importer.py b/containers/backupchecks/src/backend/app/mail_importer.py index ffa81c1..d2479c7 100644 --- a/containers/backupchecks/src/backend/app/mail_importer.py +++ b/containers/backupchecks/src/backend/app/mail_importer.py @@ -11,8 +11,9 @@ import requests from sqlalchemy import func from . import db -from .models import MailMessage, SystemSettings, Job, JobRun +from .models import MailMessage, SystemSettings, Job, JobRun, MailObject from .parsers import parse_mail_message +from .parsers.veeam import extract_vspc_active_alarms_companies from .email_utils import normalize_from_address, extract_best_html_from_eml, is_effectively_blank_html from .job_matching import find_matching_job @@ -265,6 +266,94 @@ def _store_messages(settings: SystemSettings, messages): and getattr(mail, "parse_result", None) == "ok" and not bool(getattr(mail, "approved", False)) ): + # Special case: Veeam VSPC "Active alarms summary" contains multiple companies. + bsw = (getattr(mail, "backup_software", "") or "").strip().lower() + btype = (getattr(mail, "backup_type", "") or "").strip().lower() + jname = (getattr(mail, "job_name", "") or "").strip().lower() + + if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary": + raw = (mail.text_body or "").strip() or (mail.html_body or "") + companies = extract_vspc_active_alarms_companies(raw) + + if companies: + def _is_error_status(value: str | None) -> bool: + v = (value or "").strip().lower() + return v in {"error", "failed", "critical"} or v.startswith("fail") + + created_any = False + first_job = None + mapped_count = 0 + + for company in companies: + # Build a temp message using the per-company job name + tmp = MailMessage( + from_address=mail.from_address, + backup_software=mail.backup_software, + backup_type=mail.backup_type, + job_name=f"{(mail.job_name or 'Active alarms summary').strip()} | {company}".strip(), + ) + job = find_matching_job(tmp) + if not job: + continue + + # Respect per-job flags. + if hasattr(job, "active") and not bool(job.active): + continue + if hasattr(job, "auto_approve") and not bool(job.auto_approve): + continue + + mapped_count += 1 + + objs = ( + MailObject.query.filter(MailObject.mail_message_id == mail.id) + .filter(MailObject.object_name.ilike(f"{company} | %")) + .all() + ) + saw_error = any(_is_error_status(o.status) for o in objs) + saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs) + status = "Error" if saw_error else ("Warning" if saw_warning else (mail.overall_status or "Success")) + + run = JobRun( + job_id=job.id, + mail_message_id=mail.id, + run_at=mail.received_at, + status=status or None, + missed=False, + ) + + # Optional storage metrics + if hasattr(run, "storage_used_bytes") and hasattr(mail, "storage_used_bytes"): + run.storage_used_bytes = mail.storage_used_bytes + if hasattr(run, "storage_capacity_bytes") and hasattr(mail, "storage_capacity_bytes"): + run.storage_capacity_bytes = mail.storage_capacity_bytes + if hasattr(run, "storage_free_bytes") and hasattr(mail, "storage_free_bytes"): + run.storage_free_bytes = mail.storage_free_bytes + if hasattr(run, "storage_free_percent") and hasattr(mail, "storage_free_percent"): + run.storage_free_percent = mail.storage_free_percent + + db.session.add(run) + db.session.flush() + + auto_approved_runs.append((job.customer_id, job.id, run.id, mail.id)) + created_any = True + + if not first_job: + first_job = job + + # If all companies are mapped, mark the mail as fully approved and move to history. + if created_any and mapped_count == len(companies): + mail.job_id = first_job.id if first_job else None + if hasattr(mail, "approved"): + mail.approved = True + if hasattr(mail, "approved_at"): + mail.approved_at = datetime.utcnow() + if hasattr(mail, "location"): + mail.location = "history" + auto_approved += 1 + + # Do not fall back to single-job matching for VSPC summary. + continue + job = find_matching_job(mail) if job: # Respect per-job flags. diff --git a/containers/backupchecks/src/backend/app/main/routes_inbox.py b/containers/backupchecks/src/backend/app/main/routes_inbox.py index 7ec40b5..5ed206f 100644 --- a/containers/backupchecks/src/backend/app/main/routes_inbox.py +++ b/containers/backupchecks/src/backend/app/main/routes_inbox.py @@ -2,50 +2,14 @@ from .routes_shared import * # noqa: F401,F403 from .routes_shared import _format_datetime, _log_admin_event, _send_mail_message_eml_download from ..email_utils import extract_best_html_from_eml, is_effectively_blank_html +from ..parsers.veeam import extract_vspc_active_alarms_companies +from ..models import MailObject import time import re import html as _html -def _extract_vspc_active_alarms_companies(raw: str) -> list[str]: - """Best-effort extraction of company names from VSPC "Active alarms summary" bodies. - - Only returns companies with alarms > 0, as the email is expected to list failing customers. - """ - if not raw: - return [] - - txt = raw - # Best-effort HTML to text - if "<" in txt and ">" in txt: - txt = re.sub(r"<[^>]+>", " ", txt) - txt = _html.unescape(txt) - txt = re.sub(r"\s+", " ", txt).strip() - - seen: set[str] = set() - out: list[str] = [] - - for m in re.finditer( - r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)", - txt, - flags=re.IGNORECASE, - ): - cname = (m.group(1) or "").strip() - try: - alarms = int(m.group(2)) - except Exception: - alarms = 0 - - if not cname or alarms <= 0: - continue - if cname in seen: - continue - seen.add(cname) - out.append(cname) - - return out - @main_bp.route("/inbox") @login_required @roles_required("admin", "operator", "viewer") @@ -201,7 +165,7 @@ def inbox_message_detail(message_id: int): if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary": raw = text_body if not _is_blank_text(text_body) else (html_body or "") - vspc_companies = _extract_vspc_active_alarms_companies(raw) + vspc_companies = extract_vspc_active_alarms_companies(raw) # For each company, prefill the UI with the existing customer mapping if we already have a job for it. # This avoids re-mapping known companies and keeps the message actionable in the Inbox. @@ -402,7 +366,7 @@ def inbox_message_approve_vspc_companies(message_id: int): html_body = getattr(msg, "html_body", None) text_body = getattr(msg, "text_body", None) raw_for_companies = text_body if (text_body and str(text_body).strip()) else (html_body or "") - companies_present = _extract_vspc_active_alarms_companies(raw_for_companies) + companies_present = extract_vspc_active_alarms_companies(raw_for_companies) if not companies_present: flash("No companies could be detected in this VSPC summary email.", "danger") @@ -550,7 +514,7 @@ def inbox_message_approve_vspc_companies(message_id: int): objs = ( MailObject.query.filter(MailObject.mail_message_id == msg.id) - .filter(MailObject.object_name.like(f"{company} | %")) + .filter(MailObject.object_name.ilike(f"{company} | %")) .all() ) saw_error = any(_is_error_status(o.status) for o in objs) @@ -641,9 +605,130 @@ def inbox_message_approve_vspc_companies(message_id: int): _log_admin_event("inbox_approve_error", f"Failed to finalize VSPC approval for message {msg.id}: {exc}") return redirect(url_for("main.inbox")) + # Best-effort: now that company jobs are mapped, auto-approve other inbox + # messages of the same VSPC summary type whose companies are now all mapped. + retro_approved_msgs = 0 + try: + q = MailMessage.query + if hasattr(MailMessage, "location"): + q = q.filter(MailMessage.location == "inbox") + q = q.filter(MailMessage.parse_result == "ok") + q = q.filter(MailMessage.job_id.is_(None)) + q = q.filter(MailMessage.backup_software == "Veeam") + q = q.filter(MailMessage.backup_type == "Service Provider Console") + q = q.filter(MailMessage.job_name == (msg.job_name or "Active alarms summary")) + q = q.filter(MailMessage.id != msg.id) + candidates = q.order_by(MailMessage.received_at.desc().nullslast(), MailMessage.id.desc()).limit(25).all() + + for other in candidates: + nested = db.session.begin_nested() + try: + raw_other = (other.text_body or "").strip() or (other.html_body or "") + companies = extract_vspc_active_alarms_companies(raw_other) + if not companies: + nested.commit() + continue + + jobs_by_company: dict[str, Job] = {} + all_mapped = True + for company in companies: + norm_from, store_backup, store_type, _store_job = build_job_match_key(other) + company_job_name = f"{(other.job_name or 'Active alarms summary').strip()} | {company}".strip() + tmp = MailMessage( + from_address=norm_from, + backup_software=store_backup, + backup_type=store_type, + job_name=company_job_name, + ) + with db.session.no_autoflush: + j = find_matching_job(tmp) + if not j or not getattr(j, "customer_id", None): + all_mapped = False + break + if hasattr(j, "active") and not bool(j.active): + all_mapped = False + break + if hasattr(j, "auto_approve") and not bool(j.auto_approve): + all_mapped = False + break + jobs_by_company[company] = j + + if not all_mapped: + nested.commit() + continue + + first_job2: Job | None = None + for company, job2 in jobs_by_company.items(): + if not first_job2: + first_job2 = job2 + + objs2 = ( + MailObject.query.filter(MailObject.mail_message_id == other.id) + .filter(MailObject.object_name.ilike(f"{company} | %")) + .all() + ) + saw_error2 = any(_is_error_status(o.status) for o in objs2) + saw_warning2 = any((o.status or "").strip().lower() == "warning" for o in objs2) + status2 = "Error" if saw_error2 else ("Warning" if saw_warning2 else (other.overall_status or "Success")) + + run2 = JobRun.query.filter(JobRun.job_id == job2.id, JobRun.mail_message_id == other.id).first() + if not run2: + run2 = JobRun( + job_id=job2.id, + mail_message_id=other.id, + run_at=(other.received_at or getattr(other, "parsed_at", None) or datetime.utcnow()), + status=status2 or None, + missed=False, + ) + if hasattr(run2, "remark"): + run2.remark = getattr(other, "overall_message", None) + db.session.add(run2) + db.session.flush() + + # Persist objects per company + try: + persist_objects_for_approved_run_filtered( + int(job2.customer_id), + int(job2.id), + int(run2.id), + int(other.id), + object_name_prefix=company, + strip_prefix=True, + ) + except Exception as exc: + _log_admin_event( + "object_persist_error", + f"Filtered object persistence failed for message {other.id} (company '{company}', job {job2.id}, run {run2.id}): {exc}", + ) + + other.job_id = first_job2.id if first_job2 else None + if hasattr(other, "approved"): + other.approved = True + if hasattr(other, "approved_at"): + other.approved_at = datetime.utcnow() + if hasattr(other, "approved_by_id"): + other.approved_by_id = current_user.id + if hasattr(other, "location"): + other.location = "history" + + nested.commit() + retro_approved_msgs += 1 + except Exception: + try: + nested.rollback() + except Exception: + db.session.rollback() + + db.session.commit() + except Exception: + try: + db.session.rollback() + except Exception: + pass + _log_admin_event( "inbox_approve_vspc", - f"Approved VSPC message {msg.id} into {processed_total} run(s) (job_id={msg.job_id})", + f"Approved VSPC message {msg.id} into {processed_total} run(s) (job_id={msg.job_id}), retro_approved={retro_approved_msgs}", ) flash(f"Approved VSPC summary into {processed_total} run(s).", "success") return redirect(url_for("main.inbox")) @@ -897,6 +982,95 @@ def inbox_reparse_all(): and getattr(msg, "parse_result", None) == "ok" and getattr(msg, "job_id", None) is None ): + # Special case: VSPC Active Alarms summary can contain multiple companies. + bsw = (getattr(msg, "backup_software", "") or "").strip().lower() + btype = (getattr(msg, "backup_type", "") or "").strip().lower() + jname = (getattr(msg, "job_name", "") or "").strip().lower() + + if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary": + raw = (getattr(msg, "text_body", None) or "").strip() or (getattr(msg, "html_body", None) or "") + companies = extract_vspc_active_alarms_companies(raw) + + if companies: + def _is_error_status(value: str | None) -> bool: + v = (value or "").strip().lower() + return v in {"error", "failed", "critical"} or v.startswith("fail") + + first_job = None + mapped_count = 0 + created_any = False + + for company in companies: + tmp_msg = MailMessage( + from_address=msg.from_address, + backup_software=msg.backup_software, + backup_type=msg.backup_type, + job_name=f"{(msg.job_name or 'Active alarms summary').strip()} | {company}".strip(), + ) + with db.session.no_autoflush: + job = find_matching_job(tmp_msg) + + if not job: + continue + if hasattr(job, "active") and not bool(job.active): + continue + if hasattr(job, "auto_approve") and not bool(job.auto_approve): + continue + + mapped_count += 1 + + objs = ( + MailObject.query.filter(MailObject.mail_message_id == msg.id) + .filter(MailObject.object_name.ilike(f"{company} | %")) + .all() + ) + saw_error = any(_is_error_status(o.status) for o in objs) + saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs) + status = "Error" if saw_error else ("Warning" if saw_warning else (msg.overall_status or "Success")) + + run = JobRun( + job_id=job.id, + mail_message_id=msg.id, + run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()), + status=status or None, + missed=False, + ) + + if hasattr(run, "remark"): + run.remark = getattr(msg, "overall_message", None) + + if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"): + run.storage_used_bytes = msg.storage_used_bytes + if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"): + run.storage_capacity_bytes = msg.storage_capacity_bytes + if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"): + run.storage_free_bytes = msg.storage_free_bytes + if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"): + run.storage_free_percent = msg.storage_free_percent + + db.session.add(run) + db.session.flush() + auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id)) + created_any = True + + if not first_job: + first_job = job + + if created_any and mapped_count == len(companies): + msg.job_id = first_job.id if first_job else None + if hasattr(msg, "approved"): + msg.approved = True + if hasattr(msg, "approved_at"): + msg.approved_at = datetime.utcnow() + if hasattr(msg, "approved_by_id"): + msg.approved_by_id = None + if hasattr(msg, "location"): + msg.location = "history" + auto_approved += 1 + + # Do not fall back to single-job matching for VSPC summary. + continue + # Match approved job on: From + Backup + Type + Job name # Prevent session autoflush for every match lookup while we # are still updating many messages in a loop. @@ -1077,7 +1251,7 @@ def inbox_reparse_all(): persisted_errors = 0 for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs: try: - persisted_objects += persist_objects_for_approved_run( + persisted_objects += persist_objects_for_auto_run( customer_id, job_id, run_id, mail_message_id ) except Exception as exc: diff --git a/containers/backupchecks/src/backend/app/main/routes_settings.py b/containers/backupchecks/src/backend/app/main/routes_settings.py index a8dffd0..7018135 100644 --- a/containers/backupchecks/src/backend/app/main/routes_settings.py +++ b/containers/backupchecks/src/backend/app/main/routes_settings.py @@ -169,7 +169,7 @@ def settings_objects_backfill(): for r in rows: try: - repaired_objects += persist_objects_for_approved_run( + repaired_objects += persist_objects_for_auto_run( int(r[2]), int(r[1]), int(r[0]), int(r[3]) ) repaired_runs += 1 @@ -1057,7 +1057,7 @@ def settings_mail_import(): persisted_errors = 0 for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs: try: - persisted_objects += persist_objects_for_approved_run( + persisted_objects += persist_objects_for_auto_run( int(customer_id), int(job_id), int(run_id), int(mail_message_id) ) except Exception as exc: diff --git a/containers/backupchecks/src/backend/app/main/routes_shared.py b/containers/backupchecks/src/backend/app/main/routes_shared.py index 4975c4b..e53fab2 100644 --- a/containers/backupchecks/src/backend/app/main/routes_shared.py +++ b/containers/backupchecks/src/backend/app/main/routes_shared.py @@ -60,7 +60,7 @@ from ..models import ( ) from ..mail_importer import run_manual_import, MailImportError from ..parsers import parse_mail_message -from ..object_persistence import persist_objects_for_approved_run +from ..object_persistence import persist_objects_for_approved_run, persist_objects_for_auto_run main_bp = Blueprint("main", __name__) diff --git a/containers/backupchecks/src/backend/app/object_persistence.py b/containers/backupchecks/src/backend/app/object_persistence.py index 1d60178..c6a94c9 100644 --- a/containers/backupchecks/src/backend/app/object_persistence.py +++ b/containers/backupchecks/src/backend/app/object_persistence.py @@ -195,14 +195,16 @@ def persist_objects_for_approved_run_filtered( status = r[2] error_message = r[3] - # 1) Upsert customer_objects and get id + # 1) Upsert customer_objects and get id (schema uses UNIQUE(customer_id, object_name)) customer_object_id = conn.execute( text( """ - INSERT INTO customer_objects (customer_id, object_name, object_type) - VALUES (:customer_id, :object_name, :object_type) - ON CONFLICT (customer_id, object_name, COALESCE(object_type, '')) - DO UPDATE SET object_type = EXCLUDED.object_type + INSERT INTO customer_objects (customer_id, object_name, object_type, first_seen_at, last_seen_at) + VALUES (:customer_id, :object_name, :object_type, NOW(), NOW()) + ON CONFLICT (customer_id, object_name) + DO UPDATE SET + last_seen_at = NOW(), + object_type = COALESCE(EXCLUDED.object_type, customer_objects.object_type) RETURNING id """ ), @@ -213,13 +215,14 @@ def persist_objects_for_approved_run_filtered( }, ).scalar() - # 2) Upsert job_object_links + # 2) Upsert job_object_links (keep timestamps fresh) conn.execute( text( """ - INSERT INTO job_object_links (job_id, customer_object_id) - VALUES (:job_id, :customer_object_id) - ON CONFLICT (job_id, customer_object_id) DO NOTHING + INSERT INTO job_object_links (job_id, customer_object_id, first_seen_at, last_seen_at) + VALUES (:job_id, :customer_object_id, NOW(), NOW()) + ON CONFLICT (job_id, customer_object_id) + DO UPDATE SET last_seen_at = NOW() """ ), { @@ -254,3 +257,45 @@ def persist_objects_for_approved_run_filtered( _update_override_applied_for_run(job_id, run_id) return processed + +def persist_objects_for_auto_run(customer_id: int, job_id: int, run_id: int, mail_message_id: int) -> int: + """Persist objects for a run created by auto-approve logic. + + For VSPC Active Alarms summary, objects are stored on the mail_message with + a " | " prefix. Auto-approved runs are created per-company + job ("Active alarms summary | "). In that case we persist only the + matching subset and strip the prefix so objects are correctly linked. + """ + + try: + # Lazy import to avoid circular dependencies. + from .models import Job # noqa + + job = Job.query.get(int(job_id)) + if not job: + return persist_objects_for_approved_run(customer_id, job_id, run_id, mail_message_id) + + bsw = (getattr(job, "backup_software", "") or "").strip().lower() + btype = (getattr(job, "backup_type", "") or "").strip().lower() + jname = (getattr(job, "job_name", "") or "").strip() + + if bsw == "veeam" and btype == "service provider console": + # Expected format: "Active alarms summary | " + parts = [p.strip() for p in jname.split("|", 1)] + if len(parts) == 2 and parts[0].strip().lower() == "active alarms summary" and parts[1]: + company = parts[1] + return persist_objects_for_approved_run_filtered( + customer_id, + job_id, + run_id, + mail_message_id, + object_name_prefix=company, + strip_prefix=True, + ) + + except Exception: + # Fall back to the generic behavior. + pass + + return persist_objects_for_approved_run(customer_id, job_id, run_id, mail_message_id) + diff --git a/containers/backupchecks/src/backend/app/parsers/veeam.py b/containers/backupchecks/src/backend/app/parsers/veeam.py index 92ee4a9..c3437b5 100644 --- a/containers/backupchecks/src/backend/app/parsers/veeam.py +++ b/containers/backupchecks/src/backend/app/parsers/veeam.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import html as _html from typing import Dict, Tuple, List, Optional from ..models import MailMessage @@ -22,6 +23,54 @@ VEEAM_BACKUP_TYPES = [ ] +def normalize_vspc_company_name(name: str) -> str: + """Normalize a VSPC company name so it matches across HTML/text extraction and parsing.""" + n = _strip_html_tags(name or "") + n = _html.unescape(n) + n = n.replace("\xa0", " ") + n = re.sub(r"\s+", " ", n).strip() + return n + + +def extract_vspc_active_alarms_companies(raw: str) -> List[str]: + """Best-effort extraction of company names from VSPC "Active alarms summary" bodies. + + Only returns companies with alarms > 0. + """ + if not raw: + return [] + + txt = raw + if "<" in txt and ">" in txt: + txt = re.sub(r"<[^>]+>", " ", txt) + txt = _html.unescape(txt) + txt = txt.replace("\xa0", " ") + txt = re.sub(r"\s+", " ", txt).strip() + + seen: set[str] = set() + out: List[str] = [] + + for m in re.finditer( + r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)", + txt, + flags=re.IGNORECASE, + ): + cname = normalize_vspc_company_name((m.group(1) or "").strip()) + try: + alarms = int(m.group(2)) + except Exception: + alarms = 0 + + if not cname or alarms <= 0: + continue + if cname in seen: + continue + seen.add(cname) + out.append(cname) + + return out + + def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Optional[str]]: """Parse Veeam Service Provider Console (VSPC) Active Alarms summary emails. @@ -57,6 +106,7 @@ def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Opt saw_warning = False for idx, (h_start, h_end, company_name, alarms_raw) in enumerate(headers): + company_name = normalize_vspc_company_name(company_name) seg_start = h_end seg_end = headers[idx + 1][0] if idx + 1 < len(headers) else len(html) segment_html = html[seg_start:seg_end] @@ -178,6 +228,44 @@ def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Opt return objects, overall_status, overall_message +def extract_vspc_active_alarms_companies(raw: str) -> List[str]: + """Extract company names with alarms > 0 from a VSPC Active Alarms summary body.""" + if not raw: + return [] + + txt = raw + if "<" in txt and ">" in txt: + txt = re.sub(r"<[^>]+>", " ", txt) + txt = _html.unescape(txt) + txt = txt.replace("\xa0", " ") + txt = re.sub(r"\s+", " ", txt).strip() + + seen: set[str] = set() + out: List[str] = [] + + for m in re.finditer( + r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)", + txt, + flags=re.IGNORECASE, + ): + cname = (m.group(1) or "").strip() + cname = cname.replace("\xa0", " ") + cname = re.sub(r"\s+", " ", cname).strip() + try: + alarms = int(m.group(2)) + except Exception: + alarms = 0 + + if not cname or alarms <= 0: + continue + if cname in seen: + continue + seen.add(cname) + out.append(cname) + + return out + + def _parse_cloud_connect_report_from_html(html: str) -> Tuple[List[Dict], str]: """Parse Veeam Cloud Connect daily report (provider) HTML. diff --git a/docs/changelog.md b/docs/changelog.md index d60086f..3f9c492 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -51,6 +51,17 @@ - Corrected backend deletion logic to prevent exceptions during job removal. - Ensured related records are handled safely to avoid constraint or reference errors. +--- + +## v20260113-08-vspc-object-linking +- Fixed VSPC company name normalization so company detection and object prefixing match consistently. +- Fixed filtered object persistence to respect UNIQUE(customer_id, object_name) and to update last_seen timestamps correctly. +- Added auto object persistence routing for VSPC per-company runs so objects are linked to the correct customer/job (prefix stripped). +- Improved auto-approval for VSPC Active Alarms summary: + - Creates per-company runs automatically when company jobs are mapped (new imports and inbox re-parse). + - Uses case-insensitive matching for " | " mail objects. +- Added best-effort retroactive processing after approving VSPC company mappings to automatically link older inbox messages that are now fully mapped. + *** ## v0.1.20