Auto-commit local changes before build (2026-01-13 15:07:59) #113

Merged
ivooskamp merged 1 commits from v20260113-08-vspc-object-linking into main 2026-01-13 16:45:38 +01:00
9 changed files with 466 additions and 59 deletions
Showing only changes of commit 934a495867 - Show all commits

View File

@ -1 +1 @@
v20260113-07-job-delete-fix v20260113-08-vspc-object-linking

View File

@ -7,7 +7,7 @@ from datetime import datetime
from .admin_logging import log_admin_event from .admin_logging import log_admin_event
from .mail_importer import MailImportError, run_auto_import from .mail_importer import MailImportError, run_auto_import
from .models import SystemSettings from .models import SystemSettings
from .object_persistence import persist_objects_for_approved_run from .object_persistence import persist_objects_for_auto_run
_AUTO_IMPORTER_THREAD_NAME = "auto_importer" _AUTO_IMPORTER_THREAD_NAME = "auto_importer"
@ -80,7 +80,7 @@ def start_auto_importer(app) -> None:
persisted_errors = 0 persisted_errors = 0
for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs: for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs:
try: try:
persisted_objects += persist_objects_for_approved_run( persisted_objects += persist_objects_for_auto_run(
int(customer_id), int(job_id), int(run_id), int(mail_message_id) int(customer_id), int(job_id), int(run_id), int(mail_message_id)
) )
except Exception as exc: except Exception as exc:

View File

@ -11,8 +11,9 @@ import requests
from sqlalchemy import func from sqlalchemy import func
from . import db from . import db
from .models import MailMessage, SystemSettings, Job, JobRun from .models import MailMessage, SystemSettings, Job, JobRun, MailObject
from .parsers import parse_mail_message from .parsers import parse_mail_message
from .parsers.veeam import extract_vspc_active_alarms_companies
from .email_utils import normalize_from_address, extract_best_html_from_eml, is_effectively_blank_html from .email_utils import normalize_from_address, extract_best_html_from_eml, is_effectively_blank_html
from .job_matching import find_matching_job from .job_matching import find_matching_job
@ -265,6 +266,94 @@ def _store_messages(settings: SystemSettings, messages):
and getattr(mail, "parse_result", None) == "ok" and getattr(mail, "parse_result", None) == "ok"
and not bool(getattr(mail, "approved", False)) and not bool(getattr(mail, "approved", False))
): ):
# Special case: Veeam VSPC "Active alarms summary" contains multiple companies.
bsw = (getattr(mail, "backup_software", "") or "").strip().lower()
btype = (getattr(mail, "backup_type", "") or "").strip().lower()
jname = (getattr(mail, "job_name", "") or "").strip().lower()
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
raw = (mail.text_body or "").strip() or (mail.html_body or "")
companies = extract_vspc_active_alarms_companies(raw)
if companies:
def _is_error_status(value: str | None) -> bool:
v = (value or "").strip().lower()
return v in {"error", "failed", "critical"} or v.startswith("fail")
created_any = False
first_job = None
mapped_count = 0
for company in companies:
# Build a temp message using the per-company job name
tmp = MailMessage(
from_address=mail.from_address,
backup_software=mail.backup_software,
backup_type=mail.backup_type,
job_name=f"{(mail.job_name or 'Active alarms summary').strip()} | {company}".strip(),
)
job = find_matching_job(tmp)
if not job:
continue
# Respect per-job flags.
if hasattr(job, "active") and not bool(job.active):
continue
if hasattr(job, "auto_approve") and not bool(job.auto_approve):
continue
mapped_count += 1
objs = (
MailObject.query.filter(MailObject.mail_message_id == mail.id)
.filter(MailObject.object_name.ilike(f"{company} | %"))
.all()
)
saw_error = any(_is_error_status(o.status) for o in objs)
saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs)
status = "Error" if saw_error else ("Warning" if saw_warning else (mail.overall_status or "Success"))
run = JobRun(
job_id=job.id,
mail_message_id=mail.id,
run_at=mail.received_at,
status=status or None,
missed=False,
)
# Optional storage metrics
if hasattr(run, "storage_used_bytes") and hasattr(mail, "storage_used_bytes"):
run.storage_used_bytes = mail.storage_used_bytes
if hasattr(run, "storage_capacity_bytes") and hasattr(mail, "storage_capacity_bytes"):
run.storage_capacity_bytes = mail.storage_capacity_bytes
if hasattr(run, "storage_free_bytes") and hasattr(mail, "storage_free_bytes"):
run.storage_free_bytes = mail.storage_free_bytes
if hasattr(run, "storage_free_percent") and hasattr(mail, "storage_free_percent"):
run.storage_free_percent = mail.storage_free_percent
db.session.add(run)
db.session.flush()
auto_approved_runs.append((job.customer_id, job.id, run.id, mail.id))
created_any = True
if not first_job:
first_job = job
# If all companies are mapped, mark the mail as fully approved and move to history.
if created_any and mapped_count == len(companies):
mail.job_id = first_job.id if first_job else None
if hasattr(mail, "approved"):
mail.approved = True
if hasattr(mail, "approved_at"):
mail.approved_at = datetime.utcnow()
if hasattr(mail, "location"):
mail.location = "history"
auto_approved += 1
# Do not fall back to single-job matching for VSPC summary.
continue
job = find_matching_job(mail) job = find_matching_job(mail)
if job: if job:
# Respect per-job flags. # Respect per-job flags.

View File

@ -2,50 +2,14 @@ from .routes_shared import * # noqa: F401,F403
from .routes_shared import _format_datetime, _log_admin_event, _send_mail_message_eml_download from .routes_shared import _format_datetime, _log_admin_event, _send_mail_message_eml_download
from ..email_utils import extract_best_html_from_eml, is_effectively_blank_html from ..email_utils import extract_best_html_from_eml, is_effectively_blank_html
from ..parsers.veeam import extract_vspc_active_alarms_companies
from ..models import MailObject
import time import time
import re import re
import html as _html import html as _html
def _extract_vspc_active_alarms_companies(raw: str) -> list[str]:
"""Best-effort extraction of company names from VSPC "Active alarms summary" bodies.
Only returns companies with alarms > 0, as the email is expected to list failing customers.
"""
if not raw:
return []
txt = raw
# Best-effort HTML to text
if "<" in txt and ">" in txt:
txt = re.sub(r"<[^>]+>", " ", txt)
txt = _html.unescape(txt)
txt = re.sub(r"\s+", " ", txt).strip()
seen: set[str] = set()
out: list[str] = []
for m in re.finditer(
r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)",
txt,
flags=re.IGNORECASE,
):
cname = (m.group(1) or "").strip()
try:
alarms = int(m.group(2))
except Exception:
alarms = 0
if not cname or alarms <= 0:
continue
if cname in seen:
continue
seen.add(cname)
out.append(cname)
return out
@main_bp.route("/inbox") @main_bp.route("/inbox")
@login_required @login_required
@roles_required("admin", "operator", "viewer") @roles_required("admin", "operator", "viewer")
@ -201,7 +165,7 @@ def inbox_message_detail(message_id: int):
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary": if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
raw = text_body if not _is_blank_text(text_body) else (html_body or "") raw = text_body if not _is_blank_text(text_body) else (html_body or "")
vspc_companies = _extract_vspc_active_alarms_companies(raw) vspc_companies = extract_vspc_active_alarms_companies(raw)
# For each company, prefill the UI with the existing customer mapping if we already have a job for it. # For each company, prefill the UI with the existing customer mapping if we already have a job for it.
# This avoids re-mapping known companies and keeps the message actionable in the Inbox. # This avoids re-mapping known companies and keeps the message actionable in the Inbox.
@ -402,7 +366,7 @@ def inbox_message_approve_vspc_companies(message_id: int):
html_body = getattr(msg, "html_body", None) html_body = getattr(msg, "html_body", None)
text_body = getattr(msg, "text_body", None) text_body = getattr(msg, "text_body", None)
raw_for_companies = text_body if (text_body and str(text_body).strip()) else (html_body or "") raw_for_companies = text_body if (text_body and str(text_body).strip()) else (html_body or "")
companies_present = _extract_vspc_active_alarms_companies(raw_for_companies) companies_present = extract_vspc_active_alarms_companies(raw_for_companies)
if not companies_present: if not companies_present:
flash("No companies could be detected in this VSPC summary email.", "danger") flash("No companies could be detected in this VSPC summary email.", "danger")
@ -550,7 +514,7 @@ def inbox_message_approve_vspc_companies(message_id: int):
objs = ( objs = (
MailObject.query.filter(MailObject.mail_message_id == msg.id) MailObject.query.filter(MailObject.mail_message_id == msg.id)
.filter(MailObject.object_name.like(f"{company} | %")) .filter(MailObject.object_name.ilike(f"{company} | %"))
.all() .all()
) )
saw_error = any(_is_error_status(o.status) for o in objs) saw_error = any(_is_error_status(o.status) for o in objs)
@ -641,9 +605,130 @@ def inbox_message_approve_vspc_companies(message_id: int):
_log_admin_event("inbox_approve_error", f"Failed to finalize VSPC approval for message {msg.id}: {exc}") _log_admin_event("inbox_approve_error", f"Failed to finalize VSPC approval for message {msg.id}: {exc}")
return redirect(url_for("main.inbox")) return redirect(url_for("main.inbox"))
# Best-effort: now that company jobs are mapped, auto-approve other inbox
# messages of the same VSPC summary type whose companies are now all mapped.
retro_approved_msgs = 0
try:
q = MailMessage.query
if hasattr(MailMessage, "location"):
q = q.filter(MailMessage.location == "inbox")
q = q.filter(MailMessage.parse_result == "ok")
q = q.filter(MailMessage.job_id.is_(None))
q = q.filter(MailMessage.backup_software == "Veeam")
q = q.filter(MailMessage.backup_type == "Service Provider Console")
q = q.filter(MailMessage.job_name == (msg.job_name or "Active alarms summary"))
q = q.filter(MailMessage.id != msg.id)
candidates = q.order_by(MailMessage.received_at.desc().nullslast(), MailMessage.id.desc()).limit(25).all()
for other in candidates:
nested = db.session.begin_nested()
try:
raw_other = (other.text_body or "").strip() or (other.html_body or "")
companies = extract_vspc_active_alarms_companies(raw_other)
if not companies:
nested.commit()
continue
jobs_by_company: dict[str, Job] = {}
all_mapped = True
for company in companies:
norm_from, store_backup, store_type, _store_job = build_job_match_key(other)
company_job_name = f"{(other.job_name or 'Active alarms summary').strip()} | {company}".strip()
tmp = MailMessage(
from_address=norm_from,
backup_software=store_backup,
backup_type=store_type,
job_name=company_job_name,
)
with db.session.no_autoflush:
j = find_matching_job(tmp)
if not j or not getattr(j, "customer_id", None):
all_mapped = False
break
if hasattr(j, "active") and not bool(j.active):
all_mapped = False
break
if hasattr(j, "auto_approve") and not bool(j.auto_approve):
all_mapped = False
break
jobs_by_company[company] = j
if not all_mapped:
nested.commit()
continue
first_job2: Job | None = None
for company, job2 in jobs_by_company.items():
if not first_job2:
first_job2 = job2
objs2 = (
MailObject.query.filter(MailObject.mail_message_id == other.id)
.filter(MailObject.object_name.ilike(f"{company} | %"))
.all()
)
saw_error2 = any(_is_error_status(o.status) for o in objs2)
saw_warning2 = any((o.status or "").strip().lower() == "warning" for o in objs2)
status2 = "Error" if saw_error2 else ("Warning" if saw_warning2 else (other.overall_status or "Success"))
run2 = JobRun.query.filter(JobRun.job_id == job2.id, JobRun.mail_message_id == other.id).first()
if not run2:
run2 = JobRun(
job_id=job2.id,
mail_message_id=other.id,
run_at=(other.received_at or getattr(other, "parsed_at", None) or datetime.utcnow()),
status=status2 or None,
missed=False,
)
if hasattr(run2, "remark"):
run2.remark = getattr(other, "overall_message", None)
db.session.add(run2)
db.session.flush()
# Persist objects per company
try:
persist_objects_for_approved_run_filtered(
int(job2.customer_id),
int(job2.id),
int(run2.id),
int(other.id),
object_name_prefix=company,
strip_prefix=True,
)
except Exception as exc:
_log_admin_event(
"object_persist_error",
f"Filtered object persistence failed for message {other.id} (company '{company}', job {job2.id}, run {run2.id}): {exc}",
)
other.job_id = first_job2.id if first_job2 else None
if hasattr(other, "approved"):
other.approved = True
if hasattr(other, "approved_at"):
other.approved_at = datetime.utcnow()
if hasattr(other, "approved_by_id"):
other.approved_by_id = current_user.id
if hasattr(other, "location"):
other.location = "history"
nested.commit()
retro_approved_msgs += 1
except Exception:
try:
nested.rollback()
except Exception:
db.session.rollback()
db.session.commit()
except Exception:
try:
db.session.rollback()
except Exception:
pass
_log_admin_event( _log_admin_event(
"inbox_approve_vspc", "inbox_approve_vspc",
f"Approved VSPC message {msg.id} into {processed_total} run(s) (job_id={msg.job_id})", f"Approved VSPC message {msg.id} into {processed_total} run(s) (job_id={msg.job_id}), retro_approved={retro_approved_msgs}",
) )
flash(f"Approved VSPC summary into {processed_total} run(s).", "success") flash(f"Approved VSPC summary into {processed_total} run(s).", "success")
return redirect(url_for("main.inbox")) return redirect(url_for("main.inbox"))
@ -897,6 +982,95 @@ def inbox_reparse_all():
and getattr(msg, "parse_result", None) == "ok" and getattr(msg, "parse_result", None) == "ok"
and getattr(msg, "job_id", None) is None and getattr(msg, "job_id", None) is None
): ):
# Special case: VSPC Active Alarms summary can contain multiple companies.
bsw = (getattr(msg, "backup_software", "") or "").strip().lower()
btype = (getattr(msg, "backup_type", "") or "").strip().lower()
jname = (getattr(msg, "job_name", "") or "").strip().lower()
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
raw = (getattr(msg, "text_body", None) or "").strip() or (getattr(msg, "html_body", None) or "")
companies = extract_vspc_active_alarms_companies(raw)
if companies:
def _is_error_status(value: str | None) -> bool:
v = (value or "").strip().lower()
return v in {"error", "failed", "critical"} or v.startswith("fail")
first_job = None
mapped_count = 0
created_any = False
for company in companies:
tmp_msg = MailMessage(
from_address=msg.from_address,
backup_software=msg.backup_software,
backup_type=msg.backup_type,
job_name=f"{(msg.job_name or 'Active alarms summary').strip()} | {company}".strip(),
)
with db.session.no_autoflush:
job = find_matching_job(tmp_msg)
if not job:
continue
if hasattr(job, "active") and not bool(job.active):
continue
if hasattr(job, "auto_approve") and not bool(job.auto_approve):
continue
mapped_count += 1
objs = (
MailObject.query.filter(MailObject.mail_message_id == msg.id)
.filter(MailObject.object_name.ilike(f"{company} | %"))
.all()
)
saw_error = any(_is_error_status(o.status) for o in objs)
saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs)
status = "Error" if saw_error else ("Warning" if saw_warning else (msg.overall_status or "Success"))
run = JobRun(
job_id=job.id,
mail_message_id=msg.id,
run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()),
status=status or None,
missed=False,
)
if hasattr(run, "remark"):
run.remark = getattr(msg, "overall_message", None)
if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"):
run.storage_used_bytes = msg.storage_used_bytes
if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"):
run.storage_capacity_bytes = msg.storage_capacity_bytes
if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"):
run.storage_free_bytes = msg.storage_free_bytes
if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"):
run.storage_free_percent = msg.storage_free_percent
db.session.add(run)
db.session.flush()
auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id))
created_any = True
if not first_job:
first_job = job
if created_any and mapped_count == len(companies):
msg.job_id = first_job.id if first_job else None
if hasattr(msg, "approved"):
msg.approved = True
if hasattr(msg, "approved_at"):
msg.approved_at = datetime.utcnow()
if hasattr(msg, "approved_by_id"):
msg.approved_by_id = None
if hasattr(msg, "location"):
msg.location = "history"
auto_approved += 1
# Do not fall back to single-job matching for VSPC summary.
continue
# Match approved job on: From + Backup + Type + Job name # Match approved job on: From + Backup + Type + Job name
# Prevent session autoflush for every match lookup while we # Prevent session autoflush for every match lookup while we
# are still updating many messages in a loop. # are still updating many messages in a loop.
@ -1077,7 +1251,7 @@ def inbox_reparse_all():
persisted_errors = 0 persisted_errors = 0
for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs: for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs:
try: try:
persisted_objects += persist_objects_for_approved_run( persisted_objects += persist_objects_for_auto_run(
customer_id, job_id, run_id, mail_message_id customer_id, job_id, run_id, mail_message_id
) )
except Exception as exc: except Exception as exc:

View File

@ -169,7 +169,7 @@ def settings_objects_backfill():
for r in rows: for r in rows:
try: try:
repaired_objects += persist_objects_for_approved_run( repaired_objects += persist_objects_for_auto_run(
int(r[2]), int(r[1]), int(r[0]), int(r[3]) int(r[2]), int(r[1]), int(r[0]), int(r[3])
) )
repaired_runs += 1 repaired_runs += 1
@ -1057,7 +1057,7 @@ def settings_mail_import():
persisted_errors = 0 persisted_errors = 0
for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs: for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs:
try: try:
persisted_objects += persist_objects_for_approved_run( persisted_objects += persist_objects_for_auto_run(
int(customer_id), int(job_id), int(run_id), int(mail_message_id) int(customer_id), int(job_id), int(run_id), int(mail_message_id)
) )
except Exception as exc: except Exception as exc:

View File

@ -60,7 +60,7 @@ from ..models import (
) )
from ..mail_importer import run_manual_import, MailImportError from ..mail_importer import run_manual_import, MailImportError
from ..parsers import parse_mail_message from ..parsers import parse_mail_message
from ..object_persistence import persist_objects_for_approved_run from ..object_persistence import persist_objects_for_approved_run, persist_objects_for_auto_run
main_bp = Blueprint("main", __name__) main_bp = Blueprint("main", __name__)

View File

@ -195,14 +195,16 @@ def persist_objects_for_approved_run_filtered(
status = r[2] status = r[2]
error_message = r[3] error_message = r[3]
# 1) Upsert customer_objects and get id # 1) Upsert customer_objects and get id (schema uses UNIQUE(customer_id, object_name))
customer_object_id = conn.execute( customer_object_id = conn.execute(
text( text(
""" """
INSERT INTO customer_objects (customer_id, object_name, object_type) INSERT INTO customer_objects (customer_id, object_name, object_type, first_seen_at, last_seen_at)
VALUES (:customer_id, :object_name, :object_type) VALUES (:customer_id, :object_name, :object_type, NOW(), NOW())
ON CONFLICT (customer_id, object_name, COALESCE(object_type, '')) ON CONFLICT (customer_id, object_name)
DO UPDATE SET object_type = EXCLUDED.object_type DO UPDATE SET
last_seen_at = NOW(),
object_type = COALESCE(EXCLUDED.object_type, customer_objects.object_type)
RETURNING id RETURNING id
""" """
), ),
@ -213,13 +215,14 @@ def persist_objects_for_approved_run_filtered(
}, },
).scalar() ).scalar()
# 2) Upsert job_object_links # 2) Upsert job_object_links (keep timestamps fresh)
conn.execute( conn.execute(
text( text(
""" """
INSERT INTO job_object_links (job_id, customer_object_id) INSERT INTO job_object_links (job_id, customer_object_id, first_seen_at, last_seen_at)
VALUES (:job_id, :customer_object_id) VALUES (:job_id, :customer_object_id, NOW(), NOW())
ON CONFLICT (job_id, customer_object_id) DO NOTHING ON CONFLICT (job_id, customer_object_id)
DO UPDATE SET last_seen_at = NOW()
""" """
), ),
{ {
@ -254,3 +257,45 @@ def persist_objects_for_approved_run_filtered(
_update_override_applied_for_run(job_id, run_id) _update_override_applied_for_run(job_id, run_id)
return processed return processed
def persist_objects_for_auto_run(customer_id: int, job_id: int, run_id: int, mail_message_id: int) -> int:
"""Persist objects for a run created by auto-approve logic.
For VSPC Active Alarms summary, objects are stored on the mail_message with
a "<company> | <object>" prefix. Auto-approved runs are created per-company
job ("Active alarms summary | <company>"). In that case we persist only the
matching subset and strip the prefix so objects are correctly linked.
"""
try:
# Lazy import to avoid circular dependencies.
from .models import Job # noqa
job = Job.query.get(int(job_id))
if not job:
return persist_objects_for_approved_run(customer_id, job_id, run_id, mail_message_id)
bsw = (getattr(job, "backup_software", "") or "").strip().lower()
btype = (getattr(job, "backup_type", "") or "").strip().lower()
jname = (getattr(job, "job_name", "") or "").strip()
if bsw == "veeam" and btype == "service provider console":
# Expected format: "Active alarms summary | <company>"
parts = [p.strip() for p in jname.split("|", 1)]
if len(parts) == 2 and parts[0].strip().lower() == "active alarms summary" and parts[1]:
company = parts[1]
return persist_objects_for_approved_run_filtered(
customer_id,
job_id,
run_id,
mail_message_id,
object_name_prefix=company,
strip_prefix=True,
)
except Exception:
# Fall back to the generic behavior.
pass
return persist_objects_for_approved_run(customer_id, job_id, run_id, mail_message_id)

View File

@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import re import re
import html as _html
from typing import Dict, Tuple, List, Optional from typing import Dict, Tuple, List, Optional
from ..models import MailMessage from ..models import MailMessage
@ -22,6 +23,54 @@ VEEAM_BACKUP_TYPES = [
] ]
def normalize_vspc_company_name(name: str) -> str:
"""Normalize a VSPC company name so it matches across HTML/text extraction and parsing."""
n = _strip_html_tags(name or "")
n = _html.unescape(n)
n = n.replace("\xa0", " ")
n = re.sub(r"\s+", " ", n).strip()
return n
def extract_vspc_active_alarms_companies(raw: str) -> List[str]:
"""Best-effort extraction of company names from VSPC "Active alarms summary" bodies.
Only returns companies with alarms > 0.
"""
if not raw:
return []
txt = raw
if "<" in txt and ">" in txt:
txt = re.sub(r"<[^>]+>", " ", txt)
txt = _html.unescape(txt)
txt = txt.replace("\xa0", " ")
txt = re.sub(r"\s+", " ", txt).strip()
seen: set[str] = set()
out: List[str] = []
for m in re.finditer(
r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)",
txt,
flags=re.IGNORECASE,
):
cname = normalize_vspc_company_name((m.group(1) or "").strip())
try:
alarms = int(m.group(2))
except Exception:
alarms = 0
if not cname or alarms <= 0:
continue
if cname in seen:
continue
seen.add(cname)
out.append(cname)
return out
def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Optional[str]]: def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Optional[str]]:
"""Parse Veeam Service Provider Console (VSPC) Active Alarms summary emails. """Parse Veeam Service Provider Console (VSPC) Active Alarms summary emails.
@ -57,6 +106,7 @@ def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Opt
saw_warning = False saw_warning = False
for idx, (h_start, h_end, company_name, alarms_raw) in enumerate(headers): for idx, (h_start, h_end, company_name, alarms_raw) in enumerate(headers):
company_name = normalize_vspc_company_name(company_name)
seg_start = h_end seg_start = h_end
seg_end = headers[idx + 1][0] if idx + 1 < len(headers) else len(html) seg_end = headers[idx + 1][0] if idx + 1 < len(headers) else len(html)
segment_html = html[seg_start:seg_end] segment_html = html[seg_start:seg_end]
@ -178,6 +228,44 @@ def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Opt
return objects, overall_status, overall_message return objects, overall_status, overall_message
def extract_vspc_active_alarms_companies(raw: str) -> List[str]:
"""Extract company names with alarms > 0 from a VSPC Active Alarms summary body."""
if not raw:
return []
txt = raw
if "<" in txt and ">" in txt:
txt = re.sub(r"<[^>]+>", " ", txt)
txt = _html.unescape(txt)
txt = txt.replace("\xa0", " ")
txt = re.sub(r"\s+", " ", txt).strip()
seen: set[str] = set()
out: List[str] = []
for m in re.finditer(
r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)",
txt,
flags=re.IGNORECASE,
):
cname = (m.group(1) or "").strip()
cname = cname.replace("\xa0", " ")
cname = re.sub(r"\s+", " ", cname).strip()
try:
alarms = int(m.group(2))
except Exception:
alarms = 0
if not cname or alarms <= 0:
continue
if cname in seen:
continue
seen.add(cname)
out.append(cname)
return out
def _parse_cloud_connect_report_from_html(html: str) -> Tuple[List[Dict], str]: def _parse_cloud_connect_report_from_html(html: str) -> Tuple[List[Dict], str]:
"""Parse Veeam Cloud Connect daily report (provider) HTML. """Parse Veeam Cloud Connect daily report (provider) HTML.

View File

@ -51,6 +51,17 @@
- Corrected backend deletion logic to prevent exceptions during job removal. - Corrected backend deletion logic to prevent exceptions during job removal.
- Ensured related records are handled safely to avoid constraint or reference errors. - Ensured related records are handled safely to avoid constraint or reference errors.
---
## v20260113-08-vspc-object-linking
- Fixed VSPC company name normalization so company detection and object prefixing match consistently.
- Fixed filtered object persistence to respect UNIQUE(customer_id, object_name) and to update last_seen timestamps correctly.
- Added auto object persistence routing for VSPC per-company runs so objects are linked to the correct customer/job (prefix stripped).
- Improved auto-approval for VSPC Active Alarms summary:
- Creates per-company runs automatically when company jobs are mapped (new imports and inbox re-parse).
- Uses case-insensitive matching for "<company> | <object>" mail objects.
- Added best-effort retroactive processing after approving VSPC company mappings to automatically link older inbox messages that are now fully mapped.
*** ***
## v0.1.20 ## v0.1.20