Auto-commit local changes before build (2026-01-13 15:07:59) #113
@ -1 +1 @@
|
||||
v20260113-07-job-delete-fix
|
||||
v20260113-08-vspc-object-linking
|
||||
|
||||
@ -7,7 +7,7 @@ from datetime import datetime
|
||||
from .admin_logging import log_admin_event
|
||||
from .mail_importer import MailImportError, run_auto_import
|
||||
from .models import SystemSettings
|
||||
from .object_persistence import persist_objects_for_approved_run
|
||||
from .object_persistence import persist_objects_for_auto_run
|
||||
|
||||
|
||||
_AUTO_IMPORTER_THREAD_NAME = "auto_importer"
|
||||
@ -80,7 +80,7 @@ def start_auto_importer(app) -> None:
|
||||
persisted_errors = 0
|
||||
for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs:
|
||||
try:
|
||||
persisted_objects += persist_objects_for_approved_run(
|
||||
persisted_objects += persist_objects_for_auto_run(
|
||||
int(customer_id), int(job_id), int(run_id), int(mail_message_id)
|
||||
)
|
||||
except Exception as exc:
|
||||
|
||||
@ -11,8 +11,9 @@ import requests
|
||||
from sqlalchemy import func
|
||||
|
||||
from . import db
|
||||
from .models import MailMessage, SystemSettings, Job, JobRun
|
||||
from .models import MailMessage, SystemSettings, Job, JobRun, MailObject
|
||||
from .parsers import parse_mail_message
|
||||
from .parsers.veeam import extract_vspc_active_alarms_companies
|
||||
from .email_utils import normalize_from_address, extract_best_html_from_eml, is_effectively_blank_html
|
||||
from .job_matching import find_matching_job
|
||||
|
||||
@ -265,6 +266,94 @@ def _store_messages(settings: SystemSettings, messages):
|
||||
and getattr(mail, "parse_result", None) == "ok"
|
||||
and not bool(getattr(mail, "approved", False))
|
||||
):
|
||||
# Special case: Veeam VSPC "Active alarms summary" contains multiple companies.
|
||||
bsw = (getattr(mail, "backup_software", "") or "").strip().lower()
|
||||
btype = (getattr(mail, "backup_type", "") or "").strip().lower()
|
||||
jname = (getattr(mail, "job_name", "") or "").strip().lower()
|
||||
|
||||
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
|
||||
raw = (mail.text_body or "").strip() or (mail.html_body or "")
|
||||
companies = extract_vspc_active_alarms_companies(raw)
|
||||
|
||||
if companies:
|
||||
def _is_error_status(value: str | None) -> bool:
|
||||
v = (value or "").strip().lower()
|
||||
return v in {"error", "failed", "critical"} or v.startswith("fail")
|
||||
|
||||
created_any = False
|
||||
first_job = None
|
||||
mapped_count = 0
|
||||
|
||||
for company in companies:
|
||||
# Build a temp message using the per-company job name
|
||||
tmp = MailMessage(
|
||||
from_address=mail.from_address,
|
||||
backup_software=mail.backup_software,
|
||||
backup_type=mail.backup_type,
|
||||
job_name=f"{(mail.job_name or 'Active alarms summary').strip()} | {company}".strip(),
|
||||
)
|
||||
job = find_matching_job(tmp)
|
||||
if not job:
|
||||
continue
|
||||
|
||||
# Respect per-job flags.
|
||||
if hasattr(job, "active") and not bool(job.active):
|
||||
continue
|
||||
if hasattr(job, "auto_approve") and not bool(job.auto_approve):
|
||||
continue
|
||||
|
||||
mapped_count += 1
|
||||
|
||||
objs = (
|
||||
MailObject.query.filter(MailObject.mail_message_id == mail.id)
|
||||
.filter(MailObject.object_name.ilike(f"{company} | %"))
|
||||
.all()
|
||||
)
|
||||
saw_error = any(_is_error_status(o.status) for o in objs)
|
||||
saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs)
|
||||
status = "Error" if saw_error else ("Warning" if saw_warning else (mail.overall_status or "Success"))
|
||||
|
||||
run = JobRun(
|
||||
job_id=job.id,
|
||||
mail_message_id=mail.id,
|
||||
run_at=mail.received_at,
|
||||
status=status or None,
|
||||
missed=False,
|
||||
)
|
||||
|
||||
# Optional storage metrics
|
||||
if hasattr(run, "storage_used_bytes") and hasattr(mail, "storage_used_bytes"):
|
||||
run.storage_used_bytes = mail.storage_used_bytes
|
||||
if hasattr(run, "storage_capacity_bytes") and hasattr(mail, "storage_capacity_bytes"):
|
||||
run.storage_capacity_bytes = mail.storage_capacity_bytes
|
||||
if hasattr(run, "storage_free_bytes") and hasattr(mail, "storage_free_bytes"):
|
||||
run.storage_free_bytes = mail.storage_free_bytes
|
||||
if hasattr(run, "storage_free_percent") and hasattr(mail, "storage_free_percent"):
|
||||
run.storage_free_percent = mail.storage_free_percent
|
||||
|
||||
db.session.add(run)
|
||||
db.session.flush()
|
||||
|
||||
auto_approved_runs.append((job.customer_id, job.id, run.id, mail.id))
|
||||
created_any = True
|
||||
|
||||
if not first_job:
|
||||
first_job = job
|
||||
|
||||
# If all companies are mapped, mark the mail as fully approved and move to history.
|
||||
if created_any and mapped_count == len(companies):
|
||||
mail.job_id = first_job.id if first_job else None
|
||||
if hasattr(mail, "approved"):
|
||||
mail.approved = True
|
||||
if hasattr(mail, "approved_at"):
|
||||
mail.approved_at = datetime.utcnow()
|
||||
if hasattr(mail, "location"):
|
||||
mail.location = "history"
|
||||
auto_approved += 1
|
||||
|
||||
# Do not fall back to single-job matching for VSPC summary.
|
||||
continue
|
||||
|
||||
job = find_matching_job(mail)
|
||||
if job:
|
||||
# Respect per-job flags.
|
||||
|
||||
@ -2,50 +2,14 @@ from .routes_shared import * # noqa: F401,F403
|
||||
from .routes_shared import _format_datetime, _log_admin_event, _send_mail_message_eml_download
|
||||
|
||||
from ..email_utils import extract_best_html_from_eml, is_effectively_blank_html
|
||||
from ..parsers.veeam import extract_vspc_active_alarms_companies
|
||||
from ..models import MailObject
|
||||
|
||||
import time
|
||||
import re
|
||||
import html as _html
|
||||
|
||||
|
||||
def _extract_vspc_active_alarms_companies(raw: str) -> list[str]:
|
||||
"""Best-effort extraction of company names from VSPC "Active alarms summary" bodies.
|
||||
|
||||
Only returns companies with alarms > 0, as the email is expected to list failing customers.
|
||||
"""
|
||||
if not raw:
|
||||
return []
|
||||
|
||||
txt = raw
|
||||
# Best-effort HTML to text
|
||||
if "<" in txt and ">" in txt:
|
||||
txt = re.sub(r"<[^>]+>", " ", txt)
|
||||
txt = _html.unescape(txt)
|
||||
txt = re.sub(r"\s+", " ", txt).strip()
|
||||
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
|
||||
for m in re.finditer(
|
||||
r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)",
|
||||
txt,
|
||||
flags=re.IGNORECASE,
|
||||
):
|
||||
cname = (m.group(1) or "").strip()
|
||||
try:
|
||||
alarms = int(m.group(2))
|
||||
except Exception:
|
||||
alarms = 0
|
||||
|
||||
if not cname or alarms <= 0:
|
||||
continue
|
||||
if cname in seen:
|
||||
continue
|
||||
seen.add(cname)
|
||||
out.append(cname)
|
||||
|
||||
return out
|
||||
|
||||
@main_bp.route("/inbox")
|
||||
@login_required
|
||||
@roles_required("admin", "operator", "viewer")
|
||||
@ -201,7 +165,7 @@ def inbox_message_detail(message_id: int):
|
||||
|
||||
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
|
||||
raw = text_body if not _is_blank_text(text_body) else (html_body or "")
|
||||
vspc_companies = _extract_vspc_active_alarms_companies(raw)
|
||||
vspc_companies = extract_vspc_active_alarms_companies(raw)
|
||||
|
||||
# For each company, prefill the UI with the existing customer mapping if we already have a job for it.
|
||||
# This avoids re-mapping known companies and keeps the message actionable in the Inbox.
|
||||
@ -402,7 +366,7 @@ def inbox_message_approve_vspc_companies(message_id: int):
|
||||
html_body = getattr(msg, "html_body", None)
|
||||
text_body = getattr(msg, "text_body", None)
|
||||
raw_for_companies = text_body if (text_body and str(text_body).strip()) else (html_body or "")
|
||||
companies_present = _extract_vspc_active_alarms_companies(raw_for_companies)
|
||||
companies_present = extract_vspc_active_alarms_companies(raw_for_companies)
|
||||
|
||||
if not companies_present:
|
||||
flash("No companies could be detected in this VSPC summary email.", "danger")
|
||||
@ -550,7 +514,7 @@ def inbox_message_approve_vspc_companies(message_id: int):
|
||||
|
||||
objs = (
|
||||
MailObject.query.filter(MailObject.mail_message_id == msg.id)
|
||||
.filter(MailObject.object_name.like(f"{company} | %"))
|
||||
.filter(MailObject.object_name.ilike(f"{company} | %"))
|
||||
.all()
|
||||
)
|
||||
saw_error = any(_is_error_status(o.status) for o in objs)
|
||||
@ -641,9 +605,130 @@ def inbox_message_approve_vspc_companies(message_id: int):
|
||||
_log_admin_event("inbox_approve_error", f"Failed to finalize VSPC approval for message {msg.id}: {exc}")
|
||||
return redirect(url_for("main.inbox"))
|
||||
|
||||
# Best-effort: now that company jobs are mapped, auto-approve other inbox
|
||||
# messages of the same VSPC summary type whose companies are now all mapped.
|
||||
retro_approved_msgs = 0
|
||||
try:
|
||||
q = MailMessage.query
|
||||
if hasattr(MailMessage, "location"):
|
||||
q = q.filter(MailMessage.location == "inbox")
|
||||
q = q.filter(MailMessage.parse_result == "ok")
|
||||
q = q.filter(MailMessage.job_id.is_(None))
|
||||
q = q.filter(MailMessage.backup_software == "Veeam")
|
||||
q = q.filter(MailMessage.backup_type == "Service Provider Console")
|
||||
q = q.filter(MailMessage.job_name == (msg.job_name or "Active alarms summary"))
|
||||
q = q.filter(MailMessage.id != msg.id)
|
||||
candidates = q.order_by(MailMessage.received_at.desc().nullslast(), MailMessage.id.desc()).limit(25).all()
|
||||
|
||||
for other in candidates:
|
||||
nested = db.session.begin_nested()
|
||||
try:
|
||||
raw_other = (other.text_body or "").strip() or (other.html_body or "")
|
||||
companies = extract_vspc_active_alarms_companies(raw_other)
|
||||
if not companies:
|
||||
nested.commit()
|
||||
continue
|
||||
|
||||
jobs_by_company: dict[str, Job] = {}
|
||||
all_mapped = True
|
||||
for company in companies:
|
||||
norm_from, store_backup, store_type, _store_job = build_job_match_key(other)
|
||||
company_job_name = f"{(other.job_name or 'Active alarms summary').strip()} | {company}".strip()
|
||||
tmp = MailMessage(
|
||||
from_address=norm_from,
|
||||
backup_software=store_backup,
|
||||
backup_type=store_type,
|
||||
job_name=company_job_name,
|
||||
)
|
||||
with db.session.no_autoflush:
|
||||
j = find_matching_job(tmp)
|
||||
if not j or not getattr(j, "customer_id", None):
|
||||
all_mapped = False
|
||||
break
|
||||
if hasattr(j, "active") and not bool(j.active):
|
||||
all_mapped = False
|
||||
break
|
||||
if hasattr(j, "auto_approve") and not bool(j.auto_approve):
|
||||
all_mapped = False
|
||||
break
|
||||
jobs_by_company[company] = j
|
||||
|
||||
if not all_mapped:
|
||||
nested.commit()
|
||||
continue
|
||||
|
||||
first_job2: Job | None = None
|
||||
for company, job2 in jobs_by_company.items():
|
||||
if not first_job2:
|
||||
first_job2 = job2
|
||||
|
||||
objs2 = (
|
||||
MailObject.query.filter(MailObject.mail_message_id == other.id)
|
||||
.filter(MailObject.object_name.ilike(f"{company} | %"))
|
||||
.all()
|
||||
)
|
||||
saw_error2 = any(_is_error_status(o.status) for o in objs2)
|
||||
saw_warning2 = any((o.status or "").strip().lower() == "warning" for o in objs2)
|
||||
status2 = "Error" if saw_error2 else ("Warning" if saw_warning2 else (other.overall_status or "Success"))
|
||||
|
||||
run2 = JobRun.query.filter(JobRun.job_id == job2.id, JobRun.mail_message_id == other.id).first()
|
||||
if not run2:
|
||||
run2 = JobRun(
|
||||
job_id=job2.id,
|
||||
mail_message_id=other.id,
|
||||
run_at=(other.received_at or getattr(other, "parsed_at", None) or datetime.utcnow()),
|
||||
status=status2 or None,
|
||||
missed=False,
|
||||
)
|
||||
if hasattr(run2, "remark"):
|
||||
run2.remark = getattr(other, "overall_message", None)
|
||||
db.session.add(run2)
|
||||
db.session.flush()
|
||||
|
||||
# Persist objects per company
|
||||
try:
|
||||
persist_objects_for_approved_run_filtered(
|
||||
int(job2.customer_id),
|
||||
int(job2.id),
|
||||
int(run2.id),
|
||||
int(other.id),
|
||||
object_name_prefix=company,
|
||||
strip_prefix=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
_log_admin_event(
|
||||
"object_persist_error",
|
||||
f"Filtered object persistence failed for message {other.id} (company '{company}', job {job2.id}, run {run2.id}): {exc}",
|
||||
)
|
||||
|
||||
other.job_id = first_job2.id if first_job2 else None
|
||||
if hasattr(other, "approved"):
|
||||
other.approved = True
|
||||
if hasattr(other, "approved_at"):
|
||||
other.approved_at = datetime.utcnow()
|
||||
if hasattr(other, "approved_by_id"):
|
||||
other.approved_by_id = current_user.id
|
||||
if hasattr(other, "location"):
|
||||
other.location = "history"
|
||||
|
||||
nested.commit()
|
||||
retro_approved_msgs += 1
|
||||
except Exception:
|
||||
try:
|
||||
nested.rollback()
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
try:
|
||||
db.session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
_log_admin_event(
|
||||
"inbox_approve_vspc",
|
||||
f"Approved VSPC message {msg.id} into {processed_total} run(s) (job_id={msg.job_id})",
|
||||
f"Approved VSPC message {msg.id} into {processed_total} run(s) (job_id={msg.job_id}), retro_approved={retro_approved_msgs}",
|
||||
)
|
||||
flash(f"Approved VSPC summary into {processed_total} run(s).", "success")
|
||||
return redirect(url_for("main.inbox"))
|
||||
@ -897,6 +982,95 @@ def inbox_reparse_all():
|
||||
and getattr(msg, "parse_result", None) == "ok"
|
||||
and getattr(msg, "job_id", None) is None
|
||||
):
|
||||
# Special case: VSPC Active Alarms summary can contain multiple companies.
|
||||
bsw = (getattr(msg, "backup_software", "") or "").strip().lower()
|
||||
btype = (getattr(msg, "backup_type", "") or "").strip().lower()
|
||||
jname = (getattr(msg, "job_name", "") or "").strip().lower()
|
||||
|
||||
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
|
||||
raw = (getattr(msg, "text_body", None) or "").strip() or (getattr(msg, "html_body", None) or "")
|
||||
companies = extract_vspc_active_alarms_companies(raw)
|
||||
|
||||
if companies:
|
||||
def _is_error_status(value: str | None) -> bool:
|
||||
v = (value or "").strip().lower()
|
||||
return v in {"error", "failed", "critical"} or v.startswith("fail")
|
||||
|
||||
first_job = None
|
||||
mapped_count = 0
|
||||
created_any = False
|
||||
|
||||
for company in companies:
|
||||
tmp_msg = MailMessage(
|
||||
from_address=msg.from_address,
|
||||
backup_software=msg.backup_software,
|
||||
backup_type=msg.backup_type,
|
||||
job_name=f"{(msg.job_name or 'Active alarms summary').strip()} | {company}".strip(),
|
||||
)
|
||||
with db.session.no_autoflush:
|
||||
job = find_matching_job(tmp_msg)
|
||||
|
||||
if not job:
|
||||
continue
|
||||
if hasattr(job, "active") and not bool(job.active):
|
||||
continue
|
||||
if hasattr(job, "auto_approve") and not bool(job.auto_approve):
|
||||
continue
|
||||
|
||||
mapped_count += 1
|
||||
|
||||
objs = (
|
||||
MailObject.query.filter(MailObject.mail_message_id == msg.id)
|
||||
.filter(MailObject.object_name.ilike(f"{company} | %"))
|
||||
.all()
|
||||
)
|
||||
saw_error = any(_is_error_status(o.status) for o in objs)
|
||||
saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs)
|
||||
status = "Error" if saw_error else ("Warning" if saw_warning else (msg.overall_status or "Success"))
|
||||
|
||||
run = JobRun(
|
||||
job_id=job.id,
|
||||
mail_message_id=msg.id,
|
||||
run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()),
|
||||
status=status or None,
|
||||
missed=False,
|
||||
)
|
||||
|
||||
if hasattr(run, "remark"):
|
||||
run.remark = getattr(msg, "overall_message", None)
|
||||
|
||||
if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"):
|
||||
run.storage_used_bytes = msg.storage_used_bytes
|
||||
if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"):
|
||||
run.storage_capacity_bytes = msg.storage_capacity_bytes
|
||||
if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"):
|
||||
run.storage_free_bytes = msg.storage_free_bytes
|
||||
if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"):
|
||||
run.storage_free_percent = msg.storage_free_percent
|
||||
|
||||
db.session.add(run)
|
||||
db.session.flush()
|
||||
auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id))
|
||||
created_any = True
|
||||
|
||||
if not first_job:
|
||||
first_job = job
|
||||
|
||||
if created_any and mapped_count == len(companies):
|
||||
msg.job_id = first_job.id if first_job else None
|
||||
if hasattr(msg, "approved"):
|
||||
msg.approved = True
|
||||
if hasattr(msg, "approved_at"):
|
||||
msg.approved_at = datetime.utcnow()
|
||||
if hasattr(msg, "approved_by_id"):
|
||||
msg.approved_by_id = None
|
||||
if hasattr(msg, "location"):
|
||||
msg.location = "history"
|
||||
auto_approved += 1
|
||||
|
||||
# Do not fall back to single-job matching for VSPC summary.
|
||||
continue
|
||||
|
||||
# Match approved job on: From + Backup + Type + Job name
|
||||
# Prevent session autoflush for every match lookup while we
|
||||
# are still updating many messages in a loop.
|
||||
@ -1077,7 +1251,7 @@ def inbox_reparse_all():
|
||||
persisted_errors = 0
|
||||
for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs:
|
||||
try:
|
||||
persisted_objects += persist_objects_for_approved_run(
|
||||
persisted_objects += persist_objects_for_auto_run(
|
||||
customer_id, job_id, run_id, mail_message_id
|
||||
)
|
||||
except Exception as exc:
|
||||
|
||||
@ -169,7 +169,7 @@ def settings_objects_backfill():
|
||||
|
||||
for r in rows:
|
||||
try:
|
||||
repaired_objects += persist_objects_for_approved_run(
|
||||
repaired_objects += persist_objects_for_auto_run(
|
||||
int(r[2]), int(r[1]), int(r[0]), int(r[3])
|
||||
)
|
||||
repaired_runs += 1
|
||||
@ -1057,7 +1057,7 @@ def settings_mail_import():
|
||||
persisted_errors = 0
|
||||
for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs:
|
||||
try:
|
||||
persisted_objects += persist_objects_for_approved_run(
|
||||
persisted_objects += persist_objects_for_auto_run(
|
||||
int(customer_id), int(job_id), int(run_id), int(mail_message_id)
|
||||
)
|
||||
except Exception as exc:
|
||||
|
||||
@ -60,7 +60,7 @@ from ..models import (
|
||||
)
|
||||
from ..mail_importer import run_manual_import, MailImportError
|
||||
from ..parsers import parse_mail_message
|
||||
from ..object_persistence import persist_objects_for_approved_run
|
||||
from ..object_persistence import persist_objects_for_approved_run, persist_objects_for_auto_run
|
||||
|
||||
|
||||
main_bp = Blueprint("main", __name__)
|
||||
|
||||
@ -195,14 +195,16 @@ def persist_objects_for_approved_run_filtered(
|
||||
status = r[2]
|
||||
error_message = r[3]
|
||||
|
||||
# 1) Upsert customer_objects and get id
|
||||
# 1) Upsert customer_objects and get id (schema uses UNIQUE(customer_id, object_name))
|
||||
customer_object_id = conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO customer_objects (customer_id, object_name, object_type)
|
||||
VALUES (:customer_id, :object_name, :object_type)
|
||||
ON CONFLICT (customer_id, object_name, COALESCE(object_type, ''))
|
||||
DO UPDATE SET object_type = EXCLUDED.object_type
|
||||
INSERT INTO customer_objects (customer_id, object_name, object_type, first_seen_at, last_seen_at)
|
||||
VALUES (:customer_id, :object_name, :object_type, NOW(), NOW())
|
||||
ON CONFLICT (customer_id, object_name)
|
||||
DO UPDATE SET
|
||||
last_seen_at = NOW(),
|
||||
object_type = COALESCE(EXCLUDED.object_type, customer_objects.object_type)
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
@ -213,13 +215,14 @@ def persist_objects_for_approved_run_filtered(
|
||||
},
|
||||
).scalar()
|
||||
|
||||
# 2) Upsert job_object_links
|
||||
# 2) Upsert job_object_links (keep timestamps fresh)
|
||||
conn.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO job_object_links (job_id, customer_object_id)
|
||||
VALUES (:job_id, :customer_object_id)
|
||||
ON CONFLICT (job_id, customer_object_id) DO NOTHING
|
||||
INSERT INTO job_object_links (job_id, customer_object_id, first_seen_at, last_seen_at)
|
||||
VALUES (:job_id, :customer_object_id, NOW(), NOW())
|
||||
ON CONFLICT (job_id, customer_object_id)
|
||||
DO UPDATE SET last_seen_at = NOW()
|
||||
"""
|
||||
),
|
||||
{
|
||||
@ -254,3 +257,45 @@ def persist_objects_for_approved_run_filtered(
|
||||
_update_override_applied_for_run(job_id, run_id)
|
||||
return processed
|
||||
|
||||
|
||||
def persist_objects_for_auto_run(customer_id: int, job_id: int, run_id: int, mail_message_id: int) -> int:
|
||||
"""Persist objects for a run created by auto-approve logic.
|
||||
|
||||
For VSPC Active Alarms summary, objects are stored on the mail_message with
|
||||
a "<company> | <object>" prefix. Auto-approved runs are created per-company
|
||||
job ("Active alarms summary | <company>"). In that case we persist only the
|
||||
matching subset and strip the prefix so objects are correctly linked.
|
||||
"""
|
||||
|
||||
try:
|
||||
# Lazy import to avoid circular dependencies.
|
||||
from .models import Job # noqa
|
||||
|
||||
job = Job.query.get(int(job_id))
|
||||
if not job:
|
||||
return persist_objects_for_approved_run(customer_id, job_id, run_id, mail_message_id)
|
||||
|
||||
bsw = (getattr(job, "backup_software", "") or "").strip().lower()
|
||||
btype = (getattr(job, "backup_type", "") or "").strip().lower()
|
||||
jname = (getattr(job, "job_name", "") or "").strip()
|
||||
|
||||
if bsw == "veeam" and btype == "service provider console":
|
||||
# Expected format: "Active alarms summary | <company>"
|
||||
parts = [p.strip() for p in jname.split("|", 1)]
|
||||
if len(parts) == 2 and parts[0].strip().lower() == "active alarms summary" and parts[1]:
|
||||
company = parts[1]
|
||||
return persist_objects_for_approved_run_filtered(
|
||||
customer_id,
|
||||
job_id,
|
||||
run_id,
|
||||
mail_message_id,
|
||||
object_name_prefix=company,
|
||||
strip_prefix=True,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
# Fall back to the generic behavior.
|
||||
pass
|
||||
|
||||
return persist_objects_for_approved_run(customer_id, job_id, run_id, mail_message_id)
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import html as _html
|
||||
from typing import Dict, Tuple, List, Optional
|
||||
|
||||
from ..models import MailMessage
|
||||
@ -22,6 +23,54 @@ VEEAM_BACKUP_TYPES = [
|
||||
]
|
||||
|
||||
|
||||
def normalize_vspc_company_name(name: str) -> str:
|
||||
"""Normalize a VSPC company name so it matches across HTML/text extraction and parsing."""
|
||||
n = _strip_html_tags(name or "")
|
||||
n = _html.unescape(n)
|
||||
n = n.replace("\xa0", " ")
|
||||
n = re.sub(r"\s+", " ", n).strip()
|
||||
return n
|
||||
|
||||
|
||||
def extract_vspc_active_alarms_companies(raw: str) -> List[str]:
|
||||
"""Best-effort extraction of company names from VSPC "Active alarms summary" bodies.
|
||||
|
||||
Only returns companies with alarms > 0.
|
||||
"""
|
||||
if not raw:
|
||||
return []
|
||||
|
||||
txt = raw
|
||||
if "<" in txt and ">" in txt:
|
||||
txt = re.sub(r"<[^>]+>", " ", txt)
|
||||
txt = _html.unescape(txt)
|
||||
txt = txt.replace("\xa0", " ")
|
||||
txt = re.sub(r"\s+", " ", txt).strip()
|
||||
|
||||
seen: set[str] = set()
|
||||
out: List[str] = []
|
||||
|
||||
for m in re.finditer(
|
||||
r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)",
|
||||
txt,
|
||||
flags=re.IGNORECASE,
|
||||
):
|
||||
cname = normalize_vspc_company_name((m.group(1) or "").strip())
|
||||
try:
|
||||
alarms = int(m.group(2))
|
||||
except Exception:
|
||||
alarms = 0
|
||||
|
||||
if not cname or alarms <= 0:
|
||||
continue
|
||||
if cname in seen:
|
||||
continue
|
||||
seen.add(cname)
|
||||
out.append(cname)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Optional[str]]:
|
||||
"""Parse Veeam Service Provider Console (VSPC) Active Alarms summary emails.
|
||||
|
||||
@ -57,6 +106,7 @@ def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Opt
|
||||
saw_warning = False
|
||||
|
||||
for idx, (h_start, h_end, company_name, alarms_raw) in enumerate(headers):
|
||||
company_name = normalize_vspc_company_name(company_name)
|
||||
seg_start = h_end
|
||||
seg_end = headers[idx + 1][0] if idx + 1 < len(headers) else len(html)
|
||||
segment_html = html[seg_start:seg_end]
|
||||
@ -178,6 +228,44 @@ def _parse_vspc_active_alarms_from_html(html: str) -> Tuple[List[Dict], str, Opt
|
||||
return objects, overall_status, overall_message
|
||||
|
||||
|
||||
def extract_vspc_active_alarms_companies(raw: str) -> List[str]:
|
||||
"""Extract company names with alarms > 0 from a VSPC Active Alarms summary body."""
|
||||
if not raw:
|
||||
return []
|
||||
|
||||
txt = raw
|
||||
if "<" in txt and ">" in txt:
|
||||
txt = re.sub(r"<[^>]+>", " ", txt)
|
||||
txt = _html.unescape(txt)
|
||||
txt = txt.replace("\xa0", " ")
|
||||
txt = re.sub(r"\s+", " ", txt).strip()
|
||||
|
||||
seen: set[str] = set()
|
||||
out: List[str] = []
|
||||
|
||||
for m in re.finditer(
|
||||
r"\bCompany:\s*([^\(\r\n]+?)\s*\(\s*alarms?\s*:\s*(\d+)\s*\)",
|
||||
txt,
|
||||
flags=re.IGNORECASE,
|
||||
):
|
||||
cname = (m.group(1) or "").strip()
|
||||
cname = cname.replace("\xa0", " ")
|
||||
cname = re.sub(r"\s+", " ", cname).strip()
|
||||
try:
|
||||
alarms = int(m.group(2))
|
||||
except Exception:
|
||||
alarms = 0
|
||||
|
||||
if not cname or alarms <= 0:
|
||||
continue
|
||||
if cname in seen:
|
||||
continue
|
||||
seen.add(cname)
|
||||
out.append(cname)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
|
||||
def _parse_cloud_connect_report_from_html(html: str) -> Tuple[List[Dict], str]:
|
||||
"""Parse Veeam Cloud Connect daily report (provider) HTML.
|
||||
|
||||
@ -51,6 +51,17 @@
|
||||
- Corrected backend deletion logic to prevent exceptions during job removal.
|
||||
- Ensured related records are handled safely to avoid constraint or reference errors.
|
||||
|
||||
---
|
||||
|
||||
## v20260113-08-vspc-object-linking
|
||||
- Fixed VSPC company name normalization so company detection and object prefixing match consistently.
|
||||
- Fixed filtered object persistence to respect UNIQUE(customer_id, object_name) and to update last_seen timestamps correctly.
|
||||
- Added auto object persistence routing for VSPC per-company runs so objects are linked to the correct customer/job (prefix stripped).
|
||||
- Improved auto-approval for VSPC Active Alarms summary:
|
||||
- Creates per-company runs automatically when company jobs are mapped (new imports and inbox re-parse).
|
||||
- Uses case-insensitive matching for "<company> | <object>" mail objects.
|
||||
- Added best-effort retroactive processing after approving VSPC company mappings to automatically link older inbox messages that are now fully mapped.
|
||||
|
||||
***
|
||||
|
||||
## v0.1.20
|
||||
|
||||
Loading…
Reference in New Issue
Block a user