From 6998b3aa7dc9f094b89ec4b9f793c927f20183a3 Mon Sep 17 00:00:00 2001 From: Ivo Oskamp Date: Fri, 20 Mar 2026 15:51:54 +0100 Subject: [PATCH] Auto-commit local changes before build (2026-03-20 15:51:54) --- .../src/backend/app/main/routes_inbox.py | 249 ++++++++++++++++++ .../src/templates/main/inbox.html | 109 +++++++- docs/changelog-claude.md | 7 + 3 files changed, 362 insertions(+), 3 deletions(-) diff --git a/containers/backupchecks/src/backend/app/main/routes_inbox.py b/containers/backupchecks/src/backend/app/main/routes_inbox.py index a79331b..9da9d33 100644 --- a/containers/backupchecks/src/backend/app/main/routes_inbox.py +++ b/containers/backupchecks/src/backend/app/main/routes_inbox.py @@ -1401,3 +1401,252 @@ def inbox_reparse_all(): ) return redirect(url_for("main.inbox")) + + +@main_bp.route("/inbox/reparse-batch", methods=["POST"]) +@login_required +@roles_required("admin", "operator") +def inbox_reparse_batch(): + """Process one batch of inbox messages and return JSON progress info. + + Expects JSON body: {"last_id": , "total": } + Returns JSON: {processed, total, parsed_ok, auto_approved, no_match, errors, last_id, done} + """ + from flask import jsonify + + data = request.get_json(silent=True) or {} + last_id = data.get("last_id") # keyset cursor (id < last_id) + total_known = data.get("total") # total passed from client so we don't recount + + base_q = MailMessage.query + if hasattr(MailMessage, "location"): + base_q = base_q.filter(MailMessage.location == "inbox") + + if total_known is None: + total_known = base_q.count() + + batch_size = 50 + time_budget_s = 8.0 + + started_at = time.monotonic() + processed = 0 + parsed_ok = 0 + auto_approved = 0 + auto_approved_runs = [] + no_match = 0 + errors = 0 + + q = base_q + if last_id is not None: + q = q.filter(MailMessage.id < last_id) + + batch = q.order_by(MailMessage.id.desc()).limit(batch_size).all() + + new_last_id = last_id + for msg in batch: + if (time.monotonic() - started_at) >= time_budget_s: + break + + new_last_id = msg.id + processed += 1 + + try: + parse_mail_message(msg) + + try: + if ( + getattr(msg, "location", "inbox") == "inbox" + and getattr(msg, "parse_result", None) == "ok" + and getattr(msg, "job_id", None) is None + ): + bsw = (getattr(msg, "backup_software", "") or "").strip().lower() + btype = (getattr(msg, "backup_type", "") or "").strip().lower() + jname = (getattr(msg, "job_name", "") or "").strip().lower() + + if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary": + raw = (getattr(msg, "text_body", None) or "").strip() or (getattr(msg, "html_body", None) or "") + companies = extract_vspc_active_alarms_companies(raw) + + if companies: + def _is_error_status(value): + v = (value or "").strip().lower() + return v in {"error", "failed", "critical"} or v.startswith("fail") + + first_job = None + mapped_count = 0 + created_any = False + + for company in companies: + tmp_msg = MailMessage( + from_address=msg.from_address, + backup_software=msg.backup_software, + backup_type=msg.backup_type, + job_name=f"{(msg.job_name or 'Active alarms summary').strip()} | {company}".strip(), + ) + with db.session.no_autoflush: + job = find_matching_job(tmp_msg) + + if not job: + continue + if hasattr(job, "active") and not bool(job.active): + continue + if hasattr(job, "auto_approve") and not bool(job.auto_approve): + continue + + mapped_count += 1 + objs = ( + MailObject.query.filter(MailObject.mail_message_id == msg.id) + .filter(MailObject.object_name.ilike(f"{company} | %")) + .all() + ) + saw_error = any(_is_error_status(o.status) for o in objs) + saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs) + status = "Error" if saw_error else ("Warning" if saw_warning else (msg.overall_status or "Success")) + + run = JobRun( + job_id=job.id, + mail_message_id=msg.id, + run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()), + status=status or None, + missed=False, + ) + if hasattr(run, "remark"): + run.remark = getattr(msg, "overall_message", None) + if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"): + run.storage_used_bytes = msg.storage_used_bytes + if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"): + run.storage_capacity_bytes = msg.storage_capacity_bytes + if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"): + run.storage_free_bytes = msg.storage_free_bytes + if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"): + run.storage_free_percent = msg.storage_free_percent + + db.session.add(run) + db.session.flush() + try: + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass + auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id)) + created_any = True + if not first_job: + first_job = job + + if created_any and mapped_count == len(companies): + msg.job_id = first_job.id if first_job else None + if hasattr(msg, "approved"): + msg.approved = True + if hasattr(msg, "approved_at"): + msg.approved_at = datetime.utcnow() + if hasattr(msg, "approved_by_id"): + msg.approved_by_id = None + if hasattr(msg, "location"): + msg.location = "history" + auto_approved += 1 + + # Do not fall back to single-job matching for VSPC summary. + if msg.parse_result == "ok": + parsed_ok += 1 + elif msg.parse_result == "no_match": + no_match += 1 + else: + errors += 1 + continue + + with db.session.no_autoflush: + job = find_matching_job(msg) + + if job: + if hasattr(job, "active") and not bool(job.active): + raise Exception("job not active") + if hasattr(job, "auto_approve") and not bool(job.auto_approve): + raise Exception("job auto_approve disabled") + + run = JobRun( + job_id=job.id, + mail_message_id=msg.id, + run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()), + status=msg.overall_status or None, + missed=False, + ) + if hasattr(run, "remark"): + run.remark = getattr(msg, "overall_message", None) + if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"): + run.storage_used_bytes = msg.storage_used_bytes + if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"): + run.storage_capacity_bytes = msg.storage_capacity_bytes + if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"): + run.storage_free_bytes = msg.storage_free_bytes + if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"): + run.storage_free_percent = msg.storage_free_percent + + db.session.add(run) + db.session.flush() + try: + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass + auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id)) + + msg.job_id = job.id + if hasattr(msg, "approved"): + msg.approved = True + if hasattr(msg, "approved_at"): + msg.approved_at = datetime.utcnow() + if hasattr(msg, "approved_by_id"): + msg.approved_by_id = None + if hasattr(msg, "location"): + msg.location = "history" + auto_approved += 1 + + except Exception as _exc: + current_app.logger.exception( + f"Auto-approve during reparse-batch failed for message {getattr(msg,'id',None)}: {_exc}" + ) + + if msg.parse_result == "ok": + parsed_ok += 1 + elif msg.parse_result == "no_match": + no_match += 1 + else: + errors += 1 + + except Exception as exc: + errors += 1 + msg.parse_result = "error" + msg.parse_error = str(exc)[:500] + + try: + db.session.commit() + except Exception: + db.session.rollback() + + # Persist objects for auto-approved runs + if auto_approved_runs: + for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs: + try: + persist_objects_for_auto_run(customer_id, job_id, run_id, mail_message_id) + except Exception as exc: + _log_admin_event( + "object_persist_error", + f"Object persistence failed for auto-approved message {mail_message_id} (job {job_id}, run {run_id}): {exc}", + ) + + # Determine if we are done: no more messages below new_last_id + done = False + if processed < batch_size: + done = True + elif new_last_id is not None: + remaining_q = base_q.filter(MailMessage.id < new_last_id) + done = remaining_q.count() == 0 + + return jsonify({ + "processed": processed, + "total": total_known, + "parsed_ok": parsed_ok, + "auto_approved": auto_approved, + "no_match": no_match, + "errors": errors, + "last_id": new_last_id, + "done": done, + }) diff --git a/containers/backupchecks/src/templates/main/inbox.html b/containers/backupchecks/src/templates/main/inbox.html index 0e7fd44..9c8ab9e 100644 --- a/containers/backupchecks/src/templates/main/inbox.html +++ b/containers/backupchecks/src/templates/main/inbox.html @@ -26,9 +26,7 @@ {% if current_user.is_authenticated and active_role in ["admin", "operator"] %} -
- -
+ {% endif %}
@@ -209,6 +207,27 @@
+ + +