From 46cc5b10abfb4b291752758d29f33889ea78d2fe Mon Sep 17 00:00:00 2001 From: Ivo Oskamp Date: Fri, 16 Jan 2026 16:15:43 +0100 Subject: [PATCH] Auto-commit local changes before build (2026-01-16 16:15:43) --- .last-branch | 2 +- .../src/backend/app/mail_importer.py | 13 ++ .../src/backend/app/main/routes_inbox.py | 38 ++++ .../src/backend/app/main/routes_run_checks.py | 91 ++++++++- .../src/backend/app/ticketing_utils.py | 189 ++++++++++++++++++ docs/changelog.md | 9 + 6 files changed, 336 insertions(+), 6 deletions(-) create mode 100644 containers/backupchecks/src/backend/app/ticketing_utils.py diff --git a/.last-branch b/.last-branch index 5d34faf..5f6e439 100644 --- a/.last-branch +++ b/.last-branch @@ -1 +1 @@ -v20260116-10-autotask-ticket-sync-internal-ticketjobrun +v20260116-11-autotask-ticket-sync-legacy diff --git a/containers/backupchecks/src/backend/app/mail_importer.py b/containers/backupchecks/src/backend/app/mail_importer.py index d2479c7..3867d81 100644 --- a/containers/backupchecks/src/backend/app/mail_importer.py +++ b/containers/backupchecks/src/backend/app/mail_importer.py @@ -16,6 +16,7 @@ from .parsers import parse_mail_message from .parsers.veeam import extract_vspc_active_alarms_companies from .email_utils import normalize_from_address, extract_best_html_from_eml, is_effectively_blank_html from .job_matching import find_matching_job +from .ticketing_utils import link_open_internal_tickets_to_run GRAPH_TOKEN_URL_TEMPLATE = "https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" @@ -334,6 +335,12 @@ def _store_messages(settings: SystemSettings, messages): db.session.add(run) db.session.flush() + # Legacy ticket behavior: inherit any open internal tickets for this job. + try: + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass + auto_approved_runs.append((job.customer_id, job.id, run.id, mail.id)) created_any = True @@ -384,6 +391,12 @@ def _store_messages(settings: SystemSettings, messages): db.session.add(run) db.session.flush() # ensure run.id is available + # Legacy ticket behavior: inherit any open internal tickets for this job. + try: + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass + # Update mail message to reflect approval mail.job_id = job.id if hasattr(mail, "approved"): diff --git a/containers/backupchecks/src/backend/app/main/routes_inbox.py b/containers/backupchecks/src/backend/app/main/routes_inbox.py index 5ed206f..5c074d1 100644 --- a/containers/backupchecks/src/backend/app/main/routes_inbox.py +++ b/containers/backupchecks/src/backend/app/main/routes_inbox.py @@ -4,6 +4,7 @@ from .routes_shared import _format_datetime, _log_admin_event, _send_mail_messag from ..email_utils import extract_best_html_from_eml, is_effectively_blank_html from ..parsers.veeam import extract_vspc_active_alarms_companies from ..models import MailObject +from ..ticketing_utils import link_open_internal_tickets_to_run import time import re @@ -295,6 +296,13 @@ def inbox_message_approve(message_id: int): run.storage_free_percent = msg.storage_free_percent db.session.add(run) + # Legacy ticket behavior: inherit any open internal tickets for this job. + try: + db.session.flush() # ensure run.id is available + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass + # Update mail message to reflect approval msg.job_id = job.id if hasattr(msg, "approved"): @@ -538,6 +546,12 @@ def inbox_message_approve_vspc_companies(message_id: int): db.session.add(run) db.session.flush() + + # Legacy ticket behavior: inherit any open internal tickets for this job. + try: + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass created_runs.append(run) # Persist objects for reporting (idempotent upsert; safe to repeat). @@ -685,6 +699,12 @@ def inbox_message_approve_vspc_companies(message_id: int): db.session.add(run2) db.session.flush() + # Legacy ticket behavior: inherit any open internal tickets for this job. + try: + link_open_internal_tickets_to_run(run=run2, job=job2) + except Exception: + pass + # Persist objects per company try: persist_objects_for_approved_run_filtered( @@ -1050,6 +1070,12 @@ def inbox_reparse_all(): db.session.add(run) db.session.flush() + + # Legacy ticket behavior: inherit any open internal tickets for this job. + try: + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id)) created_any = True @@ -1110,6 +1136,12 @@ def inbox_reparse_all(): db.session.add(run) db.session.flush() # ensure run.id is available + + # Legacy ticket behavior: inherit any open internal tickets for this job. + try: + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id)) msg.job_id = job.id @@ -1209,6 +1241,12 @@ def inbox_reparse_all(): db.session.add(run) db.session.flush() + + # Legacy ticket behavior: inherit any open internal tickets for this job. + try: + link_open_internal_tickets_to_run(run=run, job=job) + except Exception: + pass auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id)) msg.job_id = job.id diff --git a/containers/backupchecks/src/backend/app/main/routes_run_checks.py b/containers/backupchecks/src/backend/app/main/routes_run_checks.py index 191867c..830ce87 100644 --- a/containers/backupchecks/src/backend/app/main/routes_run_checks.py +++ b/containers/backupchecks/src/backend/app/main/routes_run_checks.py @@ -39,6 +39,12 @@ from ..models import ( User, ) +from ..ticketing_utils import ( + ensure_internal_ticket_for_job, + ensure_ticket_jobrun_links, + link_open_internal_tickets_to_run, +) + def _build_autotask_client_from_settings(): """Build an AutotaskClient from stored settings or raise a user-safe exception.""" @@ -310,6 +316,11 @@ def _ensure_missed_runs_for_job(job: Job, start_from: date, end_inclusive: date) mail_message_id=None, ) db.session.add(miss) + try: + db.session.flush() # ensure miss.id is available + link_open_internal_tickets_to_run(run=miss, job=job) + except Exception: + pass inserted += 1 d = d + timedelta(days=1) @@ -391,6 +402,11 @@ def _ensure_missed_runs_for_job(job: Job, start_from: date, end_inclusive: date) mail_message_id=None, ) db.session.add(miss) + try: + db.session.flush() # ensure miss.id is available + link_open_internal_tickets_to_run(run=miss, job=job) + except Exception: + pass inserted += 1 # Next month @@ -882,12 +898,13 @@ def run_checks_details(): @login_required @roles_required("admin", "operator") def api_run_checks_autotask_ticket_poll(): - """Read-only polling of Autotask ticket state for Run Checks. + """Poll Autotask ticket state for Run Checks. - Important: - - No Backupchecks state is modified. - - No mutations are performed in Autotask. - - This endpoint is intended to be called only from the Run Checks page. + Notes: + - This endpoint does not mutate Autotask. + - As part of the legacy ticket workflow restoration, it may backfill + missing local metadata (ticket numbers) and internal Ticket/TicketJobRun + relations when those are absent. """ include_reviewed = False @@ -979,6 +996,70 @@ def api_run_checks_autotask_ticket_poll(): } ) + # Backfill local ticket numbers and internal Ticket/TicketJobRun records when missing. + # This is intentionally best-effort and must not break the Run Checks page. + try: + id_to_number = {} + id_to_title = {} + for item in out: + tid = item.get("id") + num = (item.get("ticketNumber") or "").strip() + if tid and num: + id_to_number[int(tid)] = num + id_to_title[int(tid)] = (item.get("title") or "").strip() or None + + if id_to_number: + # Update JobRun.autotask_ticket_number if empty, and ensure internal ticket workflow exists. + jobs_seen = set() + for r in runs: + try: + tid = int(getattr(r, "autotask_ticket_id", None) or 0) + except Exception: + tid = 0 + if tid <= 0 or tid not in id_to_number: + continue + + number = id_to_number.get(tid) + if number and not ((getattr(r, "autotask_ticket_number", None) or "").strip()): + r.autotask_ticket_number = number + db.session.add(r) + + # Ensure internal Ticket + scope + links exist (per job) + if r.job_id and int(r.job_id) not in jobs_seen: + jobs_seen.add(int(r.job_id)) + job = Job.query.get(r.job_id) + if not job: + continue + + ticket = ensure_internal_ticket_for_job( + ticket_code=number, + title=id_to_title.get(tid), + description=f"Autotask ticket {number}", + job=job, + active_from_dt=getattr(r, "run_at", None), + start_dt=datetime.utcnow(), + ) + + # Link all currently active (unreviewed) runs for this job. + run_ids = [ + int(x) + for (x,) in ( + JobRun.query.filter(JobRun.job_id == job.id) + .filter(JobRun.reviewed_at.is_(None)) + .with_entities(JobRun.id) + .all() + ) + if x is not None + ] + ensure_ticket_jobrun_links(ticket_id=ticket.id, run_ids=run_ids, link_source="autotask") + + db.session.commit() + except Exception: + try: + db.session.rollback() + except Exception: + pass + return jsonify({"status": "ok", "tickets": out, "autotask_enabled": True}) @main_bp.post("/api/run-checks/autotask-ticket") @login_required diff --git a/containers/backupchecks/src/backend/app/ticketing_utils.py b/containers/backupchecks/src/backend/app/ticketing_utils.py new file mode 100644 index 0000000..02b8aed --- /dev/null +++ b/containers/backupchecks/src/backend/app/ticketing_utils.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Iterable, Optional + +from sqlalchemy import text + +from .database import db +from .models import Job, JobRun, Ticket, TicketJobRun, TicketScope +from .main.routes_shared import _get_ui_timezone_name, _to_amsterdam_date + + +def ensure_internal_ticket_for_job( + *, + ticket_code: str, + title: Optional[str], + description: str, + job: Job, + active_from_dt: Optional[datetime], + start_dt: Optional[datetime] = None, +) -> Ticket: + """Create/reuse an internal Ticket and ensure a job scope exists. + + This mirrors the legacy manual ticket workflow but allows arbitrary ticket codes + (e.g. Autotask ticket numbers). + """ + + now = datetime.utcnow() + start_dt = start_dt or now + + code = (ticket_code or "").strip().upper() + if not code: + raise ValueError("ticket_code is required") + + ticket = Ticket.query.filter_by(ticket_code=code).first() + if not ticket: + ticket = Ticket( + ticket_code=code, + title=title, + description=description, + active_from_date=_to_amsterdam_date(active_from_dt) or _to_amsterdam_date(start_dt) or start_dt.date(), + start_date=start_dt, + resolved_at=None, + ) + db.session.add(ticket) + db.session.flush() + + # Ensure an open job scope exists + scope = TicketScope.query.filter_by(ticket_id=ticket.id, scope_type="job", job_id=job.id).first() + if not scope: + scope = TicketScope( + ticket_id=ticket.id, + scope_type="job", + customer_id=job.customer_id, + backup_software=job.backup_software, + backup_type=job.backup_type, + job_id=job.id, + job_name_match=job.job_name, + job_name_match_mode="exact", + resolved_at=None, + ) + db.session.add(scope) + else: + # Re-open and refresh scope metadata (legacy behavior) + scope.resolved_at = None + scope.customer_id = job.customer_id + scope.backup_software = job.backup_software + scope.backup_type = job.backup_type + scope.job_name_match = job.job_name + scope.job_name_match_mode = "exact" + + return ticket + + +def ensure_ticket_jobrun_links( + *, + ticket_id: int, + run_ids: Iterable[int], + link_source: str, +) -> None: + """Idempotently ensure TicketJobRun links exist for all provided run IDs.""" + + run_ids_list = [int(x) for x in (run_ids or []) if x is not None] + if not run_ids_list: + return + + existing = set() + try: + rows = ( + db.session.execute( + text( + """ + SELECT job_run_id + FROM ticket_job_runs + WHERE ticket_id = :ticket_id + AND job_run_id = ANY(:run_ids) + """ + ), + {"ticket_id": int(ticket_id), "run_ids": run_ids_list}, + ) + .fetchall() + ) + existing = {int(rid) for (rid,) in rows if rid is not None} + except Exception: + existing = set() + + for rid in run_ids_list: + if rid in existing: + continue + db.session.add(TicketJobRun(ticket_id=int(ticket_id), job_run_id=int(rid), link_source=link_source)) + + +def link_open_internal_tickets_to_run(*, run: JobRun, job: Job) -> None: + """When a new run is created, link any currently open internal tickets for the job. + + This restores legacy behavior where a ticket stays visible for new runs until resolved. + Additionally (best-effort), if the job already has Autotask linkage on previous runs, + propagate that to the new run so PSA polling remains consistent. + """ + + if not run or not getattr(run, "id", None) or not job or not getattr(job, "id", None): + return + + ui_tz = _get_ui_timezone_name() + run_date = _to_amsterdam_date(getattr(run, "run_at", None)) or _to_amsterdam_date(datetime.utcnow()) + + # Find open tickets scoped to this job for the run date window. + # This matches the logic used by Job Details and Run Checks indicators. + rows = [] + try: + rows = ( + db.session.execute( + text( + """ + SELECT t.id, t.ticket_code + FROM tickets t + JOIN ticket_scopes ts ON ts.ticket_id = t.id + WHERE ts.job_id = :job_id + AND t.active_from_date <= :run_date + AND ( + COALESCE(ts.resolved_at, t.resolved_at) IS NULL + OR ((COALESCE(ts.resolved_at, t.resolved_at) AT TIME ZONE 'UTC' AT TIME ZONE :ui_tz)::date) >= :run_date + ) + ORDER BY t.start_date DESC, t.id DESC + """ + ), + {"job_id": int(job.id), "run_date": run_date, "ui_tz": ui_tz}, + ) + .fetchall() + ) + except Exception: + rows = [] + + if not rows: + return + + # Link all open tickets to this run (idempotent) + for tid, _code in rows: + if not TicketJobRun.query.filter_by(ticket_id=int(tid), job_run_id=int(run.id)).first(): + db.session.add(TicketJobRun(ticket_id=int(tid), job_run_id=int(run.id), link_source="inherit")) + + # Best-effort: propagate Autotask linkage if present on prior runs for the same ticket code. + # This allows new runs to keep the PSA ticket reference without requiring UI changes. + try: + if getattr(run, "autotask_ticket_id", None): + return + except Exception: + pass + + try: + # Use the newest ticket code to find a matching prior Autotask-linked run. + newest_code = (rows[0][1] or "").strip() + if not newest_code: + return + + prior = ( + JobRun.query.filter(JobRun.job_id == job.id) + .filter(JobRun.autotask_ticket_id.isnot(None)) + .filter(JobRun.autotask_ticket_number == newest_code) + .order_by(JobRun.id.desc()) + .first() + ) + if prior and getattr(prior, "autotask_ticket_id", None): + run.autotask_ticket_id = prior.autotask_ticket_id + run.autotask_ticket_number = prior.autotask_ticket_number + run.autotask_ticket_created_at = getattr(prior, "autotask_ticket_created_at", None) + run.autotask_ticket_created_by_user_id = getattr(prior, "autotask_ticket_created_by_user_id", None) + except Exception: + return diff --git a/docs/changelog.md b/docs/changelog.md index 2be3bbb..d7a846d 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -214,6 +214,15 @@ Changes: - Implemented idempotent behavior so repeated ticket creation or re-polling does not create duplicate tickets or links. - Prepared the ticket model for future scenarios where Autotask integration can be disabled and tickets can be managed manually again. +## v20260116-11-autotask-ticket-sync-legacy + +- Restored legacy internal ticket workflow for Autotask-created tickets by ensuring internal Ticket records are created when missing. +- Implemented automatic creation and linking of TicketJobRun records for all active job_runs (reviewed_at IS NULL) that already contain Autotask ticket data. +- Ensured 1:1 mapping between an Autotask ticket and a single internal Ticket, identical to manual ticket behavior. +- Added inheritance logic so newly created job_runs automatically link to an existing open internal Ticket until it is resolved. +- Aligned Autotask ticket creation and polling paths with the legacy manual ticket creation flow, without changing any UI behavior. +- Ensured solution works consistently with Autotask integration enabled or disabled by relying exclusively on internal Ticket and TicketJobRun structures. + *** ## v0.1.21