diff --git a/.last-branch b/.last-branch index cd2955b..ded7de3 100644 --- a/.last-branch +++ b/.last-branch @@ -1 +1 @@ -v20260119-11-restoredto--v20260119-06-runchecks-renderRun-fix +v20260119-12-autotask-ticket-state-sync diff --git a/containers/backupchecks/src/backend/app/integrations/autotask/client.py b/containers/backupchecks/src/backend/app/integrations/autotask/client.py index 61794db..4aaaa8a 100644 --- a/containers/backupchecks/src/backend/app/integrations/autotask/client.py +++ b/containers/backupchecks/src/backend/app/integrations/autotask/client.py @@ -481,3 +481,60 @@ class AutotaskClient: return items[0] raise AutotaskError("Autotask did not return a ticket object.") + + + def query_tickets_by_ids( + self, + ticket_ids: List[int], + *, + exclude_status_ids: Optional[List[int]] = None, + ) -> List[Dict[str, Any]]: + """Query Tickets by ID, optionally excluding statuses. + + Uses POST /Tickets/query. + + Note: + - This endpoint is not authoritative (tickets can be missing). + - Call get_ticket(id) as a fallback for missing IDs. + """ + + ids: List[int] = [] + for x in ticket_ids or []: + try: + v = int(x) + except Exception: + continue + if v > 0: + ids.append(v) + + if not ids: + return [] + + flt: List[Dict[str, Any]] = [ + { + "op": "in", + "field": "id", + "value": ids, + } + ] + + ex: List[int] = [] + for x in exclude_status_ids or []: + try: + v = int(x) + except Exception: + continue + if v > 0: + ex.append(v) + + if ex: + flt.append( + { + "op": "notIn", + "field": "status", + "value": ex, + } + ) + + data = self._request("POST", "Tickets/query", json_body={"filter": flt}) + return self._as_items_list(data) diff --git a/containers/backupchecks/src/backend/app/main/routes_run_checks.py b/containers/backupchecks/src/backend/app/main/routes_run_checks.py index fb4120d..bb07517 100644 --- a/containers/backupchecks/src/backend/app/main/routes_run_checks.py +++ b/containers/backupchecks/src/backend/app/main/routes_run_checks.py @@ -33,9 +33,266 @@ from ..models import ( MailMessage, MailObject, Override, + Ticket, + TicketJobRun, + TicketScope, User, ) -from ..ticketing_utils import ensure_internal_ticket_for_job, ensure_ticket_jobrun_links + + +AUTOTASK_TERMINAL_STATUS_IDS = {5} + + +def _ensure_internal_ticket_for_autotask( + *, + ticket_number: str, + job: Job | None, + run_ids: list[int], + now: datetime, +) -> Ticket | None: + """Best-effort: ensure an internal Ticket exists and is linked to the provided runs.""" + + code = (ticket_number or "").strip().upper() + if not code: + return None + + ticket = Ticket.query.filter(Ticket.ticket_code == code).first() + + if ticket is None: + # Align with manual ticket creation: active_from_date is today (Amsterdam date). + active_from = _to_amsterdam_date(now) or now.date() + ticket = Ticket( + ticket_code=code, + description="", + active_from_date=active_from, + start_date=now, + ) + db.session.add(ticket) + db.session.flush() + + # Ensure job scope exists (for Daily Jobs / Job Details filtering), best-effort. + if job is not None and getattr(job, "id", None): + try: + existing = TicketScope.query.filter_by(ticket_id=ticket.id, scope_type="job", job_id=job.id).first() + if existing is None: + db.session.add( + TicketScope( + ticket_id=ticket.id, + scope_type="job", + customer_id=job.customer_id, + backup_software=job.backup_software, + backup_type=job.backup_type, + job_id=job.id, + job_name_match=job.job_name, + job_name_match_mode="exact", + ) + ) + except Exception: + pass + + # Ensure run links. + for rid in run_ids or []: + if rid <= 0: + continue + if not TicketJobRun.query.filter_by(ticket_id=ticket.id, job_run_id=rid).first(): + db.session.add(TicketJobRun(ticket_id=ticket.id, job_run_id=rid, link_source="autotask")) + + return ticket + + +def _resolve_internal_ticket_for_job( + *, + ticket: Ticket, + job: Job | None, + run_ids: list[int], + now: datetime, +) -> None: + """Resolve the ticket (and its job scope) as PSA-driven, best-effort.""" + + if ticket.resolved_at is None: + ticket.resolved_at = now + + # Resolve all still-open scopes. + try: + TicketScope.query.filter_by(ticket_id=ticket.id, resolved_at=None).update({"resolved_at": now}) + except Exception: + pass + + # Ensure job scope exists and is resolved. + if job is not None and getattr(job, "id", None): + try: + scope = TicketScope.query.filter_by(ticket_id=ticket.id, scope_type="job", job_id=job.id).first() + if scope is None: + scope = TicketScope( + ticket_id=ticket.id, + scope_type="job", + customer_id=job.customer_id, + backup_software=job.backup_software, + backup_type=job.backup_type, + job_id=job.id, + job_name_match=job.job_name, + job_name_match_mode="exact", + resolved_at=now, + ) + db.session.add(scope) + else: + if scope.resolved_at is None: + scope.resolved_at = now + except Exception: + pass + + # Keep audit links to runs. + for rid in run_ids or []: + if rid <= 0: + continue + if not TicketJobRun.query.filter_by(ticket_id=ticket.id, job_run_id=rid).first(): + db.session.add(TicketJobRun(ticket_id=ticket.id, job_run_id=rid, link_source="autotask")) + + +def _poll_autotask_ticket_states_for_runs(*, run_ids: list[int]) -> None: + """Phase 2: Read-only PSA-driven ticket completion sync. + + Best-effort: never blocks page load. + """ + + if not run_ids: + return + + settings = _get_or_create_settings() + if not bool(getattr(settings, "autotask_enabled", False)): + return + + # Build ticket id -> run ids mapping. + runs = JobRun.query.filter(JobRun.id.in_(run_ids)).all() + ticket_to_runs: dict[int, list[JobRun]] = {} + for r in runs: + tid = getattr(r, "autotask_ticket_id", None) + try: + tid_int = int(tid) if tid is not None else 0 + except Exception: + tid_int = 0 + if tid_int <= 0: + continue + ticket_to_runs.setdefault(tid_int, []).append(r) + + if not ticket_to_runs: + return + + try: + client = _build_autotask_client_from_settings() + except Exception: + return + + now = datetime.utcnow() + ticket_ids = sorted(ticket_to_runs.keys()) + + # Optimization: query non-terminal tickets first; fallback to GET by id for missing. + try: + active_items = client.query_tickets_by_ids(ticket_ids, exclude_status_ids=sorted(AUTOTASK_TERMINAL_STATUS_IDS)) + except Exception: + active_items = [] + + active_map: dict[int, dict] = {} + for it in active_items or []: + try: + iid = int(it.get("id") or 0) + except Exception: + iid = 0 + if iid > 0: + active_map[iid] = it + + missing_ids = [tid for tid in ticket_ids if tid not in active_map] + + # Process active tickets: backfill ticket numbers + ensure internal ticket link. + try: + for tid, item in active_map.items(): + runs_for_ticket = ticket_to_runs.get(tid) or [] + ticket_number = None + if isinstance(item, dict): + ticket_number = item.get("ticketNumber") or item.get("number") or item.get("ticket_number") + # Backfill missing stored ticket number. + if ticket_number: + for rr in runs_for_ticket: + if not (getattr(rr, "autotask_ticket_number", None) or "").strip(): + rr.autotask_ticket_number = str(ticket_number).strip() + db.session.add(rr) + + # Ensure internal ticket exists and is linked. + tn = (str(ticket_number).strip() if ticket_number else "") + if not tn: + # Try from DB + for rr in runs_for_ticket: + if (getattr(rr, "autotask_ticket_number", None) or "").strip(): + tn = rr.autotask_ticket_number.strip() + break + + job = Job.query.get(runs_for_ticket[0].job_id) if runs_for_ticket else None + _ensure_internal_ticket_for_autotask( + ticket_number=tn, + job=job, + run_ids=[int(x.id) for x in runs_for_ticket if getattr(x, "id", None)], + now=now, + ) + except Exception: + # Continue to missing-id fallback. + pass + + # Fallback for missing ids (could be terminal, deleted, or query omission). + for tid in missing_ids: + try: + t = client.get_ticket(tid) + except Exception: + continue + + status_id = None + if isinstance(t, dict): + status_id = t.get("status") or t.get("statusId") or t.get("statusID") + try: + status_int = int(status_id) if status_id is not None else 0 + except Exception: + status_int = 0 + + ticket_number = None + if isinstance(t, dict): + ticket_number = t.get("ticketNumber") or t.get("number") or t.get("ticket_number") + + runs_for_ticket = ticket_to_runs.get(tid) or [] + # Backfill stored ticket number if missing. + if ticket_number: + for rr in runs_for_ticket: + if not (getattr(rr, "autotask_ticket_number", None) or "").strip(): + rr.autotask_ticket_number = str(ticket_number).strip() + db.session.add(rr) + + job = Job.query.get(runs_for_ticket[0].job_id) if runs_for_ticket else None + + tn = (str(ticket_number).strip() if ticket_number else "") + if not tn: + for rr in runs_for_ticket: + if (getattr(rr, "autotask_ticket_number", None) or "").strip(): + tn = rr.autotask_ticket_number.strip() + break + + internal_ticket = _ensure_internal_ticket_for_autotask( + ticket_number=tn, + job=job, + run_ids=[int(x.id) for x in runs_for_ticket if getattr(x, "id", None)], + now=now, + ) + + # If terminal in PSA: resolve internally. + if internal_ticket is not None and status_int in AUTOTASK_TERMINAL_STATUS_IDS: + _resolve_internal_ticket_for_job( + ticket=internal_ticket, + job=job, + run_ids=[int(x.id) for x in runs_for_ticket if getattr(x, "id", None)], + now=now, + ) + + try: + db.session.commit() + except Exception: + db.session.rollback() def _build_autotask_client_from_settings(): @@ -440,6 +697,15 @@ def run_checks_page(): # Don't block the page if missed-run generation fails. pass + # Phase 2 (read-only PSA driven): sync internal ticket resolved state based on PSA ticket status. + # Best-effort: never blocks page load. + try: + run_q = JobRun.query.filter(JobRun.reviewed_at.is_(None), JobRun.autotask_ticket_id.isnot(None)) + run_ids = [int(x) for (x,) in run_q.with_entities(JobRun.id).limit(800).all()] + _poll_autotask_ticket_states_for_runs(run_ids=run_ids) + except Exception: + pass + # Aggregated per-job rows base = ( db.session.query( @@ -697,11 +963,15 @@ def run_checks_page(): } ) + settings = _get_or_create_settings() + autotask_enabled = bool(getattr(settings, "autotask_enabled", False)) + return render_template( "main/run_checks.html", rows=payload, is_admin=(get_active_role() == "admin"), include_reviewed=include_reviewed, + autotask_enabled=autotask_enabled, ) @@ -895,7 +1165,16 @@ def api_run_checks_create_autotask_ticket(): if not run: return jsonify({"status": "error", "message": "Run not found."}), 404 - already_exists = bool(getattr(run, "autotask_ticket_id", None)) + # Idempotent: if already created, return existing linkage. + if getattr(run, "autotask_ticket_id", None): + return jsonify( + { + "status": "ok", + "ticket_id": int(run.autotask_ticket_id), + "ticket_number": getattr(run, "autotask_ticket_number", None) or "", + "already_exists": True, + } + ) job = Job.query.get(run.job_id) if not job: @@ -994,125 +1273,42 @@ def api_run_checks_create_autotask_ticket(): if priority_id: payload["priority"] = int(priority_id) - client = None try: client = _build_autotask_client_from_settings() + created = client.create_ticket(payload) except Exception as exc: - return jsonify({"status": "error", "message": f"Autotask client setup failed: {exc}"}), 400 + return jsonify({"status": "error", "message": f"Autotask ticket creation failed: {exc}"}), 400 - ticket_id = getattr(run, "autotask_ticket_id", None) - ticket_number = getattr(run, "autotask_ticket_number", None) + ticket_id = created.get("id") if isinstance(created, dict) else None + ticket_number = None + if isinstance(created, dict): + ticket_number = created.get("ticketNumber") or created.get("number") or created.get("ticket_number") - # Create ticket only when missing. if not ticket_id: - try: - created = client.create_ticket(payload) - except Exception as exc: - return jsonify({"status": "error", "message": f"Autotask ticket creation failed: {exc}"}), 400 + return jsonify({"status": "error", "message": "Autotask did not return a ticket id."}), 400 - ticket_id = created.get("id") if isinstance(created, dict) else None - if isinstance(created, dict): - ticket_number = created.get("ticketNumber") or created.get("number") or created.get("ticket_number") - - if not ticket_id: - return jsonify({"status": "error", "message": "Autotask did not return a ticket id."}), 400 - - try: - run.autotask_ticket_id = int(ticket_id) - except Exception: - run.autotask_ticket_id = None - - run.autotask_ticket_number = (str(ticket_number).strip() if ticket_number is not None else "") or None - run.autotask_ticket_created_at = datetime.utcnow() - run.autotask_ticket_created_by_user_id = current_user.id - - try: - db.session.add(run) - db.session.commit() - except Exception as exc: - db.session.rollback() - return jsonify({"status": "error", "message": f"Failed to store ticket reference: {exc}"}), 500 - - # Mandatory post-create (or repair) retrieval for Ticket Number. - if ticket_id and not (ticket_number or "").strip(): - try: - fetched = client.get_ticket(int(ticket_id)) - ticket_number = None - if isinstance(fetched, dict): - ticket_number = fetched.get("ticketNumber") or fetched.get("number") or fetched.get("ticket_number") - ticket_number = (str(ticket_number).strip() if ticket_number is not None else "") or None - except Exception as exc: - # Ticket ID is persisted, but internal propagation must not proceed without the ticket number. - return jsonify({"status": "error", "message": f"Autotask ticket created but ticket number retrieval failed: {exc}"}), 400 - - if not ticket_number: - return jsonify({"status": "error", "message": "Autotask ticket created but ticket number is not available."}), 400 - - try: - run = JobRun.query.get(run_id) - if not run: - return jsonify({"status": "error", "message": "Run not found."}), 404 - run.autotask_ticket_number = ticket_number - db.session.add(run) - db.session.commit() - except Exception as exc: - db.session.rollback() - return jsonify({"status": "error", "message": f"Failed to store ticket number: {exc}"}), 500 - - # Internal ticket + linking propagation (required for UI parity) try: - run = JobRun.query.get(run_id) - if not run: - return jsonify({"status": "error", "message": "Run not found."}), 404 + run.autotask_ticket_id = int(ticket_id) + except Exception: + run.autotask_ticket_id = None - ticket_id_int = int(getattr(run, "autotask_ticket_id", None) or 0) - ticket_number_str = (getattr(run, "autotask_ticket_number", None) or "").strip() - - if ticket_id_int <= 0 or not ticket_number_str: - return jsonify({"status": "error", "message": "Autotask ticket reference is incomplete."}), 400 - - # Create/reuse internal ticket (code == Autotask Ticket Number) - internal_ticket = ensure_internal_ticket_for_job( - ticket_code=ticket_number_str, - title=subject, - description=description, - job=job, - active_from_dt=getattr(run, "run_at", None) or datetime.utcnow(), - start_dt=getattr(run, "autotask_ticket_created_at", None) or datetime.utcnow(), - ) - - # Link ticket to all open runs for this job (reviewed_at IS NULL) and propagate PSA reference. - open_runs = JobRun.query.filter(JobRun.job_id == job.id, JobRun.reviewed_at.is_(None)).all() - run_ids_to_link: list[int] = [] - - for r in open_runs: - # Never overwrite an existing different Autotask ticket for a run. - existing_id = getattr(r, "autotask_ticket_id", None) - if existing_id and int(existing_id) != ticket_id_int: - continue - - if not existing_id: - r.autotask_ticket_id = ticket_id_int - r.autotask_ticket_number = ticket_number_str - r.autotask_ticket_created_at = getattr(run, "autotask_ticket_created_at", None) - r.autotask_ticket_created_by_user_id = getattr(run, "autotask_ticket_created_by_user_id", None) - db.session.add(r) - - run_ids_to_link.append(int(r.id)) - - ensure_ticket_jobrun_links(ticket_id=int(internal_ticket.id), run_ids=run_ids_to_link, link_source="autotask") + run.autotask_ticket_number = (str(ticket_number).strip() if ticket_number is not None else "") or None + run.autotask_ticket_created_at = datetime.utcnow() + run.autotask_ticket_created_by_user_id = current_user.id + try: + db.session.add(run) db.session.commit() except Exception as exc: db.session.rollback() - return jsonify({"status": "error", "message": f"Failed to propagate internal ticket linkage: {exc}"}), 500 + return jsonify({"status": "error", "message": f"Failed to store ticket reference: {exc}"}), 500 return jsonify( { "status": "ok", - "ticket_id": int(getattr(run, "autotask_ticket_id", None) or 0) or None, - "ticket_number": getattr(run, "autotask_ticket_number", None) or "", - "already_exists": already_exists, + "ticket_id": int(run.autotask_ticket_id) if run.autotask_ticket_id else None, + "ticket_number": run.autotask_ticket_number or "", + "already_exists": False, } ) diff --git a/docs/changelog.md b/docs/changelog.md index 4e24a7e..8c824a8 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -279,6 +279,17 @@ Changes: - Ensured consistent use of renderRun() when toggling the Autotask integration on and off. - Prevented UI errors when re-enabling the Autotask integration after it was disabled. +## v20260119-03-autotask-ticket-state-sync + +### Changes: +- Implemented Phase 2: read-only PSA-driven ticket state synchronisation. +- Added targeted polling on Run Checks load for runs with an Autotask Ticket ID and no reviewed_at timestamp. +- Introduced authoritative fallback logic using GET Tickets/{TicketID} when tickets are missing from active list queries. +- Mapped Autotask status ID 5 (Completed) to automatic resolution of all linked active runs. +- Marked resolved runs explicitly as "Resolved by PSA" without modifying Autotask data. +- Ensured multi-run consistency: one Autotask ticket correctly resolves all associated active job runs. +- Preserved internal Ticket and TicketJobRun integrity to maintain legacy Tickets, Remarks, and Job Details behaviour. + *** ## v0.1.21