Auto-commit local changes before build (2026-01-15 11:10:13)

.last-branch

@@ -1 +1 @@
-v20260115-03-autotask-settings-ui
+v20260115-04-autotask-reference-data-fix

containers/backupchecks/src/backend/app/integrations/autotask/client.py

@@ -1,7 +1,6 @@
 import json
 from dataclasses import dataclass
-from typing import Any, Dict, List, Optional, Tuple
-from urllib.parse import urlencode
+from typing import Any, Dict, List, Optional
 
 import requests
 
@@ -82,7 +81,7 @@ class AutotaskClient:
             "Accept": "application/json",
         }
 
-    def _request(self, method: str, path: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    def _request(self, method: str, path: str, params: Optional[Dict[str, Any]] = None) -> Any:
         zone = self.get_zone_info()
         base = zone.api_url.rstrip("/")
         url = f"{base}/v1.0/{path.lstrip('/')}"
@@ -112,18 +111,84 @@ class AutotaskClient:
         except Exception as exc:
             raise AutotaskError("Autotask API response is not valid JSON.") from exc
 
-    def _query_all_first_page(self, entity_name: str) -> List[Dict[str, Any]]:
-        # Use a simple 'exist' filter on id to return the first page (up to 500 items).
-        search = {"filter": [{"op": "exist", "field": "id"}]}
-        params = {"search": json.dumps(search)}
-        data = self._request("GET", f"{entity_name}/query", params=params)
-        items = data.get("items") or []
-        if not isinstance(items, list):
+    def _as_items_list(self, payload: Any) -> List[Dict[str, Any]]:
+        """Normalize common Autotask REST payload shapes to a list of dicts."""
+        if payload is None:
             return []
-        return items
+
+        if isinstance(payload, list):
+            return [x for x in payload if isinstance(x, dict)]
+
+        if isinstance(payload, dict):
+            items = payload.get("items")
+            if isinstance(items, list):
+                return [x for x in items if isinstance(x, dict)]
+
+            # Some endpoints may return a single object.
+            if "id" in payload:
+                return [payload]
+
+        return []
+
+    def _get_collection(self, resource_name: str) -> List[Dict[str, Any]]:
+        """Fetch a reference collection via GET /<resource>.
+
+        Note: Not all Autotask entities support /query. Reference data like Queues and
+        TicketSources is typically retrieved via a simple collection GET.
+        """
+        data = self._request("GET", resource_name)
+        return self._as_items_list(data)
+
+    def _get_entity_fields(self, entity_name: str) -> List[Dict[str, Any]]:
+        data = self._request("GET", f"{entity_name}/entityInformation/fields")
+        return self._as_items_list(data)
+
+    def _call_picklist_values(self, picklist_values_path: str) -> List[Dict[str, Any]]:
+        # picklistValues path can be returned as a full URL or as a relative path.
+        path = (picklist_values_path or "").strip()
+        if not path:
+            return []
+
+        # If a full URL is returned, strip everything up to /v1.0/
+        if "/v1.0/" in path:
+            path = path.split("/v1.0/", 1)[1]
+        # If it includes the base API URL without /v1.0, strip to resource path.
+        if "/atservicesrest/" in path and "/v1.0/" not in picklist_values_path:
+            # Fallback: attempt to strip after atservicesrest/
+            path = path.split("/atservicesrest/", 1)[1]
+            if path.startswith("v1.0/"):
+                path = path.split("v1.0/", 1)[1]
+
+        data = self._request("GET", path)
+        return self._as_items_list(data)
 
     def get_queues(self) -> List[Dict[str, Any]]:
-        return self._query_all_first_page("Queues")
+        return self._get_collection("Queues")
 
     def get_ticket_sources(self) -> List[Dict[str, Any]]:
-        return self._query_all_first_page("TicketSources")
+        return self._get_collection("TicketSources")
+
+    def get_ticket_priorities(self) -> List[Dict[str, Any]]:
+        """Return Ticket Priority picklist values.
+
+        We intentionally retrieve this from entity metadata to prevent hardcoded priority IDs.
+        """
+        fields = self._get_entity_fields("Tickets")
+        priority_field: Optional[Dict[str, Any]] = None
+        for f in fields:
+            name = str(f.get("name") or "").strip().lower()
+            if name == "priority":
+                priority_field = f
+                break
+
+        if not priority_field:
+            raise AutotaskError("Unable to locate Tickets.priority field metadata for picklist retrieval.")
+
+        if not bool(priority_field.get("isPickList")):
+            raise AutotaskError("Tickets.priority is not marked as a picklist in Autotask metadata.")
+
+        picklist_path = priority_field.get("picklistValues")
+        if not isinstance(picklist_path, str) or not picklist_path.strip():
+            raise AutotaskError("Tickets.priority metadata did not include a picklistValues endpoint.")
+
+        return self._call_picklist_values(picklist_path)

containers/backupchecks/src/backend/app/main/routes_settings.py

@@ -656,6 +656,7 @@ def settings():
     # Autotask cached reference data for dropdowns
     autotask_queues = []
     autotask_ticket_sources = []
+    autotask_priorities = []
     autotask_last_sync_at = getattr(settings, "autotask_reference_last_sync_at", None)
 
     try:
@@ -670,6 +671,12 @@ def settings():
     except Exception:
         autotask_ticket_sources = []
 
+    try:
+        if getattr(settings, "autotask_cached_priorities_json", None):
+            autotask_priorities = json.loads(settings.autotask_cached_priorities_json) or []
+    except Exception:
+        autotask_priorities = []
+
     return render_template(
         "main/settings.html",
         settings=settings,
@@ -684,6 +691,7 @@ def settings():
         section=section,
         autotask_queues=autotask_queues,
         autotask_ticket_sources=autotask_ticket_sources,
+        autotask_priorities=autotask_priorities,
         autotask_last_sync_at=autotask_last_sync_at,
         news_admin_items=news_admin_items,
         news_admin_stats=news_admin_stats,
@@ -1272,13 +1280,22 @@ def settings_autotask_test_connection():
             environment=(settings.autotask_environment or "production"),
         )
         zone = client.get_zone_info()
-        # Lightweight authenticated call to validate credentials
+        # Lightweight authenticated calls to validate credentials and basic API access
+        _ = client.get_queues()
         _ = client.get_ticket_sources()
         flash(f"Autotask connection OK. Zone: {zone.zone_name or 'unknown'}.", "success")
-        _log_admin_event("autotask_test_connection", details={"zone": zone.zone_name, "api_url": zone.api_url})
+        _log_admin_event(
+            "autotask_test_connection",
+            "Autotask test connection succeeded.",
+            details=json.dumps({"zone": zone.zone_name, "api_url": zone.api_url}),
+        )
     except Exception as exc:
         flash(f"Autotask connection failed: {exc}", "danger")
-        _log_admin_event("autotask_test_connection_failed", details={"error": str(exc)})
+        _log_admin_event(
+            "autotask_test_connection_failed",
+            "Autotask test connection failed.",
+            details=json.dumps({"error": str(exc)}),
+        )
 
     return redirect(url_for("main.settings", section="integrations"))
 
@@ -1304,6 +1321,7 @@ def settings_autotask_refresh_reference_data():
 
         queues = client.get_queues()
         sources = client.get_ticket_sources()
+        priorities = client.get_ticket_priorities()
 
         # Store a minimal subset for dropdowns (id + name/label)
         def _norm(items):
@@ -1324,17 +1342,43 @@ def settings_autotask_refresh_reference_data():
 
         settings.autotask_cached_queues_json = json.dumps(_norm(queues))
         settings.autotask_cached_ticket_sources_json = json.dumps(_norm(sources))
+
+        # Priorities are returned as picklist values (value/label)
+        pr_out = []
+        for it in priorities or []:
+            if not isinstance(it, dict):
+                continue
+            if it.get("isActive") is False:
+                continue
+            val = it.get("value")
+            label = it.get("label") or it.get("name") or ""
+            try:
+                val_int = int(val)
+            except Exception:
+                continue
+            pr_out.append({"id": val_int, "name": str(label)})
+        pr_out.sort(key=lambda x: (x.get("name") or "").lower())
+
+        settings.autotask_cached_priorities_json = json.dumps(pr_out)
         settings.autotask_reference_last_sync_at = datetime.utcnow()
 
         db.session.commit()
 
-        flash(f"Autotask reference data refreshed. Queues: {len(queues)}. Ticket Sources: {len(sources)}.", "success")
+        flash(
+            f"Autotask reference data refreshed. Queues: {len(queues)}. Ticket Sources: {len(sources)}. Priorities: {len(pr_out)}.",
+            "success",
+        )
         _log_admin_event(
             "autotask_refresh_reference_data",
-            details={"queues": len(queues or []), "ticket_sources": len(sources or [])},
+            "Autotask reference data refreshed.",
+            details=json.dumps({"queues": len(queues or []), "ticket_sources": len(sources or []), "priorities": len(pr_out)}),
         )
     except Exception as exc:
         flash(f"Failed to refresh Autotask reference data: {exc}", "danger")
-        _log_admin_event("autotask_refresh_reference_data_failed", details={"error": str(exc)})
+        _log_admin_event(
+            "autotask_refresh_reference_data_failed",
+            "Autotask reference data refresh failed.",
+            details=json.dumps({"error": str(exc)}),
+        )
 
     return redirect(url_for("main.settings", section="integrations"))

containers/backupchecks/src/backend/app/migrations.py

@@ -167,6 +167,7 @@ def migrate_system_settings_autotask_integration() -> None:
         ("autotask_priority_error", "INTEGER NULL"),
         ("autotask_cached_queues_json", "TEXT NULL"),
         ("autotask_cached_ticket_sources_json", "TEXT NULL"),
+        ("autotask_cached_priorities_json", "TEXT NULL"),
         ("autotask_reference_last_sync_at", "TIMESTAMP NULL"),
     ]
 

containers/backupchecks/src/backend/app/models.py

@@ -126,6 +126,7 @@ class SystemSettings(db.Model):
     # Cached reference data (for dropdowns)
     autotask_cached_queues_json = db.Column(db.Text, nullable=True)
     autotask_cached_ticket_sources_json = db.Column(db.Text, nullable=True)
+    autotask_cached_priorities_json = db.Column(db.Text, nullable=True)
     autotask_reference_last_sync_at = db.Column(db.DateTime, nullable=True)
     created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False)
     updated_at = db.Column(

containers/backupchecks/src/templates/main/settings.html

@@ -399,15 +399,27 @@
 
         <div class="col-md-6">
           <label for="autotask_priority_warning" class="form-label">Priority for Warning</label>
-          <input type="number" min="1" class="form-control" id="autotask_priority_warning" name="autotask_priority_warning" value="{{ settings.autotask_priority_warning or '' }}" />
+          <select class="form-select" id="autotask_priority_warning" name="autotask_priority_warning">
+            <option value="" {% if not settings.autotask_priority_warning %}selected{% endif %}>Select...</option>
+            {% for p in autotask_priorities %}
+              <option value="{{ p.id }}" {% if settings.autotask_priority_warning == p.id %}selected{% endif %}>{{ p.name }}</option>
+            {% endfor %}
+          </select>
+          <div class="form-text">Requires refreshed reference data.</div>
         </div>
 
         <div class="col-md-6">
           <label for="autotask_priority_error" class="form-label">Priority for Error</label>
-          <input type="number" min="1" class="form-control" id="autotask_priority_error" name="autotask_priority_error" value="{{ settings.autotask_priority_error or '' }}" />
+          <select class="form-select" id="autotask_priority_error" name="autotask_priority_error">
+            <option value="" {% if not settings.autotask_priority_error %}selected{% endif %}>Select...</option>
+            {% for p in autotask_priorities %}
+              <option value="{{ p.id }}" {% if settings.autotask_priority_error == p.id %}selected{% endif %}>{{ p.name }}</option>
+            {% endfor %}
+          </select>
+          <div class="form-text">Requires refreshed reference data.</div>
         </div>
       </div>
-      <div class="form-text mt-2">Priority values are Autotask priority IDs.</div>
+      <div class="form-text mt-2">Priorities are loaded from Autotask to avoid manual ID mistakes.</div>
     </div>
   </div>
 
@@ -431,7 +443,8 @@
         </div>
         <div class="text-muted small mt-2">
           Cached Queues: {{ autotask_queues|length }}<br />
-          Cached Ticket Sources: {{ autotask_ticket_sources|length }}
+          Cached Ticket Sources: {{ autotask_ticket_sources|length }}<br />
+          Cached Priorities: {{ autotask_priorities|length }}
         </div>
       </div>
       <div class="col-md-6">
@@ -443,7 +456,7 @@
             <button type="submit" class="btn btn-outline-primary">Refresh reference data</button>
           </form>
         </div>
-        <div class="form-text mt-2 text-md-end">Refresh loads Queues and Ticket Sources from Autotask for dropdown usage.</div>
+        <div class="form-text mt-2 text-md-end">Refresh loads Queues, Ticket Sources, and Priorities from Autotask for dropdown usage.</div>
       </div>
     </div>
   </div>

docs/changelog.md

@@ -25,6 +25,18 @@
 - Added Diagnostics & Reference Data section with actions for testing the Autotask connection and refreshing reference data.
 - Kept all functionality strictly within Phase 1 scope without introducing ticket or customer logic.
 
+## v20260115-04-autotask-reference-data-fix
+
+### Changes:
+- Fixed Autotask API client to use correct endpoints for reference data instead of invalid `/query` routes.
+- Implemented proper retrieval of Autotask Queues and Ticket Sources via collection endpoints.
+- Added dynamic retrieval of Autotask Priorities using ticket entity metadata and picklist values.
+- Cached queues, ticket sources, and priorities in system settings for safe reuse in the UI.
+- Updated Autotask settings UI to use dropdowns backed by live Autotask reference data.
+- Improved “Test connection” to validate authentication and reference data access reliably.
+- Fixed admin event logging to prevent secondary exceptions during error handling.
+
+
 ***
 
 ## v0.1.21