diff --git a/containers/backupchecks/src/backend/app/cove_importer.py b/containers/backupchecks/src/backend/app/cove_importer.py index 7b4740f..a8168a4 100644 --- a/containers/backupchecks/src/backend/app/cove_importer.py +++ b/containers/backupchecks/src/backend/app/cove_importer.py @@ -1,7 +1,13 @@ """Cove Data Protection API importer. -Fetches backup job run data from the Cove (N-able) API and creates -JobRun records in the local database. +Fetches backup job run data from the Cove (N-able) API. + +Flow (mirrors the mail Inbox flow): + 1. All Cove accounts are upserted into the `cove_accounts` staging table. + 2. Accounts without a linked job appear on the Cove Accounts page where + an admin can create or link a job (same as approving a mail from Inbox). + 3. For accounts that have a linked job, a JobRun is created per new session + (deduplicated via external_id). """ from __future__ import annotations @@ -20,12 +26,12 @@ COVE_DEFAULT_URL = "https://api.backup.management/jsonapi" # Columns to request from EnumerateAccountStatistics COVE_COLUMNS = [ - "I1", # Account name - "I18", # Account ID + "I1", # Account/device name + "I18", # Computer name "I8", # Customer / partner name - "I78", # Backup type / product + "I78", # Active datasource label "D09F00", # Overall last session status - "D09F09", # Last session start timestamp + "D09F09", # Last successful session timestamp "D09F15", # Last session end timestamp "D09F08", # 28-day colorbar # Datasource-specific status (F00) and last session time (F15) @@ -40,20 +46,20 @@ COVE_COLUMNS = [ # Mapping from Cove status code to Backupchecks status string STATUS_MAP: dict[int, str] = { - 1: "Warning", # Running + 1: "Warning", # In process 2: "Error", # Failed - 3: "Error", # Interrupted + 3: "Error", # Aborted 5: "Success", # Completed - 6: "Error", # Overdue - 7: "Warning", # No recent backup - 8: "Warning", # Idle - 9: "Warning", # Unknown - 10: "Error", # Account error - 11: "Warning", # Quota exceeded - 12: "Warning", # License issue + 6: "Error", # Interrupted + 7: "Warning", # NotStarted + 8: "Warning", # CompletedWithErrors + 9: "Warning", # InProgressWithFaults + 10: "Error", # OverQuota + 11: "Warning", # NoSelection + 12: "Warning", # Restarted } -# Datasource label mapping (prefix → human-readable label) +# Datasource label mapping (column prefix → human-readable label) DATASOURCE_LABELS: dict[str, str] = { "D1": "Files & Folders", "D10": "VssMsSql", @@ -167,7 +173,7 @@ def _cove_enumerate( if result is None: return [] - # Unwrap possible nested result (same as test script) + # Unwrap possible nested result if isinstance(result, dict) and "result" in result: result = result["result"] @@ -184,7 +190,6 @@ def _flatten_settings(account: dict) -> dict: Cove returns settings as a list of single-key dicts, e.g.: [{"D09F00": "5"}, {"I1": "device name"}, ...] - This flattens them so we can do `flat['D09F00']` instead of iterating. """ flat: dict[str, Any] = {} settings_list = account.get("Settings") or [] @@ -205,8 +210,25 @@ def _map_status(code: Any) -> str: return "Warning" +def _ts_to_dt(value: Any) -> datetime | None: + """Convert a Unix timestamp (int or str) to a naive UTC datetime.""" + if value is None: + return None + try: + ts = int(value) + if ts <= 0: + return None + return datetime.fromtimestamp(ts, tz=timezone.utc).replace(tzinfo=None) + except (ValueError, TypeError, OSError): + return None + + def run_cove_import(settings) -> tuple[int, int, int, int]: - """Fetch Cove account statistics and create/update JobRun records. + """Fetch Cove account statistics and update the staging table + JobRuns. + + For every account: + - Upsert into cove_accounts (always) + - If the account has a linked job → create a JobRun if not already seen Args: settings: SystemSettings ORM object with cove_* fields. @@ -217,8 +239,6 @@ def run_cove_import(settings) -> tuple[int, int, int, int]: Raises: CoveImportError if the API login fails. """ - from .models import Job, JobRun # lazy import to avoid circular refs - url = (getattr(settings, "cove_api_url", None) or "").strip() or COVE_DEFAULT_URL username = (getattr(settings, "cove_api_username", None) or "").strip() password = (getattr(settings, "cove_api_password", None) or "").strip() @@ -258,8 +278,8 @@ def run_cove_import(settings) -> tuple[int, int, int, int]: for account in accounts: total += 1 try: - created_this = _process_account(account, settings) - if created_this: + run_created = _process_account(account) + if run_created: created += 1 else: skipped += 1 @@ -285,17 +305,16 @@ def run_cove_import(settings) -> tuple[int, int, int, int]: return total, created, skipped, errors -def _process_account(account: dict, settings) -> bool: - """Process a single Cove account and create a JobRun if needed. +def _process_account(account: dict) -> bool: + """Upsert a Cove account into the staging table and create a JobRun if linked. - Returns True if a new JobRun was created, False if skipped (duplicate or no job match). + Returns True if a new JobRun was created, False otherwise. """ - from .models import Job, JobRun - from sqlalchemy import text + from .models import CoveAccount, JobRun flat = _flatten_settings(account) - # AccountId is a top-level field (not in Settings) + # AccountId is a top-level field account_id = account.get("AccountId") or account.get("AccountID") if not account_id: return False @@ -304,45 +323,69 @@ def _process_account(account: dict, settings) -> bool: except (ValueError, TypeError): return False - # Last session end timestamp (D09F15) – used as run_at - run_ts_raw = flat.get("D09F15") - if not run_ts_raw: - # No last run timestamp – skip this account - return False - + # Extract metadata from flat settings + account_name = (flat.get("I1") or "").strip() or None + computer_name = (flat.get("I18") or "").strip() or None + customer_name = (flat.get("I8") or "").strip() or None + datasource_types = (flat.get("I78") or "").strip() or None + last_run_at = _ts_to_dt(flat.get("D09F15")) + colorbar_28d = (flat.get("D09F08") or "").strip() or None try: - run_ts = int(run_ts_raw) + last_status_code = int(flat["D09F00"]) if flat.get("D09F00") is not None else None except (ValueError, TypeError): + last_status_code = None + + # Upsert into cove_accounts staging table + cove_acc = CoveAccount.query.filter_by(account_id=account_id).first() + if cove_acc is None: + cove_acc = CoveAccount( + account_id=account_id, + first_seen_at=datetime.utcnow(), + ) + db.session.add(cove_acc) + + cove_acc.account_name = account_name + cove_acc.computer_name = computer_name + cove_acc.customer_name = customer_name + cove_acc.datasource_types = datasource_types + cove_acc.last_status_code = last_status_code + cove_acc.last_run_at = last_run_at + cove_acc.colorbar_28d = colorbar_28d + cove_acc.last_seen_at = datetime.utcnow() + + db.session.flush() # ensure cove_acc.id is set + + # If not linked to a job yet, nothing more to do (shows up in Cove Accounts page) + if not cove_acc.job_id: + db.session.commit() return False - if run_ts <= 0: + # Account is linked: create a JobRun if the last session is new + if not last_run_at: + db.session.commit() return False - # Deduplication key + run_ts = int(flat.get("D09F15", 0)) external_id = f"cove-{account_id}-{run_ts}" - # Check for duplicate existing = JobRun.query.filter_by(external_id=external_id).first() if existing: + db.session.commit() return False - # Find corresponding Job - job = Job.query.filter_by(cove_account_id=account_id).first() + # Fetch the linked job + from .models import Job + job = Job.query.get(cove_acc.job_id) if not job: + db.session.commit() return False - # Convert Unix timestamp to datetime (UTC) - run_at = datetime.fromtimestamp(run_ts, tz=timezone.utc).replace(tzinfo=None) + status = _map_status(last_status_code) - # Map overall status - status_code = flat.get("D09F00") - status = _map_status(status_code) - - # Create JobRun run = JobRun( job_id=job.id, mail_message_id=None, - run_at=run_at, + run_at=last_run_at, status=status, remark=None, missed=False, @@ -354,9 +397,8 @@ def _process_account(account: dict, settings) -> bool: db.session.flush() # get run.id # Persist per-datasource objects - customer_id = job.customer_id - if customer_id: - _persist_datasource_objects(flat, customer_id, job.id, run.id, run_at) + if job.customer_id: + _persist_datasource_objects(flat, job.customer_id, job.id, run.id, last_run_at) db.session.commit() return True diff --git a/containers/backupchecks/src/backend/app/main/routes.py b/containers/backupchecks/src/backend/app/main/routes.py index 53fc870..d6a777d 100644 --- a/containers/backupchecks/src/backend/app/main/routes.py +++ b/containers/backupchecks/src/backend/app/main/routes.py @@ -27,5 +27,6 @@ from . import routes_api # noqa: F401 from . import routes_reporting_api # noqa: F401 from . import routes_user_settings # noqa: F401 from . import routes_search # noqa: F401 +from . import routes_cove # noqa: F401 __all__ = ["main_bp", "roles_required"] diff --git a/containers/backupchecks/src/backend/app/main/routes_cove.py b/containers/backupchecks/src/backend/app/main/routes_cove.py new file mode 100644 index 0000000..7da261e --- /dev/null +++ b/containers/backupchecks/src/backend/app/main/routes_cove.py @@ -0,0 +1,179 @@ +"""Cove Data Protection – account review routes. + +Mirrors the Inbox flow for mail messages: + /cove/accounts – list all Cove accounts (unmatched first) + /cove/accounts//link – link an account to an existing or new job + /cove/accounts//unlink – remove the job link +""" +from .routes_shared import * # noqa: F401,F403 +from .routes_shared import _log_admin_event + +from ..models import CoveAccount, Customer, Job, SystemSettings + + +@main_bp.route("/cove/accounts") +@login_required +@roles_required("admin", "operator") +def cove_accounts(): + settings = SystemSettings.query.first() + if not settings or not getattr(settings, "cove_enabled", False): + flash("Cove integration is not enabled.", "warning") + return redirect(url_for("main.settings", section="integrations")) + + # Unmatched accounts (no job linked) – shown first, like Inbox items + unmatched = ( + CoveAccount.query + .filter(CoveAccount.job_id.is_(None)) + .order_by(CoveAccount.customer_name.asc().nullslast(), CoveAccount.account_name.asc()) + .all() + ) + + # Matched accounts + matched = ( + CoveAccount.query + .filter(CoveAccount.job_id.isnot(None)) + .order_by(CoveAccount.customer_name.asc().nullslast(), CoveAccount.account_name.asc()) + .all() + ) + + customers = Customer.query.filter_by(active=True).order_by(Customer.name.asc()).all() + jobs = Job.query.filter_by(archived=False).order_by(Job.job_name.asc()).all() + + return render_template( + "main/cove_accounts.html", + unmatched=unmatched, + matched=matched, + customers=customers, + jobs=jobs, + settings=settings, + STATUS_LABELS={ + 1: "In process", 2: "Failed", 3: "Aborted", 5: "Completed", + 6: "Interrupted", 7: "Not started", 8: "Completed with errors", + 9: "In progress with faults", 10: "Over quota", + 11: "No selection", 12: "Restarted", + }, + STATUS_CLASS={ + 1: "warning", 2: "danger", 3: "danger", 5: "success", + 6: "danger", 7: "secondary", 8: "warning", 9: "warning", + 10: "danger", 11: "warning", 12: "warning", + }, + ) + + +@main_bp.route("/cove/accounts//link", methods=["POST"]) +@login_required +@roles_required("admin", "operator") +def cove_account_link(cove_account_db_id: int): + """Link a Cove account to a job (create a new one or select existing).""" + cove_acc = CoveAccount.query.get_or_404(cove_account_db_id) + + action = (request.form.get("action") or "").strip() # "create" or "link" + + if action == "create": + # Create a new job from the Cove account data + customer_id_raw = (request.form.get("customer_id") or "").strip() + if not customer_id_raw: + flash("Please select a customer.", "danger") + return redirect(url_for("main.cove_accounts")) + + try: + customer_id = int(customer_id_raw) + except ValueError: + flash("Invalid customer selection.", "danger") + return redirect(url_for("main.cove_accounts")) + + customer = Customer.query.get(customer_id) + if not customer: + flash("Customer not found.", "danger") + return redirect(url_for("main.cove_accounts")) + + job_name = (request.form.get("job_name") or cove_acc.account_name or "").strip() + backup_type = (request.form.get("backup_type") or cove_acc.datasource_types or "Backup").strip() + + job = Job( + customer_id=customer.id, + backup_software="Cove Data Protection", + backup_type=backup_type, + job_name=job_name, + cove_account_id=cove_acc.account_id, + active=True, + auto_approve=True, + ) + db.session.add(job) + db.session.flush() + + cove_acc.job_id = job.id + db.session.commit() + + _log_admin_event( + "cove_account_linked", + f"Created job {job.id} and linked Cove account {cove_acc.account_id} ({cove_acc.account_name})", + details=f"customer={customer.name}, job_name={job_name}", + ) + flash( + f"Job '{job_name}' created for customer '{customer.name}'. " + "Runs will appear after the next Cove import.", + "success", + ) + + elif action == "link": + # Link to an existing job + job_id_raw = (request.form.get("job_id") or "").strip() + if not job_id_raw: + flash("Please select a job.", "danger") + return redirect(url_for("main.cove_accounts")) + + try: + job_id = int(job_id_raw) + except ValueError: + flash("Invalid job selection.", "danger") + return redirect(url_for("main.cove_accounts")) + + job = Job.query.get(job_id) + if not job: + flash("Job not found.", "danger") + return redirect(url_for("main.cove_accounts")) + + job.cove_account_id = cove_acc.account_id + cove_acc.job_id = job.id + db.session.commit() + + _log_admin_event( + "cove_account_linked", + f"Linked Cove account {cove_acc.account_id} ({cove_acc.account_name}) to existing job {job.id}", + details=f"job_name={job.job_name}", + ) + flash( + f"Cove account linked to job '{job.job_name}'. " + "Runs will appear after the next Cove import.", + "success", + ) + + else: + flash("Unknown action.", "warning") + + return redirect(url_for("main.cove_accounts")) + + +@main_bp.route("/cove/accounts//unlink", methods=["POST"]) +@login_required +@roles_required("admin", "operator") +def cove_account_unlink(cove_account_db_id: int): + """Remove the job link from a Cove account (puts it back in the unmatched list).""" + cove_acc = CoveAccount.query.get_or_404(cove_account_db_id) + + old_job_id = cove_acc.job_id + if old_job_id: + job = Job.query.get(old_job_id) + if job and job.cove_account_id == cove_acc.account_id: + job.cove_account_id = None + + cove_acc.job_id = None + db.session.commit() + + _log_admin_event( + "cove_account_unlinked", + f"Unlinked Cove account {cove_acc.account_id} ({cove_acc.account_name}) from job {old_job_id}", + ) + flash("Cove account unlinked.", "success") + return redirect(url_for("main.cove_accounts")) diff --git a/containers/backupchecks/src/backend/app/migrations.py b/containers/backupchecks/src/backend/app/migrations.py index c2ea17c..e28963a 100644 --- a/containers/backupchecks/src/backend/app/migrations.py +++ b/containers/backupchecks/src/backend/app/migrations.py @@ -1070,6 +1070,47 @@ def migrate_rename_admin_logs_to_audit_logs() -> None: print("[migrations] audit_logs table will be created by db.create_all()") +def migrate_cove_accounts_table() -> None: + """Create the cove_accounts staging table if it does not exist. + + This table stores all accounts returned by Cove EnumerateAccountStatistics. + Unlinked accounts (job_id IS NULL) appear in the Cove Accounts review page. + """ + try: + engine = db.get_engine() + except Exception as exc: + print(f"[migrations] Could not get engine for cove_accounts migration: {exc}") + return + + try: + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS cove_accounts ( + id SERIAL PRIMARY KEY, + account_id INTEGER NOT NULL UNIQUE, + account_name VARCHAR(512) NULL, + computer_name VARCHAR(512) NULL, + customer_name VARCHAR(255) NULL, + datasource_types VARCHAR(255) NULL, + last_status_code INTEGER NULL, + last_run_at TIMESTAMP NULL, + colorbar_28d VARCHAR(64) NULL, + job_id INTEGER NULL REFERENCES jobs(id) ON DELETE SET NULL, + first_seen_at TIMESTAMP NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMP NOT NULL DEFAULT NOW() + ) + """)) + conn.execute(text( + "CREATE INDEX IF NOT EXISTS idx_cove_accounts_account_id ON cove_accounts (account_id)" + )) + conn.execute(text( + "CREATE INDEX IF NOT EXISTS idx_cove_accounts_job_id ON cove_accounts (job_id)" + )) + print("[migrations] migrate_cove_accounts_table completed.") + except Exception as exc: + print(f"[migrations] Failed to migrate cove_accounts table: {exc}") + + def migrate_cove_integration() -> None: """Add Cove Data Protection integration columns if missing. @@ -1177,6 +1218,7 @@ def run_migrations() -> None: migrate_system_settings_require_daily_dashboard_visit() migrate_rename_admin_logs_to_audit_logs() migrate_cove_integration() + migrate_cove_accounts_table() print("[migrations] All migrations completed.") diff --git a/containers/backupchecks/src/backend/app/models.py b/containers/backupchecks/src/backend/app/models.py index 27e9bd1..69b719d 100644 --- a/containers/backupchecks/src/backend/app/models.py +++ b/containers/backupchecks/src/backend/app/models.py @@ -252,8 +252,8 @@ class Job(db.Model): auto_approve = db.Column(db.Boolean, nullable=False, default=True) active = db.Column(db.Boolean, nullable=False, default=True) - # Cove Data Protection integration - cove_account_id = db.Column(db.Integer, nullable=True) # Cove AccountId mapping + # Cove Data Protection integration (legacy: account ID stored directly on job) + cove_account_id = db.Column(db.Integer, nullable=True) # kept for backwards compat # Archived jobs are excluded from Daily Jobs and Run Checks. # JobRuns remain in the database and are still included in reporting. @@ -331,6 +331,41 @@ class JobRun(db.Model): autotask_ticket_created_by = db.relationship("User", foreign_keys=[autotask_ticket_created_by_user_id]) +class CoveAccount(db.Model): + """Staging table for Cove Data Protection accounts. + + All accounts returned by EnumerateAccountStatistics are upserted here. + Unlinked accounts (job_id IS NULL) appear in the Cove Accounts page + where an admin can create or link a job – the same flow as the mail Inbox. + Once linked, the importer creates JobRuns for each new session. + """ + __tablename__ = "cove_accounts" + + id = db.Column(db.Integer, primary_key=True) + + # Cove account identifier (unique, from AccountId field) + account_id = db.Column(db.Integer, nullable=False, unique=True) + + # Account/device info from Cove columns + account_name = db.Column(db.String(512), nullable=True) # I1 – device/backup name + computer_name = db.Column(db.String(512), nullable=True) # I18 – computer name + customer_name = db.Column(db.String(255), nullable=True) # I8 – Cove customer/partner name + datasource_types = db.Column(db.String(255), nullable=True) # I78 – active datasource label + + # Last known status + last_status_code = db.Column(db.Integer, nullable=True) # D09F00 + last_run_at = db.Column(db.DateTime, nullable=True) # D09F15 (converted from Unix ts) + colorbar_28d = db.Column(db.String(64), nullable=True) # D09F08 + + # Link to a Backupchecks job (NULL = unmatched, needs review) + job_id = db.Column(db.Integer, db.ForeignKey("jobs.id"), nullable=True) + + first_seen_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + last_seen_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + + job = db.relationship("Job", backref=db.backref("cove_account", uselist=False)) + + class JobRunReviewEvent(db.Model): __tablename__ = "job_run_review_events" diff --git a/containers/backupchecks/src/templates/layout/base.html b/containers/backupchecks/src/templates/layout/base.html index 9710971..bf87e06 100644 --- a/containers/backupchecks/src/templates/layout/base.html +++ b/containers/backupchecks/src/templates/layout/base.html @@ -95,6 +95,11 @@ + {% if system_settings and system_settings.cove_enabled and active_role in ('admin', 'operator') %} + + {% endif %} {% if active_role == 'admin' %}