Add Veeam Cloud Connect importer (inbox-style staging flow)
- cloud_connect_importer.py: parse Cloud Connect daily report HTML, upsert tenant rows into cloud_connect_accounts, create JobRuns for linked accounts (deduped via external_id) - routes_cloud_connect.py + cloud_connect_accounts.html: inbox-style review page with create/link/unlink actions (mirrors Cove flow) - CloudConnectAccount model: staging table unique on user × section - migrate_cloud_connect_accounts_table(): creates table + indexes, registered in run_all_migrations() - mail_importer.py: detect btype=="cloud connect report", call upsert_cloud_connect_report(), auto-approve on success - base.html: sidebar link "Cloud Connect" for admin/operator Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
7a8f1aa4e5
commit
ea134f49f3
@ -0,0 +1,306 @@
|
|||||||
|
"""Veeam Cloud Connect daily report importer.
|
||||||
|
|
||||||
|
Parses the HTML body of a Veeam Cloud Connect provider daily report email
|
||||||
|
and upserts each tenant (User row) into the cloud_connect_accounts staging
|
||||||
|
table — identical in spirit to the Cove Data Protection importer.
|
||||||
|
|
||||||
|
Flow:
|
||||||
|
1. When the mail-importer receives a Cloud Connect daily report it calls
|
||||||
|
``upsert_cloud_connect_report(mail_message_id)``.
|
||||||
|
2. Every User × section combination is upserted into cloud_connect_accounts.
|
||||||
|
3. Unlinked accounts appear on the new "Cloud Connect" review page where an
|
||||||
|
admin can create or link a Backupchecks job (same UX as Cove Accounts).
|
||||||
|
4. For linked accounts a JobRun is created/updated; the mail_message_id is
|
||||||
|
attached so the mail body is available in the popup.
|
||||||
|
|
||||||
|
Status mapping (row background colour in the HTML report):
|
||||||
|
#fb9895 → Failed
|
||||||
|
#ffd96c → Warning
|
||||||
|
#ffffff → Success
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from .database import db
|
||||||
|
from .models import CloudConnectAccount, Customer, Job, JobRun
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# HTML parsing helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _strip_tags(html: str) -> str:
|
||||||
|
"""Strip HTML tags and normalise whitespace."""
|
||||||
|
if not html:
|
||||||
|
return ""
|
||||||
|
text = re.sub(r"<br\s*/?>", " ", html, flags=re.IGNORECASE)
|
||||||
|
text = re.sub(r"<[^>]+>", "", text)
|
||||||
|
text = re.sub(r"\s+", " ", text)
|
||||||
|
return text.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _row_status(row_style: str) -> str:
|
||||||
|
"""Map Veeam row background colour to a Backupchecks status string."""
|
||||||
|
m = re.search(r"background-color\s*:\s*([^;\"'\s]+)", row_style, re.IGNORECASE)
|
||||||
|
if not m:
|
||||||
|
return "Success"
|
||||||
|
colour = m.group(1).strip().lower()
|
||||||
|
if colour in {"#fb9895", "#ff9999", "#f4cccc", "#ffb3b3"}:
|
||||||
|
return "Failed"
|
||||||
|
if colour in {"#ffd96c", "#fff2cc", "#ffe599", "#f9cb9c"}:
|
||||||
|
return "Warning"
|
||||||
|
return "Success"
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_last_active(raw: str) -> Optional[datetime]:
|
||||||
|
"""Convert a 'Last active' string like '14 hours ago' to a UTC datetime.
|
||||||
|
|
||||||
|
Returns None when the value is 'never' or cannot be parsed.
|
||||||
|
"""
|
||||||
|
s = (raw or "").strip().lower()
|
||||||
|
if not s or s == "never":
|
||||||
|
return None
|
||||||
|
now = datetime.utcnow()
|
||||||
|
m = re.match(r"(\d+)\s+(hour|day|week|month)s?\s+ago", s)
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
n = int(m.group(1))
|
||||||
|
unit = m.group(2)
|
||||||
|
if unit == "hour":
|
||||||
|
return now - timedelta(hours=n)
|
||||||
|
if unit == "day":
|
||||||
|
return now - timedelta(days=n)
|
||||||
|
if unit == "week":
|
||||||
|
return now - timedelta(weeks=n)
|
||||||
|
if unit == "month":
|
||||||
|
return now - timedelta(days=n * 30)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_report_tables(html: str) -> list[dict]:
|
||||||
|
"""Extract all tenant rows from a Cloud Connect daily report HTML body.
|
||||||
|
|
||||||
|
Returns a list of dicts with keys:
|
||||||
|
section, user, repo_name, repo_type, num_items,
|
||||||
|
total_quota, used_space, free_space, last_active_raw,
|
||||||
|
last_active_dt, status
|
||||||
|
"""
|
||||||
|
if not html:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Section headers are <p> tags with font-size 18px just before each table.
|
||||||
|
# We walk the HTML top-to-bottom, tracking the current section name.
|
||||||
|
section_pattern = re.compile(
|
||||||
|
r'<p[^>]*font-size:\s*18px[^>]*>\s*(Backup|Replication|Agent)\s*</p>',
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
table_pattern = re.compile(r'<table[^>]*>(.*?)</table>', re.IGNORECASE | re.DOTALL)
|
||||||
|
row_pattern = re.compile(r'<tr([^>]*)>(.*?)</tr>', re.IGNORECASE | re.DOTALL)
|
||||||
|
cell_pattern = re.compile(r'<t[dh][^>]*>(.*?)</t[dh]>', re.IGNORECASE | re.DOTALL)
|
||||||
|
|
||||||
|
results: list[dict] = []
|
||||||
|
|
||||||
|
# Find positions of section headers and tables to interleave them.
|
||||||
|
section_positions = [(m.start(), m.group(1)) for m in section_pattern.finditer(html)]
|
||||||
|
table_positions = [(m.start(), m.group(1)) for m in table_pattern.finditer(html)]
|
||||||
|
|
||||||
|
def _section_for_table(table_start: int) -> str:
|
||||||
|
"""Return the section name of the nearest preceding section header."""
|
||||||
|
current = "Backup"
|
||||||
|
for pos, name in section_positions:
|
||||||
|
if pos < table_start:
|
||||||
|
current = name
|
||||||
|
return current
|
||||||
|
|
||||||
|
for table_start, table_inner in table_positions:
|
||||||
|
section = _section_for_table(table_start)
|
||||||
|
|
||||||
|
rows = row_pattern.findall(table_inner)
|
||||||
|
if not rows:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Determine column positions from header row.
|
||||||
|
first_row_cells = [_strip_tags(c).strip() for _, c in [rows[0]]]
|
||||||
|
# Re-parse properly:
|
||||||
|
header_cells = [_strip_tags(c).strip() for c in cell_pattern.findall(rows[0][1])]
|
||||||
|
if not header_cells or header_cells[0].lower() != "user":
|
||||||
|
continue # not a tenant table (e.g. the version footer table)
|
||||||
|
|
||||||
|
# Column index lookup (graceful — different tables have different columns)
|
||||||
|
col = {name.lower(): i for i, name in enumerate(header_cells)}
|
||||||
|
|
||||||
|
for row_attr, row_inner in rows[1:]:
|
||||||
|
cells_raw = cell_pattern.findall(row_inner)
|
||||||
|
cells = [_strip_tags(c).strip() for c in cells_raw]
|
||||||
|
if not cells:
|
||||||
|
continue
|
||||||
|
|
||||||
|
user = cells[0] if len(cells) > 0 else ""
|
||||||
|
if not user or user.upper() == "TOTAL":
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Column indices differ between Backup and Agent tables.
|
||||||
|
# Backup: User | #VM | Repo Name | Repo | Quota | Used | Free | Last active | Expiry
|
||||||
|
# Agent: User | #WS | #Server | Repo Name | Repo | Quota | Used | Free | Last active | Expiry
|
||||||
|
is_agent = section.lower() == "agent"
|
||||||
|
|
||||||
|
if is_agent:
|
||||||
|
num_ws = cells[1] if len(cells) > 1 else ""
|
||||||
|
num_srv = cells[2] if len(cells) > 2 else ""
|
||||||
|
num_items = f"{num_ws} WS / {num_srv} Server"
|
||||||
|
repo_name = cells[3] if len(cells) > 3 else ""
|
||||||
|
repo_type = cells[4] if len(cells) > 4 else ""
|
||||||
|
total_quota = cells[5] if len(cells) > 5 else ""
|
||||||
|
used_space = cells[6] if len(cells) > 6 else ""
|
||||||
|
free_space = cells[7] if len(cells) > 7 else ""
|
||||||
|
last_active_raw = cells[8] if len(cells) > 8 else ""
|
||||||
|
else:
|
||||||
|
num_items = cells[1] if len(cells) > 1 else ""
|
||||||
|
repo_name = cells[2] if len(cells) > 2 else ""
|
||||||
|
repo_type = cells[3] if len(cells) > 3 else ""
|
||||||
|
total_quota = cells[4] if len(cells) > 4 else ""
|
||||||
|
used_space = cells[5] if len(cells) > 5 else ""
|
||||||
|
free_space = cells[6] if len(cells) > 6 else ""
|
||||||
|
last_active_raw = cells[7] if len(cells) > 7 else ""
|
||||||
|
|
||||||
|
status = _row_status(row_attr)
|
||||||
|
|
||||||
|
# Downgrade to Warning when last active is suspiciously old but row is white.
|
||||||
|
last_active_dt = _parse_last_active(last_active_raw)
|
||||||
|
if status == "Success" and last_active_raw.lower() == "never":
|
||||||
|
status = "Warning"
|
||||||
|
elif (
|
||||||
|
status == "Success"
|
||||||
|
and last_active_dt
|
||||||
|
and (datetime.utcnow() - last_active_dt) > timedelta(days=3)
|
||||||
|
):
|
||||||
|
status = "Warning"
|
||||||
|
|
||||||
|
results.append({
|
||||||
|
"section": section,
|
||||||
|
"user": user,
|
||||||
|
"repo_name": repo_name,
|
||||||
|
"repo_type": repo_type,
|
||||||
|
"num_items": num_items,
|
||||||
|
"total_quota": total_quota,
|
||||||
|
"used_space": used_space,
|
||||||
|
"free_space": free_space,
|
||||||
|
"last_active_raw": last_active_raw,
|
||||||
|
"last_active_dt": last_active_dt,
|
||||||
|
"status": status,
|
||||||
|
})
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Public import entry point
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def upsert_cloud_connect_report(mail_message_id: int, html_body: str) -> dict:
|
||||||
|
"""Parse a Cloud Connect daily report and upsert all tenant rows.
|
||||||
|
|
||||||
|
Called by the mail importer when it detects a Cloud Connect daily report.
|
||||||
|
Returns a summary dict: {total, linked, unlinked, created, skipped}.
|
||||||
|
"""
|
||||||
|
rows = _parse_report_tables(html_body)
|
||||||
|
if not rows:
|
||||||
|
return {"total": 0, "linked": 0, "unlinked": 0, "created": 0, "skipped": 0}
|
||||||
|
|
||||||
|
now = datetime.utcnow()
|
||||||
|
counters = {"total": len(rows), "linked": 0, "unlinked": 0, "created": 0, "skipped": 0}
|
||||||
|
|
||||||
|
for row in rows:
|
||||||
|
user = row["user"]
|
||||||
|
section = row["section"]
|
||||||
|
|
||||||
|
# Upsert the staging record — keyed on (user, section).
|
||||||
|
acc = CloudConnectAccount.query.filter_by(user=user, section=section).first()
|
||||||
|
if acc is None:
|
||||||
|
acc = CloudConnectAccount(
|
||||||
|
user=user,
|
||||||
|
section=section,
|
||||||
|
first_seen_at=now,
|
||||||
|
)
|
||||||
|
db.session.add(acc)
|
||||||
|
|
||||||
|
acc.repo_name = row["repo_name"]
|
||||||
|
acc.repo_type = row["repo_type"]
|
||||||
|
acc.num_items = row["num_items"]
|
||||||
|
acc.total_quota = row["total_quota"]
|
||||||
|
acc.used_space = row["used_space"]
|
||||||
|
acc.free_space = row["free_space"]
|
||||||
|
acc.last_active_raw = row["last_active_raw"]
|
||||||
|
acc.last_active_dt = row["last_active_dt"]
|
||||||
|
acc.last_status = row["status"]
|
||||||
|
acc.last_seen_at = now
|
||||||
|
acc.last_mail_message_id = mail_message_id
|
||||||
|
|
||||||
|
db.session.flush()
|
||||||
|
|
||||||
|
if not acc.job_id:
|
||||||
|
counters["unlinked"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Account is linked — create a JobRun if not already present for today.
|
||||||
|
job = Job.query.get(acc.job_id)
|
||||||
|
if not job:
|
||||||
|
counters["skipped"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Deduplicate: one run per job per calendar day (report is daily).
|
||||||
|
run_date = now.date().isoformat()
|
||||||
|
external_id = f"vcc-{user}-{section}-{run_date}".lower().replace(" ", "_")
|
||||||
|
|
||||||
|
existing = JobRun.query.filter_by(job_id=job.id, external_id=external_id).first()
|
||||||
|
if existing:
|
||||||
|
# Update status in case re-import happens same day with different result.
|
||||||
|
existing.status = row["status"]
|
||||||
|
existing.run_at = now
|
||||||
|
db.session.add(existing)
|
||||||
|
counters["skipped"] += 1
|
||||||
|
counters["linked"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
error_message = _build_error_message(row)
|
||||||
|
|
||||||
|
run = JobRun(
|
||||||
|
job_id=job.id,
|
||||||
|
mail_message_id=mail_message_id,
|
||||||
|
run_at=now,
|
||||||
|
status=row["status"],
|
||||||
|
remark=error_message or None,
|
||||||
|
missed=False,
|
||||||
|
override_applied=False,
|
||||||
|
source_type="cloud_connect",
|
||||||
|
external_id=external_id,
|
||||||
|
)
|
||||||
|
db.session.add(run)
|
||||||
|
counters["created"] += 1
|
||||||
|
counters["linked"] += 1
|
||||||
|
|
||||||
|
db.session.commit()
|
||||||
|
return counters
|
||||||
|
|
||||||
|
|
||||||
|
def _build_error_message(row: dict) -> str:
|
||||||
|
"""Build a human-readable remark for a Cloud Connect run."""
|
||||||
|
parts = [
|
||||||
|
f"Repository: {row['repo_name']} ({row['repo_type']})",
|
||||||
|
f"Used: {row['used_space']} / {row['total_quota']}",
|
||||||
|
f"Free: {row['free_space']}",
|
||||||
|
f"Last active: {row['last_active_raw'] or 'unknown'}",
|
||||||
|
]
|
||||||
|
if row["status"] == "Failed":
|
||||||
|
parts.append("⚠ Repository appears to be full or near full")
|
||||||
|
elif row["status"] == "Warning" and row["last_active_raw"].lower() in ("never", ""):
|
||||||
|
parts.append("⚠ Backup has never run")
|
||||||
|
elif row["status"] == "Warning" and "days ago" in row["last_active_raw"].lower():
|
||||||
|
parts.append(f"⚠ No recent activity: {row['last_active_raw']}")
|
||||||
|
return " | ".join(parts)
|
||||||
@ -14,6 +14,7 @@ from . import db
|
|||||||
from .models import MailMessage, SystemSettings, Job, JobRun, MailObject
|
from .models import MailMessage, SystemSettings, Job, JobRun, MailObject
|
||||||
from .parsers import parse_mail_message
|
from .parsers import parse_mail_message
|
||||||
from .parsers.veeam import extract_vspc_active_alarms_companies
|
from .parsers.veeam import extract_vspc_active_alarms_companies
|
||||||
|
from .cloud_connect_importer import upsert_cloud_connect_report
|
||||||
from .email_utils import normalize_from_address, extract_best_html_from_eml, is_effectively_blank_html
|
from .email_utils import normalize_from_address, extract_best_html_from_eml, is_effectively_blank_html
|
||||||
from .job_matching import find_matching_job
|
from .job_matching import find_matching_job
|
||||||
from .ticketing_utils import link_open_internal_tickets_to_run
|
from .ticketing_utils import link_open_internal_tickets_to_run
|
||||||
@ -272,6 +273,37 @@ def _store_messages(settings: SystemSettings, messages):
|
|||||||
btype = (getattr(mail, "backup_type", "") or "").strip().lower()
|
btype = (getattr(mail, "backup_type", "") or "").strip().lower()
|
||||||
jname = (getattr(mail, "job_name", "") or "").strip().lower()
|
jname = (getattr(mail, "job_name", "") or "").strip().lower()
|
||||||
|
|
||||||
|
# ── Veeam Cloud Connect daily report ──────────────────────
|
||||||
|
# One report contains all tenants. Upsert each into the
|
||||||
|
# cloud_connect_accounts staging table; linked accounts get
|
||||||
|
# a JobRun automatically — same flow as Cove Data Protection.
|
||||||
|
if bsw == "veeam" and btype == "cloud connect report":
|
||||||
|
try:
|
||||||
|
result = upsert_cloud_connect_report(
|
||||||
|
mail_message_id=mail.id,
|
||||||
|
html_body=(mail.html_body or ""),
|
||||||
|
)
|
||||||
|
logger.debug(
|
||||||
|
"Cloud Connect import: total=%s linked=%s unlinked=%s "
|
||||||
|
"created=%s skipped=%s",
|
||||||
|
result.get("total"), result.get("linked"),
|
||||||
|
result.get("unlinked"), result.get("created"),
|
||||||
|
result.get("skipped"),
|
||||||
|
)
|
||||||
|
if result.get("created", 0) > 0 or result.get("linked", 0) > 0:
|
||||||
|
if hasattr(mail, "approved"):
|
||||||
|
mail.approved = True
|
||||||
|
if hasattr(mail, "approved_at"):
|
||||||
|
mail.approved_at = datetime.utcnow()
|
||||||
|
if hasattr(mail, "location"):
|
||||||
|
mail.location = "history"
|
||||||
|
auto_approved += 1
|
||||||
|
except Exception as cc_exc:
|
||||||
|
logger.warning("Cloud Connect import failed: %s", cc_exc)
|
||||||
|
db.session.commit()
|
||||||
|
continue
|
||||||
|
# ── end Cloud Connect ──────────────────────────────────────
|
||||||
|
|
||||||
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
|
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
|
||||||
raw = (mail.text_body or "").strip() or (mail.html_body or "")
|
raw = (mail.text_body or "").strip() or (mail.html_body or "")
|
||||||
companies = extract_vspc_active_alarms_companies(raw)
|
companies = extract_vspc_active_alarms_companies(raw)
|
||||||
|
|||||||
@ -28,5 +28,6 @@ from . import routes_reporting_api # noqa: F401
|
|||||||
from . import routes_user_settings # noqa: F401
|
from . import routes_user_settings # noqa: F401
|
||||||
from . import routes_search # noqa: F401
|
from . import routes_search # noqa: F401
|
||||||
from . import routes_cove # noqa: F401
|
from . import routes_cove # noqa: F401
|
||||||
|
from . import routes_cloud_connect # noqa: F401
|
||||||
|
|
||||||
__all__ = ["main_bp", "roles_required"]
|
__all__ = ["main_bp", "roles_required"]
|
||||||
|
|||||||
@ -0,0 +1,119 @@
|
|||||||
|
"""Veeam Cloud Connect accounts review routes.
|
||||||
|
|
||||||
|
Mirrors the Cove Accounts flow:
|
||||||
|
/cloud-connect/accounts – list all accounts (unmatched first)
|
||||||
|
/cloud-connect/accounts/<id>/link – link to existing job or create new job
|
||||||
|
/cloud-connect/accounts/<id>/unlink – remove the job link
|
||||||
|
"""
|
||||||
|
from .routes_shared import * # noqa: F401,F403
|
||||||
|
from .routes_shared import _log_admin_event
|
||||||
|
from ..models import CloudConnectAccount, Customer, Job, JobRun
|
||||||
|
|
||||||
|
|
||||||
|
@main_bp.route("/cloud-connect/accounts")
|
||||||
|
@login_required
|
||||||
|
@roles_required("admin", "operator")
|
||||||
|
def cloud_connect_accounts():
|
||||||
|
# Unmatched accounts shown first, then matched — same as Cove Accounts
|
||||||
|
unmatched = (
|
||||||
|
CloudConnectAccount.query
|
||||||
|
.filter(CloudConnectAccount.job_id.is_(None))
|
||||||
|
.order_by(CloudConnectAccount.user.asc(), CloudConnectAccount.section.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
matched = (
|
||||||
|
CloudConnectAccount.query
|
||||||
|
.filter(CloudConnectAccount.job_id.isnot(None))
|
||||||
|
.order_by(CloudConnectAccount.user.asc(), CloudConnectAccount.section.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
customers = Customer.query.filter_by(active=True).order_by(Customer.name.asc()).all()
|
||||||
|
jobs = Job.query.filter_by(archived=False).order_by(Job.job_name.asc()).all()
|
||||||
|
|
||||||
|
# Attach derived fields for the template
|
||||||
|
for acc in unmatched + matched:
|
||||||
|
acc.derived_backup_software = "Veeam"
|
||||||
|
acc.derived_backup_type = (
|
||||||
|
"Cloud Connect Agent" if acc.section == "Agent" else "Cloud Connect Backup"
|
||||||
|
)
|
||||||
|
acc.derived_job_name = acc.user
|
||||||
|
|
||||||
|
return render_template(
|
||||||
|
"main/cloud_connect_accounts.html",
|
||||||
|
unmatched=unmatched,
|
||||||
|
matched=matched,
|
||||||
|
customers=customers,
|
||||||
|
jobs=jobs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@main_bp.route("/cloud-connect/accounts/<int:cc_account_db_id>/link", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
@roles_required("admin", "operator")
|
||||||
|
def cloud_connect_account_link(cc_account_db_id: int):
|
||||||
|
acc = CloudConnectAccount.query.get_or_404(cc_account_db_id)
|
||||||
|
action = (request.form.get("action") or "").strip() # "create" or "link"
|
||||||
|
|
||||||
|
if action == "create":
|
||||||
|
customer_id = request.form.get("customer_id", type=int)
|
||||||
|
if not customer_id:
|
||||||
|
flash("Please select a customer.", "danger")
|
||||||
|
return redirect(url_for("main.cloud_connect_accounts"))
|
||||||
|
|
||||||
|
customer = Customer.query.get_or_404(customer_id)
|
||||||
|
|
||||||
|
job_name = (request.form.get("job_name") or acc.user).strip()
|
||||||
|
backup_type = (request.form.get("backup_type") or acc.derived_backup_type).strip()
|
||||||
|
|
||||||
|
job = Job(
|
||||||
|
customer_id=customer.id,
|
||||||
|
backup_software="Veeam",
|
||||||
|
backup_type=backup_type,
|
||||||
|
job_name=job_name,
|
||||||
|
)
|
||||||
|
db.session.add(job)
|
||||||
|
db.session.flush()
|
||||||
|
|
||||||
|
acc.job_id = job.id
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
_log_admin_event(
|
||||||
|
event_type="cloud_connect_account_linked",
|
||||||
|
message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to new job '{job_name}'",
|
||||||
|
details=f"customer={customer.name}, job_name={job_name}",
|
||||||
|
)
|
||||||
|
flash(f"Job '{job_name}' created and linked to '{acc.user}' ({acc.section}).", "success")
|
||||||
|
|
||||||
|
elif action == "link":
|
||||||
|
job_id = request.form.get("job_id", type=int)
|
||||||
|
if not job_id:
|
||||||
|
flash("Please select a job.", "danger")
|
||||||
|
return redirect(url_for("main.cloud_connect_accounts"))
|
||||||
|
|
||||||
|
job = Job.query.get_or_404(job_id)
|
||||||
|
acc.job_id = job.id
|
||||||
|
db.session.commit()
|
||||||
|
|
||||||
|
_log_admin_event(
|
||||||
|
event_type="cloud_connect_account_linked",
|
||||||
|
message=f"Cloud Connect account '{acc.user}' ({acc.section}) linked to existing job '{job.job_name}'",
|
||||||
|
details=f"job_id={job.id}, job_name={job.job_name}",
|
||||||
|
)
|
||||||
|
flash(f"Linked '{acc.user}' ({acc.section}) to job '{job.job_name}'.", "success")
|
||||||
|
|
||||||
|
else:
|
||||||
|
flash("Unknown action.", "danger")
|
||||||
|
|
||||||
|
return redirect(url_for("main.cloud_connect_accounts"))
|
||||||
|
|
||||||
|
|
||||||
|
@main_bp.route("/cloud-connect/accounts/<int:cc_account_db_id>/unlink", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
@roles_required("admin", "operator")
|
||||||
|
def cloud_connect_account_unlink(cc_account_db_id: int):
|
||||||
|
acc = CloudConnectAccount.query.get_or_404(cc_account_db_id)
|
||||||
|
acc.job_id = None
|
||||||
|
db.session.commit()
|
||||||
|
flash(f"Unlinked '{acc.user}' ({acc.section}).", "success")
|
||||||
|
return redirect(url_for("main.cloud_connect_accounts"))
|
||||||
@ -1270,6 +1270,48 @@ def migrate_entra_sso_settings() -> None:
|
|||||||
print(f"[migrations] Failed to migrate Entra SSO columns: {exc}")
|
print(f"[migrations] Failed to migrate Entra SSO columns: {exc}")
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_cloud_connect_accounts_table() -> None:
|
||||||
|
"""Create the cloud_connect_accounts staging table if it does not exist."""
|
||||||
|
try:
|
||||||
|
engine = db.get_engine()
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"[migrations] Could not get engine for cloud_connect_accounts migration: {exc}")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
with engine.begin() as conn:
|
||||||
|
conn.execute(text("""
|
||||||
|
CREATE TABLE IF NOT EXISTS cloud_connect_accounts (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
"user" VARCHAR(255) NOT NULL,
|
||||||
|
section VARCHAR(32) NOT NULL,
|
||||||
|
repo_name VARCHAR(512) NULL,
|
||||||
|
repo_type VARCHAR(255) NULL,
|
||||||
|
num_items VARCHAR(64) NULL,
|
||||||
|
total_quota VARCHAR(32) NULL,
|
||||||
|
used_space VARCHAR(32) NULL,
|
||||||
|
free_space VARCHAR(32) NULL,
|
||||||
|
last_active_raw VARCHAR(64) NULL,
|
||||||
|
last_active_dt TIMESTAMP NULL,
|
||||||
|
last_status VARCHAR(32) NULL,
|
||||||
|
last_mail_message_id INTEGER NULL REFERENCES mail_messages(id) ON DELETE SET NULL,
|
||||||
|
job_id INTEGER NULL REFERENCES jobs(id) ON DELETE SET NULL,
|
||||||
|
first_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
last_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
|
CONSTRAINT uq_cloud_connect_accounts_user_section UNIQUE ("user", section)
|
||||||
|
)
|
||||||
|
"""))
|
||||||
|
conn.execute(text(
|
||||||
|
'CREATE INDEX IF NOT EXISTS idx_cc_accounts_user ON cloud_connect_accounts ("user")'
|
||||||
|
))
|
||||||
|
conn.execute(text(
|
||||||
|
"CREATE INDEX IF NOT EXISTS idx_cc_accounts_job_id ON cloud_connect_accounts (job_id)"
|
||||||
|
))
|
||||||
|
print("[migrations] migrate_cloud_connect_accounts_table completed.")
|
||||||
|
except Exception as exc:
|
||||||
|
print(f"[migrations] Failed to migrate cloud_connect_accounts table: {exc}")
|
||||||
|
|
||||||
|
|
||||||
def run_migrations() -> None:
|
def run_migrations() -> None:
|
||||||
print("[migrations] Starting migrations...")
|
print("[migrations] Starting migrations...")
|
||||||
migrate_add_username_to_users()
|
migrate_add_username_to_users()
|
||||||
@ -1315,6 +1357,7 @@ def run_migrations() -> None:
|
|||||||
migrate_rename_admin_logs_to_audit_logs()
|
migrate_rename_admin_logs_to_audit_logs()
|
||||||
migrate_cove_integration()
|
migrate_cove_integration()
|
||||||
migrate_cove_accounts_table()
|
migrate_cove_accounts_table()
|
||||||
|
migrate_cloud_connect_accounts_table()
|
||||||
migrate_entra_sso_settings()
|
migrate_entra_sso_settings()
|
||||||
print("[migrations] All migrations completed.")
|
print("[migrations] All migrations completed.")
|
||||||
|
|
||||||
|
|||||||
@ -382,6 +382,46 @@ class CoveAccount(db.Model):
|
|||||||
job = db.relationship("Job", backref=db.backref("cove_account", uselist=False))
|
job = db.relationship("Job", backref=db.backref("cove_account", uselist=False))
|
||||||
|
|
||||||
|
|
||||||
|
class CloudConnectAccount(db.Model):
|
||||||
|
"""Staging table for Veeam Cloud Connect tenant accounts.
|
||||||
|
|
||||||
|
Each row represents one User × section (Backup / Agent) combination
|
||||||
|
as found in the Veeam Cloud Connect daily report email.
|
||||||
|
|
||||||
|
Unlinked accounts (job_id IS NULL) appear on the Cloud Connect Accounts
|
||||||
|
review page where an admin can create or link a Backupchecks job —
|
||||||
|
identical to the Cove Accounts flow.
|
||||||
|
"""
|
||||||
|
__tablename__ = "cloud_connect_accounts"
|
||||||
|
|
||||||
|
id = db.Column(db.Integer, primary_key=True)
|
||||||
|
|
||||||
|
user = db.Column(db.String(255), nullable=False)
|
||||||
|
section = db.Column(db.String(32), nullable=False)
|
||||||
|
|
||||||
|
repo_name = db.Column(db.String(512), nullable=True)
|
||||||
|
repo_type = db.Column(db.String(255), nullable=True)
|
||||||
|
num_items = db.Column(db.String(64), nullable=True)
|
||||||
|
total_quota = db.Column(db.String(32), nullable=True)
|
||||||
|
used_space = db.Column(db.String(32), nullable=True)
|
||||||
|
free_space = db.Column(db.String(32), nullable=True)
|
||||||
|
last_active_raw = db.Column(db.String(64), nullable=True)
|
||||||
|
last_active_dt = db.Column(db.DateTime, nullable=True)
|
||||||
|
last_status = db.Column(db.String(32), nullable=True)
|
||||||
|
|
||||||
|
last_mail_message_id = db.Column(db.Integer, db.ForeignKey("mail_messages.id"), nullable=True)
|
||||||
|
job_id = db.Column(db.Integer, db.ForeignKey("jobs.id"), nullable=True)
|
||||||
|
|
||||||
|
first_seen_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False)
|
||||||
|
last_seen_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False)
|
||||||
|
|
||||||
|
job = db.relationship("Job", backref=db.backref("cloud_connect_account", uselist=False))
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
db.UniqueConstraint("user", "section", name="uq_cloud_connect_accounts_user_section"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class JobRunReviewEvent(db.Model):
|
class JobRunReviewEvent(db.Model):
|
||||||
__tablename__ = "job_run_review_events"
|
__tablename__ = "job_run_review_events"
|
||||||
|
|
||||||
|
|||||||
@ -111,6 +111,9 @@
|
|||||||
{% if system_settings and system_settings.cove_enabled and active_role in ('admin', 'operator') %}
|
{% if system_settings and system_settings.cove_enabled and active_role in ('admin', 'operator') %}
|
||||||
{{ bc_nav_item('main.cove_accounts', 'Cove Accounts', icon_cloud()) }}
|
{{ bc_nav_item('main.cove_accounts', 'Cove Accounts', icon_cloud()) }}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
{% if active_role in ('admin', 'operator') %}
|
||||||
|
{{ bc_nav_item('main.cloud_connect_accounts', 'Cloud Connect', icon_server()) }}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
<div class="bc-nav-divider"></div>
|
<div class="bc-nav-divider"></div>
|
||||||
<div class="bc-nav-label">Info</div>
|
<div class="bc-nav-label">Info</div>
|
||||||
|
|||||||
@ -0,0 +1,211 @@
|
|||||||
|
{% extends "layout/base.html" %}
|
||||||
|
{% block content %}
|
||||||
|
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||||
|
<h2 class="mb-0">Cloud Connect Accounts</h2>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{# ── Unmatched accounts ─────────────────────────────────────────────────── #}
|
||||||
|
{% if unmatched %}
|
||||||
|
<h4 class="mb-2">Unmatched <span class="badge bg-warning text-dark">{{ unmatched|length }}</span></h4>
|
||||||
|
<p class="text-muted small mb-3">These accounts have no linked job yet. Create a new job or link to an existing one.</p>
|
||||||
|
|
||||||
|
<div class="table-responsive mb-4">
|
||||||
|
<table class="table table-sm table-hover align-middle">
|
||||||
|
<thead class="table-light">
|
||||||
|
<tr>
|
||||||
|
<th>User</th>
|
||||||
|
<th>Section</th>
|
||||||
|
<th>Repository</th>
|
||||||
|
<th>Used / Quota</th>
|
||||||
|
<th>Free</th>
|
||||||
|
<th>Last active</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th>First seen</th>
|
||||||
|
<th></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for acc in unmatched %}
|
||||||
|
<tr>
|
||||||
|
<td class="fw-semibold">{{ acc.user }}</td>
|
||||||
|
<td><span class="badge bg-secondary">{{ acc.section }}</span></td>
|
||||||
|
<td class="text-muted small">{{ acc.repo_name or '—' }}<br><span class="text-muted" style="font-size:11px;">{{ acc.repo_type or '' }}</span></td>
|
||||||
|
<td class="text-muted small">{{ acc.used_space or '—' }} / {{ acc.total_quota or '—' }}</td>
|
||||||
|
<td class="text-muted small">{{ acc.free_space or '—' }}</td>
|
||||||
|
<td class="text-muted small">{{ acc.last_active_raw or '—' }}</td>
|
||||||
|
<td>
|
||||||
|
{% if acc.last_status == 'Failed' %}
|
||||||
|
<span class="badge bg-danger">Failed</span>
|
||||||
|
{% elif acc.last_status == 'Warning' %}
|
||||||
|
<span class="badge bg-warning text-dark">Warning</span>
|
||||||
|
{% else %}
|
||||||
|
<span class="badge bg-success">Success</span>
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td class="text-muted small">{{ acc.first_seen_at|local_datetime }}</td>
|
||||||
|
<td>
|
||||||
|
<button class="btn btn-sm btn-primary"
|
||||||
|
data-bs-toggle="modal"
|
||||||
|
data-bs-target="#link-modal-{{ acc.id }}">
|
||||||
|
Link / Create job
|
||||||
|
</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
|
||||||
|
{# Link modal #}
|
||||||
|
<div class="modal fade" id="link-modal-{{ acc.id }}" tabindex="-1">
|
||||||
|
<div class="modal-dialog">
|
||||||
|
<div class="modal-content">
|
||||||
|
<div class="modal-header">
|
||||||
|
<h5 class="modal-title">Link: {{ acc.user }} ({{ acc.section }})</h5>
|
||||||
|
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
|
||||||
|
</div>
|
||||||
|
<div class="modal-body">
|
||||||
|
<ul class="nav nav-tabs mb-3" role="tablist">
|
||||||
|
<li class="nav-item" role="presentation">
|
||||||
|
<button class="nav-link active" data-bs-toggle="tab"
|
||||||
|
data-bs-target="#create-{{ acc.id }}" type="button">
|
||||||
|
Create new job
|
||||||
|
</button>
|
||||||
|
</li>
|
||||||
|
<li class="nav-item" role="presentation">
|
||||||
|
<button class="nav-link" data-bs-toggle="tab"
|
||||||
|
data-bs-target="#existing-{{ acc.id }}" type="button">
|
||||||
|
Link to existing job
|
||||||
|
</button>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<div class="tab-content">
|
||||||
|
{# Tab 1: Create new job #}
|
||||||
|
<div class="tab-pane fade show active" id="create-{{ acc.id }}">
|
||||||
|
<form method="post" action="{{ url_for('main.cloud_connect_account_link', cc_account_db_id=acc.id) }}">
|
||||||
|
<input type="hidden" name="action" value="create" />
|
||||||
|
<div class="mb-3">
|
||||||
|
<label class="form-label">Customer <span class="text-danger">*</span></label>
|
||||||
|
<select class="form-select" name="customer_id" required>
|
||||||
|
<option value="">Select customer…</option>
|
||||||
|
{% for c in customers %}
|
||||||
|
<option value="{{ c.id }}">{{ c.name }}</option>
|
||||||
|
{% endfor %}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
<label class="form-label">Job name</label>
|
||||||
|
<input type="text" class="form-control" name="job_name" value="{{ acc.user }}" />
|
||||||
|
<div class="form-text">Defaults to the user name from the report.</div>
|
||||||
|
</div>
|
||||||
|
<div class="mb-3">
|
||||||
|
<label class="form-label">Backup type</label>
|
||||||
|
<input type="text" class="form-control" name="backup_type"
|
||||||
|
value="{{ acc.derived_backup_type }}" />
|
||||||
|
<div class="form-text">Cloud Connect Backup or Cloud Connect Agent.</div>
|
||||||
|
</div>
|
||||||
|
<div class="d-flex justify-content-end gap-2">
|
||||||
|
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||||
|
<button type="submit" class="btn btn-primary">Create job & link</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{# Tab 2: Link to existing job #}
|
||||||
|
<div class="tab-pane fade" id="existing-{{ acc.id }}">
|
||||||
|
<form method="post" action="{{ url_for('main.cloud_connect_account_link', cc_account_db_id=acc.id) }}">
|
||||||
|
<input type="hidden" name="action" value="link" />
|
||||||
|
<div class="mb-3">
|
||||||
|
<label class="form-label">Job <span class="text-danger">*</span></label>
|
||||||
|
<select class="form-select" name="job_id" required>
|
||||||
|
<option value="">Select job…</option>
|
||||||
|
{% for j in jobs %}
|
||||||
|
<option value="{{ j.id }}">
|
||||||
|
{{ j.customer.name ~ ' – ' if j.customer else '' }}{{ j.backup_software }} / {{ j.job_name }}
|
||||||
|
</option>
|
||||||
|
{% endfor %}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div class="d-flex justify-content-end gap-2">
|
||||||
|
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||||
|
<button type="submit" class="btn btn-primary">Link to job</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{% else %}
|
||||||
|
<div class="alert alert-success mb-4">
|
||||||
|
<strong>All accounts matched.</strong> No unmatched Cloud Connect accounts.
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{# ── Matched accounts ───────────────────────────────────────────────────── #}
|
||||||
|
{% if matched %}
|
||||||
|
<h4 class="mb-2">Linked <span class="badge bg-success">{{ matched|length }}</span></h4>
|
||||||
|
<div class="table-responsive">
|
||||||
|
<table class="table table-sm table-hover align-middle">
|
||||||
|
<thead class="table-light">
|
||||||
|
<tr>
|
||||||
|
<th>User</th>
|
||||||
|
<th>Section</th>
|
||||||
|
<th>Repository</th>
|
||||||
|
<th>Used / Quota</th>
|
||||||
|
<th>Free</th>
|
||||||
|
<th>Last active</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th>Linked job</th>
|
||||||
|
<th></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for acc in matched %}
|
||||||
|
<tr>
|
||||||
|
<td class="fw-semibold">{{ acc.user }}</td>
|
||||||
|
<td><span class="badge bg-secondary">{{ acc.section }}</span></td>
|
||||||
|
<td class="text-muted small">{{ acc.repo_name or '—' }}<br><span style="font-size:11px;">{{ acc.repo_type or '' }}</span></td>
|
||||||
|
<td class="text-muted small">{{ acc.used_space or '—' }} / {{ acc.total_quota or '—' }}</td>
|
||||||
|
<td class="text-muted small">{{ acc.free_space or '—' }}</td>
|
||||||
|
<td class="text-muted small">{{ acc.last_active_raw or '—' }}</td>
|
||||||
|
<td>
|
||||||
|
{% if acc.last_status == 'Failed' %}
|
||||||
|
<span class="badge bg-danger">Failed</span>
|
||||||
|
{% elif acc.last_status == 'Warning' %}
|
||||||
|
<span class="badge bg-warning text-dark">Warning</span>
|
||||||
|
{% else %}
|
||||||
|
<span class="badge bg-success">Success</span>
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if acc.job %}
|
||||||
|
<a href="{{ url_for('main.job_detail', job_id=acc.job.id) }}">
|
||||||
|
{{ acc.job.customer.name ~ ' – ' if acc.job.customer else '' }}{{ acc.job.job_name }}
|
||||||
|
</a>
|
||||||
|
{% else %}—{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<form method="post"
|
||||||
|
action="{{ url_for('main.cloud_connect_account_unlink', cc_account_db_id=acc.id) }}"
|
||||||
|
onsubmit="return confirm('Remove link for {{ acc.user }} ({{ acc.section }})?');"
|
||||||
|
class="mb-0">
|
||||||
|
<button type="submit" class="btn btn-sm btn-outline-secondary">Unlink</button>
|
||||||
|
</form>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% if not unmatched and not matched %}
|
||||||
|
<div class="alert alert-info">
|
||||||
|
No Cloud Connect accounts found yet. They appear here automatically after the first daily report email is imported.
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% endblock %}
|
||||||
@ -2,6 +2,18 @@
|
|||||||
|
|
||||||
This file documents all changes made to this project via Claude Code.
|
This file documents all changes made to this project via Claude Code.
|
||||||
|
|
||||||
|
## [2026-03-19] (2)
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Veeam Cloud Connect importer — same inbox-style staging flow as Cove Data Protection:
|
||||||
|
- `app/cloud_connect_importer.py` — HTML parser for Cloud Connect daily report emails, upserts tenant rows into `cloud_connect_accounts`, creates `JobRun` records for linked accounts
|
||||||
|
- `app/main/routes_cloud_connect.py` — `/cloud-connect/accounts` page with link/unlink actions (create new job or link to existing)
|
||||||
|
- `templates/main/cloud_connect_accounts.html` — inbox-style page: unmatched accounts first, matched accounts below
|
||||||
|
- `CloudConnectAccount` model added to `models.py` (staging table, unique on user × section)
|
||||||
|
- `migrate_cloud_connect_accounts_table()` added to `migrations.py`, registered in `run_all_migrations()`
|
||||||
|
- `mail_importer.py` — Cloud Connect hook: detects `backup_type == "cloud connect report"`, calls `upsert_cloud_connect_report()`, auto-approves mail on success
|
||||||
|
- Sidebar link "Cloud Connect" added for admin/operator roles
|
||||||
|
|
||||||
## [2026-03-19]
|
## [2026-03-19]
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user