v20260104-14-reports-stats-total-runs-success-rate-fix #28

Merged
ivooskamp merged 22 commits from v20260104-14-reports-stats-total-runs-success-rate-fix into main 2026-01-13 10:59:39 +01:00
3 changed files with 217 additions and 37 deletions
Showing only changes of commit c880121cd3 - Show all commits

View File

@ -1 +1 @@
v20260104-05-reports-html-jobs-columns-selection v20260104-06-reports-html-jobs-summary-success-rate

View File

@ -223,22 +223,23 @@ def build_report_columns_meta():
], ],
"jobs": [ "jobs": [
"object_name", "object_name",
"customer_name",
"job_name", "job_name",
"backup_software", "backup_software",
"backup_type", "backup_type",
"run_at", "total_runs",
"status", "success_count",
"missed", "warning_count",
"override_applied", "failed_count",
"ticket_number", "missed_count",
"remark", "success_rate",
], ],
}, },
"groups": [ "groups": [
{ {
"name": "Summary metrics", "name": "Summary metrics",
"items": [ "items": [
{"key": "object_name", "label": "Object", "views": ["summary"]}, {"key": "object_name", "label": "Text", "views": ["summary"]},
{"key": "total_runs", "label": "Total runs", "views": ["summary"]}, {"key": "total_runs", "label": "Total runs", "views": ["summary"]},
{"key": "success_count", "label": "Success", "views": ["summary"]}, {"key": "success_count", "label": "Success", "views": ["summary"]},
{"key": "success_override_count", "label": "Success (override)", "views": ["summary"]}, {"key": "success_override_count", "label": "Success (override)", "views": ["summary"]},
@ -248,6 +249,18 @@ def build_report_columns_meta():
{"key": "success_rate", "label": "Success rate", "views": ["summary"]}, {"key": "success_rate", "label": "Success rate", "views": ["summary"]},
], ],
}, },
{
"name": "Job summary",
"items": [
{"key": "total_runs", "label": "Total runs", "views": ["jobs"]},
{"key": "success_count", "label": "Success", "views": ["jobs"]},
{"key": "success_override_count", "label": "Success (override)", "views": ["jobs"]},
{"key": "warning_count", "label": "Warning", "views": ["jobs"]},
{"key": "failed_count", "label": "Failed", "views": ["jobs"]},
{"key": "missed_count", "label": "Missed", "views": ["jobs"]},
{"key": "success_rate", "label": "Success rate", "views": ["jobs"]},
],
},
{ {
"name": "Job Information", "name": "Job Information",
"items": [ "items": [
@ -277,9 +290,9 @@ def build_report_columns_meta():
], ],
}, },
{ {
"name": "Object", "name": "Text",
"items": [ "items": [
{"key": "object_name", "label": "Object", "views": ["snapshot", "summary", "jobs"]}, {"key": "object_name", "label": "Text", "views": ["snapshot", "summary", "jobs"]},
], ],
}, },
{ {
@ -614,7 +627,7 @@ def api_reports_data(report_id: int):
include_keys = _get_success_rate_keys_from_report(report) include_keys = _get_success_rate_keys_from_report(report)
view = (request.args.get("view") or "summary").strip().lower() view = (request.args.get("view") or "summary").strip().lower()
if view not in ("summary", "snapshot"): if view not in ("summary", "snapshot", "jobs"):
view = "summary" view = "summary"
limit = _clamp_int(request.args.get("limit"), default=100, min_v=1, max_v=500) limit = _clamp_int(request.args.get("limit"), default=100, min_v=1, max_v=500)
@ -660,6 +673,86 @@ def api_reports_data(report_id: int):
], ],
} }
if view == "jobs":
# Aggregated per-job summary over the reporting period (not latest snapshot).
include_keys_jobs = _get_success_rate_keys_from_report(report, view_key="jobs")
with db.engine.connect() as conn:
rows = conn.execute(
text(
"""
SELECT
COALESCE(object_name, '') AS object_name,
COALESCE(customer_name, '') AS customer_name,
COALESCE(job_name, '') AS job_name,
COALESCE(backup_software, '') AS backup_software,
COALESCE(backup_type, '') AS backup_type,
COUNT(*)::INTEGER AS total_runs,
SUM(CASE WHEN ((COALESCE(status,'') ILIKE 'Success%%' OR override_applied = TRUE) AND missed = FALSE) THEN 1 ELSE 0 END)::INTEGER AS success_count,
SUM(CASE WHEN (override_applied = TRUE AND missed = FALSE) THEN 1 ELSE 0 END)::INTEGER AS success_override_count,
SUM(CASE WHEN (COALESCE(status,'') ILIKE 'Warning%%' AND missed = FALSE) THEN 1 ELSE 0 END)::INTEGER AS warning_count,
SUM(CASE WHEN (COALESCE(status,'') ILIKE 'Fail%%' AND missed = FALSE) THEN 1 ELSE 0 END)::INTEGER AS failed_count,
SUM(CASE WHEN missed = TRUE THEN 1 ELSE 0 END)::INTEGER AS missed_count
FROM report_object_snapshots
WHERE report_id = :rid
GROUP BY 1,2,3,4,5
ORDER BY customer_name ASC, object_name ASC, job_name ASC
LIMIT :limit
OFFSET :offset
"""
),
{"rid": report_id, "limit": limit, "offset": offset},
).fetchall()
total_rows = conn.execute(
text(
"""
SELECT COUNT(*)::INTEGER AS cnt FROM (
SELECT 1
FROM report_object_snapshots
WHERE report_id = :rid
GROUP BY COALESCE(object_name,''), COALESCE(customer_name,''), COALESCE(job_name,''), COALESCE(backup_software,''), COALESCE(backup_type,'')
) x
"""
),
{"rid": report_id},
).scalar() or 0
items = []
for r in rows or []:
rate = _compute_success_rate(
success=int(r.success_count or 0),
warning=int(r.warning_count or 0),
failed=int(r.failed_count or 0),
missed=int(r.missed_count or 0),
include_keys=include_keys_jobs,
)
items.append(
{
"object_name": r.object_name or "",
"customer_name": r.customer_name or "",
"job_name": r.job_name or "",
"backup_software": r.backup_software or "",
"backup_type": r.backup_type or "",
"total_runs": int(r.total_runs or 0),
"success_count": int(r.success_count or 0),
"success_override_count": int(r.success_override_count or 0),
"warning_count": int(r.warning_count or 0),
"failed_count": int(r.failed_count or 0),
"missed_count": int(r.missed_count or 0),
"success_rate": round(rate, 2),
}
)
return {
"view": "jobs",
"total": int(total_rows),
"limit": int(limit),
"offset": int(offset),
"items": items,
}
q = db.session.query(ReportObjectSnapshot).filter(ReportObjectSnapshot.report_id == report_id) q = db.session.query(ReportObjectSnapshot).filter(ReportObjectSnapshot.report_id == report_id)
total = q.count() total = q.count()
rows = ( rows = (
@ -709,7 +802,7 @@ def _normalize_status_row(status: str, missed: bool) -> str:
return "unknown" return "unknown"
def _get_success_rate_keys_from_report(report: ReportDefinition) -> set[str]: def _get_success_rate_keys_from_report(report: ReportDefinition, view_key: str = "summary") -> set[str]:
"""Return which status count keys should contribute to success rate. """Return which status count keys should contribute to success rate.
The report UI allows choosing summary columns. Success rate should be The report UI allows choosing summary columns. Success rate should be
@ -719,11 +812,11 @@ def _get_success_rate_keys_from_report(report: ReportDefinition) -> set[str]:
rc = _safe_json_dict(getattr(report, "report_config", None)) rc = _safe_json_dict(getattr(report, "report_config", None))
cols = (rc.get("columns") or {}) if isinstance(rc, dict) else {} cols = (rc.get("columns") or {}) if isinstance(rc, dict) else {}
summary_cols = cols.get("summary") if isinstance(cols, dict) else None view_cols = cols.get(view_key) if isinstance(cols, dict) else None
selected = set() selected = set()
if isinstance(summary_cols, list): if isinstance(view_cols, list):
for k in summary_cols: for k in view_cols:
if not isinstance(k, str): if not isinstance(k, str):
continue continue
kk = k.strip() kk = k.strip()
@ -957,7 +1050,7 @@ def api_reports_stats(report_id: int):
def _export_csv_response(report: ReportDefinition, report_id: int, view: str): def _export_csv_response(report: ReportDefinition, report_id: int, view: str):
view = (view or "summary").strip().lower() view = (view or "summary").strip().lower()
if view not in ("summary", "snapshot"): if view not in ("summary", "snapshot", "jobs"):
view = "summary" view = "summary"
include_keys = _get_success_rate_keys_from_report(report) include_keys = _get_success_rate_keys_from_report(report)
@ -1182,33 +1275,32 @@ def _export_html_response(report: ReportDefinition, report_id: int, view: str):
want_snapshot_table = (view or "summary").strip().lower() == "snapshot" want_snapshot_table = (view or "summary").strip().lower() == "snapshot"
jobs_table_html = "" jobs_table_html = ""
jobs_chart_html = ""
if include_jobs and not want_snapshot_table: if include_jobs and not want_snapshot_table:
# Job table (latest snapshot per job/object) for a single-customer report. # Job table (aggregated per job) for a single-customer report.
include_keys_jobs = _get_success_rate_keys_from_report(report, view_key="jobs")
jobs_rows = [] jobs_rows = []
with db.engine.connect() as conn: with db.engine.connect() as conn:
rows = conn.execute( rows = conn.execute(
text( text(
""" """
SELECT DISTINCT ON (COALESCE(object_name,''), COALESCE(job_name,''), COALESCE(backup_software,''), COALESCE(backup_type,'')) SELECT
COALESCE(object_name,'') AS object_name, COALESCE(object_name,'') AS object_name,
COALESCE(customer_name,'') AS customer_name, COALESCE(customer_name,'') AS customer_name,
COALESCE(job_name,'') AS job_name, COALESCE(job_name,'') AS job_name,
COALESCE(backup_software,'') AS backup_software, COALESCE(backup_software,'') AS backup_software,
COALESCE(backup_type,'') AS backup_type, COALESCE(backup_type,'') AS backup_type,
run_at, COUNT(*)::INTEGER AS total_runs,
COALESCE(status,'') AS status, SUM(CASE WHEN ((COALESCE(status,'') ILIKE 'Success%%' OR override_applied = TRUE) AND missed = FALSE) THEN 1 ELSE 0 END)::INTEGER AS success_count,
missed, SUM(CASE WHEN (override_applied = TRUE AND missed = FALSE) THEN 1 ELSE 0 END)::INTEGER AS success_override_count,
override_applied, SUM(CASE WHEN (COALESCE(status,'') ILIKE 'Warning%%' AND missed = FALSE) THEN 1 ELSE 0 END)::INTEGER AS warning_count,
COALESCE(ticket_number,'') AS ticket_number, SUM(CASE WHEN (COALESCE(status,'') ILIKE 'Fail%%' AND missed = FALSE) THEN 1 ELSE 0 END)::INTEGER AS failed_count,
COALESCE(remark,'') AS remark SUM(CASE WHEN missed = TRUE THEN 1 ELSE 0 END)::INTEGER AS missed_count
FROM report_object_snapshots FROM report_object_snapshots
WHERE report_id = :rid WHERE report_id = :rid
ORDER BY GROUP BY 1,2,3,4,5
COALESCE(object_name,''), ORDER BY customer_name ASC, object_name ASC, job_name ASC
COALESCE(job_name,''),
COALESCE(backup_software,''),
COALESCE(backup_type,''),
run_at DESC NULLS LAST
LIMIT 1000 LIMIT 1000
""" """
), ),
@ -1216,6 +1308,13 @@ def _export_html_response(report: ReportDefinition, report_id: int, view: str):
).fetchall() ).fetchall()
for r in rows or []: for r in rows or []:
rate = _compute_success_rate(
success=int(r.success_count or 0),
warning=int(r.warning_count or 0),
failed=int(r.failed_count or 0),
missed=int(r.missed_count or 0),
include_keys=include_keys_jobs,
)
jobs_rows.append( jobs_rows.append(
{ {
"object_name": r.object_name or "", "object_name": r.object_name or "",
@ -1223,12 +1322,13 @@ def _export_html_response(report: ReportDefinition, report_id: int, view: str):
"job_name": r.job_name or "", "job_name": r.job_name or "",
"backup_software": r.backup_software or "", "backup_software": r.backup_software or "",
"backup_type": r.backup_type or "", "backup_type": r.backup_type or "",
"run_at": r.run_at.isoformat() if r.run_at else "", "total_runs": str(int(r.total_runs or 0)),
"status": r.status or "", "success_count": str(int(r.success_count or 0)),
"missed": bool(r.missed), "success_override_count": str(int(r.success_override_count or 0)),
"override_applied": bool(r.override_applied), "warning_count": str(int(r.warning_count or 0)),
"ticket_number": r.ticket_number or "", "failed_count": str(int(r.failed_count or 0)),
"remark": (r.remark or "").replace("\r", " ").replace("\n", " ").strip(), "missed_count": str(int(r.missed_count or 0)),
"success_rate": f"{round(rate, 2)}%",
} }
) )
@ -1239,7 +1339,7 @@ def _export_html_response(report: ReportDefinition, report_id: int, view: str):
<div class="card shadow-sm"> <div class="card shadow-sm">
<div class="card-header bg-white"> <div class="card-header bg-white">
<div class="fw-semibold">Jobs</div> <div class="fw-semibold">Jobs</div>
<div class="small-muted">Latest snapshot per job</div> <div class="small-muted">Aggregated per job over the report period</div>
</div> </div>
<div class="card-body"> <div class="card-body">
<div class="table-responsive"> <div class="table-responsive">
@ -1256,6 +1356,72 @@ def _export_html_response(report: ReportDefinition, report_id: int, view: str):
</div> </div>
</div>""" </div>"""
# Per-job success rate chart (top 20 by total runs)
chart_rows = []
for it in jobs_rows:
try:
tr = int(it.get("total_runs") or 0)
except Exception:
tr = 0
try:
sr = float(str(it.get("success_rate") or "0").replace("%", ""))
except Exception:
sr = 0.0
label = (it.get("job_name") or it.get("object_name") or "(job)").strip()
chart_rows.append((tr, sr, label))
chart_rows.sort(key=lambda x: (-x[0], x[2]))
chart_rows = chart_rows[:20]
job_labels = [c[2] for c in chart_rows]
job_rates = [round(float(c[1]), 2) for c in chart_rows]
jobs_chart_html = f"""
<div class="row g-3 mb-3">
<div class="col-12">
<div class="card shadow-sm">
<div class="card-header bg-white">
<div class="fw-semibold">Success rate per job</div>
<div class="small-muted">Top {len(job_labels)} jobs by total runs</div>
</div>
<div class="card-body">
<canvas id="jobSuccessRateChart" height="120"></canvas>
</div>
</div>
</div>
</div>
<script>
(function() {{
const labels = {_json.dumps(job_labels)};
const data = {_json.dumps(job_rates)};
const ctx = document.getElementById('jobSuccessRateChart');
if (!ctx) return;
new Chart(ctx, {{
type: 'bar',
data: {{
labels: labels,
datasets: [{{
label: 'Success rate (%)',
data: data,
}}]
}},
options: {{
responsive: true,
plugins: {{
legend: {{ display: false }}
}},
scales: {{
y: {{
beginAtZero: true,
max: 100
}}
}}
}}
}});
}})();
</script>
"""
snapshot_table_html = "" snapshot_table_html = ""
if want_snapshot_table: if want_snapshot_table:
snap_rows = [] snap_rows = []
@ -1463,6 +1629,8 @@ def _export_html_response(report: ReportDefinition, report_id: int, view: str):
{perf_table_html} {perf_table_html}
{jobs_chart_html}
{jobs_table_html} {jobs_table_html}
{snapshot_table_html} {snapshot_table_html}

View File

@ -226,6 +226,18 @@
- Ensured the HTML report Jobs table only renders columns explicitly selected in the report configuration. - Ensured the HTML report Jobs table only renders columns explicitly selected in the report configuration.
- Aligned Jobs table rendering logic with Snapshot and Summary column selection behavior. - Aligned Jobs table rendering logic with Snapshot and Summary column selection behavior.
---
## v20260104-06-reports-html-jobs-summary-success-rate
- Restored the **Text** column label in the report column selector (previously shown as Object).
- Added job-level metrics to the Jobs view, including total runs, success, warning, failed, missed counts, and success rate.
- Made **success rate per job** selectable as a column in job-based reports.
- Changed the HTML Jobs section from showing the **latest snapshot per job** to an **aggregated summary per job** over the selected reporting period.
- Removed the “Latest snapshot per job” subtitle from HTML reports.
- Added a **per-job success rate chart** to the HTML report to visualize backup performance per job.
- Prevented job status from being presented as a summary value, as it represents a momentary state rather than a stable metric.
================================================================================================================================================ ================================================================================================================================================
## v0.1.15 ## v0.1.15