Auto-commit local changes before build (2026-03-20 15:51:54)
This commit is contained in:
parent
7ae04c59fe
commit
6998b3aa7d
@ -1401,3 +1401,252 @@ def inbox_reparse_all():
|
||||
)
|
||||
|
||||
return redirect(url_for("main.inbox"))
|
||||
|
||||
|
||||
@main_bp.route("/inbox/reparse-batch", methods=["POST"])
|
||||
@login_required
|
||||
@roles_required("admin", "operator")
|
||||
def inbox_reparse_batch():
|
||||
"""Process one batch of inbox messages and return JSON progress info.
|
||||
|
||||
Expects JSON body: {"last_id": <int|null>, "total": <int|null>}
|
||||
Returns JSON: {processed, total, parsed_ok, auto_approved, no_match, errors, last_id, done}
|
||||
"""
|
||||
from flask import jsonify
|
||||
|
||||
data = request.get_json(silent=True) or {}
|
||||
last_id = data.get("last_id") # keyset cursor (id < last_id)
|
||||
total_known = data.get("total") # total passed from client so we don't recount
|
||||
|
||||
base_q = MailMessage.query
|
||||
if hasattr(MailMessage, "location"):
|
||||
base_q = base_q.filter(MailMessage.location == "inbox")
|
||||
|
||||
if total_known is None:
|
||||
total_known = base_q.count()
|
||||
|
||||
batch_size = 50
|
||||
time_budget_s = 8.0
|
||||
|
||||
started_at = time.monotonic()
|
||||
processed = 0
|
||||
parsed_ok = 0
|
||||
auto_approved = 0
|
||||
auto_approved_runs = []
|
||||
no_match = 0
|
||||
errors = 0
|
||||
|
||||
q = base_q
|
||||
if last_id is not None:
|
||||
q = q.filter(MailMessage.id < last_id)
|
||||
|
||||
batch = q.order_by(MailMessage.id.desc()).limit(batch_size).all()
|
||||
|
||||
new_last_id = last_id
|
||||
for msg in batch:
|
||||
if (time.monotonic() - started_at) >= time_budget_s:
|
||||
break
|
||||
|
||||
new_last_id = msg.id
|
||||
processed += 1
|
||||
|
||||
try:
|
||||
parse_mail_message(msg)
|
||||
|
||||
try:
|
||||
if (
|
||||
getattr(msg, "location", "inbox") == "inbox"
|
||||
and getattr(msg, "parse_result", None) == "ok"
|
||||
and getattr(msg, "job_id", None) is None
|
||||
):
|
||||
bsw = (getattr(msg, "backup_software", "") or "").strip().lower()
|
||||
btype = (getattr(msg, "backup_type", "") or "").strip().lower()
|
||||
jname = (getattr(msg, "job_name", "") or "").strip().lower()
|
||||
|
||||
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
|
||||
raw = (getattr(msg, "text_body", None) or "").strip() or (getattr(msg, "html_body", None) or "")
|
||||
companies = extract_vspc_active_alarms_companies(raw)
|
||||
|
||||
if companies:
|
||||
def _is_error_status(value):
|
||||
v = (value or "").strip().lower()
|
||||
return v in {"error", "failed", "critical"} or v.startswith("fail")
|
||||
|
||||
first_job = None
|
||||
mapped_count = 0
|
||||
created_any = False
|
||||
|
||||
for company in companies:
|
||||
tmp_msg = MailMessage(
|
||||
from_address=msg.from_address,
|
||||
backup_software=msg.backup_software,
|
||||
backup_type=msg.backup_type,
|
||||
job_name=f"{(msg.job_name or 'Active alarms summary').strip()} | {company}".strip(),
|
||||
)
|
||||
with db.session.no_autoflush:
|
||||
job = find_matching_job(tmp_msg)
|
||||
|
||||
if not job:
|
||||
continue
|
||||
if hasattr(job, "active") and not bool(job.active):
|
||||
continue
|
||||
if hasattr(job, "auto_approve") and not bool(job.auto_approve):
|
||||
continue
|
||||
|
||||
mapped_count += 1
|
||||
objs = (
|
||||
MailObject.query.filter(MailObject.mail_message_id == msg.id)
|
||||
.filter(MailObject.object_name.ilike(f"{company} | %"))
|
||||
.all()
|
||||
)
|
||||
saw_error = any(_is_error_status(o.status) for o in objs)
|
||||
saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs)
|
||||
status = "Error" if saw_error else ("Warning" if saw_warning else (msg.overall_status or "Success"))
|
||||
|
||||
run = JobRun(
|
||||
job_id=job.id,
|
||||
mail_message_id=msg.id,
|
||||
run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()),
|
||||
status=status or None,
|
||||
missed=False,
|
||||
)
|
||||
if hasattr(run, "remark"):
|
||||
run.remark = getattr(msg, "overall_message", None)
|
||||
if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"):
|
||||
run.storage_used_bytes = msg.storage_used_bytes
|
||||
if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"):
|
||||
run.storage_capacity_bytes = msg.storage_capacity_bytes
|
||||
if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"):
|
||||
run.storage_free_bytes = msg.storage_free_bytes
|
||||
if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"):
|
||||
run.storage_free_percent = msg.storage_free_percent
|
||||
|
||||
db.session.add(run)
|
||||
db.session.flush()
|
||||
try:
|
||||
link_open_internal_tickets_to_run(run=run, job=job)
|
||||
except Exception:
|
||||
pass
|
||||
auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id))
|
||||
created_any = True
|
||||
if not first_job:
|
||||
first_job = job
|
||||
|
||||
if created_any and mapped_count == len(companies):
|
||||
msg.job_id = first_job.id if first_job else None
|
||||
if hasattr(msg, "approved"):
|
||||
msg.approved = True
|
||||
if hasattr(msg, "approved_at"):
|
||||
msg.approved_at = datetime.utcnow()
|
||||
if hasattr(msg, "approved_by_id"):
|
||||
msg.approved_by_id = None
|
||||
if hasattr(msg, "location"):
|
||||
msg.location = "history"
|
||||
auto_approved += 1
|
||||
|
||||
# Do not fall back to single-job matching for VSPC summary.
|
||||
if msg.parse_result == "ok":
|
||||
parsed_ok += 1
|
||||
elif msg.parse_result == "no_match":
|
||||
no_match += 1
|
||||
else:
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
with db.session.no_autoflush:
|
||||
job = find_matching_job(msg)
|
||||
|
||||
if job:
|
||||
if hasattr(job, "active") and not bool(job.active):
|
||||
raise Exception("job not active")
|
||||
if hasattr(job, "auto_approve") and not bool(job.auto_approve):
|
||||
raise Exception("job auto_approve disabled")
|
||||
|
||||
run = JobRun(
|
||||
job_id=job.id,
|
||||
mail_message_id=msg.id,
|
||||
run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()),
|
||||
status=msg.overall_status or None,
|
||||
missed=False,
|
||||
)
|
||||
if hasattr(run, "remark"):
|
||||
run.remark = getattr(msg, "overall_message", None)
|
||||
if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"):
|
||||
run.storage_used_bytes = msg.storage_used_bytes
|
||||
if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"):
|
||||
run.storage_capacity_bytes = msg.storage_capacity_bytes
|
||||
if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"):
|
||||
run.storage_free_bytes = msg.storage_free_bytes
|
||||
if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"):
|
||||
run.storage_free_percent = msg.storage_free_percent
|
||||
|
||||
db.session.add(run)
|
||||
db.session.flush()
|
||||
try:
|
||||
link_open_internal_tickets_to_run(run=run, job=job)
|
||||
except Exception:
|
||||
pass
|
||||
auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id))
|
||||
|
||||
msg.job_id = job.id
|
||||
if hasattr(msg, "approved"):
|
||||
msg.approved = True
|
||||
if hasattr(msg, "approved_at"):
|
||||
msg.approved_at = datetime.utcnow()
|
||||
if hasattr(msg, "approved_by_id"):
|
||||
msg.approved_by_id = None
|
||||
if hasattr(msg, "location"):
|
||||
msg.location = "history"
|
||||
auto_approved += 1
|
||||
|
||||
except Exception as _exc:
|
||||
current_app.logger.exception(
|
||||
f"Auto-approve during reparse-batch failed for message {getattr(msg,'id',None)}: {_exc}"
|
||||
)
|
||||
|
||||
if msg.parse_result == "ok":
|
||||
parsed_ok += 1
|
||||
elif msg.parse_result == "no_match":
|
||||
no_match += 1
|
||||
else:
|
||||
errors += 1
|
||||
|
||||
except Exception as exc:
|
||||
errors += 1
|
||||
msg.parse_result = "error"
|
||||
msg.parse_error = str(exc)[:500]
|
||||
|
||||
try:
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
|
||||
# Persist objects for auto-approved runs
|
||||
if auto_approved_runs:
|
||||
for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs:
|
||||
try:
|
||||
persist_objects_for_auto_run(customer_id, job_id, run_id, mail_message_id)
|
||||
except Exception as exc:
|
||||
_log_admin_event(
|
||||
"object_persist_error",
|
||||
f"Object persistence failed for auto-approved message {mail_message_id} (job {job_id}, run {run_id}): {exc}",
|
||||
)
|
||||
|
||||
# Determine if we are done: no more messages below new_last_id
|
||||
done = False
|
||||
if processed < batch_size:
|
||||
done = True
|
||||
elif new_last_id is not None:
|
||||
remaining_q = base_q.filter(MailMessage.id < new_last_id)
|
||||
done = remaining_q.count() == 0
|
||||
|
||||
return jsonify({
|
||||
"processed": processed,
|
||||
"total": total_known,
|
||||
"parsed_ok": parsed_ok,
|
||||
"auto_approved": auto_approved,
|
||||
"no_match": no_match,
|
||||
"errors": errors,
|
||||
"last_id": new_last_id,
|
||||
"done": done,
|
||||
})
|
||||
|
||||
@ -26,9 +26,7 @@
|
||||
</div>
|
||||
|
||||
{% if current_user.is_authenticated and active_role in ["admin", "operator"] %}
|
||||
<form method="POST" action="{{ url_for('main.inbox_reparse_all') }}" class="me-3 mb-0">
|
||||
<button type="submit" class="btn btn-outline-secondary btn-sm">Re-parse all</button>
|
||||
</form>
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm me-3" id="btnReparseAll" data-bs-toggle="modal" data-bs-target="#reparseProgressModal">Re-parse all</button>
|
||||
{% endif %}
|
||||
|
||||
<div class="d-flex align-items-center">
|
||||
@ -209,6 +207,27 @@
|
||||
</div>
|
||||
|
||||
|
||||
<!-- Re-parse progress modal -->
|
||||
<div class="modal fade" id="reparseProgressModal" tabindex="-1" aria-labelledby="reparseProgressModalLabel" aria-hidden="true" data-bs-backdrop="static" data-bs-keyboard="false">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="reparseProgressModalLabel">Re-parse all messages</h5>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<div id="reparseStatusText" class="mb-2 text-center">Starting…</div>
|
||||
<div class="progress mb-2" style="height: 22px;">
|
||||
<div id="reparseProgressBar" class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" style="width: 0%;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">0%</div>
|
||||
</div>
|
||||
<div id="reparseStatsText" class="small text-muted text-center"></div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary d-none" id="reparseCloseBtn" data-bs-dismiss="modal">Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- VSPC company mapping modal (for multi-company summary emails) -->
|
||||
<div class="modal fade" id="vspcCompanyMapModal" tabindex="-1" aria-labelledby="vspcCompanyMapModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg modal-dialog-scrollable">
|
||||
@ -680,4 +699,88 @@ function findCustomerIdByName(name) {
|
||||
})();
|
||||
</script>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
var reparseModal = null;
|
||||
var reparseRunning = false;
|
||||
|
||||
function initReparseModal() {
|
||||
var modalEl = document.getElementById("reparseProgressModal");
|
||||
if (!modalEl) return;
|
||||
reparseModal = new bootstrap.Modal(modalEl);
|
||||
|
||||
modalEl.addEventListener("show.bs.modal", function () {
|
||||
if (reparseRunning) return;
|
||||
resetReparseUI();
|
||||
startReparse();
|
||||
});
|
||||
}
|
||||
|
||||
function resetReparseUI() {
|
||||
setProgress(0, 0);
|
||||
document.getElementById("reparseStatusText").textContent = "Starting…";
|
||||
document.getElementById("reparseStatsText").textContent = "";
|
||||
document.getElementById("reparseCloseBtn").classList.add("d-none");
|
||||
}
|
||||
|
||||
function setProgress(done, total) {
|
||||
var bar = document.getElementById("reparseProgressBar");
|
||||
var pct = total > 0 ? Math.round((done / total) * 100) : 0;
|
||||
bar.style.width = pct + "%";
|
||||
bar.setAttribute("aria-valuenow", pct);
|
||||
bar.textContent = pct + "%";
|
||||
}
|
||||
|
||||
function startReparse() {
|
||||
reparseRunning = true;
|
||||
runBatch(null, null, 0, 0, 0, 0, 0);
|
||||
}
|
||||
|
||||
function runBatch(lastId, total, totalProcessed, totalOk, totalApproved, totalNoMatch, totalErrors) {
|
||||
var payload = { last_id: lastId, total: total };
|
||||
fetch("{{ url_for('main.inbox_reparse_batch') }}", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (data) {
|
||||
var newTotal = data.total || total || 0;
|
||||
var newProcessed = totalProcessed + (data.processed || 0);
|
||||
var newOk = totalOk + (data.parsed_ok || 0);
|
||||
var newApproved = totalApproved + (data.auto_approved || 0);
|
||||
var newNoMatch = totalNoMatch + (data.no_match || 0);
|
||||
var newErrors = totalErrors + (data.errors || 0);
|
||||
|
||||
setProgress(newProcessed, newTotal);
|
||||
document.getElementById("reparseStatusText").textContent =
|
||||
"Processing… " + newProcessed + " / " + newTotal;
|
||||
document.getElementById("reparseStatsText").textContent =
|
||||
"Parsed: " + newOk + " | Auto-approved: " + newApproved +
|
||||
" | No match: " + newNoMatch + " | Errors: " + newErrors;
|
||||
|
||||
if (data.done) {
|
||||
reparseRunning = false;
|
||||
setProgress(newTotal, newTotal);
|
||||
document.getElementById("reparseStatusText").textContent = "Finished!";
|
||||
document.getElementById("reparseCloseBtn").classList.remove("d-none");
|
||||
var bar = document.getElementById("reparseProgressBar");
|
||||
bar.classList.remove("progress-bar-animated");
|
||||
} else {
|
||||
setTimeout(function () {
|
||||
runBatch(data.last_id, newTotal, newProcessed, newOk, newApproved, newNoMatch, newErrors);
|
||||
}, 100);
|
||||
}
|
||||
})
|
||||
.catch(function (err) {
|
||||
reparseRunning = false;
|
||||
document.getElementById("reparseStatusText").textContent = "Error: " + err;
|
||||
document.getElementById("reparseCloseBtn").classList.remove("d-none");
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", initReparseModal);
|
||||
})();
|
||||
</script>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
@ -2,6 +2,13 @@
|
||||
|
||||
This file documents all changes made to this project via Claude Code.
|
||||
|
||||
## [2026-03-20] (9)
|
||||
|
||||
### Added
|
||||
- Inbox: "Re-parse all" now shows a progress modal with a live progress bar instead of blocking the page:
|
||||
- New `POST /inbox/reparse-batch` JSON endpoint processes 50 messages per call (8 s time budget) and returns `{processed, total, parsed_ok, auto_approved, no_match, errors, last_id, done}`
|
||||
- The Re-parse all button now opens a Bootstrap modal that calls the batch endpoint in a loop (via `fetch`) until `done: true`, updating a progress bar and live stats (Parsed / Auto-approved / No match / Errors) after each batch
|
||||
|
||||
## [2026-03-20] (8)
|
||||
|
||||
### Added
|
||||
|
||||
Loading…
Reference in New Issue
Block a user