Auto-commit local changes before build (2026-03-20 15:51:54)
This commit is contained in:
parent
7ae04c59fe
commit
6998b3aa7d
@ -1401,3 +1401,252 @@ def inbox_reparse_all():
|
|||||||
)
|
)
|
||||||
|
|
||||||
return redirect(url_for("main.inbox"))
|
return redirect(url_for("main.inbox"))
|
||||||
|
|
||||||
|
|
||||||
|
@main_bp.route("/inbox/reparse-batch", methods=["POST"])
|
||||||
|
@login_required
|
||||||
|
@roles_required("admin", "operator")
|
||||||
|
def inbox_reparse_batch():
|
||||||
|
"""Process one batch of inbox messages and return JSON progress info.
|
||||||
|
|
||||||
|
Expects JSON body: {"last_id": <int|null>, "total": <int|null>}
|
||||||
|
Returns JSON: {processed, total, parsed_ok, auto_approved, no_match, errors, last_id, done}
|
||||||
|
"""
|
||||||
|
from flask import jsonify
|
||||||
|
|
||||||
|
data = request.get_json(silent=True) or {}
|
||||||
|
last_id = data.get("last_id") # keyset cursor (id < last_id)
|
||||||
|
total_known = data.get("total") # total passed from client so we don't recount
|
||||||
|
|
||||||
|
base_q = MailMessage.query
|
||||||
|
if hasattr(MailMessage, "location"):
|
||||||
|
base_q = base_q.filter(MailMessage.location == "inbox")
|
||||||
|
|
||||||
|
if total_known is None:
|
||||||
|
total_known = base_q.count()
|
||||||
|
|
||||||
|
batch_size = 50
|
||||||
|
time_budget_s = 8.0
|
||||||
|
|
||||||
|
started_at = time.monotonic()
|
||||||
|
processed = 0
|
||||||
|
parsed_ok = 0
|
||||||
|
auto_approved = 0
|
||||||
|
auto_approved_runs = []
|
||||||
|
no_match = 0
|
||||||
|
errors = 0
|
||||||
|
|
||||||
|
q = base_q
|
||||||
|
if last_id is not None:
|
||||||
|
q = q.filter(MailMessage.id < last_id)
|
||||||
|
|
||||||
|
batch = q.order_by(MailMessage.id.desc()).limit(batch_size).all()
|
||||||
|
|
||||||
|
new_last_id = last_id
|
||||||
|
for msg in batch:
|
||||||
|
if (time.monotonic() - started_at) >= time_budget_s:
|
||||||
|
break
|
||||||
|
|
||||||
|
new_last_id = msg.id
|
||||||
|
processed += 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
parse_mail_message(msg)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if (
|
||||||
|
getattr(msg, "location", "inbox") == "inbox"
|
||||||
|
and getattr(msg, "parse_result", None) == "ok"
|
||||||
|
and getattr(msg, "job_id", None) is None
|
||||||
|
):
|
||||||
|
bsw = (getattr(msg, "backup_software", "") or "").strip().lower()
|
||||||
|
btype = (getattr(msg, "backup_type", "") or "").strip().lower()
|
||||||
|
jname = (getattr(msg, "job_name", "") or "").strip().lower()
|
||||||
|
|
||||||
|
if bsw == "veeam" and btype == "service provider console" and jname == "active alarms summary":
|
||||||
|
raw = (getattr(msg, "text_body", None) or "").strip() or (getattr(msg, "html_body", None) or "")
|
||||||
|
companies = extract_vspc_active_alarms_companies(raw)
|
||||||
|
|
||||||
|
if companies:
|
||||||
|
def _is_error_status(value):
|
||||||
|
v = (value or "").strip().lower()
|
||||||
|
return v in {"error", "failed", "critical"} or v.startswith("fail")
|
||||||
|
|
||||||
|
first_job = None
|
||||||
|
mapped_count = 0
|
||||||
|
created_any = False
|
||||||
|
|
||||||
|
for company in companies:
|
||||||
|
tmp_msg = MailMessage(
|
||||||
|
from_address=msg.from_address,
|
||||||
|
backup_software=msg.backup_software,
|
||||||
|
backup_type=msg.backup_type,
|
||||||
|
job_name=f"{(msg.job_name or 'Active alarms summary').strip()} | {company}".strip(),
|
||||||
|
)
|
||||||
|
with db.session.no_autoflush:
|
||||||
|
job = find_matching_job(tmp_msg)
|
||||||
|
|
||||||
|
if not job:
|
||||||
|
continue
|
||||||
|
if hasattr(job, "active") and not bool(job.active):
|
||||||
|
continue
|
||||||
|
if hasattr(job, "auto_approve") and not bool(job.auto_approve):
|
||||||
|
continue
|
||||||
|
|
||||||
|
mapped_count += 1
|
||||||
|
objs = (
|
||||||
|
MailObject.query.filter(MailObject.mail_message_id == msg.id)
|
||||||
|
.filter(MailObject.object_name.ilike(f"{company} | %"))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
saw_error = any(_is_error_status(o.status) for o in objs)
|
||||||
|
saw_warning = any((o.status or "").strip().lower() == "warning" for o in objs)
|
||||||
|
status = "Error" if saw_error else ("Warning" if saw_warning else (msg.overall_status or "Success"))
|
||||||
|
|
||||||
|
run = JobRun(
|
||||||
|
job_id=job.id,
|
||||||
|
mail_message_id=msg.id,
|
||||||
|
run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()),
|
||||||
|
status=status or None,
|
||||||
|
missed=False,
|
||||||
|
)
|
||||||
|
if hasattr(run, "remark"):
|
||||||
|
run.remark = getattr(msg, "overall_message", None)
|
||||||
|
if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"):
|
||||||
|
run.storage_used_bytes = msg.storage_used_bytes
|
||||||
|
if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"):
|
||||||
|
run.storage_capacity_bytes = msg.storage_capacity_bytes
|
||||||
|
if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"):
|
||||||
|
run.storage_free_bytes = msg.storage_free_bytes
|
||||||
|
if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"):
|
||||||
|
run.storage_free_percent = msg.storage_free_percent
|
||||||
|
|
||||||
|
db.session.add(run)
|
||||||
|
db.session.flush()
|
||||||
|
try:
|
||||||
|
link_open_internal_tickets_to_run(run=run, job=job)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id))
|
||||||
|
created_any = True
|
||||||
|
if not first_job:
|
||||||
|
first_job = job
|
||||||
|
|
||||||
|
if created_any and mapped_count == len(companies):
|
||||||
|
msg.job_id = first_job.id if first_job else None
|
||||||
|
if hasattr(msg, "approved"):
|
||||||
|
msg.approved = True
|
||||||
|
if hasattr(msg, "approved_at"):
|
||||||
|
msg.approved_at = datetime.utcnow()
|
||||||
|
if hasattr(msg, "approved_by_id"):
|
||||||
|
msg.approved_by_id = None
|
||||||
|
if hasattr(msg, "location"):
|
||||||
|
msg.location = "history"
|
||||||
|
auto_approved += 1
|
||||||
|
|
||||||
|
# Do not fall back to single-job matching for VSPC summary.
|
||||||
|
if msg.parse_result == "ok":
|
||||||
|
parsed_ok += 1
|
||||||
|
elif msg.parse_result == "no_match":
|
||||||
|
no_match += 1
|
||||||
|
else:
|
||||||
|
errors += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
with db.session.no_autoflush:
|
||||||
|
job = find_matching_job(msg)
|
||||||
|
|
||||||
|
if job:
|
||||||
|
if hasattr(job, "active") and not bool(job.active):
|
||||||
|
raise Exception("job not active")
|
||||||
|
if hasattr(job, "auto_approve") and not bool(job.auto_approve):
|
||||||
|
raise Exception("job auto_approve disabled")
|
||||||
|
|
||||||
|
run = JobRun(
|
||||||
|
job_id=job.id,
|
||||||
|
mail_message_id=msg.id,
|
||||||
|
run_at=(msg.received_at or getattr(msg, "parsed_at", None) or datetime.utcnow()),
|
||||||
|
status=msg.overall_status or None,
|
||||||
|
missed=False,
|
||||||
|
)
|
||||||
|
if hasattr(run, "remark"):
|
||||||
|
run.remark = getattr(msg, "overall_message", None)
|
||||||
|
if hasattr(run, "storage_used_bytes") and hasattr(msg, "storage_used_bytes"):
|
||||||
|
run.storage_used_bytes = msg.storage_used_bytes
|
||||||
|
if hasattr(run, "storage_capacity_bytes") and hasattr(msg, "storage_capacity_bytes"):
|
||||||
|
run.storage_capacity_bytes = msg.storage_capacity_bytes
|
||||||
|
if hasattr(run, "storage_free_bytes") and hasattr(msg, "storage_free_bytes"):
|
||||||
|
run.storage_free_bytes = msg.storage_free_bytes
|
||||||
|
if hasattr(run, "storage_free_percent") and hasattr(msg, "storage_free_percent"):
|
||||||
|
run.storage_free_percent = msg.storage_free_percent
|
||||||
|
|
||||||
|
db.session.add(run)
|
||||||
|
db.session.flush()
|
||||||
|
try:
|
||||||
|
link_open_internal_tickets_to_run(run=run, job=job)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
auto_approved_runs.append((job.customer_id, job.id, run.id, msg.id))
|
||||||
|
|
||||||
|
msg.job_id = job.id
|
||||||
|
if hasattr(msg, "approved"):
|
||||||
|
msg.approved = True
|
||||||
|
if hasattr(msg, "approved_at"):
|
||||||
|
msg.approved_at = datetime.utcnow()
|
||||||
|
if hasattr(msg, "approved_by_id"):
|
||||||
|
msg.approved_by_id = None
|
||||||
|
if hasattr(msg, "location"):
|
||||||
|
msg.location = "history"
|
||||||
|
auto_approved += 1
|
||||||
|
|
||||||
|
except Exception as _exc:
|
||||||
|
current_app.logger.exception(
|
||||||
|
f"Auto-approve during reparse-batch failed for message {getattr(msg,'id',None)}: {_exc}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if msg.parse_result == "ok":
|
||||||
|
parsed_ok += 1
|
||||||
|
elif msg.parse_result == "no_match":
|
||||||
|
no_match += 1
|
||||||
|
else:
|
||||||
|
errors += 1
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
errors += 1
|
||||||
|
msg.parse_result = "error"
|
||||||
|
msg.parse_error = str(exc)[:500]
|
||||||
|
|
||||||
|
try:
|
||||||
|
db.session.commit()
|
||||||
|
except Exception:
|
||||||
|
db.session.rollback()
|
||||||
|
|
||||||
|
# Persist objects for auto-approved runs
|
||||||
|
if auto_approved_runs:
|
||||||
|
for (customer_id, job_id, run_id, mail_message_id) in auto_approved_runs:
|
||||||
|
try:
|
||||||
|
persist_objects_for_auto_run(customer_id, job_id, run_id, mail_message_id)
|
||||||
|
except Exception as exc:
|
||||||
|
_log_admin_event(
|
||||||
|
"object_persist_error",
|
||||||
|
f"Object persistence failed for auto-approved message {mail_message_id} (job {job_id}, run {run_id}): {exc}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Determine if we are done: no more messages below new_last_id
|
||||||
|
done = False
|
||||||
|
if processed < batch_size:
|
||||||
|
done = True
|
||||||
|
elif new_last_id is not None:
|
||||||
|
remaining_q = base_q.filter(MailMessage.id < new_last_id)
|
||||||
|
done = remaining_q.count() == 0
|
||||||
|
|
||||||
|
return jsonify({
|
||||||
|
"processed": processed,
|
||||||
|
"total": total_known,
|
||||||
|
"parsed_ok": parsed_ok,
|
||||||
|
"auto_approved": auto_approved,
|
||||||
|
"no_match": no_match,
|
||||||
|
"errors": errors,
|
||||||
|
"last_id": new_last_id,
|
||||||
|
"done": done,
|
||||||
|
})
|
||||||
|
|||||||
@ -26,9 +26,7 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
{% if current_user.is_authenticated and active_role in ["admin", "operator"] %}
|
{% if current_user.is_authenticated and active_role in ["admin", "operator"] %}
|
||||||
<form method="POST" action="{{ url_for('main.inbox_reparse_all') }}" class="me-3 mb-0">
|
<button type="button" class="btn btn-outline-secondary btn-sm me-3" id="btnReparseAll" data-bs-toggle="modal" data-bs-target="#reparseProgressModal">Re-parse all</button>
|
||||||
<button type="submit" class="btn btn-outline-secondary btn-sm">Re-parse all</button>
|
|
||||||
</form>
|
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
<div class="d-flex align-items-center">
|
<div class="d-flex align-items-center">
|
||||||
@ -209,6 +207,27 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<!-- Re-parse progress modal -->
|
||||||
|
<div class="modal fade" id="reparseProgressModal" tabindex="-1" aria-labelledby="reparseProgressModalLabel" aria-hidden="true" data-bs-backdrop="static" data-bs-keyboard="false">
|
||||||
|
<div class="modal-dialog modal-dialog-centered">
|
||||||
|
<div class="modal-content">
|
||||||
|
<div class="modal-header">
|
||||||
|
<h5 class="modal-title" id="reparseProgressModalLabel">Re-parse all messages</h5>
|
||||||
|
</div>
|
||||||
|
<div class="modal-body">
|
||||||
|
<div id="reparseStatusText" class="mb-2 text-center">Starting…</div>
|
||||||
|
<div class="progress mb-2" style="height: 22px;">
|
||||||
|
<div id="reparseProgressBar" class="progress-bar progress-bar-striped progress-bar-animated" role="progressbar" style="width: 0%;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">0%</div>
|
||||||
|
</div>
|
||||||
|
<div id="reparseStatsText" class="small text-muted text-center"></div>
|
||||||
|
</div>
|
||||||
|
<div class="modal-footer">
|
||||||
|
<button type="button" class="btn btn-secondary d-none" id="reparseCloseBtn" data-bs-dismiss="modal">Close</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- VSPC company mapping modal (for multi-company summary emails) -->
|
<!-- VSPC company mapping modal (for multi-company summary emails) -->
|
||||||
<div class="modal fade" id="vspcCompanyMapModal" tabindex="-1" aria-labelledby="vspcCompanyMapModalLabel" aria-hidden="true">
|
<div class="modal fade" id="vspcCompanyMapModal" tabindex="-1" aria-labelledby="vspcCompanyMapModalLabel" aria-hidden="true">
|
||||||
<div class="modal-dialog modal-lg modal-dialog-scrollable">
|
<div class="modal-dialog modal-lg modal-dialog-scrollable">
|
||||||
@ -680,4 +699,88 @@ function findCustomerIdByName(name) {
|
|||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
(function () {
|
||||||
|
var reparseModal = null;
|
||||||
|
var reparseRunning = false;
|
||||||
|
|
||||||
|
function initReparseModal() {
|
||||||
|
var modalEl = document.getElementById("reparseProgressModal");
|
||||||
|
if (!modalEl) return;
|
||||||
|
reparseModal = new bootstrap.Modal(modalEl);
|
||||||
|
|
||||||
|
modalEl.addEventListener("show.bs.modal", function () {
|
||||||
|
if (reparseRunning) return;
|
||||||
|
resetReparseUI();
|
||||||
|
startReparse();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetReparseUI() {
|
||||||
|
setProgress(0, 0);
|
||||||
|
document.getElementById("reparseStatusText").textContent = "Starting…";
|
||||||
|
document.getElementById("reparseStatsText").textContent = "";
|
||||||
|
document.getElementById("reparseCloseBtn").classList.add("d-none");
|
||||||
|
}
|
||||||
|
|
||||||
|
function setProgress(done, total) {
|
||||||
|
var bar = document.getElementById("reparseProgressBar");
|
||||||
|
var pct = total > 0 ? Math.round((done / total) * 100) : 0;
|
||||||
|
bar.style.width = pct + "%";
|
||||||
|
bar.setAttribute("aria-valuenow", pct);
|
||||||
|
bar.textContent = pct + "%";
|
||||||
|
}
|
||||||
|
|
||||||
|
function startReparse() {
|
||||||
|
reparseRunning = true;
|
||||||
|
runBatch(null, null, 0, 0, 0, 0, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function runBatch(lastId, total, totalProcessed, totalOk, totalApproved, totalNoMatch, totalErrors) {
|
||||||
|
var payload = { last_id: lastId, total: total };
|
||||||
|
fetch("{{ url_for('main.inbox_reparse_batch') }}", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
})
|
||||||
|
.then(function (r) { return r.json(); })
|
||||||
|
.then(function (data) {
|
||||||
|
var newTotal = data.total || total || 0;
|
||||||
|
var newProcessed = totalProcessed + (data.processed || 0);
|
||||||
|
var newOk = totalOk + (data.parsed_ok || 0);
|
||||||
|
var newApproved = totalApproved + (data.auto_approved || 0);
|
||||||
|
var newNoMatch = totalNoMatch + (data.no_match || 0);
|
||||||
|
var newErrors = totalErrors + (data.errors || 0);
|
||||||
|
|
||||||
|
setProgress(newProcessed, newTotal);
|
||||||
|
document.getElementById("reparseStatusText").textContent =
|
||||||
|
"Processing… " + newProcessed + " / " + newTotal;
|
||||||
|
document.getElementById("reparseStatsText").textContent =
|
||||||
|
"Parsed: " + newOk + " | Auto-approved: " + newApproved +
|
||||||
|
" | No match: " + newNoMatch + " | Errors: " + newErrors;
|
||||||
|
|
||||||
|
if (data.done) {
|
||||||
|
reparseRunning = false;
|
||||||
|
setProgress(newTotal, newTotal);
|
||||||
|
document.getElementById("reparseStatusText").textContent = "Finished!";
|
||||||
|
document.getElementById("reparseCloseBtn").classList.remove("d-none");
|
||||||
|
var bar = document.getElementById("reparseProgressBar");
|
||||||
|
bar.classList.remove("progress-bar-animated");
|
||||||
|
} else {
|
||||||
|
setTimeout(function () {
|
||||||
|
runBatch(data.last_id, newTotal, newProcessed, newOk, newApproved, newNoMatch, newErrors);
|
||||||
|
}, 100);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(function (err) {
|
||||||
|
reparseRunning = false;
|
||||||
|
document.getElementById("reparseStatusText").textContent = "Error: " + err;
|
||||||
|
document.getElementById("reparseCloseBtn").classList.remove("d-none");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener("DOMContentLoaded", initReparseModal);
|
||||||
|
})();
|
||||||
|
</script>
|
||||||
|
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|||||||
@ -2,6 +2,13 @@
|
|||||||
|
|
||||||
This file documents all changes made to this project via Claude Code.
|
This file documents all changes made to this project via Claude Code.
|
||||||
|
|
||||||
|
## [2026-03-20] (9)
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Inbox: "Re-parse all" now shows a progress modal with a live progress bar instead of blocking the page:
|
||||||
|
- New `POST /inbox/reparse-batch` JSON endpoint processes 50 messages per call (8 s time budget) and returns `{processed, total, parsed_ok, auto_approved, no_match, errors, last_id, done}`
|
||||||
|
- The Re-parse all button now opens a Bootstrap modal that calls the batch endpoint in a loop (via `fetch`) until `done: true`, updating a progress bar and live stats (Parsed / Auto-approved / No match / Errors) after each batch
|
||||||
|
|
||||||
## [2026-03-20] (8)
|
## [2026-03-20] (8)
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user