Add duplicate detection, Convert warning, and performance TODO

- Convert: warn when title+author already exists in library (preload check)
- Library: Duplicates sidebar section with grouped view and live counter
- Fix: Duplicates view cover loading now uses same canvas/two-pass pattern as renderBooksGrid
- Docs: add TODO-PERF-library-load.md with four identified bottlenecks

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Ivo Oskamp 2026-03-27 16:22:02 +01:00
parent 2d672ff7bc
commit 00e75a6106
8 changed files with 271 additions and 5 deletions

View File

@ -216,9 +216,27 @@ async def preload(request: Request):
book = await scraper.fetch_book_info(client, url)
series = book.get("series", "")
hint = int(book.get("series_index_hint", 0) or 0)
title = book.get("title", "")
author = book.get("author", "")
existing_books = []
if title or author:
with get_db_conn() as conn:
with conn.cursor() as cur:
cur.execute(
"""SELECT filename, title, author FROM library
WHERE LOWER(TRIM(title)) = LOWER(TRIM(%s))
AND LOWER(TRIM(author)) = LOWER(TRIM(%s))""",
(title, author),
)
existing_books = [
{"filename": r[0], "title": r[1] or "", "author": r[2] or ""}
for r in cur.fetchall()
]
return {
"title": book.get("title", ""),
"author": book.get("author", ""),
"title": title,
"author": author,
"publisher": book.get("publisher", ""),
"series": series,
"series_index_next": hint if hint else _next_series_index(series),
@ -228,6 +246,8 @@ async def preload(request: Request):
"description": book.get("description", ""),
"updated_date": book.get("updated_date", ""),
"publication_status": book.get("publication_status", ""),
"already_exists": bool(existing_books),
"existing_books": existing_books,
}

View File

@ -59,6 +59,18 @@ html, body {
padding: 4rem 2rem;
}
.group-heading {
font-family: var(--mono);
font-size: 0.72rem;
letter-spacing: 0.08em;
text-transform: uppercase;
color: var(--text-dim);
padding: 1.5rem 0 0.5rem;
border-bottom: 1px solid var(--border);
margin-bottom: 0.75rem;
}
.group-heading:first-child { padding-top: 0; }
.import-dropzone {
border: 1px dashed var(--border);
background: rgba(34, 31, 27, 0.45);

View File

@ -138,6 +138,10 @@ function updateCounts() {
const ratedCount = active.filter(b => b.rating > 0).length;
const ratedEl = document.getElementById('count-rated');
if (ratedEl) ratedEl.textContent = ratedCount || '';
const dupGroups = _duplicateGroups(active);
const dupCount = dupGroups.reduce((s, g) => s + g.books.length, 0);
const dupEl = document.getElementById('count-duplicates');
if (dupEl) dupEl.textContent = dupCount || '';
}
function _filenameBase(filename) {
@ -184,6 +188,7 @@ function _viewUrl(view, param) {
if (view === 'archived') return '/library#archived';
if (view === 'bookmarks') return '/library#bookmarks';
if (view === 'rated') return '/library#rated';
if (view === 'duplicates') return '/library#duplicates';
if (view === 'new') return '/library#new';
if (view === 'genre') return '/library#genre/' + encodeURIComponent(param || '');
return '/library';
@ -199,7 +204,7 @@ function _applyView(view, param) {
if (si) { si.value = ''; document.getElementById('search-clear').style.display = 'none'; }
}
['nav-all','nav-wtr','nav-new','nav-series','nav-authors','nav-publishers','nav-archived','nav-bookmarks','nav-rated'].forEach(id => {
['nav-all','nav-wtr','nav-new','nav-series','nav-authors','nav-publishers','nav-archived','nav-bookmarks','nav-rated','nav-duplicates'].forEach(id => {
const el = document.getElementById(id);
if (el) el.classList.remove('active');
});
@ -212,6 +217,7 @@ function _applyView(view, param) {
'archived': 'nav-archived',
'bookmarks': 'nav-bookmarks',
'rated': 'nav-rated',
'duplicates': 'nav-duplicates',
};
const el = document.getElementById(activeMap[view]);
if (el) el.classList.add('active');
@ -229,6 +235,7 @@ function _applyView(view, param) {
view === 'archived' ? 'Archived' :
view === 'bookmarks' ? 'Bookmarks' :
view === 'rated' ? 'Rated' :
view === 'duplicates' ? 'Duplicates' :
view === 'genre' ? `Genre: ${param || ''}` :
view === 'search' ? `Search: "${param || ''}"` : '';
@ -279,6 +286,7 @@ function renderGrid() {
else if (currentView === 'search') renderSearchResults(currentParam);
else if (currentView === 'bookmarks') renderBookmarksView();
else if (currentView === 'rated') renderRatedView();
else if (currentView === 'duplicates') renderDuplicatesView();
}
// ── New view (bulk review + list/grid toggle) ─────────────────────────────
@ -1473,6 +1481,121 @@ function renderRatedView() {
renderBooksGrid(books);
}
// ── Duplicates view ────────────────────────────────────────────────────────
function _duplicateGroups(books) {
const map = new Map();
books.forEach(b => {
const key = (bookTitle(b).trim().toLowerCase()) + '|' + (bookAuthor(b).trim().toLowerCase());
if (!map.has(key)) map.set(key, []);
map.get(key).push(b);
});
return Array.from(map.values())
.filter(g => g.length >= 2)
.sort((a, b) => bookTitle(a[0]).localeCompare(bookTitle(b[0])));
}
function renderDuplicatesView() {
const container = document.getElementById('grid-container');
const groups = _duplicateGroups(activeBooks());
if (!groups.length) {
container.innerHTML = '<div class="empty">No duplicate books found.</div>';
return;
}
const idxSeries = indexedSeriesSet();
container.innerHTML = '';
groups.forEach(groupBooks => {
const heading = document.createElement('div');
heading.className = 'group-heading';
heading.textContent = `${bookTitle(groupBooks[0])}${bookAuthor(groupBooks[0])} (${groupBooks.length} copies)`;
container.appendChild(heading);
const grid = document.createElement('div');
grid.className = 'cover-grid';
groupBooks.forEach(b => {
const author = bookAuthor(b);
const title = bookTitle(b);
const card = document.createElement('div');
card.className = 'book-card';
card.id = `card-${cssId(b.filename)}`;
const st = (b.publication_status || '').toLowerCase();
let statusBadge = '';
if (st === 'complete') {
statusBadge = `<div class="badge-status badge-complete" title="Complete">
<svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3"><polyline points="20 6 9 17 4 12"/></svg>
</div>`;
} else if (st === 'ongoing') {
statusBadge = `<div class="badge-status badge-ongoing" title="Ongoing">
<svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3"><circle cx="12" cy="12" r="10"/><polyline points="12 6 12 12 16 14"/></svg>
</div>`;
} else if (st === 'hiatus') {
statusBadge = `<div class="badge-status badge-hiatus" title="Hiatus">
<svg width="10" height="10" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3"><line x1="10" y1="9" x2="10" y2="15"/><line x1="14" y1="9" x2="14" y2="15"/><circle cx="12" cy="12" r="10"/></svg>
</div>`;
}
const starClass = b.want_to_read ? 'btn-star starred' : 'btn-star';
card.innerHTML = `
<div class="cover-wrap" id="wrap-${cssId(b.filename)}">
<canvas class="cover-canvas" id="canvas-${cssId(b.filename)}"></canvas>
<button class="${starClass}" id="star-${cssId(b.filename)}"
onclick="event.stopPropagation();toggleWtr('${jsEsc(b.filename)}')" title="Want to Read">
<svg width="11" height="11" viewBox="0 0 24 24" fill="${b.want_to_read ? 'currentColor' : 'none'}" stroke="currentColor" stroke-width="2.5" id="star-svg-${cssId(b.filename)}">
<polygon points="12 2 15.09 8.26 22 9.27 17 14.14 18.18 21.02 12 17.77 5.82 21.02 7 14.14 2 9.27 8.91 8.26 12 2"/>
</svg>
</button>
${statusBadge}
${b.read_count > 0 ? `<div class="read-pill">${b.read_count}\u00d7</div>` : ''}
${b.progress > 0 ? `<div class="progress-mini"><div class="progress-mini-fill" style="width:${b.progress}%"></div></div>` : ''}
</div>
${starsHtml(b.filename, b.rating)}
<div class="book-info">
<div class="book-title">${esc(title)}</div>
<div class="book-author">${esc(author)}</div>
</div>`;
card.onclick = () => { location.href = `/library/book/${encodeURIComponent(b.filename)}`; };
grid.appendChild(card);
});
container.appendChild(grid);
// Second pass: load covers (same as renderBooksGrid)
groupBooks.forEach(b => {
const author = bookAuthor(b);
const title = bookTitle(b);
const wrap = document.getElementById(`wrap-${cssId(b.filename)}`);
const canvas = document.getElementById(`canvas-${cssId(b.filename)}`);
if (b.has_cover) {
const img = document.createElement('img');
img.className = 'cover-img';
img.style.cssText = 'position:absolute;inset:0;width:100%;height:100%;object-fit:cover';
img.src = `/library/cover-cached/${encodeURIComponent(b.filename)}`;
img.alt = title;
if (b.has_cached_cover) {
canvas.style.display = 'none';
}
img.onload = () => { canvas.style.display = 'none'; };
img.onerror = () => {
canvas.style.display = 'block';
makePlaceholderCover(canvas, title, author);
};
wrap.insertBefore(img, wrap.firstChild);
}
if (!b.has_cover || !b.has_cached_cover) {
requestAnimationFrame(() => makePlaceholderCover(canvas, title, author));
}
});
});
}
// ── Author detail ──────────────────────────────────────────────────────────
function renderAuthorDetail(authorName) {
@ -1702,7 +1825,7 @@ document.getElementById('search-input').addEventListener('input', function() {
if (q) {
currentView = 'search';
currentParam = q;
['nav-all','nav-wtr','nav-new','nav-series','nav-authors','nav-publishers','nav-archived','nav-bookmarks','nav-rated'].forEach(id => {
['nav-all','nav-wtr','nav-new','nav-series','nav-authors','nav-publishers','nav-archived','nav-bookmarks','nav-rated','nav-duplicates'].forEach(id => {
const el = document.getElementById(id);
if (el) el.classList.remove('active');
});
@ -1764,6 +1887,7 @@ loadLibrary().then(() => {
else if (hash === 'new') view = 'new';
else if (hash === 'bookmarks') view = 'bookmarks';
else if (hash === 'rated') view = 'rated';
else if (hash === 'duplicates') view = 'duplicates';
else if (hash.startsWith('genre/')) { view = 'genre'; param = decodeURIComponent(hash.slice(6)); }
history.replaceState({ view, param }, '', _viewUrl(view, param));
_applyView(view, param);

View File

@ -129,6 +129,16 @@
<span class="sidebar-count" id="count-rated"></span>
</a>
</li>
<li>
<a href="{% if active == 'library' %}#{% else %}/library#duplicates{% endif %}"
{% if active == 'library' %}id="nav-duplicates" onclick="switchView('duplicates'); return false;"{% endif %}>
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
<rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 01-2-2V4a2 2 0 012-2h9a2 2 0 012 2v1"/>
</svg>
Duplicates
<span class="sidebar-count" id="count-duplicates"></span>
</a>
</li>
<li>
<a href="/stats"{% if active == 'stats' %} class="active"{% endif %}>
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
@ -236,6 +246,15 @@
const publisherCount = new Set(active.map(b => b.publisher).filter(Boolean)).size;
const archivedCount = books.filter(b => b.archived).length;
const ratedCount = active.filter(b => b.rating > 0).length;
const dupMap = new Map();
active.forEach(b => {
const key = (b.title || '').trim().toLowerCase() + '|' + (b.author || '').trim().toLowerCase();
dupMap.set(key, (dupMap.get(key) || 0) + 1);
});
const dupCount = active.filter(b => {
const key = (b.title || '').trim().toLowerCase() + '|' + (b.author || '').trim().toLowerCase();
return dupMap.get(key) >= 2;
}).length;
const setCount = (id, value) => {
const el = document.getElementById(id);
@ -250,6 +269,7 @@
setCount('count-publishers', publisherCount);
setCount('count-rated', ratedCount);
setCount('count-archived', archivedCount);
setCount('count-duplicates', dupCount);
}
async function refreshLibraryCounts() {

View File

@ -227,6 +227,17 @@
border: 1px solid var(--border);
}
.btn-outline:hover { background: var(--surface); color: var(--text); border-color: var(--text-faint); }
.dup-warning {
display: none; width: 100%; max-width: 620px; margin-bottom: 1.5rem;
background: rgba(200,160,58,0.08); border: 1px solid rgba(200,160,58,0.35);
border-radius: var(--radius); padding: 0.85rem 1rem;
font-family: var(--mono); font-size: 0.78rem; color: var(--warning);
line-height: 1.6;
}
.dup-warning.visible { display: block; }
.dup-warning a { color: var(--accent2); text-decoration: none; }
.dup-warning a:hover { text-decoration: underline; }
</style>
</head>
<body>
@ -239,7 +250,7 @@
<div class="card">
<div class="card-title">Book URL</div>
<label for="url">Story overview page</label>
<input type="url" id="url" placeholder="https://..." oninput="checkUrlCredentials()"/>
<input type="url" id="url" placeholder="https://..." oninput="checkUrlCredentials(); clearDupWarning()"/>
<div class="cred-status" id="cred-status"></div>
<button id="load-btn" onclick="loadMeta()">
<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5">
@ -250,6 +261,9 @@
</button>
</div>
<!-- Duplicate warning -->
<div class="dup-warning" id="dup-warning"></div>
<!-- Step 2: Metadata preview + cover upload + Convert -->
<div class="card" id="meta-card">
<div class="card-title">Book info</div>
@ -362,6 +376,7 @@
}
renderMeta(d);
showDupWarning(d.already_exists ? d.existing_books : []);
document.getElementById('meta-card').classList.add('visible');
// Reset cover upload
document.getElementById('cover-file').value = '';
@ -553,6 +568,22 @@
div.scrollTop = div.scrollHeight;
}
function clearDupWarning() {
const el = document.getElementById('dup-warning');
el.classList.remove('visible');
el.innerHTML = '';
}
function showDupWarning(books) {
const el = document.getElementById('dup-warning');
if (!books || !books.length) { clearDupWarning(); return; }
const links = books.map(b =>
`<a href="/library/book/${encodeURIComponent(b.filename)}" target="_blank">${esc(b.title)}</a>`
).join(', ');
el.innerHTML = `⚠ This title already exists in your library: ${links}. You can still proceed with the conversion.`;
el.classList.add('visible');
}
function esc(s) {
return String(s ?? '')
.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');

View File

@ -254,6 +254,8 @@ Dropbox settings are managed via the web UI on `/backup`.
- `Edit EPUB` button in Book Detail is only shown for `.epub` files.
- Backup page supports: manual run, dry-run, Dropbox root, retention count, schedule (on/off + hours), status + history.
- Bookmarks: saved per book via `POST /library/bookmarks/{filename}`; shown in Library sidebar section; navigated via `?bm_ch=N&bm_scroll=F` URL params on reader page.
- Convert page: after loading metadata, if a book with the same title+author already exists in the library, a warning banner is shown (with a link to the existing book); user can still proceed with conversion. Check is done server-side in `/preload` response (`already_exists`, `existing_books`).
- Duplicates view (`#duplicates`): groups non-archived books by `(title, author)` (case-insensitive); shows only groups with ≥ 2 copies; counter in sidebar shows total number of duplicate books. Detection is entirely client-side from the existing library data.
- Book Builder (`/builder`): create EPUB books from scratch; drafts stored in `builder_drafts` (JSONB chapters); contenteditable editor with toolbar (bold/italic/underline/blockquote/author-note/scene-break/normalize); autosave every 30 s + Ctrl+S; publish normalizes HTML via `normalize_wysiwyg_html()` and builds EPUB via `build_epub()`.
---

View File

@ -0,0 +1,52 @@
# TODO: Performance improvements — Library "All Books" load speed
Observed: loading all books (especially on hard refresh CTRL+F5) is slow.
Root cause analysis identified four bottlenecks, ordered by impact.
---
### 1. Lazy cover loading (HIGH impact)
**Problem:** On every full render, a `<img>` is created immediately for every book card that has a cover. With 500+ books this fires 500+ simultaneous HTTP requests to `/library/cover-cached/`. The browser throttles these and the server gets hammered.
**Fix:** Use an `IntersectionObserver` to defer image loading until a card scrolls into the viewport. Cards outside the viewport stay as canvas placeholders until they appear.
**Affected:** `static/library.js``renderBooksGrid()` second pass (lines ~9811005)
---
### 2. HTTP caching on `/api/library` (HIGH impact)
**Problem:** Every refresh (including soft refresh) fetches the full book list JSON from the server with no caching. No `ETag`, `Last-Modified` or `Cache-Control` headers are set.
**Fix:** Add an `ETag` header based on a hash of the serialized response (or a DB row count + last `updated_at`). Browser sends `If-None-Match`; server returns `304 Not Modified` when nothing changed — zero data transfer.
**Affected:** `routers/library.py``GET /api/library` endpoint; `static/library.js``loadLibrary()`
---
### 3. Double DOM pass in renderBooksGrid (MEDIUM impact)
**Problem:** `renderBooksGrid` makes two full iterations over the book list:
1. Build all card HTML and append to the grid.
2. Query the DOM again for each `canvas-*` and `wrap-*` element to set up image loading.
This causes two reflows and 500+ `getElementById` calls after the DOM is already populated.
**Fix:** Combine both passes into one: build the card element, immediately set up the `<img>` element and canvas, then append. No second iteration needed.
**Affected:** `static/library.js``renderBooksGrid()` (lines ~9041005)
---
### 4. Tags via separate query + Python merge (LOW impact)
**Problem:** `list_library_json()` fetches book tags via a separate `SELECT * FROM book_tags` query and then merges them in Python using a dict. For large libraries this means two round-trips and an O(n) in-process merge.
**Fix:** Use a PostgreSQL JSON aggregation in the main query:
```sql
COALESCE(
json_agg(json_build_object('tag', bt.tag, 'tag_type', bt.tag_type))
FILTER (WHERE bt.tag IS NOT NULL),
'[]'
) AS tags
```
This returns tags inline per book row, eliminating the second query and the Python merge loop.
**Affected:** `routers/common.py``list_library_json()` (lines ~397458)

View File

@ -3,6 +3,11 @@
This file tracks changes on the `develop` line.
`changelog.md` can later be used for release summaries.
## 2026-03-27 (1)
- Convert page: duplicate warning shown after loading metadata when a book with the same title+author already exists in the library; warning includes a link to the existing book; user can still proceed with conversion
- Library: added Duplicates section to sidebar (between Rated and Statistics); counter shows total number of books that are part of a duplicate group (same title+author, case-insensitive); Duplicates view groups books by title+author with a subheading per group
- Fixed Duplicates view not loading covers: card renderer now uses the same canvas + two-pass img/`makePlaceholderCover` pattern as `renderBooksGrid`
## 2026-03-26 (2)
- Fixed Book Builder page showing white background: `library.css` added to `builder.html` to load `:root` CSS variables and dark `body` background; all CSS variable references in `builder.css` aligned with library theme names (`--text`, `--surface`, `--surface2`, `--text-dim`, `--border`, `--accent`, `--sidebar`)