From 0ae181706d92ea2dc7068725f6b32e9957003019 Mon Sep 17 00:00:00 2001 From: Ivo Oskamp Date: Sun, 22 Mar 2026 16:13:45 +0100 Subject: [PATCH] Bootstrap Novela 2.0 implementation and docs --- .gitignore | 3 + build-and-push.sh | 269 +++++ containers/novela/Dockerfile | 17 + .../novela/__pycache__/cbr.cpython-311.pyc | Bin 0 -> 4959 bytes .../novela/__pycache__/db.cpython-311.pyc | Bin 0 -> 2857 bytes .../novela/__pycache__/epub.cpython-311.pyc | Bin 0 -> 21492 bytes .../novela/__pycache__/main.cpython-311.pyc | Bin 0 -> 2060 bytes .../__pycache__/migrations.cpython-311.pyc | Bin 0 -> 8792 bytes .../novela/__pycache__/pdf.cpython-311.pyc | Bin 0 -> 5168 bytes .../__pycache__/security.cpython-311.pyc | Bin 0 -> 2680 bytes .../novela/__pycache__/xhtml.cpython-311.pyc | Bin 0 -> 10680 bytes containers/novela/cbr.py | 61 ++ containers/novela/db.py | 55 + containers/novela/epub.py | 355 +++++++ containers/novela/main.py | 42 + containers/novela/migrations.py | 203 ++++ containers/novela/pdf.py | 68 ++ containers/novela/requirements.txt | 14 + containers/novela/routers/__init__.py | 15 + .../__pycache__/__init__.cpython-311.pyc | Bin 0 -> 581 bytes .../__pycache__/backup.cpython-311.pyc | Bin 0 -> 21143 bytes .../__pycache__/common.cpython-311.pyc | Bin 0 -> 26981 bytes .../__pycache__/editor.cpython-311.pyc | Bin 0 -> 28284 bytes .../__pycache__/grabber.cpython-311.pyc | Bin 0 -> 32045 bytes .../__pycache__/library.cpython-311.pyc | Bin 0 -> 27427 bytes .../__pycache__/reader.cpython-311.pyc | Bin 0 -> 62202 bytes .../__pycache__/settings.cpython-311.pyc | Bin 0 -> 9088 bytes containers/novela/routers/backup.py | 359 +++++++ containers/novela/routers/common.py | 431 ++++++++ containers/novela/routers/editor.py | 434 ++++++++ containers/novela/routers/grabber.py | 473 +++++++++ containers/novela/routers/library.py | 383 +++++++ containers/novela/routers/reader.py | 993 ++++++++++++++++++ containers/novela/routers/settings.py | 104 ++ containers/novela/scrapers/__init__.py | 17 + .../__pycache__/__init__.cpython-311.pyc | Bin 0 -> 955 bytes .../__pycache__/awesomedude.cpython-311.pyc | Bin 0 -> 14055 bytes .../scrapers/__pycache__/base.cpython-311.pyc | Bin 0 -> 3389 bytes .../__pycache__/gayauthors.cpython-311.pyc | Bin 0 -> 15962 bytes containers/novela/scrapers/awesomedude.py | 265 +++++ containers/novela/scrapers/base.py | 58 + containers/novela/scrapers/gayauthors.py | 236 +++++ containers/novela/security.py | 43 + containers/novela/static/book.css | 282 +++++ containers/novela/static/book.js | 304 ++++++ containers/novela/static/break.png | Bin 0 -> 69153 bytes containers/novela/static/editor.css | 206 ++++ containers/novela/static/editor.js | 430 ++++++++ containers/novela/static/epub-style.css | 254 +++++ containers/novela/static/library.css | 456 ++++++++ containers/novela/static/library.js | 979 +++++++++++++++++ containers/novela/static/sidebar.css | 153 +++ containers/novela/templates/_sidebar.html | 250 +++++ containers/novela/templates/backup.html | 274 +++++ containers/novela/templates/book.html | 319 ++++++ containers/novela/templates/credentials.html | 406 +++++++ containers/novela/templates/debug.html | 327 ++++++ containers/novela/templates/editor.html | 97 ++ containers/novela/templates/grabber.html | 562 ++++++++++ containers/novela/templates/home.html | 422 ++++++++ containers/novela/templates/index.html | 562 ++++++++++ containers/novela/templates/library.html | 78 ++ containers/novela/templates/reader.html | 455 ++++++++ containers/novela/templates/settings.html | 441 ++++++++ containers/novela/templates/stats.html | 320 ++++++ containers/novela/xhtml.py | 169 +++ docs/BLUEPRINT.md | 420 ++++++++ docs/TECHNICAL.md | 100 ++ stack/novela.env | 7 + stack/stack.yml | 48 + 70 files changed, 13219 insertions(+) create mode 100644 .gitignore create mode 100755 build-and-push.sh create mode 100644 containers/novela/Dockerfile create mode 100644 containers/novela/__pycache__/cbr.cpython-311.pyc create mode 100644 containers/novela/__pycache__/db.cpython-311.pyc create mode 100644 containers/novela/__pycache__/epub.cpython-311.pyc create mode 100644 containers/novela/__pycache__/main.cpython-311.pyc create mode 100644 containers/novela/__pycache__/migrations.cpython-311.pyc create mode 100644 containers/novela/__pycache__/pdf.cpython-311.pyc create mode 100644 containers/novela/__pycache__/security.cpython-311.pyc create mode 100644 containers/novela/__pycache__/xhtml.cpython-311.pyc create mode 100644 containers/novela/cbr.py create mode 100644 containers/novela/db.py create mode 100644 containers/novela/epub.py create mode 100644 containers/novela/main.py create mode 100644 containers/novela/migrations.py create mode 100644 containers/novela/pdf.py create mode 100644 containers/novela/requirements.txt create mode 100644 containers/novela/routers/__init__.py create mode 100644 containers/novela/routers/__pycache__/__init__.cpython-311.pyc create mode 100644 containers/novela/routers/__pycache__/backup.cpython-311.pyc create mode 100644 containers/novela/routers/__pycache__/common.cpython-311.pyc create mode 100644 containers/novela/routers/__pycache__/editor.cpython-311.pyc create mode 100644 containers/novela/routers/__pycache__/grabber.cpython-311.pyc create mode 100644 containers/novela/routers/__pycache__/library.cpython-311.pyc create mode 100644 containers/novela/routers/__pycache__/reader.cpython-311.pyc create mode 100644 containers/novela/routers/__pycache__/settings.cpython-311.pyc create mode 100644 containers/novela/routers/backup.py create mode 100644 containers/novela/routers/common.py create mode 100644 containers/novela/routers/editor.py create mode 100644 containers/novela/routers/grabber.py create mode 100644 containers/novela/routers/library.py create mode 100644 containers/novela/routers/reader.py create mode 100644 containers/novela/routers/settings.py create mode 100644 containers/novela/scrapers/__init__.py create mode 100644 containers/novela/scrapers/__pycache__/__init__.cpython-311.pyc create mode 100644 containers/novela/scrapers/__pycache__/awesomedude.cpython-311.pyc create mode 100644 containers/novela/scrapers/__pycache__/base.cpython-311.pyc create mode 100644 containers/novela/scrapers/__pycache__/gayauthors.cpython-311.pyc create mode 100644 containers/novela/scrapers/awesomedude.py create mode 100644 containers/novela/scrapers/base.py create mode 100644 containers/novela/scrapers/gayauthors.py create mode 100644 containers/novela/security.py create mode 100644 containers/novela/static/book.css create mode 100644 containers/novela/static/book.js create mode 100644 containers/novela/static/break.png create mode 100644 containers/novela/static/editor.css create mode 100644 containers/novela/static/editor.js create mode 100644 containers/novela/static/epub-style.css create mode 100644 containers/novela/static/library.css create mode 100644 containers/novela/static/library.js create mode 100644 containers/novela/static/sidebar.css create mode 100644 containers/novela/templates/_sidebar.html create mode 100644 containers/novela/templates/backup.html create mode 100644 containers/novela/templates/book.html create mode 100644 containers/novela/templates/credentials.html create mode 100644 containers/novela/templates/debug.html create mode 100644 containers/novela/templates/editor.html create mode 100644 containers/novela/templates/grabber.html create mode 100644 containers/novela/templates/home.html create mode 100644 containers/novela/templates/index.html create mode 100644 containers/novela/templates/library.html create mode 100644 containers/novela/templates/reader.html create mode 100644 containers/novela/templates/settings.html create mode 100644 containers/novela/templates/stats.html create mode 100644 containers/novela/xhtml.py create mode 100644 docs/BLUEPRINT.md create mode 100644 docs/TECHNICAL.md create mode 100644 stack/novela.env create mode 100644 stack/stack.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2620661 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.files/ + +.last-branch diff --git a/build-and-push.sh b/build-and-push.sh new file mode 100755 index 0000000..b024899 --- /dev/null +++ b/build-and-push.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================ +# build-and-push.sh +# Location: repo root (e.g. /docker/develop/novela) +# +# Purpose: +# - Automatic version bump: +# 1 = patch, 2 = minor, 3 = major, t = test +# - Test builds: only update :dev (no commit/tag) +# - Release builds: update version.txt, commit, tag, push (to the current branch) +# - Build & push Docker images for each service under ./containers/* +# - Preflight checks: Docker daemon up, logged in to registry, valid names/tags +# - Summary: show all images + tags built and pushed +# - Branch visibility: +# - Shows currently checked out branch (authoritative) +# - Reads .last-branch for info (if present) when BRANCH is not set +# - Writes the current branch back to .last-branch at the end +# +# Usage: +# BRANCH= ./build-and-push.sh [bump] # BRANCH is optional; informative only +# ./build-and-push.sh [bump] +# If [bump] is omitted, you will be prompted (default = t). +# ============================================================================ + +DOCKER_REGISTRY="gitea.oskamp.info" +DOCKER_NAMESPACE="ivooskamp" + +VERSION_FILE="version.txt" +START_VERSION="v0.1.0" +COMPOSE_DIR="containers" +LAST_BRANCH_FILE=".last-branch" # stored in repo root + +# --- Input: prompt if missing ------------------------------------------------ +BUMP="${1:-}" +if [[ -z "${BUMP}" ]]; then + echo "Select bump type: [1] patch, [2] minor, [3] major, [t] test (default: t)" + read -r BUMP + BUMP="${BUMP:-t}" +fi + +if [[ "$BUMP" != "1" && "$BUMP" != "2" && "$BUMP" != "3" && "$BUMP" != "t" ]]; then + echo "[ERROR] Unknown bump type '$BUMP' (use 1, 2, 3, or t)." + exit 1 +fi + +# --- Helpers ----------------------------------------------------------------- +read_version() { + if [[ -f "$VERSION_FILE" ]]; then + tr -d ' \t\n\r' < "$VERSION_FILE" + else + echo "$START_VERSION" + fi +} + +write_version() { + echo "$1" > "$VERSION_FILE" +} + +bump_version() { + local cur="$1" + local kind="$2" + local core="${cur#v}" + IFS='.' read -r MA MI PA <<< "$core" + case "$kind" in + 1) PA=$((PA + 1));; + 2) MI=$((MI + 1)); PA=0;; + 3) MA=$((MA + 1)); MI=0; PA=0;; + *) echo "[ERROR] Unknown bump kind"; exit 1;; + esac + echo "v${MA}.${MI}.${PA}" +} + +check_docker_ready() { + if ! docker info >/dev/null 2>&1; then + echo "[ERROR] Docker daemon not reachable. Is Docker running and do you have permission to use it?" + exit 1 + fi +} + +ensure_registry_login() { + local cfg="${HOME}/.docker/config.json" + if [[ ! -f "$cfg" ]]; then + echo "[ERROR] Docker config not found at $cfg. Please login: docker login ${DOCKER_REGISTRY}" + exit 1 + fi + if ! grep -q "\"${DOCKER_REGISTRY}\"" "$cfg"; then + echo "[ERROR] No registry auth found for ${DOCKER_REGISTRY}. Please run: docker login ${DOCKER_REGISTRY}" + exit 1 + fi +} + +validate_repo_component() { + local comp="$1" + if [[ ! "$comp" =~ ^[a-z0-9]+([._-][a-z0-9]+)*$ ]]; then + echo "[ERROR] Invalid repository component '$comp'." + echo " Must match: ^[a-z0-9]+([._-][a-z0-9]+)*$ (lowercase, digits, ., _, - as separators)." + return 1 + fi +} + +validate_tag() { + local tag="$1" + local len="${#tag}" + if (( len < 1 || len > 128 )); then + echo "[ERROR] Invalid tag length ($len). Must be between 1 and 128 characters." + return 1 + fi + if [[ ! "$tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*$ ]]; then + echo "[ERROR] Invalid tag '$tag'. Allowed: [A-Za-z0-9_.-], must start with alphanumeric or underscore." + return 1 + fi +} + +# --- Preflight --------------------------------------------------------------- +if [[ ! -d ".git" ]]; then + echo "[ERROR] Not a git repository (.git missing)." + exit 1 +fi + +if [[ ! -d "$COMPOSE_DIR" ]]; then + echo "[ERROR] '$COMPOSE_DIR' directory missing. Expected ./containers// with a Dockerfile." + exit 1 +fi + +check_docker_ready +ensure_registry_login +validate_repo_component "$DOCKER_NAMESPACE" + +# Detect currently checked out branch (authoritative for this script) +DETECTED_BRANCH="$(git branch --show-current 2>/dev/null || true)" +if [[ -z "$DETECTED_BRANCH" ]]; then + DETECTED_BRANCH="$(git symbolic-ref --quiet --short HEAD 2>/dev/null || true)" +fi +if [[ -z "$DETECTED_BRANCH" ]]; then + # Try to derive from upstream + UPSTREAM_REF_DERIVED="$(git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || true)" + if [[ -n "$UPSTREAM_REF_DERIVED" ]]; then + DETECTED_BRANCH="${UPSTREAM_REF_DERIVED#origin/}" + fi +fi +if [[ -z "$DETECTED_BRANCH" ]]; then + DETECTED_BRANCH="main" +fi + +# Optional signals: BRANCH env and .last-branch (informational only) +ENV_BRANCH="${BRANCH:-}" +LAST_BRANCH_FILE_PATH="$(pwd)/$LAST_BRANCH_FILE" +LAST_BRANCH_VALUE="" +if [[ -z "$ENV_BRANCH" && -f "$LAST_BRANCH_FILE_PATH" ]]; then + LAST_BRANCH_VALUE="$(tr -d ' \t\n\r' < "$LAST_BRANCH_FILE_PATH")" +fi + +UPSTREAM_REF="$(git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "origin/$DETECTED_BRANCH")" +HEAD_SHA="$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")" + +echo "[INFO] Repo: $(pwd)" +echo "[INFO] Current branch: $DETECTED_BRANCH" +echo "[INFO] Upstream: $UPSTREAM_REF" +echo "[INFO] HEAD (sha): $HEAD_SHA" + +if [[ -n "$ENV_BRANCH" && "$ENV_BRANCH" != "$DETECTED_BRANCH" ]]; then + echo "[WARNING] BRANCH='$ENV_BRANCH' differs from checked out branch '$DETECTED_BRANCH'." + echo "[WARNING] This script does not switch branches; continuing on '$DETECTED_BRANCH'." +fi + +if [[ -n "$LAST_BRANCH_VALUE" && "$LAST_BRANCH_VALUE" != "$DETECTED_BRANCH" && -z "$ENV_BRANCH" ]]; then + echo "[INFO] .last-branch suggests '$LAST_BRANCH_VALUE', but current checkout is '$DETECTED_BRANCH'." + echo "[INFO] If you intended to build '$LAST_BRANCH_VALUE', switch branches first (use update-and-build.sh)." +fi + +# --- Versioning -------------------------------------------------------------- +CURRENT_VERSION="$(read_version)" +NEW_VERSION="$CURRENT_VERSION" +DO_TAG_AND_BUMP=true + +if [[ "$BUMP" == "t" ]]; then + echo "[INFO] Test build: keeping version $CURRENT_VERSION; will only update :dev." + DO_TAG_AND_BUMP=false +else + NEW_VERSION="$(bump_version "$CURRENT_VERSION" "$BUMP")" + echo "[INFO] New version: $NEW_VERSION" +fi + +if $DO_TAG_AND_BUMP; then + validate_tag "$NEW_VERSION" +fi +validate_tag "latest" + +# --- Version update + VCS ops (release builds only) -------------------------- +if $DO_TAG_AND_BUMP; then + echo "[INFO] Writing $NEW_VERSION to $VERSION_FILE" + write_version "$NEW_VERSION" + + echo "[INFO] Git add + commit (branch: $DETECTED_BRANCH)" + git add "$VERSION_FILE" + git commit -m "Release $NEW_VERSION on branch $DETECTED_BRANCH (bump type $BUMP)" + + echo "[INFO] Git tag $NEW_VERSION" + git tag -a "$NEW_VERSION" -m "Release $NEW_VERSION" + + echo "[INFO] Git push + tags" + git push origin "$DETECTED_BRANCH" + git push --tags +else + echo "[INFO] Skipping commit/tagging (test build)." +fi + +# --- Build & push per service ------------------------------------------------ +shopt -s nullglob +services=( "$COMPOSE_DIR"/* ) +if [[ ${#services[@]} -eq 0 ]]; then + echo "[ERROR] No services found under $COMPOSE_DIR" + exit 1 +fi + +BUILT_IMAGES=() + +for svc_path in "${services[@]}"; do + [[ -d "$svc_path" ]] || continue + svc="$(basename "$svc_path")" + dockerfile="$svc_path/Dockerfile" + + validate_repo_component "$svc" + + if [[ ! -f "$dockerfile" ]]; then + echo "[WARNING] Skipping '${svc}': Dockerfile not found in ${svc_path}" + continue + fi + + IMAGE_BASE="${DOCKER_REGISTRY}/${DOCKER_NAMESPACE}/${svc}" + + if $DO_TAG_AND_BUMP; then + echo "============================================================" + echo "[INFO] Building ${svc} -> tags: ${NEW_VERSION}, latest" + echo "============================================================" + docker build -t "${IMAGE_BASE}:${NEW_VERSION}" -t "${IMAGE_BASE}:dev" "$svc_path" + docker push "${IMAGE_BASE}:${NEW_VERSION}" + docker push "${IMAGE_BASE}:dev" + BUILT_IMAGES+=("${IMAGE_BASE}:${NEW_VERSION}" "${IMAGE_BASE}:dev") + else + echo "============================================================" + echo "[INFO] Test build ${svc} -> tag: latest" + echo "============================================================" + docker build -t "${IMAGE_BASE}:dev" "$svc_path" + docker push "${IMAGE_BASE}:dev" + BUILT_IMAGES+=("${IMAGE_BASE}:dev") + fi +done + +# --- Persist current branch to .last-branch ---------------------------------- +# (This helps script 1 to preselect next time, and is informative if you run script 2 standalone) +echo "$DETECTED_BRANCH" > "$LAST_BRANCH_FILE_PATH" + +# --- Summary ----------------------------------------------------------------- +echo "" +echo "============================================================" +echo "[SUMMARY] Build & push complete (branch: $DETECTED_BRANCH)" +if $DO_TAG_AND_BUMP; then + echo "[INFO] Release version: $NEW_VERSION" +else + echo "[INFO] Test build (no version bump)" +fi +echo "[INFO] Images pushed:" +for img in "${BUILT_IMAGES[@]}"; do + echo " - $img" +done +echo "============================================================" diff --git a/containers/novela/Dockerfile b/containers/novela/Dockerfile new file mode 100644 index 0000000..7477bfb --- /dev/null +++ b/containers/novela/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + libmagic1 \ + unrar-free \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app + +EXPOSE 8000 +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/containers/novela/__pycache__/cbr.cpython-311.pyc b/containers/novela/__pycache__/cbr.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd13a81548587ea891c53161ea562cd90d5cef19 GIT binary patch literal 4959 zcmd5=U2Idy6`r|2zW#~rgd~K7Z2p`j7?GqY*`m_Gmi#oLC6I<)ZLwNKj_-AF@L%U% zCn2&|S;RwFxgxBn;)g!mZ7B<4rBZ3B_NA&8c&%hdmawiA38~r#-m+r1MLhM)_$R(5 zS+x(Xc6{%dzjJ2Joip>DGycTsv?EA={pa68oP^LnXr-L2rNWE9Kw$&PNXB6l!9I*> znG16f-p8{t9~OK9wDDolCqgcSEfL8lMXWw6MwgH*{sH@JvPI!#NwEh7*?LXzIZE|9 z#o1b(5ka2~`dp{@kE^FF!23!Vn2W@L&WD0P7vl3k%d@Ojts=F7}9J=odp5X1RZsh z;~K?*0fT~_ZCWF8DB$5tXJ`uQ_Z$B_JUY0@1hkibD$Of|IkXK_9~?$XI@9JG=I)y1gdl+yf4B) zHqbmQ>92-T*klZB9%P|n7QM1arTJK9USy4$xs(>sp9YioCYS#H1#}PH6NmsEq@I!n zAcph8yVKL>XWp3@pFJ}*d5<%=f$teOs?uq?x(LDkOXIcHDa*|UwX&4=~50<)RUed74N z<74M-Cm1l-bTQv_acyForz}OrBK?X9ea&cf`(EXjFJJE2^^FC&n!h{xeCcRdfAjju z^vSGnK=r^H-tIi@=D%`5Keh^j$SLc$r}}acdRK zMi6G0zxFCl!iYQ0KgDZQn^H>hX6^;L^;y!Z#w@qhdLF$B6nA*!nlV#?On;(mktNwG z+kR)!t2Ee=O9GeKXQf1q0%|GMt`r&5C@z-L0t?G%rIru8sU>OA9c9kyy!QK(Vvq~$ z;Xf~+-TkMwv$$xX;iJ1t}y|JctFy6sjProo$Z#S&UqEZ9Z-(Rt++9$2x5 zX!JHngR&J{u?%5axtRuORH=qQ zA===TWz7&Gp@>5ID7P1gN<(i7*)$blyN(`Wn1^{*@;3BS|Dw|U4oA4IHPf4`>&(}6 zu1*vr_j>o$S z2h6{IBz+`rKls$%y=m{x**$r?=c)b3rv1p>V>$a^-afcG_RM{-P~To~x0R^7ZQCi; zS+)_7f4PukzZzKctR-`n_PnKi+l@rqH~Wy)UG^uO*!P+{=>u=l*t~K4qvM(3n&V+WfKl}5Jq35ejinEl z*0t-YbSfif-90d|=wZD(1SS?eF!5_0J1neJocLR~RJs=){jg>14gT>P!dSQTxEE78 z(lXw~Kj{+2d!;9R7^ulbRG{_=7`hf}Wo!nRU&3ig;S^U5vO8h92`Fo#M>A{!#PE9r z(Y0X6C;5d6Y|7jsCe08~?+S>o0jYV8@@*3Wa#_&mxi6)KN;tm*il2&iz;WKMTqjaO z_1BqW-ISOV_5i99h<|Fk1CHW8yv2l3Qq<}7>9u4Y`M-tMYEYvdduDRf5P+N}r(q^9 z7di??4m|<_w^U0+(JsYgLwIjubo%)WFg&LV?d1(iFh(MNhy#C}@P|X%6*5NKoeUHR z^>c<0i$gF#SBFp;!=)^Tg0mVxry7r`3cyWFhR`w)i!LfeGx%UgGwd^p>W{?3A;3fF z?CEo3zVjChLG>>}=n1&9=np3puca7Z3bJ4G8+g{>LJ>pME+ry!1`qhG(%Bejl~Bt~ zQ7aIGS!Nj*c@G$d;Z+6rN&qfbTV_~q$hkaum*<1Y)$z6NEt@Od_S?zK_|2(XQ#o5t z-UflEBzEX?Th8XZvsa(VIS=QZhu64*!LPI)3n2W{2#VdrsE(<)k zDCW%!K2(bRz+#Cw+$h93`j<#?+fcg{FEjKOQoPhzM_`*I!pbS)hc>VmQV#(Il*G7z z_Gisc0UgMip8|5Nvfs0Ep@2HGyFJh9N3-l%b#7ZR_OAB=rebeqm}RxZD&f|Q0IQ8# z^(H-VyOkAdNfXAsnIlc<}D$V-I}!!5_1FEjen# R9T^w`GW{)(MNkdwPlok|WDNQTV2w8h^WaN|C zJLjV~Q~^IEA#IFFaSADzQj(gy__1&8pCEw>=0FfAf#yxcP)wdWb0;Y}C$xm_?aj{Z z%+2o1Z+7k;hQmPu?Zn@|=BpARf8ay2DGg@xM_^XS8Nvvo1(L>5D9~9UEl_VxOp7ol z7Q|U8Ex{bk%7!!(Xc}VlE3joKOgaivOg7XD0t+lk=?>P#I*cF-!Ywm;A;7dnIjym7 z76NIAg%?RW3{MewiWmaxTqF;`!lR~PlI~8}>Xmb1gH#sBnvLZ+=eES-VX(12bM*6`EZ8$S4yYfaFn2YBxg% zYOiETWRl;~16cb}YyDTUC9{Cv)+@cO{I9oF*5O~91@FrMkrsEtiPOo{smYVc%*Vhu z>Ri#Xr@3LZ2@;AhJ4 zyKHKB?y3{au&E3r)rI^t-whHxG}cuhCGte?z8hSLEJyBry&kOS<5hjUruTgJ;oVPG z;>+>XQ6D_p5kU4Wj1fZ22EhHSCGlIvJu316us;Q#-H7# zy*5f3ZKm}W6Bft;e*#xQsm17UJwo zUF1fV8SEq2pOLfkMKb}%(v2wJ3mO)#zGn~bN9~ThnYY~o%lE@`6y=A3K**I)=~_*X zR`sE>{^k0q-_F7S{r5z=k~mjQobw)U$avL7qV6bNbK~VZ@wk>7lQ=N`m|P!dKP#)P z^9PEp=8psx!sn4#qw$#FI_{{sLeVm^g@PLncNd;}L091U_cmBXJgpcI*T1sTTT{b# zBDW$HwYRGFmet-3J=*X{Ch9Iw_!pPXgB`z1rARx>+c)oXx0M25q}BXyBnf{LL^07r z7x8fvI6atn+Z|P59?y=+hd}LhxUE4TO&Ha#o7!#dhE~(`JI8JvyLtTf@v_wC`PLqI zgvy001&rUf>w^pA`k>E;_BTP_YYW6*P7yROb#e4OYQUz<&AZO5!KVZFZlS?OVurX; zFtV26IW+f&2*Rn)$!+E;G&t#3m9xvi}& z3%|$9(p%(V*AX!Xe5(b3bMpZICH#?j{-I_xPi%~1qxoZoWeYw=Lfk7+B7707tGVLb z^dUzg3`ZRhftdUUqP<_pFLd<;}D!K^Nzb&*oKK{`r}YlFN|_V>r6 zv*cbIWLMeWA0HSg$tAn0>|IS&lvtUJK(COIdO)FprHML$VLOS6^gOK-q)V1J{@mk@ z>&zQpLPHYWQ73*9k!U~8HAqCDyX(YHf<(~kf{ZFa{u1GFanPI4DLqgpFA^8Ke*h}( BEvNth literal 0 HcmV?d00001 diff --git a/containers/novela/__pycache__/epub.cpython-311.pyc b/containers/novela/__pycache__/epub.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..58ad15375c926655b9e0f5459d75f0e057d6fcf0 GIT binary patch literal 21492 zcmc(HYit|Y)z}Q5!#5>S56YH2q9seRMAGs@mSu^%vSiEJwdIH8^ +
+
+ + {% if has_cover %} + {{ title }} + {% endif %} +
+ + +
+ + +
+
{{ title or filename }}
+ {% if author %}
{{ author }}
{% endif %} + +
+ {% if series %} +
+ Series + {{ series }}{% if series_index %} [{{ series_index }}]{% endif %} +
+ {% endif %} +
+ Publisher + {% if publisher %} + {{ publisher }} + {% else %} + No publisher + {% endif %} +
+ {% if publication_status %} +
+ Status + + {% set st = publication_status | lower %} + + {{ publication_status }} + + +
+ {% endif %} + {% if publish_date %} +
+ Updated + {{ publish_date }} +
+ {% endif %} + {% if genres %} +
+ Genres + {% for g in genres %}{{ g }}{% endfor %} +
+ {% endif %} + {% if subgenres %} +
+ Sub-genres + {% for g in subgenres %}{{ g }}{% endfor %} +
+ {% endif %} + {% if tags %} +
+ Tags + {% for g in tags %}{{ g }}{% endfor %} +
+ {% endif %} + {% if description %} +
+ Description +
{{ description }}
+
+ {% endif %} + {% if source_url %} +
+ Bron + + + {{ source_url }} + + +
+ {% endif %} +
+ + {% if progress > 0 %} +
+
Reading progress
+
+
+
+
{{ progress }}% complete
+
+ {% endif %} + + {% if read_count > 0 %} +
+ Read {{ read_count }}× + {% if last_read %} + · Last read {{ last_read[:10] }} + {% endif %} +
+ {% endif %} + +
+ + + + + + {% if progress > 0 %}Continue reading{% else %}Start reading{% endif %} + + {% if progress > 0 %} + + {% endif %} + + + + + + + Download + + + + + + + + + + Edit EPUB + + + + +
+
+ + + +
+
+
+ Edit metadata + +
+
+
+
+
+
+
+
+
+ + +
+
+
+
+
+ +
+
+ +
+ +
+
+
+ +
+
+ +
+ +
+
+
+ +
+
+ +
+ +
+
+
+ + +
+
+ + + + + + + + + diff --git a/containers/novela/templates/credentials.html b/containers/novela/templates/credentials.html new file mode 100644 index 0000000..9ef2564 --- /dev/null +++ b/containers/novela/templates/credentials.html @@ -0,0 +1,406 @@ + + + + + + Novela — Credentials + + + + + + + +{% include "_sidebar.html" %} + +
+ + +
+
Saved Credentials
+
    +
  • No credentials saved yet.
    • +
+
+ + +
+
Add Credentials
+ + +
Tip: use dropbox with the access token in Password.
+ + +
+
+ + +
+
+ +
+ + +
+
+
+ + + +
+ +
+ + + + diff --git a/containers/novela/templates/debug.html b/containers/novela/templates/debug.html new file mode 100644 index 0000000..97f276c --- /dev/null +++ b/containers/novela/templates/debug.html @@ -0,0 +1,327 @@ + + + + + + Novela — Debug + + + + + + + +{% include "_sidebar.html" %} + +
+ +
+
Inspect URL
+ + + +
+ +
+ +
+ + + + diff --git a/containers/novela/templates/editor.html b/containers/novela/templates/editor.html new file mode 100644 index 0000000..46d794c --- /dev/null +++ b/containers/novela/templates/editor.html @@ -0,0 +1,97 @@ + + + + + + Novela — Edit {{ title or filename }} + + + + + + + +
+ + + + + {{ (title or filename) | truncate(30, True) }} + +
—
+
+ + + + + + + +
+
+ + +
+ +
+
+ + + + + + + + + diff --git a/containers/novela/templates/grabber.html b/containers/novela/templates/grabber.html new file mode 100644 index 0000000..109612a --- /dev/null +++ b/containers/novela/templates/grabber.html @@ -0,0 +1,562 @@ + + + + + + Novela + + + + + + + +{% include "_sidebar.html" %} + +
+ + +
+
Book URL
+ + +
+ +
+ + +
+
Book info
+ +
+ +
+ + +
+ + cover preview +
+ Click to select a cover image +
+
+
+ + +
+ + +
+
Progress
+
Connecting...
+
+
+
+
    +
    +
    + + +
    +
    Done
    +
    +
    + + +
    +
    + +
    + + + + diff --git a/containers/novela/templates/home.html b/containers/novela/templates/home.html new file mode 100644 index 0000000..fff06a5 --- /dev/null +++ b/containers/novela/templates/home.html @@ -0,0 +1,422 @@ + + + + + + Novela — Home + + + + + + + +{% include "_sidebar.html" %} + +
    + + +
    + + + + + + + + + + + +
    + + + + +
    + + + + diff --git a/containers/novela/templates/index.html b/containers/novela/templates/index.html new file mode 100644 index 0000000..109612a --- /dev/null +++ b/containers/novela/templates/index.html @@ -0,0 +1,562 @@ + + + + + + Novela + + + + + + + +{% include "_sidebar.html" %} + +
    + + +
    +
    Book URL
    + + +
    + +
    + + +
    +
    Book info
    + +
    + +
    + + +
    + + cover preview +
    + Click to select a cover image +
    +
    +
    + + +
    + + +
    +
    Progress
    +
    Connecting...
    +
    +
    +
    +
      +
      +
      + + +
      +
      Done
      +
      +
      + + +
      +
      + +
      + + + + diff --git a/containers/novela/templates/library.html b/containers/novela/templates/library.html new file mode 100644 index 0000000..9995902 --- /dev/null +++ b/containers/novela/templates/library.html @@ -0,0 +1,78 @@ + + + + + + Novela — Library + + + + + + + +{% include "_sidebar.html" %} + + +
      +
      +
      + +
      All books
      +
      +
      + + + + + +
      +
      +
      + +
      Drop EPUB files here
      +
      or click to choose files
      +
      +
      +
      Loading…
      +
      +
      + + +
      +
      +
      Delete book
      +

      Delete ?
      This cannot be undone.

      +
      + + +
      +
      +
      + + +
      +
      +
      Add cover
      +

      +
      + + preview +
      Click to select a cover image
      +
      +
      + + +
      +
      +
      + + + + diff --git a/containers/novela/templates/reader.html b/containers/novela/templates/reader.html new file mode 100644 index 0000000..02e2ee5 --- /dev/null +++ b/containers/novela/templates/reader.html @@ -0,0 +1,455 @@ + + + + + + Novela — {{ title }} + + + + + + + +
      + + Loading… +
      + + +
      +
      +
      Reading settings
      +
      +
      + Content width + 65% +
      + +
      +
      + + +
      + + + + + + {{ title | truncate(30, True) }} + +
      +
      + +
      +
      + + +
      +
      +
      + + + +
      +
      + + +
      +
      + +
      + +
      + + + + diff --git a/containers/novela/templates/settings.html b/containers/novela/templates/settings.html new file mode 100644 index 0000000..f0933a1 --- /dev/null +++ b/containers/novela/templates/settings.html @@ -0,0 +1,441 @@ + + + + + + Novela — Settings + + + + + + + +{% include "_sidebar.html" %} + +
      +
      Settings
      + + +
      +
      Reading history
      +
      + Reset all recorded reading sessions. This will permanently delete the entire reading history + and reset all counters on library cards and the Statistics page.
      + This action cannot be undone. +
      + + +
      + +
      +
      Break detection
      +
      + Patronen die herkend worden als scèneovergang tijdens het converteren. + Wijzigingen zijn actief bij de eerstvolgende conversie. +
      + +
      +
      Regex patronen
      +
      +
      + + +
      +
      + + +
      +
      + +
      + +
      +
      CSS classes
      +
      +
      + + +
      +
      + +
      + +
      +
      Test
      +
      + + +
      + +
      + + +
      +
      + + +
      +
      +
      Reset reading history
      +

      + This will permanently delete all reading sessions from the database. + Statistics will be cleared and all read counts on library cards will reset to zero.

      + Are you sure you want to continue? +

      +
      + + +
      +
      +
      + + + + diff --git a/containers/novela/templates/stats.html b/containers/novela/templates/stats.html new file mode 100644 index 0000000..96dfed8 --- /dev/null +++ b/containers/novela/templates/stats.html @@ -0,0 +1,320 @@ + + + + + + Novela — Statistics + + + + + + + + +{% include "_sidebar.html" %} + +
      +
      Reading Statistics
      + +
      +
      +
      Total reads
      +
      —
      +
      +
      +
      Books read
      +
      —
      +
      unique titles
      +
      +
      +
      Favourite genre
      +
      —
      +
      +
      +
      +
      Publisher
      +
      —
      +
      +
      +
      + + +
      +
      +
      Reads per month — last 12 months
      +
      +
      +
      + + +
      +
      +
      Day of the week
      +
      +
      +
      +
      Hour of the day
      +
      +
      +
      + + +
      +
      +
      Genre distribution (library)
      +
      +
      +
      +
      Most read books
      +
      +
      +
      + + +
      +
      Reading history — last 50 sessions
      +
      +
      Loading…
      +
      +
      +
      + + + + diff --git a/containers/novela/xhtml.py b/containers/novela/xhtml.py new file mode 100644 index 0000000..0db7fde --- /dev/null +++ b/containers/novela/xhtml.py @@ -0,0 +1,169 @@ +import re +from html import escape as he + +from bs4 import NavigableString, Tag + +BREAK_PATTERNS = [ + re.compile(r"^\s*[\*\-]{3,}\s*$"), # *** of --- + re.compile(r"^\s*[·•◦‣⁃]\s*[·•◦‣⁃]\s*[·•◦‣⁃]\s*$"), # • • • + re.compile(r"^\s*~{2,}\s*$"), # ~~ + re.compile(r"^\s*={3,}\s*$"), # === + re.compile(r"^\s*#{3,}\s*$"), # ### + re.compile(r"^\s*[oO0]{1,3}\s*$"), # oOo + re.compile(r"^\s*[-–—]\s*[oO0]\s*[-–—]\s*$"), # -o- / —O— + re.compile(r"^\s*[<>]+\s*[·•*]\s*[<>]+\s*$"), # <<<<<·>>>>> +] + +BREAK_CSS_CLASSES = [ + "hr", "separator", "section-break", "divider", "break", + "chapterbreak", "scene-break", "scenebreak", +] +# Normalised set (hyphens removed, lowercase) for exact-match checking. +# Substring matching caused false positives: e.g. "ipsPageBreak" contains +# "break" but is a layout class, not a scene-break marker. +_BREAK_CSS_NORM = frozenset(b.replace("-", "") for b in BREAK_CSS_CLASSES) + +# --------------------------------------------------------------------------- +# Runtime-configurable overrides (populated from DB by main.py before scraping) +# --------------------------------------------------------------------------- + +_active_patterns: list | None = None # None → fall back to BREAK_PATTERNS +_active_css_norm: frozenset | None = None # None → fall back to _BREAK_CSS_NORM + + +def configure_break_patterns(regex_strings: list[str], css_classes: list[str]) -> None: + """Override the active break patterns with values loaded from the database. + + Called by main.py before each scrape so user-edited patterns take effect + without requiring a server restart. + """ + global _active_patterns, _active_css_norm + compiled = [] + for p in regex_strings: + try: + compiled.append(re.compile(p)) + except re.error: + pass + _active_patterns = compiled + _active_css_norm = frozenset(c.lower().replace("-", "") for c in css_classes) + + +def _get_patterns() -> list: + return _active_patterns if _active_patterns is not None else BREAK_PATTERNS + + +def _get_css_norm() -> frozenset: + return _active_css_norm if _active_css_norm is not None else _BREAK_CSS_NORM + + +def is_break_element(el, empty_p_is_spacer: bool = False) -> bool: + """Detect scene breaks based on tag, class, or text pattern.""" + patterns = _get_patterns() + css_norm = _get_css_norm() + if isinstance(el, Tag): + if el.name == "hr": + return True + classes = el.get("class", []) + for cls in classes: + if cls.lower().replace("-", "") in css_norm: + return True + # Empty paragraph (whitespace or   only) counts as a break, + # unless the content uses them as spacers between every paragraph. + if el.name == "p" and not empty_p_is_spacer: + child_tags = [c for c in el.children if isinstance(c, Tag)] + if not child_tags and not el.get_text().replace("\xa0", "").strip(): + return True + # Image that represents a break + if el.name == "img": + src = el.get("src", "").lower() + alt = el.get("alt", "").lower() + if any(b in src or b in alt for b in ["break", "divider", "separator", "hr"]): + return True + # Element containing only a single break image + children = [c for c in el.children if not (isinstance(c, NavigableString) and not c.strip())] + if len(children) == 1 and isinstance(children[0], Tag) and children[0].name == "img": + return is_break_element(children[0]) + # Text pattern + text = el.get_text() + for pat in patterns: + if pat.match(text): + return True + elif isinstance(el, NavigableString): + for pat in patterns: + if pat.match(str(el)): + return True + return False + + +def element_to_xhtml(el, break_img_path: str = "../Images/break.png", empty_p_is_spacer: bool = False) -> str: + """Convert a BeautifulSoup element to an XHTML fragment.""" + if is_break_element(el, empty_p_is_spacer): + result = f'
      ' + # HTML parsers (notably html.parser) can nest subsequent siblings inside + # void elements like
      , so a break element may contain actual content + # as children. Process those children so no text is silently discarded. + if isinstance(el, Tag): + trailer = "".join( + element_to_xhtml(c, break_img_path, empty_p_is_spacer) + for c in el.children + ) + if trailer.strip(): + result += "\n" + trailer + return result + + if isinstance(el, NavigableString): + text = str(el) + if text.strip(): + return he(text) + return "" + + if el.name in ("p", "div"): + inner = "".join(element_to_xhtml(c, break_img_path, empty_p_is_spacer) for c in el.children) + inner = inner.strip() + if not inner: + return "" + return f"

      {inner}

      \n" + + if el.name in ("em", "i"): + inner = "".join(element_to_xhtml(c, break_img_path, empty_p_is_spacer) for c in el.children) + return f"{inner}" + + if el.name in ("strong", "b"): + inner = "".join(element_to_xhtml(c, break_img_path, empty_p_is_spacer) for c in el.children) + return f"{inner}" + + if el.name in ("h1", "h2", "h3", "h4"): + inner = "".join(element_to_xhtml(c, break_img_path, empty_p_is_spacer) for c in el.children) + return f"<{el.name}>{inner}\n" + + if el.name == "br": + return "
      " + + if el.name in ("sup", "sub"): + inner = "".join(element_to_xhtml(c, break_img_path, empty_p_is_spacer) for c in el.children) + return inner + + if el.name == "a": + inner = "".join(element_to_xhtml(c, break_img_path, empty_p_is_spacer) for c in el.children) + return inner # strip links, keep text + + if el.name == "img": + src = el.get("src", "") + alt = he(el.get("alt", "")) + if src: + return f'{alt}\n' + return "" + + if el.name == "figure": + parts = [] + for c in el.children: + if isinstance(c, Tag) and c.name == "figcaption": + continue + parts.append(element_to_xhtml(c, break_img_path, empty_p_is_spacer)) + return "".join(parts) + + # Other tags: recurse + parts = [] + for c in el.children: + parts.append(element_to_xhtml(c, break_img_path, empty_p_is_spacer)) + return "".join(parts) diff --git a/docs/BLUEPRINT.md b/docs/BLUEPRINT.md new file mode 100644 index 0000000..5917940 --- /dev/null +++ b/docs/BLUEPRINT.md @@ -0,0 +1,420 @@ +# Novela 2.0 - Blauwdruk + +> Vervangt repository `story-grabber`. Nieuwe repo: **Novela**. +> Stack: FastAPI · Jinja2 · plain JS · PostgreSQL 16 · Docker / Portainer + +--- + +## 1. Doelstelling + +Novela 2.0 is een volledig zelfgehoste media-bibliotheek en e-reader voor epub, pdf en cbr/cbz. +Het vervangt Kavita (library), Calibre (metadata), en Sigil (epub editor) in een web-applicatie. + +Kernprincipe: **de database is de snelle index, het bestand is de bron van waarheid.** +Elke schrijfactie raakt altijd beide: eerst het bestand, dan de database. Lezen gaat altijd via de database. + +--- + +## 2. Wat behouden blijft uit v1 + +| Module | Bestand | Toelichting | +|---|---|---| +| EPUB bouw | `epub.py` | `make_epub`, `make_chapter_xhtml`, `add_cover_to_epub` | +| EPUB lezen/schrijven | `epub.py` | `read_epub_file`, `write_epub_file` | +| XHTML conversie | `xhtml.py` | `element_to_xhtml`, `is_break_element`, `configure_break_patterns` | +| Scrapers | `scrapers/` | base, awesomedude, gayauthors, plugin-patroon blijft | +| SSE job streaming | `main.py` | `JOBS` dict + `/events/{job_id}` `StreamingResponse` | +| Migrations patroon | `migrations.py` | idempotente `CREATE IF NOT EXISTS`, `run_migrations()` bij startup | +| Cover cache | DB tabel | `library_cover_cache`, WebP thumbnails 300x450 | +| Reading progress | DB tabel | CFI voor epub, paginanummer voor pdf/cbr | +| Reading sessions | DB tabel | leesgeschiedenis per boek | +| Break patterns | DB tabel | regex + css_class patronen voor scene-breaks | + +--- + +## 3. Projectstructuur + +```text +novela/ +├── containers/ +│ └── novela/ +│ ├── main.py +│ ├── migrations.py +│ ├── db.py +│ ├── epub.py +│ ├── xhtml.py +│ ├── pdf.py +│ ├── cbr.py +│ ├── routers/ +│ │ ├── __init__.py +│ │ ├── library.py +│ │ ├── reader.py +│ │ ├── editor.py +│ │ ├── grabber.py +│ │ ├── backup.py +│ │ └── settings.py +│ ├── scrapers/ +│ ├── static/ +│ ├── templates/ +│ ├── requirements.txt +│ └── Dockerfile +├── stack/ +│ ├── stack.yml +│ └── novela.env +└── docs/ + ├── BLUEPRINT.md + └── TECHNICAL.md +``` + +--- + +## 4. Bibliotheek op schijf + +`output/` wordt `library/`. + +```text +library/ +├── epub/ +│ └── {Publisher}/ +│ └── {Author}/ +│ ├── Stories/ +│ │ └── {Titel}.epub +│ └── Series/ +│ └── {Serienaam}/ +│ └── {001 - Titel}.epub +├── pdf/ +│ └── {Author}/ +│ └── {Titel}.pdf +├── comics/ +│ └── {Author of Serienaam}/ +│ └── {001 - Titel}.cbr +└── covers/ +``` + +Naamgeving-regels: +- Ongeldige tekens weg: `< > : " / \\ | ? *` en control chars +- Max 80 tekens per map-segment, 140 voor bestandsnaam +- Bij conflict: `Titel (2).epub`, `Titel (3).epub`, enz. + +Hernoemen na metadata-bewerking: +- Bestand verplaatsen op schijf +- DB-verwijzingen updaten: `library`, `book_tags`, `reading_progress`, `reading_sessions`, `library_cover_cache` +- Lege mappen opruimen + +--- + +## 5. Database schema + +### 5.1 `library` + +```sql +CREATE TABLE library ( + id SERIAL PRIMARY KEY, + filename VARCHAR(600) UNIQUE NOT NULL, + media_type VARCHAR(10) NOT NULL DEFAULT 'epub', + title VARCHAR(500), + author VARCHAR(255), + publisher VARCHAR(255), + series VARCHAR(500), + series_index INTEGER DEFAULT 0, + publication_status VARCHAR(100), + has_cover BOOLEAN DEFAULT FALSE, + description TEXT DEFAULT '', + source_url VARCHAR(1000), + publish_date DATE, + archived BOOLEAN DEFAULT FALSE, + want_to_read BOOLEAN DEFAULT FALSE, + needs_review BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); +``` + +### 5.2 `book_tags` + +```sql +CREATE TABLE book_tags ( + id SERIAL PRIMARY KEY, + filename VARCHAR(600) NOT NULL REFERENCES library(filename) ON DELETE CASCADE, + tag VARCHAR(255) NOT NULL, + tag_type VARCHAR(20) NOT NULL, + UNIQUE (filename, tag, tag_type) +); +CREATE INDEX idx_book_tags_filename ON book_tags (filename); +``` + +`tag_type`: +- `genre` +- `subgenre` +- `tag` +- `subject` + +### 5.3 `reading_progress` + +```sql +CREATE TABLE reading_progress ( + id SERIAL PRIMARY KEY, + filename VARCHAR(600) UNIQUE NOT NULL REFERENCES library(filename) ON DELETE CASCADE, + cfi TEXT, + page INTEGER, + progress INTEGER DEFAULT 0, + updated_at TIMESTAMP DEFAULT NOW() +); +``` + +### 5.4 `reading_sessions` + +```sql +CREATE TABLE reading_sessions ( + id SERIAL PRIMARY KEY, + filename VARCHAR(600) NOT NULL REFERENCES library(filename) ON DELETE CASCADE, + read_at TIMESTAMP DEFAULT NOW() +); +CREATE INDEX idx_reading_sessions_filename ON reading_sessions (filename); +``` + +### 5.5 `library_cover_cache` + +```sql +CREATE TABLE library_cover_cache ( + filename VARCHAR(600) PRIMARY KEY REFERENCES library(filename) ON DELETE CASCADE, + mime_type VARCHAR(100) NOT NULL, + thumb_webp BYTEA NOT NULL, + updated_at TIMESTAMP DEFAULT NOW() +); +``` + +### 5.6 `credentials` + +```sql +CREATE TABLE credentials ( + id SERIAL PRIMARY KEY, + site VARCHAR(255) UNIQUE NOT NULL, + username VARCHAR(255) NOT NULL, + password VARCHAR(255) NOT NULL, + updated_at TIMESTAMP DEFAULT NOW() +); +``` + +### 5.7 `break_patterns` + +```sql +CREATE TABLE break_patterns ( + id SERIAL PRIMARY KEY, + pattern_type VARCHAR(20) NOT NULL, + pattern TEXT NOT NULL, + enabled BOOLEAN DEFAULT TRUE, + is_default BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP DEFAULT NOW(), + UNIQUE (pattern_type, pattern) +); +``` + +### 5.8 `backup_log` + +```sql +CREATE TABLE backup_log ( + id SERIAL PRIMARY KEY, + status VARCHAR(20) NOT NULL, + files_count INTEGER, + size_bytes BIGINT, + error_msg TEXT, + started_at TIMESTAMP DEFAULT NOW(), + finished_at TIMESTAMP +); +``` + +--- + +## 6. Schrijfprincipe: bestand en database synchroon + +Volgorde per bewerking: +1. Bewerk bestand op schijf +2. Update database +3. Retourneer succes + +Nooit alleen DB updaten zonder bestand. + +--- + +## 7. Coverstrategie + +Opslaan: +- EPUB cover in bestand (`OEBPS/Images/cover.{ext}`) +- Thumbnail als `300x450` WebP in `library_cover_cache` + +Ontbrekende cover: +- Als geen cover: voeg tag `Cover Missing` toe +- UI upload schrijft cover in EPUB en cache + +Opvragen: +- Primair: `/library/cover-cached/{filename}` +- Fallback: `/library/cover/{filename}` + +PDF en CBR: +- PDF: eerste pagina als thumbnail +- CBR/CBZ: eerste afbeelding als thumbnail + +--- + +## 8. Verwijder-flow + +`DELETE /library/file/{filename}`: +1. Verwijder bestand +2. Prune lege mappen +3. Delete uit `library` (cascade verwijdert gerelateerde tabellen) + +--- + +## 9. Router-overzicht + +### 9.1 `routers/library.py` +- `GET /library` +- `GET /api/library` +- `POST /library/rescan` +- `POST /library/import` +- `DELETE /library/file/{filename}` +- `GET /library/cover/{filename}` +- `GET /library/cover-cached/{filename}` +- `POST /library/cover/{filename}` +- `POST /library/want-to-read/{filename}` +- `POST /library/archive/{filename}` +- `GET /home` +- `GET /api/home` +- `GET /stats` +- `GET /api/stats` + +### 9.2 `routers/reader.py` +- `GET /library/read/{filename}` +- `GET /library/book/{filename}` +- `PATCH /library/book/{filename}` +- `GET /library/epub/{filename}` +- `GET /library/chapters/{filename}` +- `GET /library/chapter/{index}/{filename}` +- `GET /library/chapter-img/{path}` +- `GET /library/pdf/{filename}` +- `GET /library/cbr/{filename}/{page}` +- `GET /library/progress/{filename}` +- `POST /library/progress/{filename}` +- `DELETE /library/progress/{filename}` +- `POST /library/mark-read/{filename}` +- `GET /api/genres` + +### 9.3 `routers/editor.py` +- `GET /library/editor/{filename}` +- `GET /api/edit/chapter/{index}/{filename}` +- `POST /api/edit/chapter/{index}/{filename}` +- `POST /api/edit/chapter/add/{filename}` +- `DELETE /api/edit/chapter/{index}/{filename}` + +### 9.4 `routers/grabber.py` +- `GET /grabber` +- `POST /preload` +- `POST /convert` +- `GET /events/{job_id}` +- `GET /debug` +- `POST /debug/run` +- `GET /credentials` +- `POST /credentials` +- `DELETE /credentials/{site}` + +### 9.5 `routers/backup.py` +- `GET /backup` +- `GET /api/backup/status` +- `POST /api/backup/run` +- `GET /api/backup/history` + +### 9.6 `routers/settings.py` +- `GET /settings` +- `GET /api/break-patterns` +- `POST /api/break-patterns` +- `PATCH /api/break-patterns/{id}` +- `DELETE /api/break-patterns/{id}` +- `DELETE /api/reading-history` + +--- + +## 10. Nieuwe modules + +### 10.1 `db.py` +Gedeelde psycopg2 connection pool (`init_pool`, `get_conn`, `release_conn`). + +### 10.2 `pdf.py` +PyMuPDF rendering (`pdf_render_page`), page count en cover thumb. + +### 10.3 `cbr.py` +RAR/ZIP paginalijst, page extract en cover thumb. + +--- + +## 11. Cover-flow per mediatype + +| Actie | EPUB | PDF | CBR/CBZ | +|---|---|---|---| +| Cover import | Uit OPF/Images | Eerste pagina render | Eerste image uit archief | +| Thumbnail | Pillow -> WebP | PyMuPDF + Pillow -> WebP | Pillow -> WebP | +| Opslag | EPUB + cache | cache | cache | +| Cover vervangen | Ja | Nee | Nee | +| Geen cover | `Cover Missing` tag | `Cover Missing` tag | `Cover Missing` tag | + +--- + +## 12. Database-opzet + +- Start met schone v2 database +- Geen migratiepad vanuit v1 data +- `run_migrations()` op startup +- `CREATE TABLE IF NOT EXISTS` overal idempotent + +--- + +## 13. Docker stack + +Zie [`stack/stack.yml`](../stack/stack.yml). + +Belangrijk: +- App container expose `8099 -> 8000` +- PostgreSQL 16 +- Adminer op `8098` +- `NOVELA_MASTER_KEY` in `stack/novela.env` en doorgifte in `stack/stack.yml` voor encrypted credentials + +--- + +## 14. Requirements + +Zie [`containers/novela/requirements.txt`](../containers/novela/requirements.txt). + +--- + +## 15. Bestanden klaarzetten + +Bron: `/docker/develop/story-grabber/containers/story-grabber`. +Doel: `/docker/develop/novela/containers/novela`. + +Overnemen: +- `epub.py` +- `xhtml.py` +- `scrapers/*` +- `static/*` +- `templates/*` + +Nieuw schrijven: +- `main.py`, `db.py`, `pdf.py`, `cbr.py`, `migrations.py` +- `routers/*` + +--- + +## 16. Bouw-volgorde + +1. `db.py` +2. `migrations.py` +3. `main.py` +4. `routers/library.py` +5. `routers/reader.py` +6. `routers/editor.py` +7. `routers/grabber.py` +8. `routers/settings.py` +9. `pdf.py` + reader uitbreiding +10. `cbr.py` + reader uitbreiding +11. `routers/backup.py` +12. `routers/library.py` uitbreiden voor pdf/cbr import diff --git a/docs/TECHNICAL.md b/docs/TECHNICAL.md new file mode 100644 index 0000000..6e09903 --- /dev/null +++ b/docs/TECHNICAL.md @@ -0,0 +1,100 @@ +# Novela 2.0 - Technical Plan + +## Scope +Dit document beschrijft de technische uitvoering van de blauwdruk in implementeerbare stappen. + +## Architectural Rules +- Bestand is source of truth. +- Database is snelle index. +- Schrijfacties: eerst bestand, dan DB. +- Lezen: primair uit DB, met scan/rescan voor recovery. + +## Data Integrity Rules +- Alle child-tabellen refereren `library(filename)` met `ON DELETE CASCADE`. +- Verwijderen van een boek is een enkel `DELETE FROM library` na file-delete. +- Rename-flow moet `filename` synchroon aanpassen in: + - `library` + - `book_tags` + - `reading_progress` + - `reading_sessions` + - `library_cover_cache` + +## Runtime Lifecycle +- Startup: + 1. `init_pool()` + 2. `run_migrations()` + 3. routers mounten +- Shutdown: + 1. `close_pool()` + +## Module Responsibilities +- `db.py`: pool ownership + connection helpers. +- `migrations.py`: schema + seeds. +- `routers/library.py`: import/scan/delete/cover/home/stats. +- `routers/reader.py`: lezen + progress + metadata patch + epub editor endpoints. +- `routers/editor.py`: uiteindelijke dedicated editor routes (kan initieel delegaten). +- `routers/grabber.py`: scraper orchestration + credentials + SSE. +- `routers/backup.py`: Dropbox sync + pg dump + logging. +- `routers/settings.py`: break patterns + cleaning endpoints. + +## Endpoint Contract Notes +- Alle file routes gebruiken veilige path-resolutie tegen traversal. +- Cover endpoint gedrag: + - cached eerst + - fallback raw extract + - anders 404 +- Progress payload: + - EPUB: `{ cfi, progress }` + - PDF/CBR: `{ page, progress }` + +## Backup Plan +- `POST /api/backup/run`: + - insert `running` in `backup_log` + - sync files naar Dropbox (incremental op mtime+size) + - draai `pg_dump` en upload `.sql` + - update `backup_log` naar `success`/`error` +- OAuth token opslag via `credentials` (`site='dropbox'`) en encrypted-at-rest (Fernet) in de database. +- Beheer via webinterface op `/credentials-manager` (site: `dropbox`, token in password veld). +- Legacy plaintext credentials worden automatisch gemigreerd naar encrypted bij uitlezen. + +## Migration Plan from Current State +1. Behoud v1 stabiele modules (`epub.py`, `xhtml.py`, scrapers, templates/static). +2. Introduceer nieuwe routers zonder bestaande frontend te breken (compat routes waar nodig). +3. Schakel library root om naar `library/`. +4. Activeer PDF/CBR scan en reader paden. +5. Split editor-routes uit reader naar dedicated `editor.py`. +6. Volledige scrape->epub flow migreren naar `grabber.py`. +7. Backup volledig afronden (Dropbox + pg_dump). + +## Test Matrix +- Import: + - EPUB met/zonder cover + - PDF 1+ pagina + - CBR/CBZ met images +- Reader: + - EPUB CFI save/load + - PDF page render + page progress + - CBR page render + page progress +- Metadata edit: + - rename path + - db references geupdate + - old row cleanup +- Delete: + - file weg + - lege dirs gepruned + - cascade records weg +- Break patterns: + - create/update/delete/enable +- Grabber: + - preload/debug + - convert job events +- Backup: + - status/history + - success/error logging + +## Deployment Notes +- Docker image bouwt vanuit `containers/novela`. +- Stack uit `stack/stack.yml` met env uit `stack/novela.env`. +- `NOVELA_MASTER_KEY` is verplicht voor encrypt/decrypt van credentials in de database en moet stabiel blijven na initiele ingebruikname. +- Postgres volume persistent. +- Library mount persistent. diff --git a/stack/novela.env b/stack/novela.env new file mode 100644 index 0000000..e7b6ffb --- /dev/null +++ b/stack/novela.env @@ -0,0 +1,7 @@ +POSTGRES_DB=novela +POSTGRES_USER=novela +POSTGRES_PASSWORD=change-me + +# Required for credential encryption/decryption (Fernet) in DB. +# Keep this stable after first use; changing it breaks decrypt of existing credentials. +NOVELA_MASTER_KEY=change-me-long-random-secret diff --git a/stack/stack.yml b/stack/stack.yml new file mode 100644 index 0000000..42d6e8e --- /dev/null +++ b/stack/stack.yml @@ -0,0 +1,48 @@ +version: "3.8" +services: + novela: + image: gitea.oskamp.info/ivooskamp/novela:dev + container_name: novela + restart: unless-stopped + ports: + - "8099:8000" + environment: + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + NOVELA_MASTER_KEY: ${NOVELA_MASTER_KEY} + volumes: + - /docker/appdata/novela/library:/app/library + - /docker/appdata/novela/config:/app/config + depends_on: + - postgres + networks: + - novela-net + + postgres: + image: postgres:16 + container_name: novela-db + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - /docker/appdata/novela/postgres:/var/lib/postgresql/data + networks: + - novela-net + + adminer: + image: adminer:latest + container_name: novela-adminer + restart: unless-stopped + ports: + - "8098:8080" + networks: + - novela-net + +networks: + novela-net: + driver: bridge