restructure project: pivot from multi-approach notebooks to focused COMICS panel search build

The original scaffold explored three separate retrieval approaches across six
numbered notebooks (text+dense, image embeddings, hybrid) with a flat utils/
package. This commit replaces that structure with a production-oriented layout
targeting the COMICS dataset specifically.

Key changes:
- Removed numbered exploration notebooks (00–06) and flat utils/ package
- Added src/ package with modules for ingest, embeddings, pinecone, and search
- ingest/: manifest building and OCR text cleaning
- embeddings/: dense image embeddings (embed_images) and sparse text (embed_sparse_text)
- pinecone/: index creation, document assembly, and panel upserting
- search/: dense, sparse, FTS, and combined search with client-side fusion
- Added notebooks/query_comics_index.ipynb as the single interactive entry point
- Added config.yaml for shared runtime configuration
- Added updated-build.md capturing the revised build spec and data source decisions
(COMICS dataset via umiacs.umd.edu, Pinecone document-schema index with
one dense + one sparse vector field + FTS-enabled string fields)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

00_data_acquisition.ipynb

@@ -1,143 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 00 · Data Acquisition\n",
-    "\n",
-    "Download public-domain golden age comics from the [Digital Comic Museum](https://digitalcomicmuseum.com/index.php?cid=110), extract their pages, and build a manifest CSV.\n",
-    "\n",
-    "**Comics used (all public domain):**\n",
-    "Add direct CBZ download URLs to `COMIC_URLS` below. You can find download links by browsing the category page and clicking through to each comic's download page."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pathlib import Path\n",
-    "import pandas as pd\n",
-    "import sys\n",
-    "sys.path.insert(0, str(Path().resolve()))\n",
-    "\n",
-    "from utils.downloader import download_comic\n",
-    "from utils.extractor import extract_comic\n",
-    "\n",
-    "RAW_DIR   = Path(\"data/raw\")\n",
-    "PAGES_DIR = Path(\"data/pages\")\n",
-    "MAX_PAGES = 30  # set lower to reduce cost during testing"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Define comics to download\n",
-    "\n",
-    "Populate `COMIC_URLS` with `(filename, url)` tuples.  \n",
-    "Direct download links look like: `https://digitalcomicmuseum.com/download.php?...`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Add (filename, url) pairs — get URLs from the DCM download pages\n",
-    "COMIC_URLS = [\n",
-    "    # (\"comic_filename.cbz\", \"https://...\"),\n",
-    "]\n",
-    "\n",
-    "if not COMIC_URLS:\n",
-    "    print(\"⚠️  No comic URLs defined. Add entries to COMIC_URLS above.\")\n",
-    "    print(\"   Browse https://digitalcomicmuseum.com/index.php?cid=110 to find download links.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Download archives\n",
-    "archive_paths = []\n",
-    "for filename, url in COMIC_URLS:\n",
-    "    path = download_comic(url, RAW_DIR, filename)\n",
-    "    archive_paths.append(path)\n",
-    "print(f\"Downloaded {len(archive_paths)} archives.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Extract pages"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "all_pages = []\n",
-    "for archive in archive_paths:\n",
-    "    pages = extract_comic(archive, PAGES_DIR)\n",
-    "    # Honour MAX_PAGES per comic to control cost\n",
-    "    pages = pages[:MAX_PAGES]\n",
-    "    all_pages.extend(pages)\n",
-    "    print(f\"  {archive.name}: {len(pages)} pages\")\n",
-    "\n",
-    "print(f\"\\nTotal pages: {len(all_pages)}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Build and save manifest"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "manifest = pd.DataFrame(all_pages)\n",
-    "manifest[\"page_id\"] = manifest[\"comic_slug\"] + \"_p\" + manifest[\"page_num\"].astype(str).str.zfill(3)\n",
-    "manifest.to_csv(\"data/manifest.csv\", index=False)\n",
-    "print(f\"Saved manifest with {len(manifest)} rows.\")\n",
-    "manifest.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Quick sanity check — display first page of each comic\n",
-    "from IPython.display import display, Image as IPyImage\n",
-    "from PIL import Image\n",
-    "\n",
-    "for slug, grp in manifest.groupby(\"comic_slug\"):\n",
-    "    first = grp.iloc[0]\n",
-    "    print(f\"\\n{first['comic_title']} — {len(grp)} pages\")\n",
-    "    img = Image.open(first[\"file_path\"])\n",
-    "    img.thumbnail((300, 400))\n",
-    "    display(img)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
-  "language_info": {"name": "python", "version": "3.11.0"}
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

01_transcription.ipynb

@@ -1,200 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 01 · Transcription Pipeline\n",
-    "\n",
-    "Run [comic-transcriber](https://github.com/nearestnabors/comic-transcriber) on each page to extract structured panel descriptions and dialogue, then parse and save as JSON for downstream embedding pipelines.\n",
-    "\n",
-    "**Output per page:**\n",
-    "```json\n",
-    "{\n",
-    "  \"page_id\": \"action_comics_001_p003\",\n",
-    "  \"comic_title\": \"Action Comics 001\",\n",
-    "  \"page_num\": 3,\n",
-    "  \"file_path\": \"data/pages/...\",\n",
-    "  \"all_dialogue\": \"Look out! The villains are here...\",\n",
-    "  \"scene_descriptions\": \"A hero leaps across rooftops...\",\n",
-    "  \"raw_transcript\": \"## PAGE 3, PANEL 1...\"\n",
-    "}\n",
-    "```"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import re\n",
-    "import subprocess\n",
-    "from pathlib import Path\n",
-    "\n",
-    "import pandas as pd\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "TRANSCRIPTS_DIR = Path(\"data/transcripts\")\n",
-    "TRANSCRIPTS_DIR.mkdir(parents=True, exist_ok=True)\n",
-    "\n",
-    "manifest = pd.read_csv(\"data/manifest.csv\")\n",
-    "print(f\"Pages to transcribe: {len(manifest)}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Install comic-transcriber\n",
-    "\n",
-    "comic-transcriber runs via `npx`. Node.js must be installed."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Verify npx is available\n",
-    "result = subprocess.run([\"npx\", \"--version\"], capture_output=True, text=True)\n",
-    "if result.returncode != 0:\n",
-    "    raise RuntimeError(\"npx not found — install Node.js from https://nodejs.org\")\n",
-    "print(f\"npx {result.stdout.strip()} ✓\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Parsing helpers"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def parse_transcript(raw: str) -> tuple[str, str]:\n",
-    "    \"\"\"Extract dialogue and scene descriptions from comic-transcriber markdown output.\"\"\"\n",
-    "    dialogue_lines = []\n",
-    "    description_lines = []\n",
-    "\n",
-    "    for line in raw.splitlines():\n",
-    "        line = line.strip()\n",
-    "        if not line or line.startswith(\"#\"):\n",
-    "            continue\n",
-    "        # Italic lines (wrapped in * or _) are scene descriptions\n",
-    "        if re.match(r'^[*_].+[*_]$', line):\n",
-    "            description_lines.append(re.sub(r'^[*_]|[*_]$', '', line))\n",
-    "        # Bold name followed by colon is dialogue: **CHARACTER**: text\n",
-    "        elif re.match(r'^\\*\\*.+\\*\\*:', line):\n",
-    "            dialogue_lines.append(re.sub(r'^\\*\\*.+\\*\\*:\\s*', '', line))\n",
-    "        # Plain lines without markup — treat as description\n",
-    "        elif not line.startswith(\"[\"):\n",
-    "            description_lines.append(line)\n",
-    "\n",
-    "    return \" \".join(dialogue_lines), \" \".join(description_lines)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Run transcription"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def transcribe_page(image_path: str) -> str:\n",
-    "    \"\"\"Run comic-transcriber on a single page image and return raw markdown.\"\"\"\n",
-    "    result = subprocess.run(\n",
-    "        [\"npx\", \"comic-transcriber\", image_path],\n",
-    "        capture_output=True, text=True, timeout=120\n",
-    "    )\n",
-    "    if result.returncode != 0:\n",
-    "        raise RuntimeError(f\"comic-transcriber failed: {result.stderr[:500]}\")\n",
-    "    return result.stdout\n",
-    "\n",
-    "\n",
-    "records = []\n",
-    "errors = []\n",
-    "\n",
-    "for _, row in tqdm(manifest.iterrows(), total=len(manifest), desc=\"Transcribing\"):\n",
-    "    out_file = TRANSCRIPTS_DIR / f\"{row['page_id']}.json\"\n",
-    "    if out_file.exists():\n",
-    "        records.append(json.loads(out_file.read_text()))\n",
-    "        continue\n",
-    "    try:\n",
-    "        raw = transcribe_page(row[\"file_path\"])\n",
-    "        dialogue, descriptions = parse_transcript(raw)\n",
-    "        record = {\n",
-    "            \"page_id\":           row[\"page_id\"],\n",
-    "            \"comic_title\":       row[\"comic_title\"],\n",
-    "            \"comic_slug\":        row[\"comic_slug\"],\n",
-    "            \"page_num\":          int(row[\"page_num\"]),\n",
-    "            \"file_path\":         row[\"file_path\"],\n",
-    "            \"all_dialogue\":      dialogue,\n",
-    "            \"scene_descriptions\": descriptions,\n",
-    "            \"raw_transcript\":    raw,\n",
-    "        }\n",
-    "        out_file.write_text(json.dumps(record, ensure_ascii=False, indent=2))\n",
-    "        records.append(record)\n",
-    "    except Exception as e:\n",
-    "        errors.append({\"page_id\": row[\"page_id\"], \"error\": str(e)})\n",
-    "        print(f\"  ✗ {row['page_id']}: {e}\")\n",
-    "\n",
-    "print(f\"\\n✓ {len(records)} transcribed  ✗ {len(errors)} errors\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Spot-check"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if records:\n",
-    "    sample = records[0]\n",
-    "    print(f\"page_id:          {sample['page_id']}\")\n",
-    "    print(f\"all_dialogue:     {sample['all_dialogue'][:300]}\")\n",
-    "    print(f\"scene_descriptions: {sample['scene_descriptions'][:300]}\")\n",
-    "    print(f\"\\nraw_transcript preview:\\n{sample['raw_transcript'][:500]}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Summary stats\n",
-    "df = pd.DataFrame(records)\n",
-    "print(df[[\"comic_title\", \"page_num\", \"all_dialogue\", \"scene_descriptions\"]]\n",
-    "      .assign(dialogue_len=df[\"all_dialogue\"].str.len(),\n",
-    "              desc_len=df[\"scene_descriptions\"].str.len())\n",
-    "      .groupby(\"comic_title\")[[\"dialogue_len\", \"desc_len\"]]\n",
-    "      .describe())"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
-  "language_info": {"name": "python", "version": "3.11.0"}
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

02_approach1_text_dense_fts.ipynb

@@ -1,289 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 02 · Approach 1: LLM Caption → Dense + FTS\n",
-    "\n",
-    "**Strategy:** Use Claude to generate rich scene descriptions from the comic-transcriber output, then embed those descriptions with Pinecone's integrated `llama-text-embed-v2`. Dialogue text is stored as a full-text search (FTS) field for exact keyword retrieval.\n",
-    "\n",
-    "**Namespace:** `ns-text-caption`\n",
-    "\n",
-    "**Blog post insight:** LLM captions add semantic richness — a query for *\"a moment of betrayal\"* can surface relevant pages even when no character says that word. FTS on dialogue nails exact quotes and sound effects.\n",
-    "\n",
-    "---\n",
-    "\n",
-    "## When to use this approach\n",
-    "- You want semantic search over *meaning* (themes, mood, setting)\n",
-    "- Your source content is primarily visual with little usable text metadata\n",
-    "- You want keyword search on transcribed dialogue/captions as a complementary signal"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import os\n",
-    "import time\n",
-    "from pathlib import Path\n",
-    "\n",
-    "import anthropic\n",
-    "from dotenv import load_dotenv\n",
-    "from pinecone import Pinecone\n",
-    "from pinecone.preview import SchemaBuilder\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "load_dotenv()\n",
-    "\n",
-    "TRANSCRIPTS_DIR = Path(\"data/transcripts\")\n",
-    "INDEX_NAME      = \"comics-multimodal\"\n",
-    "NAMESPACE       = \"ns-text-caption\"\n",
-    "BATCH_SIZE      = 50\n",
-    "\n",
-    "pc     = Pinecone(api_key=os.environ[\"PINECONE_API_KEY\"])\n",
-    "claude = anthropic.Anthropic()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Create (or reuse) the hybrid index\n",
-    "\n",
-    "This index is shared across all three approaches. It has dense + sparse + FTS fields in a single index via `SchemaBuilder`. Run this cell once — it handles the 409 case if the index already exists."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "schema = (\n",
-    "    SchemaBuilder()\n",
-    "    .add_string_field(\n",
-    "        name=\"dialogue\",\n",
-    "        full_text_search={\"language\": \"en\", \"stemming\": True, \"stop_words\": True},\n",
-    "    )\n",
-    "    .add_string_field(name=\"comic_title\", filterable=True)\n",
-    "    .add_string_field(name=\"page_id\", filterable=True)\n",
-    "    .add_dense_vector_field(name=\"embedding\", dimension=1024, metric=\"dotproduct\")\n",
-    "    .add_sparse_vector_field(name=\"sparse_embedding\")\n",
-    "    .build()\n",
-    ")\n",
-    "\n",
-    "try:\n",
-    "    index_model = pc.preview.indexes.create(\n",
-    "        name=INDEX_NAME,\n",
-    "        schema=schema,\n",
-    "        read_capacity={\"mode\": \"OnDemand\"},\n",
-    "    )\n",
-    "    print(f\"Created index: {INDEX_NAME}\")\n",
-    "except Exception as e:\n",
-    "    if \"409\" in str(e) or \"already exists\" in str(e).lower():\n",
-    "        print(f\"Index '{INDEX_NAME}' already exists — reusing.\")\n",
-    "    else:\n",
-    "        raise\n",
-    "\n",
-    "# Wait for ready\n",
-    "while True:\n",
-    "    info = pc.preview.indexes.describe(name=INDEX_NAME)\n",
-    "    if info.status.ready:\n",
-    "        INDEX_HOST = info.host\n",
-    "        break\n",
-    "    print(\"  waiting for index to be ready...\")\n",
-    "    time.sleep(3)\n",
-    "\n",
-    "print(f\"Index ready. Host: {INDEX_HOST}\")\n",
-    "index = pc.index(host=INDEX_HOST)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Generate LLM scene captions\n",
-    "\n",
-    "We use `claude-haiku-4-5-20251001` (fast and cheap) to generate a rich scene description from the comic-transcriber transcript. The prompt is included verbatim so you can reproduce or tune it."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "CAPTION_PROMPT = \"\"\"\\\n",
-    "You are describing a comic book page for a semantic search index.\n",
-    "Based on the transcript below, write a dense 2–4 sentence scene description covering:\n",
-    "- Setting and time of day\n",
-    "- Characters present and their emotional state\n",
-    "- Key action or plot moment happening\n",
-    "- Visual mood or atmosphere\n",
-    "\n",
-    "Be specific. Do NOT include meta-commentary about the comic itself.\n",
-    "Write only the description, no preamble.\n",
-    "\n",
-    "TRANSCRIPT:\n",
-    "{transcript}\"\"\"\n",
-    "\n",
-    "\n",
-    "def generate_caption(record: dict) -> str:\n",
-    "    transcript = record[\"raw_transcript\"][:3000]  # stay well within context\n",
-    "    msg = claude.messages.create(\n",
-    "        model=\"claude-haiku-4-5-20251001\",\n",
-    "        max_tokens=256,\n",
-    "        messages=[{\"role\": \"user\", \"content\": CAPTION_PROMPT.format(transcript=transcript)}],\n",
-    "    )\n",
-    "    return msg.content[0].text.strip()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "records = [json.loads(p.read_text()) for p in sorted(TRANSCRIPTS_DIR.glob(\"*.json\"))]\n",
-    "print(f\"Loaded {len(records)} transcripts\")\n",
-    "\n",
-    "# Generate captions (cached in each record's JSON to avoid re-billing)\n",
-    "for record in tqdm(records, desc=\"Generating captions\"):\n",
-    "    if record.get(\"llm_caption\"):\n",
-    "        continue\n",
-    "    record[\"llm_caption\"] = generate_caption(record)\n",
-    "    path = TRANSCRIPTS_DIR / f\"{record['page_id']}.json\"\n",
-    "    path.write_text(json.dumps(record, ensure_ascii=False, indent=2))\n",
-    "\n",
-    "print(\"Captions ready.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Upsert to Pinecone (ns-text-caption)\n",
-    "\n",
-    "We pass the LLM caption as the `embedding` text field — Pinecone's integrated `llama-text-embed-v2` inference embeds it automatically. Dialogue goes into the FTS `dialogue` field."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "documents = [\n",
-    "    {\n",
-    "        \"_id\":        r[\"page_id\"],\n",
-    "        \"embedding\":  r[\"llm_caption\"],   # integrated inference embeds this\n",
-    "        \"dialogue\":   r[\"all_dialogue\"],\n",
-    "        \"comic_title\": r[\"comic_title\"],\n",
-    "        \"page_id\":    r[\"page_id\"],\n",
-    "        \"page_num\":   r[\"page_num\"],\n",
-    "        \"file_path\":  r[\"file_path\"],\n",
-    "        \"llm_caption\": r[\"llm_caption\"],\n",
-    "    }\n",
-    "    for r in records\n",
-    "]\n",
-    "\n",
-    "index.documents.batch_upsert(\n",
-    "    namespace=NAMESPACE,\n",
-    "    documents=documents,\n",
-    "    batch_size=BATCH_SIZE,\n",
-    "    show_progress=True,\n",
-    ")\n",
-    "print(f\"Upserted {len(documents)} pages to {NAMESPACE}.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search demos"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from IPython.display import display\n",
-    "from PIL import Image\n",
-    "\n",
-    "\n",
-    "def show_results(matches, label=\"\"):\n",
-    "    if label:\n",
-    "        print(f\"\\n{'='*60}\\n{label}\\n{'='*60}\")\n",
-    "    for m in matches:\n",
-    "        caption = getattr(m, \"llm_caption\", \"\") or \"\"\n",
-    "        print(f\"[{m._score:.3f}] {m._id} — {caption[:120]}\")\n",
-    "        fp = getattr(m, \"file_path\", None)\n",
-    "        if fp and Path(fp).exists():\n",
-    "            img = Image.open(fp)\n",
-    "            img.thumbnail((250, 350))\n",
-    "            display(img)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Dense semantic search on LLM captions\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=3,\n",
-    "    score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": \"hero discovers villain's secret lair\"}],\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"llm_caption\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"Dense semantic: 'hero discovers villain's secret lair'\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# FTS on dialogue\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=3,\n",
-    "    score_by=[{\"type\": \"text\", \"fields\": [\"dialogue\"], \"query\": \"look out\"}],\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"dialogue\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"FTS on dialogue: 'look out'\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Filter by keyword in dialogue, rank by dense vector semantics\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=3,\n",
-    "    score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": \"tense confrontation\"}],\n",
-    "    filter={\"dialogue\": {\"$match_any\": [\"help\", \"stop\", \"run\"]}},\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"llm_caption\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"Keyword filter + dense rerank: 'tense confrontation' where dialogue contains help/stop/run\")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
-  "language_info": {"name": "python", "version": "3.11.0"}
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

03_approach2_image_embeddings.ipynb

@@ -1,248 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 03 · Approach 2: Direct Image Embeddings (Voyage AI)\n",
-    "\n",
-    "**Strategy:** Embed each comic page image directly using [Voyage AI's `voyage-multimodal-3`](https://docs.voyageai.com/docs/multimodal-embeddings) — no text description needed. Both images and text queries live in the same 1024-dimensional embedding space, enabling cross-modal search.\n",
-    "\n",
-    "**Namespace:** `ns-image-clip`\n",
-    "\n",
-    "**Blog post insight:** CLIP-style embeddings capture *visual* information that text descriptions miss: art style, panel composition, colour tone, action pose. But they're less precise for abstract semantic concepts — a query for *\"betrayal\"* works better with a language model.\n",
-    "\n",
-    "---\n",
-    "\n",
-    "## When to use this approach\n",
-    "- You want visual similarity search (find pages that look alike)\n",
-    "- You need cross-modal retrieval: text queries returning images, or image queries returning images\n",
-    "- You have no text transcripts and don't want to generate them"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import os\n",
-    "import time\n",
-    "from pathlib import Path\n",
-    "\n",
-    "import voyageai\n",
-    "from dotenv import load_dotenv\n",
-    "from pinecone import Pinecone\n",
-    "from PIL import Image\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "load_dotenv()\n",
-    "\n",
-    "TRANSCRIPTS_DIR = Path(\"data/transcripts\")\n",
-    "INDEX_NAME      = \"comics-multimodal\"\n",
-    "NAMESPACE       = \"ns-image-clip\"\n",
-    "BATCH_SIZE      = 10   # Voyage multimodal batches are smaller due to image payload\n",
-    "\n",
-    "pc = Pinecone(api_key=os.environ[\"PINECONE_API_KEY\"])\n",
-    "vo = voyageai.Client()  # reads VOYAGE_API_KEY from env\n",
-    "\n",
-    "# Connect to the shared index (created in notebook 02)\n",
-    "info = pc.preview.indexes.describe(name=INDEX_NAME)\n",
-    "INDEX_HOST = info.host\n",
-    "index = pc.index(host=INDEX_HOST)\n",
-    "print(f\"Connected to {INDEX_NAME} at {INDEX_HOST}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Embed page images with Voyage AI\n",
-    "\n",
-    "Each page is embedded as a PIL Image. Voyage `voyage-multimodal-3` accepts images directly — no base64 encoding step needed.\n",
-    "\n",
-    "**Cost:** ~$0.000006/image. For 150 pages ≈ $0.0009 total."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "records = [json.loads(p.read_text()) for p in sorted(TRANSCRIPTS_DIR.glob(\"*.json\"))]\n",
-    "print(f\"Pages to embed: {len(records)}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def embed_images_batch(paths: list[str]) -> list[list[float]]:\n",
-    "    \"\"\"Embed a batch of image paths. Returns list of 1024-dim vectors.\"\"\"\n",
-    "    inputs = [[Image.open(p).convert(\"RGB\")] for p in paths]\n",
-    "    result = vo.multimodal_embed(inputs, model=\"voyage-multimodal-3\", input_type=\"document\")\n",
-    "    return result.embeddings\n",
-    "\n",
-    "\n",
-    "# Embed all pages (cache vectors in the JSON records to avoid re-billing)\n",
-    "for i in tqdm(range(0, len(records), BATCH_SIZE), desc=\"Embedding images\"):\n",
-    "    batch = records[i : i + BATCH_SIZE]\n",
-    "    # Skip pages that already have a cached vector\n",
-    "    needs_embedding = [r for r in batch if not r.get(\"image_vector\")]\n",
-    "    if not needs_embedding:\n",
-    "        continue\n",
-    "    paths = [r[\"file_path\"] for r in needs_embedding]\n",
-    "    vectors = embed_images_batch(paths)\n",
-    "    for r, v in zip(needs_embedding, vectors):\n",
-    "        r[\"image_vector\"] = v\n",
-    "        path = TRANSCRIPTS_DIR / f\"{r['page_id']}.json\"\n",
-    "        path.write_text(json.dumps(r, ensure_ascii=False, indent=2))\n",
-    "\n",
-    "print(\"Image vectors ready.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Upsert to Pinecone (ns-image-clip)\n",
-    "\n",
-    "Unlike the text-caption namespace, here we **bring our own vectors** — we pass the 1024-dim Voyage vector directly rather than letting Pinecone's integrated inference embed a text field."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "documents = [\n",
-    "    {\n",
-    "        \"_id\":        r[\"page_id\"],\n",
-    "        \"embedding\":  r[\"image_vector\"],   # pre-computed Voyage vector (BYOV)\n",
-    "        \"dialogue\":   r[\"all_dialogue\"],\n",
-    "        \"comic_title\": r[\"comic_title\"],\n",
-    "        \"page_id\":    r[\"page_id\"],\n",
-    "        \"page_num\":   r[\"page_num\"],\n",
-    "        \"file_path\":  r[\"file_path\"],\n",
-    "    }\n",
-    "    for r in records\n",
-    "]\n",
-    "\n",
-    "index.documents.batch_upsert(\n",
-    "    namespace=NAMESPACE,\n",
-    "    documents=documents,\n",
-    "    batch_size=BATCH_SIZE,\n",
-    "    show_progress=True,\n",
-    ")\n",
-    "print(f\"Upserted {len(documents)} pages to {NAMESPACE}.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search demos\n",
-    "\n",
-    "All queries are embedded with Voyage AI at query time — both text and image queries go through the same model, landing in the same vector space."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from IPython.display import display\n",
-    "\n",
-    "\n",
-    "def show_results(matches, label=\"\"):\n",
-    "    if label:\n",
-    "        print(f\"\\n{'='*60}\\n{label}\\n{'='*60}\")\n",
-    "    for m in matches:\n",
-    "        comic = getattr(m, \"comic_title\", \"\")\n",
-    "        page  = getattr(m, \"page_num\", \"\")\n",
-    "        print(f\"[{m._score:.3f}] {m._id}  ({comic}, p{page})\")\n",
-    "        fp = getattr(m, \"file_path\", None)\n",
-    "        if fp and Path(fp).exists():\n",
-    "            img = Image.open(fp)\n",
-    "            img.thumbnail((250, 350))\n",
-    "            display(img)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Text-to-image: embed a text query with Voyage, search image vectors\n",
-    "text_query = \"dark alley chase scene at night\"\n",
-    "result = vo.multimodal_embed([[text_query]], model=\"voyage-multimodal-3\", input_type=\"query\")\n",
-    "query_vector = result.embeddings[0]\n",
-    "\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=3,\n",
-    "    score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": query_vector}],\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"page_num\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, f\"Text-to-image: '{text_query}'\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Image-to-image: use a page image as the query to find visually similar pages\n",
-    "query_page = records[0]  # change to any page you want to use as a query\n",
-    "query_img  = Image.open(query_page[\"file_path\"]).convert(\"RGB\")\n",
-    "result = vo.multimodal_embed([[query_img]], model=\"voyage-multimodal-3\", input_type=\"query\")\n",
-    "img_query_vector = result.embeddings[0]\n",
-    "\n",
-    "print(f\"Query image: {query_page['page_id']}\")\n",
-    "query_img.thumbnail((250, 350))\n",
-    "display(query_img)\n",
-    "\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=4,\n",
-    "    score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": img_query_vector}],\n",
-    "    # Exclude the query page itself\n",
-    "    filter={\"page_id\": {\"$ne\": query_page[\"page_id\"]}},\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"page_num\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"Image-to-image: visually similar pages\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Cross-comic visual style: find action scenes across all comics\n",
-    "result = vo.multimodal_embed([[\"superhero punching villain action scene\"]], model=\"voyage-multimodal-3\", input_type=\"query\")\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=5,\n",
-    "    score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": result.embeddings[0]}],\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"page_num\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"Cross-comic: 'superhero punching villain action scene'\")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
-  "language_info": {"name": "python", "version": "3.11.0"}
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

04_approach3_hybrid.ipynb

@@ -1,266 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 04 · Approach 3: Full Hybrid (Dense + Sparse + FTS)\n",
-    "\n",
-    "**Strategy:** Combine all three Pinecone search signals in a single namespace:\n",
-    "- **Dense vector** — LLM caption embedded with `llama-text-embed-v2` (semantic meaning)\n",
-    "- **Sparse vector** — dialogue embedded with `pinecone-sparse-english-v0` (keyword relevance)\n",
-    "- **FTS field** — raw dialogue text indexed with BM25 (exact phrase matching)\n",
-    "\n",
-    "**Namespace:** `ns-hybrid`\n",
-    "\n",
-    "**Blog post insight:** A single Pinecone index can store all three signal types together. The key design constraint: each query picks *one* ranking signal — but you can chain them: use FTS/sparse filters to narrow the candidate set, then dense vectors re-rank for semantic relevance. This gives you the precision of keyword search with the recall of semantic search.\n",
-    "\n",
-    "---\n",
-    "\n",
-    "## Signal comparison\n",
-    "\n",
-    "| Signal | Strengths | Weaknesses |\n",
-    "|---|---|---|\n",
-    "| Dense (LLM caption) | Conceptual themes, mood, setting | Misses specific names/words |\n",
-    "| Sparse (dialogue BM25) | Exact character names, sound effects (KAPOW!) | No semantic understanding |\n",
-    "| FTS (BM25 w/ stemming) | Phrase matching, stemming (\"run\" → \"running\") | Still keyword-based |\n",
-    "| Filter + dense rerank | Precision of keywords + recall of semantics | Requires both signals |\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import os\n",
-    "import time\n",
-    "from pathlib import Path\n",
-    "\n",
-    "from dotenv import load_dotenv\n",
-    "from pinecone import Pinecone\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "load_dotenv()\n",
-    "\n",
-    "TRANSCRIPTS_DIR = Path(\"data/transcripts\")\n",
-    "INDEX_NAME      = \"comics-multimodal\"\n",
-    "NAMESPACE       = \"ns-hybrid\"\n",
-    "BATCH_SIZE      = 50\n",
-    "\n",
-    "pc = Pinecone(api_key=os.environ[\"PINECONE_API_KEY\"])\n",
-    "\n",
-    "info = pc.preview.indexes.describe(name=INDEX_NAME)\n",
-    "INDEX_HOST = info.host\n",
-    "index = pc.index(host=INDEX_HOST)\n",
-    "print(f\"Connected to {INDEX_NAME}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Generate sparse vectors\n",
-    "\n",
-    "We use Pinecone's `pinecone-sparse-english-v0` model (via the inference API) to embed dialogue text into sparse vectors. These complement the dense LLM caption vectors."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "records = [json.loads(p.read_text()) for p in sorted(TRANSCRIPTS_DIR.glob(\"*.json\"))]\n",
-    "print(f\"Loaded {len(records)} transcripts\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def embed_sparse_batch(texts: list[str]) -> list[dict]:\n",
-    "    \"\"\"Embed texts with pinecone-sparse-english-v0. Returns list of {indices, values} dicts.\"\"\"\n",
-    "    result = pc.inference.embed(\n",
-    "        model=\"pinecone-sparse-english-v0\",\n",
-    "        inputs=texts,\n",
-    "        parameters={\"input_type\": \"passage\"},\n",
-    "    )\n",
-    "    return [{\"indices\": e.sparse_indices, \"values\": e.sparse_values} for e in result]\n",
-    "\n",
-    "\n",
-    "# Cache sparse vectors in transcript JSON to avoid re-billing\n",
-    "for i in tqdm(range(0, len(records), BATCH_SIZE), desc=\"Sparse embedding dialogue\"):\n",
-    "    batch = records[i : i + BATCH_SIZE]\n",
-    "    needs = [r for r in batch if not r.get(\"sparse_vector\")]\n",
-    "    if not needs:\n",
-    "        continue\n",
-    "    texts = [r[\"all_dialogue\"] or r[\"scene_descriptions\"] for r in needs]\n",
-    "    sparse_vecs = embed_sparse_batch(texts)\n",
-    "    for r, sv in zip(needs, sparse_vecs):\n",
-    "        r[\"sparse_vector\"] = sv\n",
-    "        path = TRANSCRIPTS_DIR / f\"{r['page_id']}.json\"\n",
-    "        path.write_text(json.dumps(r, ensure_ascii=False, indent=2))\n",
-    "\n",
-    "print(\"Sparse vectors ready.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Upsert to Pinecone (ns-hybrid)\n",
-    "\n",
-    "Each document carries:\n",
-    "- `embedding` — the LLM caption text (Pinecone auto-embeds with `llama-text-embed-v2`)\n",
-    "- `sparse_embedding` — pre-computed sparse vector from `pinecone-sparse-english-v0`\n",
-    "- `dialogue` — raw text for the FTS field"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "documents = [\n",
-    "    {\n",
-    "        \"_id\":              r[\"page_id\"],\n",
-    "        \"embedding\":        r[\"llm_caption\"],        # integrated inference → dense vector\n",
-    "        \"sparse_embedding\": r[\"sparse_vector\"],       # BYOV sparse vector\n",
-    "        \"dialogue\":         r[\"all_dialogue\"],        # FTS field\n",
-    "        \"comic_title\":      r[\"comic_title\"],\n",
-    "        \"page_id\":          r[\"page_id\"],\n",
-    "        \"page_num\":         r[\"page_num\"],\n",
-    "        \"file_path\":        r[\"file_path\"],\n",
-    "        \"llm_caption\":      r[\"llm_caption\"],\n",
-    "    }\n",
-    "    for r in records\n",
-    "]\n",
-    "\n",
-    "index.documents.batch_upsert(\n",
-    "    namespace=NAMESPACE,\n",
-    "    documents=documents,\n",
-    "    batch_size=BATCH_SIZE,\n",
-    "    show_progress=True,\n",
-    ")\n",
-    "print(f\"Upserted {len(documents)} pages to {NAMESPACE}.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Search demos — comparing all four query modes"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from IPython.display import display\n",
-    "from PIL import Image\n",
-    "\n",
-    "\n",
-    "def show_results(matches, label=\"\"):\n",
-    "    if label:\n",
-    "        print(f\"\\n{'='*60}\\n{label}\\n{'='*60}\")\n",
-    "    for m in matches:\n",
-    "        caption = getattr(m, \"llm_caption\", \"\") or \"\"\n",
-    "        print(f\"[{m._score:.3f}] {m._id} — {caption[:100]}\")\n",
-    "        fp = getattr(m, \"file_path\", None)\n",
-    "        if fp and Path(fp).exists():\n",
-    "            img = Image.open(fp)\n",
-    "            img.thumbnail((250, 350))\n",
-    "            display(img)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 1. Dense-only: semantic search on LLM captions\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=3,\n",
-    "    score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": \"a moment of betrayal and deception\"}],\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"llm_caption\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"Dense only: 'a moment of betrayal and deception'\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 2. Sparse-only: keyword search on dialogue\n",
-    "sparse_query = pc.inference.embed(\n",
-    "    model=\"pinecone-sparse-english-v0\",\n",
-    "    inputs=[\"KAPOW detective badge\"],\n",
-    "    parameters={\"input_type\": \"query\"},\n",
-    ")[0]\n",
-    "\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=3,\n",
-    "    score_by=[{\n",
-    "        \"type\": \"sparse_vector\",\n",
-    "        \"field\": \"sparse_embedding\",\n",
-    "        \"query\": {\"indices\": sparse_query.sparse_indices, \"values\": sparse_query.sparse_values},\n",
-    "    }],\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"dialogue\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"Sparse only: 'KAPOW detective badge'\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 3. FTS: BM25 on dialogue field (stemming catches run/running/ran)\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=3,\n",
-    "    score_by=[{\"type\": \"text\", \"fields\": [\"dialogue\"], \"query\": \"running from danger\"}],\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"dialogue\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"FTS (BM25 + stemming): 'running from danger'\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 4. Filter + dense rerank — the \"hybrid\" pattern\n",
-    "# Keyword filter narrows candidates, dense vector re-ranks by semantic relevance\n",
-    "resp = index.documents.search(\n",
-    "    namespace=NAMESPACE,\n",
-    "    top_k=3,\n",
-    "    score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": \"tense face-off between hero and villain\"}],\n",
-    "    filter={\"dialogue\": {\"$match_any\": [\"stop\", \"never\", \"finish\"]}},\n",
-    "    include_fields=[\"page_id\", \"comic_title\", \"llm_caption\", \"file_path\"],\n",
-    ")\n",
-    "show_results(resp.matches, \"Keyword filter + dense rerank: semantic 'tense face-off' where dialogue has stop/never/finish\")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
-  "language_info": {"name": "python", "version": "3.11.0"}
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

05_story_builder.ipynb

@@ -1,251 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 05 · Story Builder Demo\n",
-    "\n",
-    "Two demos in one notebook.\n",
-    "\n",
-    "**Section 1 — Cross-namespace comparison:** The same query goes to all three namespaces simultaneously. Results are shown side-by-side so you can see exactly how each embedding approach retrieves different pages for the same search.\n",
-    "\n",
-    "**Section 2 — Narrative assembler:** Chain four queries (setup → conflict → climax → resolution) to stitch pages from multiple different comics into a brand-new story arc."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import os\n",
-    "from pathlib import Path\n",
-    "\n",
-    "import matplotlib.pyplot as plt\n",
-    "import voyageai\n",
-    "from dotenv import load_dotenv\n",
-    "from PIL import Image\n",
-    "from pinecone import Pinecone\n",
-    "\n",
-    "load_dotenv()\n",
-    "\n",
-    "TRANSCRIPTS_DIR = Path(\"data/transcripts\")\n",
-    "INDEX_NAME      = \"comics-multimodal\"\n",
-    "\n",
-    "pc = Pinecone(api_key=os.environ[\"PINECONE_API_KEY\"])\n",
-    "vo = voyageai.Client()\n",
-    "\n",
-    "info = pc.preview.indexes.describe(name=INDEX_NAME)\n",
-    "index = pc.index(host=info.host)\n",
-    "print(f\"Connected to {INDEX_NAME}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Helper: embed query text into Voyage space (for ns-image-clip)\n",
-    "def voyage_query_vector(text: str) -> list[float]:\n",
-    "    result = vo.multimodal_embed([[text]], model=\"voyage-multimodal-3\", input_type=\"query\")\n",
-    "    return result.embeddings[0]\n",
-    "\n",
-    "\n",
-    "# Helper: search a namespace, return matches\n",
-    "def search_namespace(namespace: str, query: str, top_k: int = 5) -> list:\n",
-    "    if namespace == \"ns-image-clip\":\n",
-    "        # Voyage multimodal vector — text and image in the same space\n",
-    "        qvec = voyage_query_vector(query)\n",
-    "        score_by = [{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": qvec}]\n",
-    "    else:\n",
-    "        # llama-text-embed-v2 via integrated inference\n",
-    "        score_by = [{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": query}]\n",
-    "\n",
-    "    resp = index.documents.search(\n",
-    "        namespace=namespace,\n",
-    "        top_k=top_k,\n",
-    "        score_by=score_by,\n",
-    "        include_fields=[\"page_id\", \"comic_title\", \"page_num\", \"file_path\", \"llm_caption\"],\n",
-    "    )\n",
-    "    return resp.matches"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "---\n",
-    "## Section 1 — Cross-namespace comparison\n",
-    "\n",
-    "Same query, three namespaces, side-by-side."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def compare_namespaces(query: str, top_k: int = 3):\n",
-    "    namespaces = [\n",
-    "        (\"ns-text-caption\", \"LLM Caption\\n(dense)\"),\n",
-    "        (\"ns-image-clip\",   \"Voyage Image\\n(visual)\"),\n",
-    "        (\"ns-hybrid\",       \"Hybrid\\n(dense+sparse+FTS)\"),\n",
-    "    ]\n",
-    "\n",
-    "    all_results = {}\n",
-    "    for ns, _ in namespaces:\n",
-    "        all_results[ns] = search_namespace(ns, query, top_k=top_k)\n",
-    "\n",
-    "    fig, axes = plt.subplots(top_k, len(namespaces), figsize=(14, top_k * 4))\n",
-    "    fig.suptitle(f'Query: \"{query}\"', fontsize=13, fontweight=\"bold\", y=1.01)\n",
-    "\n",
-    "    for col, (ns, ns_label) in enumerate(namespaces):\n",
-    "        axes[0][col].set_title(ns_label, fontsize=11, pad=8)\n",
-    "        for row, match in enumerate(all_results[ns]):\n",
-    "            ax = axes[row][col]\n",
-    "            fp = getattr(match, \"file_path\", None)\n",
-    "            if fp and Path(fp).exists():\n",
-    "                img = Image.open(fp).convert(\"RGB\")\n",
-    "                ax.imshow(img)\n",
-    "            else:\n",
-    "                ax.text(0.5, 0.5, \"no image\", ha=\"center\", va=\"center\")\n",
-    "            comic = getattr(match, \"comic_title\", \"\") or \"\"\n",
-    "            page  = getattr(match, \"page_num\", \"\") or \"\"\n",
-    "            ax.set_xlabel(f\"{comic}\\np{page}  score={match._score:.3f}\", fontsize=8)\n",
-    "            ax.set_xticks([])\n",
-    "            ax.set_yticks([])\n",
-    "\n",
-    "    plt.tight_layout()\n",
-    "    plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "compare_namespaces(\"detective discovers a clue\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "compare_namespaces(\"KAPOW punch fight action\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "compare_namespaces(\"mysterious figure in shadows\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "---\n",
-    "## Section 2 — Narrative Assembler\n",
-    "\n",
-    "Chain four queries — one per story beat — and assemble the top matching page from each into a 2×2 comic grid.\n",
-    "\n",
-    "Pages are deduplicated: if two beats return the same page, we take the next best match."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def assemble_story(story_beats: list[tuple[str, str]], namespace: str = \"ns-hybrid\"):\n",
-    "    \"\"\"\n",
-    "    story_beats: list of (beat_label, query_text)\n",
-    "    Fetches top pages per beat (with dedup) and renders a grid.\n",
-    "    \"\"\"\n",
-    "    used_ids = set()\n",
-    "    selected = []  # list of (label, match)\n",
-    "\n",
-    "    for label, query in story_beats:\n",
-    "        matches = search_namespace(namespace, query, top_k=10)\n",
-    "        for m in matches:\n",
-    "            if m._id not in used_ids:\n",
-    "                used_ids.add(m._id)\n",
-    "                selected.append((label, m))\n",
-    "                break\n",
-    "\n",
-    "    n = len(selected)\n",
-    "    cols = 2\n",
-    "    rows = (n + 1) // cols\n",
-    "    fig, axes = plt.subplots(rows, cols, figsize=(12, rows * 5))\n",
-    "    axes = axes.flatten() if n > 1 else [axes]\n",
-    "\n",
-    "    for i, (label, match) in enumerate(selected):\n",
-    "        ax = axes[i]\n",
-    "        fp = getattr(match, \"file_path\", None)\n",
-    "        if fp and Path(fp).exists():\n",
-    "            ax.imshow(Image.open(fp).convert(\"RGB\"))\n",
-    "        comic   = getattr(match, \"comic_title\", \"\") or \"\"\n",
-    "        page    = getattr(match, \"page_num\", \"\") or \"\"\n",
-    "        caption = getattr(match, \"llm_caption\", \"\") or \"\"\n",
-    "        ax.set_title(f\"{label}\", fontsize=11, fontweight=\"bold\")\n",
-    "        ax.set_xlabel(f\"{comic}  ·  p{page}\\n{caption[:80]}…\", fontsize=8)\n",
-    "        ax.set_xticks([])\n",
-    "        ax.set_yticks([])\n",
-    "\n",
-    "    # Hide unused axes\n",
-    "    for j in range(n, len(axes)):\n",
-    "        axes[j].set_visible(False)\n",
-    "\n",
-    "    fig.suptitle(\"Assembled Story Arc\", fontsize=14, fontweight=\"bold\")\n",
-    "    plt.tight_layout()\n",
-    "    plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "assemble_story([\n",
-    "    (\"1. Setup\",      \"detective arrives at a crime scene\"),\n",
-    "    (\"2. Conflict\",   \"villain threatens the hero\"),\n",
-    "    (\"3. Climax\",     \"dramatic fight or physical confrontation\"),\n",
-    "    (\"4. Resolution\", \"hero wins and villain is defeated\"),\n",
-    "])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Try your own story arc:\n",
-    "assemble_story([\n",
-    "    (\"1. Setup\",      \"mysterious stranger arrives in town\"),\n",
-    "    (\"2. Tension\",    \"secret revealed in a dimly lit room\"),\n",
-    "    (\"3. Climax\",     \"desperate chase through the streets\"),\n",
-    "    (\"4. Twist\",      \"unexpected alliance between enemies\"),\n",
-    "])"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
-  "language_info": {"name": "python", "version": "3.11.0"}
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

06_panel_level_exploration.ipynb

@@ -1,333 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# 06 · Panel-Level Exploration (Bonus)\n",
-    "\n",
-    "**Goal:** Explore whether embedding individual *panels* rather than full *pages* produces meaningfully better retrieval.\n",
-    "\n",
-    "**Hypothesis:** A page of 6 panels covers multiple scenes and characters. Embedding the whole page averages out these signals. A panel-level embedding is more focused — a query for *\"a character looking surprised\"* should return a single panel, not a page where that moment is one of six.\n",
-    "\n",
-    "**Namespace:** `ns-panels`\n",
-    "\n",
-    "**Panel detection approach:** OpenCV contour detection. Comic panels have strong rectangular borders; edge detection + bounding box extraction works surprisingly well on classic golden-age comics without needing an ML model.\n",
-    "\n",
-    "---\n",
-    "\n",
-    "## When to use panel-level embeddings\n",
-    "- Maximum retrieval precision (the story-builder demo becomes much more compelling)\n",
-    "- Character-level queries: \"find all panels featuring a woman in a cape\"\n",
-    "- Panel count is manageable — 5 comics × 30 pages × ~6 panels = ~900 panels"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import json\n",
-    "import os\n",
-    "from pathlib import Path\n",
-    "\n",
-    "import cv2\n",
-    "import numpy as np\n",
-    "import voyageai\n",
-    "from dotenv import load_dotenv\n",
-    "from IPython.display import display\n",
-    "from PIL import Image\n",
-    "from pinecone import Pinecone\n",
-    "from tqdm import tqdm\n",
-    "\n",
-    "load_dotenv()\n",
-    "\n",
-    "TRANSCRIPTS_DIR = Path(\"data/transcripts\")\n",
-    "PANELS_DIR      = Path(\"data/panels\")\n",
-    "PANELS_DIR.mkdir(parents=True, exist_ok=True)\n",
-    "\n",
-    "INDEX_NAME  = \"comics-multimodal\"\n",
-    "NAMESPACE   = \"ns-panels\"\n",
-    "MIN_PANEL_W = 100   # pixels — ignore tiny boxes (captions, gutters)\n",
-    "MIN_PANEL_H = 100\n",
-    "\n",
-    "pc = Pinecone(api_key=os.environ[\"PINECONE_API_KEY\"])\n",
-    "vo = voyageai.Client()\n",
-    "\n",
-    "info  = pc.preview.indexes.describe(name=INDEX_NAME)\n",
-    "index = pc.index(host=info.host)\n",
-    "print(f\"Connected to {INDEX_NAME}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Panel detection with OpenCV\n",
-    "\n",
-    "Classic comics have clean rectangular panel borders. We:\n",
-    "1. Convert to grayscale and apply binary threshold\n",
-    "2. Find external contours\n",
-    "3. Filter by minimum area and aspect ratio\n",
-    "4. Return bounding boxes sorted top-to-bottom, left-to-right (reading order)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def detect_panels(image_path: str, min_w: int = MIN_PANEL_W, min_h: int = MIN_PANEL_H) -> list[tuple]:\n",
-    "    \"\"\"Return list of (x, y, w, h) bounding boxes for detected panels.\"\"\"\n",
-    "    img = cv2.imread(image_path)\n",
-    "    if img is None:\n",
-    "        return []\n",
-    "    gray   = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
-    "    _, bw  = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY_INV)\n",
-    "    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))\n",
-    "    bw     = cv2.morphologyEx(bw, cv2.MORPH_CLOSE, kernel)\n",
-    "    contours, _ = cv2.findContours(bw, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n",
-    "\n",
-    "    boxes = []\n",
-    "    h_img, w_img = img.shape[:2]\n",
-    "    for c in contours:\n",
-    "        x, y, w, h = cv2.boundingRect(c)\n",
-    "        # Skip boxes that are the whole page or too small\n",
-    "        if w < min_w or h < min_h:\n",
-    "            continue\n",
-    "        if w > w_img * 0.95 and h > h_img * 0.95:\n",
-    "            continue\n",
-    "        boxes.append((x, y, w, h))\n",
-    "\n",
-    "    # Sort reading order: top-to-bottom, then left-to-right\n",
-    "    boxes.sort(key=lambda b: (b[1] // (h_img // 4), b[0]))\n",
-    "    return boxes\n",
-    "\n",
-    "\n",
-    "def crop_panel(image_path: str, box: tuple, out_path: Path) -> str | None:\n",
-    "    \"\"\"Crop a panel bounding box and save. Returns path or None on failure.\"\"\"\n",
-    "    if out_path.exists():\n",
-    "        return str(out_path)\n",
-    "    img = Image.open(image_path).convert(\"RGB\")\n",
-    "    x, y, w, h = box\n",
-    "    panel = img.crop((x, y, x + w, y + h))\n",
-    "    panel.save(out_path, quality=90)\n",
-    "    return str(out_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Detect and crop all panels, build panel manifest\n",
-    "records    = [json.loads(p.read_text()) for p in sorted(TRANSCRIPTS_DIR.glob(\"*.json\"))]\n",
-    "panel_data = []\n",
-    "\n",
-    "for r in tqdm(records, desc=\"Detecting panels\"):\n",
-    "    boxes = detect_panels(r[\"file_path\"])\n",
-    "    if not boxes:\n",
-    "        # Fall back: treat the whole page as a single panel\n",
-    "        boxes = [(0, 0, 9999, 9999)]\n",
-    "    slug_dir = PANELS_DIR / r[\"comic_slug\"]\n",
-    "    slug_dir.mkdir(parents=True, exist_ok=True)\n",
-    "    for i, box in enumerate(boxes):\n",
-    "        panel_id = f\"{r['page_id']}_panel{i+1:02d}\"\n",
-    "        out_path = slug_dir / f\"{panel_id}.jpg\"\n",
-    "        if box == (0, 0, 9999, 9999):\n",
-    "            # Copy full page\n",
-    "            import shutil\n",
-    "            shutil.copy(r[\"file_path\"], out_path)\n",
-    "        else:\n",
-    "            crop_panel(r[\"file_path\"], box, out_path)\n",
-    "        panel_data.append({\n",
-    "            \"panel_id\":    panel_id,\n",
-    "            \"page_id\":     r[\"page_id\"],\n",
-    "            \"comic_title\": r[\"comic_title\"],\n",
-    "            \"comic_slug\":  r[\"comic_slug\"],\n",
-    "            \"page_num\":    r[\"page_num\"],\n",
-    "            \"panel_num\":   i + 1,\n",
-    "            \"file_path\":   str(out_path),\n",
-    "        })\n",
-    "\n",
-    "print(f\"Total panels detected: {len(panel_data)}  ({len(panel_data)/len(records):.1f} per page avg)\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Spot-check panel crops"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Show the first 6 panels from the first page\n",
-    "first_page_panels = [p for p in panel_data if p[\"page_id\"] == panel_data[0][\"page_id\"]][:6]\n",
-    "fig, axes = plt.subplots(1, len(first_page_panels), figsize=(14, 4))\n",
-    "if len(first_page_panels) == 1:\n",
-    "    axes = [axes]\n",
-    "for ax, p in zip(axes, first_page_panels):\n",
-    "    ax.imshow(Image.open(p[\"file_path\"]))\n",
-    "    ax.set_title(f\"Panel {p['panel_num']}\", fontsize=9)\n",
-    "    ax.axis(\"off\")\n",
-    "plt.suptitle(f\"Detected panels — {first_page_panels[0]['page_id']}\", fontsize=11)\n",
-    "plt.tight_layout()\n",
-    "plt.show()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Embed panels and upsert to ns-panels"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import matplotlib.pyplot as plt\n",
-    "\n",
-    "EMBED_BATCH = 10\n",
-    "\n",
-    "for i in tqdm(range(0, len(panel_data), EMBED_BATCH), desc=\"Embedding panels\"):\n",
-    "    batch = panel_data[i : i + EMBED_BATCH]\n",
-    "    needs = [p for p in batch if not p.get(\"image_vector\")]\n",
-    "    if not needs:\n",
-    "        continue\n",
-    "    inputs  = [[Image.open(p[\"file_path\"]).convert(\"RGB\")] for p in needs]\n",
-    "    result  = vo.multimodal_embed(inputs, model=\"voyage-multimodal-3\", input_type=\"document\")\n",
-    "    for p, vec in zip(needs, result.embeddings):\n",
-    "        p[\"image_vector\"] = vec\n",
-    "\n",
-    "print(\"Panel vectors ready.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "documents = [\n",
-    "    {\n",
-    "        \"_id\":        p[\"panel_id\"],\n",
-    "        \"embedding\":  p[\"image_vector\"],\n",
-    "        \"comic_title\": p[\"comic_title\"],\n",
-    "        \"page_id\":    p[\"page_id\"],\n",
-    "        \"page_num\":   p[\"page_num\"],\n",
-    "        \"panel_num\":  p[\"panel_num\"],\n",
-    "        \"file_path\":  p[\"file_path\"],\n",
-    "    }\n",
-    "    for p in panel_data if p.get(\"image_vector\")\n",
-    "]\n",
-    "\n",
-    "index.documents.batch_upsert(\n",
-    "    namespace=NAMESPACE,\n",
-    "    documents=documents,\n",
-    "    batch_size=50,\n",
-    "    show_progress=True,\n",
-    ")\n",
-    "print(f\"Upserted {len(documents)} panels to {NAMESPACE}.\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Compare: page-level vs panel-level retrieval\n",
-    "\n",
-    "Run the same query against `ns-image-clip` (page-level) and `ns-panels` (panel-level) and see which gives more precise results."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def compare_page_vs_panel(query: str, top_k: int = 4):\n",
-    "    qvec = vo.multimodal_embed([[query]], model=\"voyage-multimodal-3\", input_type=\"query\").embeddings[0]\n",
-    "\n",
-    "    resp_page = index.documents.search(\n",
-    "        namespace=\"ns-image-clip\",\n",
-    "        top_k=top_k,\n",
-    "        score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": qvec}],\n",
-    "        include_fields=[\"page_id\", \"comic_title\", \"page_num\", \"file_path\"],\n",
-    "    )\n",
-    "    resp_panel = index.documents.search(\n",
-    "        namespace=NAMESPACE,\n",
-    "        top_k=top_k,\n",
-    "        score_by=[{\"type\": \"dense_vector\", \"field\": \"embedding\", \"query\": qvec}],\n",
-    "        include_fields=[\"page_id\", \"comic_title\", \"page_num\", \"panel_num\", \"file_path\"],\n",
-    "    )\n",
-    "\n",
-    "    fig, axes = plt.subplots(2, top_k, figsize=(14, 8))\n",
-    "    fig.suptitle(f'Query: \"{query}\"', fontsize=12, fontweight=\"bold\")\n",
-    "\n",
-    "    for col, m in enumerate(resp_page.matches):\n",
-    "        ax = axes[0][col]\n",
-    "        fp = getattr(m, \"file_path\", None)\n",
-    "        if fp and Path(fp).exists():\n",
-    "            ax.imshow(Image.open(fp))\n",
-    "        ax.set_title(f\"Page-level\\n{getattr(m, 'comic_title', '')} p{getattr(m, 'page_num', '')}\", fontsize=8)\n",
-    "        ax.axis(\"off\")\n",
-    "\n",
-    "    for col, m in enumerate(resp_panel.matches):\n",
-    "        ax = axes[1][col]\n",
-    "        fp = getattr(m, \"file_path\", None)\n",
-    "        if fp and Path(fp).exists():\n",
-    "            ax.imshow(Image.open(fp))\n",
-    "        panel_n = getattr(m, \"panel_num\", \"\")\n",
-    "        ax.set_title(f\"Panel-level\\n{getattr(m, 'comic_title', '')} p{getattr(m, 'page_num', '')} panel{panel_n}\", fontsize=8)\n",
-    "        ax.axis(\"off\")\n",
-    "\n",
-    "    plt.tight_layout()\n",
-    "    plt.show()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "compare_page_vs_panel(\"character looking surprised or shocked\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "compare_page_vs_panel(\"close up face of a villain sneering\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "compare_page_vs_panel(\"two people talking at a table\")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"},
-  "language_info": {"name": "python", "version": "3.11.0"}
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

PLAN.md

@@ -1,311 +0,0 @@
-# Plan: Multimodal Comics Search with Pinecone
-
-## Context
-
-Build a blog post-quality project that demonstrates Pinecone's vector search capabilities using public-domain golden age comics from the Digital Comic Museum. The project showcases multimodal data handling — combining image embeddings, LLM-generated descriptions, sparse vectors, and full-text search across different Pinecone namespaces. The goal is thought leadership content showing real trade-offs between embedding approaches, culminating in a "story builder" demo that assembles comic panels from multiple comics into a narrative.
-
----
-
-## Project Structure
-
-```
-comics-pinecone/
-├── 00_data_acquisition.ipynb       # Download + extract comics
-├── 01_transcription.ipynb          # comic-transcriber pipeline
-├── 02_approach1_text_dense_fts.ipynb     # Approach 1: LLM caption → dense + FTS
-├── 03_approach2_image_embeddings.ipynb   # Approach 2: Direct image embeddings (CLIP)
-├── 04_approach3_hybrid.ipynb             # Approach 3: Dense + Sparse + FTS hybrid
-├── 05_story_builder.ipynb                # Demo: cross-namespace comparison + story assembly
-├── 06_panel_level_exploration.ipynb      # Bonus: panel detection + panel-level embeddings
-├── data/
-│   ├── raw/          # Downloaded CBZ/zip files
-│   ├── pages/        # Extracted page images (PNG)
-│   └── transcripts/  # comic-transcriber markdown output
-├── utils/
-│   ├── downloader.py
-│   ├── extractor.py
-│   └── embedders.py
-└── .env.example
-```
-
----
-
-## Notebook 00 — Data Acquisition
-
-**Goal:** Download ~5–10 comics from https://digitalcomicmuseum.com/index.php?cid=110
-
-**Steps:**
-1. Identify download links from the category page (web scraping or manual curation)
-2. Download CBZ/ZIP archives
-3. Extract to `data/pages/<comic_title>/page_001.jpg` etc.
-4. Build a manifest CSV: `comic_title, page_num, file_path, image_url`
-
-**Key decision:** Start with 5 comics (~50–150 pages total) to keep costs and runtime manageable for a blog post demo.
-
----
-
-## Notebook 01 — Transcription Pipeline
-
-**Goal:** Run comic-transcriber on each page and produce structured output.
-
-**Steps:**
-1. Install comic-transcriber via NPM: `npx comic-transcriber`
-2. Process each page image → outputs structured markdown per page:
-   ```
-   ## PAGE X, PANEL Y
-   [Scene description]
-   **CHARACTER**: Dialogue
-   ```
-3. Parse the markdown output into a structured Python dict per page:
-   - `page_id`: `{comic_title}_{page_num}`
-   - `panels`: list of panel dicts (description, dialogue)
-   - `all_dialogue`: concatenated dialogue text (for FTS)
-   - `scene_summary`: concatenated descriptions (for dense embedding)
-4. Save to `data/transcripts/{comic_title}.json`
-
-**Output schema per page:**
-```json
-{
-  "page_id": "action_comics_001_p003",
-  "comic_title": "Action Comics 001",
-  "page_num": 3,
-  "file_path": "data/pages/action_comics_001/page_003.jpg",
-  "all_dialogue": "Look out! The villains are here...",
-  "scene_descriptions": "A hero leaps across rooftops under a full moon...",
-  "raw_transcript": "## PAGE 3, PANEL 1..."
-}
-```
-
----
-
-## Namespace Strategy (Single Pinecone Index)
-
-One **hybrid index** (`comics-multimodal`) via `pc.preview.indexes.create` with `SchemaBuilder`. A single Pinecone index can hold dense + sparse + FTS fields simultaneously (confirmed in docs: https://docs.pinecone.io/guides/search/full-text-search#schema-definition). Three namespaces, each representing a different embedding strategy.
-
-**Critical constraint:** Each query uses exactly one ranking signal (dense, sparse, or BM25/text). To combine signals, use text-match filters (`$match_phrase`, `$match_all`) to narrow candidates, then rank by vector.
-
-| Namespace | Embedding source | Model | Dim | Also has |
-|---|---|---|---|---|
-| `ns-text-caption` | LLM scene description → dense text embedding | `llama-text-embed-v2` | 1024 | FTS on dialogue |
-| `ns-image-clip` | Raw comic page image → multimodal embedding | `voyage-multimodal-3` | 1024 | — |
-| `ns-hybrid` | LLM description (dense) + dialogue (sparse + FTS) | `llama-text-embed-v2` + `pinecone-sparse-english-v0` | 1024 | FTS + sparse |
-
-**Index configuration (matches Pinecone web console):**
-
-| Setting | Value |
-|---|---|
-| Search type | Search by both meaning and exact words |
-| Embeddings | Dense + sparse |
-| Setup by model | `llama-text-embed-v2` (integrated inference for text namespaces) |
-| Dimension | 1,024 |
-| Metric | dotproduct (unit vectors → equivalent to cosine) |
-| Dense field name | `embedding` |
-| Sparse field name | `sparse_embedding` |
-| FTS field | `dialogue`, language: en, stemming: ON, stop words: ON |
-
-**Index schema (via `SchemaBuilder`):**
-```python
-schema = (
-    SchemaBuilder()
-      .add_string_field(name="dialogue", full_text_search={"language": "en", "stemming": True, "stop_words": True})
-      .add_string_field(name="comic_title", filterable=True)
-      .add_string_field(name="page_id", filterable=True)
-      .add_dense_vector_field(name="embedding", dimension=1024, metric="dotproduct")
-      .add_sparse_vector_field(name="sparse_embedding")
-      .build()
-)
-```
-
-**Upsert pattern differs by namespace:**
-- `ns-text-caption` / `ns-hybrid`: pass text field → Pinecone auto-embeds via `llama-text-embed-v2` (integrated inference)
-- `ns-image-clip`: upsert raw 1024-dim Voyage AI vectors directly (bypass integrated inference)
-
----
-
-## Notebook 02 — Approach 1: Text Caption + Dense + FTS
-
-**Embedding pipeline:**
-1. Use Claude (`claude-haiku-4-5-20251001`) to generate a rich scene description per page from the comic-transcriber output. Prompt focuses on: setting, action, mood, characters present, visual style.
-2. Embed the description using Pinecone's integrated `llama-text-embed-v2` (via `create_index_for_model`)
-3. Store dialogue text as a separate FTS field using `pc.preview.indexes` with `SchemaBuilder`
-
-**Note:** Since we need both dense + FTS, this may require a hybrid index type or two separate calls. Check Pinecone's current support for combined dense + FTS in a single index. If not supported together, use two separate indexes and merge results at query time.
-
-**Metadata per vector:**
-```python
-{
-  "page_id": "action_comics_001_p003",
-  "comic_title": "Action Comics 001",
-  "page_num": 3,
-  "image_url": "...",
-  "scene_description": "...",  # the LLM caption
-  "all_dialogue": "..."         # FTS field
-}
-```
-
-**Demo queries in notebook:**
-- `"hero discovers villain's secret lair"` → semantic search on descriptions
-- `"Look out"` → FTS search on dialogue
-- Combined: FTS filter + dense reranking
-
-**Blog post insight:** LLM captions add semantic richness but lose visual detail. FTS on dialogue is great for exact quote matching.
-
----
-
-## Notebook 03 — Approach 2: Direct Image Embeddings
-
-**Model:** Voyage AI `voyage-multimodal-3` (1024-dim, cosine)
-- API: `voyageai` Python package (`pip install voyageai`)
-- Cost: ~$0.000006/image
-- Handles both images and text queries in the same embedding space — enables cross-modal search
-
-**Pipeline:**
-```python
-import voyageai
-vo = voyageai.Client()  # reads VOYAGE_API_KEY from env
-
-# Indexing: embed page image
-result = vo.multimodal_embed([[page_image]], model="voyage-multimodal-3", input_type="document")
-vector = result.embeddings[0]  # 1024-dim list
-
-# Querying: embed text query into same space
-result = vo.multimodal_embed([["dark alley chase scene"]], model="voyage-multimodal-3", input_type="query")
-query_vector = result.embeddings[0]
-```
-
-**Demo queries:**
-- Text-to-image: `"dark alley chase scene"` → embed text, search `ns-image-clip`
-- Image-to-image: Use a comic page image as the query input → find visually similar pages
-- Cross-comic visual style matching: find pages with similar composition across comics
-
-**Blog post insight:** Voyage multimodal embeddings capture visual style and composition that text descriptions miss. But they're less precise for abstract semantic concepts like "betrayal."
-
----
-
-## Notebook 04 — Approach 3: Full Hybrid (Dense + Sparse + FTS)
-
-**Pipeline:**
-1. Dense vector: Same as Approach 1 (LLM caption → `llama-text-embed-v2`)
-2. Sparse vector: Embed the dialogue text using `pinecone-sparse-english-v0`
-3. FTS field: Dialogue text with BM25 (via `SchemaBuilder`)
-
-**Index type:** Single hybrid index via `SchemaBuilder` (dense + sparse + FTS fields in one index). All in `ns-hybrid` namespace.
-
-**Query types to demonstrate (one ranking signal per query, per Pinecone constraint):**
-
-| Query mode | Ranking signal | Notes |
-|---|---|---|
-| Dense semantic | `dense_vector` | "find scenes about justice" |
-| Sparse keyword | `sparse_vector` | "KAPOW detective badge" |
-| BM25 / FTS | `text` on `dialogue` field | exact phrase in speech bubbles |
-| Filter + rank | Text `$match_phrase` filter → dense ranking | narrow by keyword, then rerank by semantics |
-
-```python
-# Filter by keyword, rank by vector (the "hybrid" pattern in Pinecone)
-index.documents.search(
-    namespace="ns-hybrid",
-    top_k=5,
-    score_by=[{"type": "dense_vector", "field": "embedding", "query": query_vector}],
-    filter={"dialogue": {"$match_phrase": "look out"}},
-)
-```
-
-**Blog post insight:** A single index holds all three signal types. The key lesson: Pinecone picks one ranker per query, but you can chain: FTS/sparse filters narrow the candidate set, then dense vector re-ranks for semantic relevance. Dense retrieves conceptually similar scenes; sparse/FTS catches exact names, sound effects (KAPOW!), specific dialogue.
-
----
-
-## Notebook 05 — Story Builder Demo
-
-**Core demo:** Two sections in one notebook.
-
-**Section 1 — Cross-namespace comparison view:**
-- User types one query: e.g. `"detective discovers a clue"`
-- Same query sent to all 3 namespaces simultaneously (parallel calls)
-- Results displayed side-by-side: 3 columns (ns-text-caption | ns-image-clip | ns-hybrid), top-5 each
-- Each result shows: thumbnail, comic title, page num, match score
-- Purpose: show readers concretely how the same query returns different results depending on embedding strategy
-
-**Section 2 — Narrative assembler:**
-```
-Query 1 (setup):      "detective arrives at crime scene"
-Query 2 (conflict):   "villain threatens the hero"
-Query 3 (climax):     "dramatic fight or confrontation"
-Query 4 (resolution): "hero wins, villain defeated"
-```
-- Each query fetches top-1 page from `ns-hybrid` (best all-round namespace)
-- Deduplicate: if two queries return the same page, fetch next best
-- Display the 4 pages in sequence as a 2×2 image grid with caption: comic title + page num
-- Output: a brand new "story" assembled from panels across multiple comics
-
-**Output format:** Inline IPython `display(Image(...))` grid — no external UI needed, renders directly in the notebook.
-
----
-
-## Notebook 06 — Panel-Level Exploration (Bonus)
-
-**Goal:** Explore whether panel-level granularity produces meaningfully better results.
-
-**Panel detection approach:**
-- Option A (simple): Use `comic-panels` Python library or OpenCV edge detection + contour finding
-- Option B (ML): Use a pre-trained comic panel segmentation model (e.g. from HuggingFace)
-
-**Pipeline:**
-1. Detect panel bounding boxes per page
-2. Crop individual panels (100–400 panels per comic)
-3. Embed each panel (both image CLIP + LLM caption for the panel's transcribed text)
-4. Upsert to a `ns-panels` namespace
-
-**Demo:** "Find all panels of a character running" — much more precise than page-level.
-
-**Blog post insight:** Panel-level is more powerful for the story-builder, but requires significant additional pipeline complexity. Page-level is a good starting point for most use cases.
-
----
-
-## Key Technical Decisions
-
-1. **Pinecone index type for dense + FTS:** ✅ Confirmed — a single hybrid index via `SchemaBuilder` supports dense + sparse + FTS fields. One ranking signal per query; use text-match filters + vector ranking as the "hybrid" pattern.
-
-2. **Image model:** ✅ Confirmed — Voyage AI `voyage-multimodal-3`. Requires `VOYAGE_API_KEY`. Include Jina CLIP v2 as a free open-source footnote for readers who want no extra API key.
-
-3. **LLM for captions:** Use `claude-haiku-4-5-20251001` (fast + cheap). Prompt should be consistent and reproducible — include it verbatim in the notebook for transparency.
-
-4. **Rate limiting / cost controls:** Comic Transcriber + Claude captions + Voyage AI embeddings will have associated costs. Add a `MAX_PAGES` constant to each notebook so readers can test with fewer pages.
-
----
-
-## Environment Variables
-
-```
-PINECONE_API_KEY=
-ANTHROPIC_API_KEY=     # for LLM captions
-VOYAGE_API_KEY=        # for image embeddings
-```
-
----
-
-## Verification Plan
-
-1. **Notebook 00:** Verify `data/pages/` populated with images; manifest CSV has correct row count
-2. **Notebook 01:** Spot-check 3 transcripts; confirm dialogue and description fields are non-empty
-3. **Notebook 02:** Run 5 test queries; verify semantic matches are qualitatively relevant
-4. **Notebook 03:** Run text-to-image query; verify returned images visually match query intent
-5. **Notebook 04:** Compare hybrid vs. dense-only recall on 3 test queries
-6. **Notebook 05:** Build one complete story arc end-to-end; display assembled panel grid
-7. **Notebook 06 (bonus):** Verify panel crops look correct; compare panel-level vs. page-level for a test query
-
----
-
-## Critical Files to Create
-
-- `00_data_acquisition.ipynb`
-- `01_transcription.ipynb`
-- `02_approach1_text_dense_fts.ipynb`
-- `03_approach2_image_embeddings.ipynb`
-- `04_approach3_hybrid.ipynb`
-- `05_story_builder.ipynb`
-- `06_panel_level_exploration.ipynb`
-- `utils/downloader.py` — comic download helpers
-- `utils/extractor.py` — CBZ extraction + page listing
-- `utils/embedders.py` — shared embedding functions (Voyage, llama-text-embed-v2, sparse)
-- `.env.example`
-- `requirements.txt`

README.md

@@ -0,0 +1,127 @@
+# COMICS Panel Search
+
+Panel-level vector search over the [COMICS dataset](https://github.com/miyyer/comics) using Pinecone. Supports dense image search (OpenCLIP), sparse keyword search, full-text search over OCR text, and combined retrieval with Reciprocal Rank Fusion.
+
+## Setup
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+```
+
+Copy `.env` and fill in your keys:
+
+```bash
+PINECONE_API_KEY="..."
+PINECONE_INDEX_NAME="comic-panels"
+PINECONE_NAMESPACE="comics-v1"
+COMICS_DATA_DIR="data/comics"
+```
+
+## Step 1: Download COMICS data
+
+```bash
+mkdir -p data/comics/raw
+cd data/comics/raw
+
+wget -c https://obj.umiacs.umd.edu/comics/raw_panel_images.tar.gz
+wget -c https://obj.umiacs.umd.edu/comics/COMICS_ocr_file.csv
+wget -c https://obj.umiacs.umd.edu/comics/predadpages.txt
+
+tar -xzf raw_panel_images.tar.gz
+cd ../../..
+```
+
+> Optional — only needed if you want page-level reconstruction later:
+> ```bash
+> wget -c https://obj.umiacs.umd.edu/comics/raw_page_images.tar.gz
+> ```
+
+## Step 2: Run the ingestion pipeline
+
+```bash
+python -m src.ingest.build_manifest
+python -m src.embeddings.embed_images
+python -m src.embeddings.embed_sparse_text
+python -m src.pinecone.create_index
+python -m src.pinecone.build_documents
+python -m src.pinecone.upsert_panels
+```
+
+Each script is idempotent and saves intermediate results to `data/comics/processed/` before the next step depends on them.
+
+> **First run note:** `build_manifest` will print the OCR CSV column names. If the auto-detected column mapping looks wrong, update `build_ocr_lookup()` in `src/ingest/build_manifest.py` with the correct column names and re-run.
+
+## Step 3: Query the index
+
+```bash
+jupyter lab notebooks/query_comics_index.ipynb
+```
+
+Or open the notebook directly:
+
+```bash
+jupyter notebook notebooks/query_comics_index.ipynb
+```
+
+## Search smoke tests
+
+```bash
+python -m src.search.search_fts --query '"secret formula"'
+python -m src.search.search_dense --query "a detective in a dark alley"
+python -m src.search.search_sparse --query "masked villain laboratory"
+python -m src.search.search_combined --query "a laboratory panel containing formula"
+```
+
+## Project structure
+
+```
+src/
+  ingest/
+    clean_text.py         # Conservative OCR cleaning
+    build_manifest.py     # Panel manifest from images + OCR CSV
+  embeddings/
+    embed_images.py       # OpenCLIP image → dense vectors
+    embed_sparse_text.py  # pinecone-sparse-english-v0 → sparse vectors
+  pinecone/
+    create_index.py       # Create document-schema Pinecone index
+    build_documents.py    # Join manifest + vectors → JSONL
+    upsert_panels.py      # Batch upsert to Pinecone with retry
+  search/
+    search_dense.py       # Dense image vector search
+    search_sparse.py      # Sparse text vector search
+    search_fts.py         # Full-text search (BM25)
+    search_combined.py    # All three signals + RRF merge
+    fusion.py             # Reciprocal Rank Fusion
+    router.py             # Route query to signal(s)
+notebooks/
+  query_comics_index.ipynb  # Interactive search playground
+data/
+  comics/
+    raw/                  # Downloaded COMICS files
+    processed/            # Manifest, vectors, and Pinecone documents
+config.yaml               # Model and field configuration
+```
+
+## Configuration
+
+`config.yaml` controls the embedding model, index name, namespace, and field names. If you change `dense_embedding.dimension`, update it before running `create_index.py`.
+
+Default dense model: `open_clip_ViT-B-32` (512-dim, cosine).
+
+## Data model
+
+Each Pinecone document = one comic panel:
+
+| Field | Type | Description |
+|---|---|---|
+| `_id` | string | `{comic_id}:{page_num}:{panel_num}` |
+| `image_dense` | dense vector | 512-dim OpenCLIP embedding |
+| `text_sparse` | sparse vector | pinecone-sparse-english-v0 on OCR text |
+| `ocr_text` | string (FTS) | Raw cleaned OCR text |
+| `search_text` | string (FTS) | Search-optimized text (= ocr_text for v1) |
+| `comic_id` | metadata | Book identifier |
+| `page_num` | metadata | Page number within book |
+| `panel_num` | metadata | Panel number on the page |
+| `is_ad_page` | metadata | True if page is an advertisement |

config.yaml

@@ -0,0 +1,28 @@
+pinecone:
+  index_name: comic-panels
+  namespace: comics-v1
+
+dense_embedding:
+  model_name: open_clip_ViT-B-32
+  field_name: image_dense
+  dimension: 512
+  metric: cosine
+
+sparse_embedding:
+  model_name: pinecone-sparse-english-v0
+  field_name: text_sparse
+  source_text_field: search_text
+
+fields:
+  dense_vector: image_dense
+  sparse_vector: text_sparse
+  fts_text:
+    - ocr_text
+    - search_text
+
+data:
+  comics_data_dir: data/comics
+  manifest_path: data/comics/processed/panels_manifest.parquet
+  dense_vectors_path: data/comics/processed/image_dense_vectors.parquet
+  sparse_vectors_path: data/comics/processed/text_sparse_vectors.parquet
+  pinecone_documents_path: data/comics/processed/pinecone_documents.jsonl

notebooks/query_comics_index.ipynb

@@ -0,0 +1,394 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a1b2c3d4-0001",
+   "metadata": {},
+   "source": ["# COMICS Panel Search — Query Playground\n", "\n", "Developer-facing notebook for validating search quality and comparing dense, sparse, FTS, and combined retrieval over the Pinecone COMICS index."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0002",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "from pathlib import Path\n",
+    "\n",
+    "from dotenv import load_dotenv\n",
+    "from pinecone import Pinecone\n",
+    "from PIL import Image\n",
+    "import pandas as pd\n",
+    "from IPython.display import display, Markdown\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "PROJECT_ROOT = Path.cwd().parent if Path.cwd().name == \"notebooks\" else Path.cwd()\n",
+    "sys.path.insert(0, str(PROJECT_ROOT))\n",
+    "\n",
+    "PINECONE_API_KEY = os.environ[\"PINECONE_API_KEY\"]\n",
+    "PINECONE_INDEX_NAME = os.environ.get(\"PINECONE_INDEX_NAME\", \"comic-panels\")\n",
+    "PINECONE_NAMESPACE = os.environ.get(\"PINECONE_NAMESPACE\", \"comics-v1\")\n",
+    "COMICS_DATA_DIR = Path(os.environ.get(\"COMICS_DATA_DIR\", \"data/comics\"))\n",
+    "\n",
+    "pc = Pinecone(api_key=PINECONE_API_KEY)\n",
+    "index = pc.preview.index(name=PINECONE_INDEX_NAME)\n",
+    "\n",
+    "print(\"Connected to:\", PINECONE_INDEX_NAME)\n",
+    "print(\"Namespace:   \", PINECONE_NAMESPACE)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0003",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.search.search_dense import search_dense, search_dense_with_phrase_filter\n",
+    "from src.search.search_sparse import search_sparse\n",
+    "from src.search.search_fts import search_fts\n",
+    "from src.search.search_combined import search_combined\n",
+    "from src.search.router import route_query"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0004",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.embeddings.embed_images import embed_text_query\n",
+    "from src.embeddings.embed_sparse_text import embed_sparse_query\n",
+    "\n",
+    "\n",
+    "def embed_query_dense(query: str) -> list[float]:\n",
+    "    \"\"\"Embed a text query using the text side of the OpenCLIP model.\"\"\"\n",
+    "    return embed_text_query(query)\n",
+    "\n",
+    "\n",
+    "def embed_query_sparse(query: str) -> dict | None:\n",
+    "    \"\"\"Embed a text query using pinecone-sparse-english-v0 in query mode.\"\"\"\n",
+    "    return embed_sparse_query(query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0005",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def display_results(results, max_items=10, image_width=300):\n",
+    "    for i, hit in enumerate(results[:max_items], start=1):\n",
+    "        fields = hit.get(\"fields\", hit)\n",
+    "\n",
+    "        panel_id = hit.get(\"_id\") or hit.get(\"panel_id\")\n",
+    "        score = hit.get(\"_score\") or hit.get(\"score\") or hit.get(\"rrf_score\")\n",
+    "        sources = hit.get(\"sources\", fields.get(\"sources\", []))\n",
+    "\n",
+    "        image_path = fields.get(\"image_path\") or hit.get(\"image_path\")\n",
+    "        ocr_text = fields.get(\"ocr_text\") or hit.get(\"ocr_text\", \"\")\n",
+    "        comic_id = fields.get(\"comic_id\") or hit.get(\"comic_id\")\n",
+    "        page_num = fields.get(\"page_num\") or hit.get(\"page_num\")\n",
+    "        panel_num = fields.get(\"panel_num\") or hit.get(\"panel_num\")\n",
+    "\n",
+    "        display(Markdown(f\"### {i}. `{panel_id}`\"))\n",
+    "        display(Markdown(\n",
+    "            f\"**Score:** `{score}`  \\n\"\n",
+    "            f\"**Sources:** `{sources}`  \\n\"\n",
+    "            f\"**Comic:** `{comic_id}`  \\n\"\n",
+    "            f\"**Page:** `{page_num}`  \\n\"\n",
+    "            f\"**Panel:** `{panel_num}`\"\n",
+    "        ))\n",
+    "\n",
+    "        if image_path and Path(image_path).exists():\n",
+    "            img = Image.open(image_path)\n",
+    "            display(img.resize((image_width, int(image_width * img.height / img.width))))\n",
+    "        else:\n",
+    "            display(Markdown(f\"`Image not found: {image_path}`\"))\n",
+    "\n",
+    "        if ocr_text:\n",
+    "            display(Markdown(f\"**OCR:** {ocr_text}\"))\n",
+    "\n",
+    "        display(Markdown(\"---\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1b2c3d4-0006",
+   "metadata": {},
+   "source": ["## Dense Search\n", "\n", "Semantic / visual search — best for descriptive scene queries."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0007",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"a detective in a dark alley\"\n",
+    "top_k = 10\n",
+    "\n",
+    "dense_vector = embed_query_dense(query)\n",
+    "\n",
+    "dense_response = search_dense(\n",
+    "    index=index,\n",
+    "    namespace=PINECONE_NAMESPACE,\n",
+    "    query_vector=dense_vector,\n",
+    "    top_k=top_k,\n",
+    "    filters={\"is_ad_page\": False},\n",
+    ")\n",
+    "\n",
+    "dense_hits = dense_response[\"result\"][\"hits\"]\n",
+    "display_results(dense_hits, max_items=top_k)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1b2c3d4-0008",
+   "metadata": {},
+   "source": ["## Sparse Search\n", "\n", "Keyword/lexical search — best for names, specific terms, and short phrases."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0009",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"masked villain laboratory\"\n",
+    "top_k = 10\n",
+    "\n",
+    "sparse_vector = embed_query_sparse(query)\n",
+    "\n",
+    "sparse_response = search_sparse(\n",
+    "    index=index,\n",
+    "    namespace=PINECONE_NAMESPACE,\n",
+    "    query_sparse=sparse_vector,\n",
+    "    top_k=top_k,\n",
+    "    filters={\"is_ad_page\": False},\n",
+    ")\n",
+    "\n",
+    "sparse_hits = sparse_response[\"result\"][\"hits\"]\n",
+    "display_results(sparse_hits, max_items=top_k)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1b2c3d4-0010",
+   "metadata": {},
+   "source": ["## Full-Text Search\n", "\n", "BM25 over OCR/dialogue — best for exact phrases, quoted dialogue, sound effects, Boolean operators."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0011",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = '\"secret formula\"'\n",
+    "top_k = 10\n",
+    "\n",
+    "fts_response = search_fts(\n",
+    "    index=index,\n",
+    "    namespace=PINECONE_NAMESPACE,\n",
+    "    query=query,\n",
+    "    top_k=top_k,\n",
+    "    filters={\"is_ad_page\": False},\n",
+    ")\n",
+    "\n",
+    "fts_hits = fts_response[\"result\"][\"hits\"]\n",
+    "display_results(fts_hits, max_items=top_k)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1b2c3d4-0012",
+   "metadata": {},
+   "source": ["## Combined Search (RRF)\n", "\n", "Runs dense + sparse + FTS (based on router) and merges results with Reciprocal Rank Fusion."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0013",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"a laboratory panel containing formula\"\n",
+    "top_k = 10\n",
+    "\n",
+    "route = route_query(query)\n",
+    "print(\"Route:\", route)\n",
+    "\n",
+    "dense_vector = embed_query_dense(query) if route[\"dense\"] else None\n",
+    "sparse_vector = embed_query_sparse(query) if route[\"sparse\"] else None\n",
+    "\n",
+    "combined_hits = search_combined(\n",
+    "    index=index,\n",
+    "    namespace=PINECONE_NAMESPACE,\n",
+    "    query_text=query,\n",
+    "    dense_query_vector=dense_vector,\n",
+    "    sparse_query_vector=sparse_vector,\n",
+    "    top_k=top_k,\n",
+    "    filters={\"is_ad_page\": False},\n",
+    "    run_fts=route[\"fts\"],\n",
+    ")\n",
+    "\n",
+    "display_results(combined_hits, max_items=top_k)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1b2c3d4-0014",
+   "metadata": {},
+   "source": ["## Compare Modes Side-by-Side\n", "\n", "Run the same query across all three signals and compare results in a DataFrame."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0015",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = \"secret formula laboratory\"\n",
+    "top_k = 10\n",
+    "\n",
+    "dense_vector = embed_query_dense(query)\n",
+    "sparse_vector = embed_query_sparse(query)\n",
+    "\n",
+    "dense_hits = search_dense(\n",
+    "    index=index, namespace=PINECONE_NAMESPACE,\n",
+    "    query_vector=dense_vector, top_k=top_k,\n",
+    "    filters={\"is_ad_page\": False},\n",
+    ")[\"result\"][\"hits\"]\n",
+    "\n",
+    "sparse_hits = search_sparse(\n",
+    "    index=index, namespace=PINECONE_NAMESPACE,\n",
+    "    query_sparse=sparse_vector, top_k=top_k,\n",
+    "    filters={\"is_ad_page\": False},\n",
+    ")[\"result\"][\"hits\"]\n",
+    "\n",
+    "fts_hits = search_fts(\n",
+    "    index=index, namespace=PINECONE_NAMESPACE,\n",
+    "    query=query, top_k=top_k,\n",
+    "    filters={\"is_ad_page\": False},\n",
+    ")[\"result\"][\"hits\"]\n",
+    "\n",
+    "\n",
+    "def hits_to_rows(mode, hits):\n",
+    "    rows = []\n",
+    "    for rank, hit in enumerate(hits, start=1):\n",
+    "        fields = hit.get(\"fields\", {})\n",
+    "        rows.append({\n",
+    "            \"mode\": mode,\n",
+    "            \"rank\": rank,\n",
+    "            \"panel_id\": hit.get(\"_id\"),\n",
+    "            \"score\": hit.get(\"_score\"),\n",
+    "            \"ocr_text\": str(fields.get(\"ocr_text\", \"\"))[:60],\n",
+    "            \"comic_id\": fields.get(\"comic_id\"),\n",
+    "            \"page_num\": fields.get(\"page_num\"),\n",
+    "            \"panel_num\": fields.get(\"panel_num\"),\n",
+    "        })\n",
+    "    return rows\n",
+    "\n",
+    "\n",
+    "df = pd.DataFrame(\n",
+    "    hits_to_rows(\"dense\", dense_hits)\n",
+    "    + hits_to_rows(\"sparse\", sparse_hits)\n",
+    "    + hits_to_rows(\"fts\", fts_hits)\n",
+    ")\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1b2c3d4-0016",
+   "metadata": {},
+   "source": ["## Metadata Filters\n", "\n", "Examples of filtering by comic, page range, or ad-page exclusion."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0017",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Exclude ad pages (applied by default in all cells above)\n",
+    "filters = {\"is_ad_page\": False}\n",
+    "\n",
+    "# Filter to a specific comic\n",
+    "# filters = {\"comic_id\": \"some_comic_id\", \"is_ad_page\": False}\n",
+    "\n",
+    "# Filter to a page range\n",
+    "# filters = {\"page_num\": {\"$gte\": 5, \"$lte\": 20}, \"is_ad_page\": False}\n",
+    "\n",
+    "query = \"hero battles villain\"\n",
+    "dense_vector = embed_query_dense(query)\n",
+    "response = search_dense(index=index, namespace=PINECONE_NAMESPACE,\n",
+    "                        query_vector=dense_vector, top_k=5, filters=filters)\n",
+    "display_results(response[\"result\"][\"hits\"], max_items=5)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a1b2c3d4-0018",
+   "metadata": {},
+   "source": ["## Manual Evaluation\n", "\n", "Run a battery of test queries across all modes and inspect results."]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1b2c3d4-0019",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_queries = [\n",
+    "    '\"secret formula\"',\n",
+    "    \"BANG\",\n",
+    "    \"a detective in a dark alley\",\n",
+    "    \"a superhero flying through the sky\",\n",
+    "    \"masked villain laboratory\",\n",
+    "    \"a laboratory panel containing formula\",\n",
+    "]\n",
+    "\n",
+    "for q in test_queries:\n",
+    "    display(Markdown(f\"---\\n### Query: `{q}`\"))\n",
+    "    route = route_query(q)\n",
+    "    display(Markdown(f\"Route: `{route}`\"))\n",
+    "\n",
+    "    dense_vector = embed_query_dense(q) if route[\"dense\"] else None\n",
+    "    sparse_vector = embed_query_sparse(q) if route[\"sparse\"] else None\n",
+    "\n",
+    "    hits = search_combined(\n",
+    "        index=index,\n",
+    "        namespace=PINECONE_NAMESPACE,\n",
+    "        query_text=q,\n",
+    "        dense_query_vector=dense_vector,\n",
+    "        sparse_query_vector=sparse_vector,\n",
+    "        top_k=5,\n",
+    "        filters={\"is_ad_page\": False},\n",
+    "        run_fts=route[\"fts\"],\n",
+    "    )\n",
+    "\n",
+    "    display_results(hits, max_items=5)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "multi-modal",
+   "language": "python",
+   "name": "multi-modal"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.13.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -1,11 +1,13 @@
 pinecone>=6.0.0
-anthropic>=0.40.0
-voyageai>=0.3.0
+open-clip-torch>=2.20.0
+torch>=2.0.0
 pillow>=10.0.0
 requests>=2.31.0
 python-dotenv>=1.0.0
 pandas>=2.0.0
+pyarrow>=14.0.0
 tqdm>=4.66.0
 ipython>=8.0.0
 matplotlib>=3.7.0
 jupyter>=1.0.0
+pyyaml>=6.0

src/embeddings/embed_images.py

@@ -0,0 +1,72 @@
+"""Generate dense image embeddings for each panel using OpenCLIP."""
+
+from pathlib import Path
+import os
+import pandas as pd
+from PIL import Image
+from tqdm import tqdm
+import torch
+import open_clip
+
+
+MANIFEST_PATH = Path("data/comics/processed/panels_manifest.parquet")
+OUTPUT_PATH = Path("data/comics/processed/image_dense_vectors.parquet")
+FAILURES_PATH = Path("data/comics/processed/image_embedding_failures.csv")
+
+_MODEL_NAME = os.environ.get("IMAGE_EMBED_MODEL", "open_clip_ViT-B-32")
+_ARCH = _MODEL_NAME.replace("open_clip_", "")
+_DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+_model, _, _preprocess = open_clip.create_model_and_transforms(_ARCH, pretrained="openai")
+_model.eval()
+_model = _model.to(_DEVICE)
+_tokenizer = open_clip.get_tokenizer(_ARCH)
+
+
+def embed_image(image: Image.Image) -> list[float]:
+    tensor = _preprocess(image).unsqueeze(0).to(_DEVICE)
+    with torch.no_grad():
+        features = _model.encode_image(tensor)
+        features = features / features.norm(dim=-1, keepdim=True)
+    return features[0].cpu().tolist()
+
+
+def embed_text_query(text: str) -> list[float]:
+    """Embed a text query into the same vector space as panel images."""
+    tokens = _tokenizer([text]).to(_DEVICE)
+    with torch.no_grad():
+        features = _model.encode_text(tokens)
+        features = features / features.norm(dim=-1, keepdim=True)
+    return features[0].cpu().tolist()
+
+
+def main():
+    manifest = pd.read_parquet(MANIFEST_PATH)
+    print(f"Embedding {len(manifest)} panels with {_ARCH} on {_DEVICE}")
+
+    rows = []
+    failures = []
+
+    for row in tqdm(manifest.itertuples(index=False), total=len(manifest)):
+        try:
+            image = Image.open(row.image_path).convert("RGB")
+            vector = embed_image(image)
+            rows.append({"panel_id": row.panel_id, "image_dense": vector})
+        except Exception as exc:
+            failures.append({
+                "panel_id": row.panel_id,
+                "image_path": row.image_path,
+                "error": str(exc),
+            })
+
+    OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    pd.DataFrame(rows).to_parquet(OUTPUT_PATH, index=False)
+    print(f"Saved {len(rows)} vectors → {OUTPUT_PATH}")
+
+    if failures:
+        pd.DataFrame(failures).to_csv(FAILURES_PATH, index=False)
+        print(f"Failed: {len(failures)} panels → {FAILURES_PATH}")
+
+
+if __name__ == "__main__":
+    main()

src/embeddings/embed_sparse_text.py

@@ -0,0 +1,74 @@
+"""Generate sparse text embeddings for each panel using Pinecone inference."""
+
+from pathlib import Path
+import os
+import pandas as pd
+from tqdm import tqdm
+from pinecone import Pinecone
+
+
+MANIFEST_PATH = Path("data/comics/processed/panels_manifest.parquet")
+OUTPUT_PATH = Path("data/comics/processed/text_sparse_vectors.parquet")
+
+SPARSE_MODEL = os.environ.get("SPARSE_EMBED_MODEL", "pinecone-sparse-english-v0")
+BATCH_SIZE = 96  # Stay within Pinecone inference batch limits
+
+_pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+
+
+def embed_sparse_batch(texts: list[str], input_type: str = "passage") -> list[dict | None]:
+    """Embed a batch of texts using pinecone-sparse-english-v0."""
+    non_empty = [(i, t) for i, t in enumerate(texts) if t.strip()]
+    results: list[dict | None] = [None] * len(texts)
+
+    if not non_empty:
+        return results
+
+    indices, batch_texts = zip(*non_empty)
+    response = _pc.inference.embed(
+        model=SPARSE_MODEL,
+        inputs=list(batch_texts),
+        parameters={"input_type": input_type},
+    )
+
+    for idx, embedding in zip(indices, response):
+        if embedding.sparse_indices and embedding.sparse_values:
+            results[idx] = {
+                "indices": embedding.sparse_indices,
+                "values": embedding.sparse_values,
+            }
+
+    return results
+
+
+def embed_sparse_query(text: str) -> dict | None:
+    """Embed a single query string for search time."""
+    if not text.strip():
+        return None
+    results = embed_sparse_batch([text], input_type="query")
+    return results[0]
+
+
+def main():
+    manifest = pd.read_parquet(MANIFEST_PATH)
+    print(f"Sparse-embedding {len(manifest)} panels with {SPARSE_MODEL}")
+
+    texts = manifest["search_text"].fillna("").tolist()
+    panel_ids = manifest["panel_id"].tolist()
+
+    all_sparse: list[dict | None] = []
+    for start in tqdm(range(0, len(texts), BATCH_SIZE)):
+        batch = texts[start : start + BATCH_SIZE]
+        all_sparse.extend(embed_sparse_batch(batch))
+
+    rows = [{"panel_id": pid, "text_sparse": sparse} for pid, sparse in zip(panel_ids, all_sparse)]
+
+    OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    pd.DataFrame(rows).to_parquet(OUTPUT_PATH, index=False)
+
+    filled = sum(1 for r in rows if r["text_sparse"] is not None)
+    print(f"Saved {len(rows)} rows ({filled} with sparse vectors) → {OUTPUT_PATH}")
+
+
+if __name__ == "__main__":
+    main()

src/ingest/build_manifest.py

@@ -0,0 +1,223 @@
+"""Build a panel manifest from COMICS raw panel images and OCR CSV."""
+
+from pathlib import Path
+import hashlib
+import re
+import pandas as pd
+
+from src.ingest.clean_text import clean_ocr, build_search_text
+
+
+COMICS_DATA_DIR = Path("data/comics")
+RAW_DIR = COMICS_DATA_DIR / "raw"
+PROCESSED_DIR = COMICS_DATA_DIR / "processed"
+PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def stable_hash(value: str) -> str:
+    return hashlib.sha1(value.encode("utf-8")).hexdigest()[:16]
+
+
+def parse_panel_path(image_path: Path) -> dict:
+    """
+    Infer comic_id, page_num, panel_num from an image path.
+
+    COMICS raw_panel_images layout (from dataset inspection):
+      raw_panel_images/{book_id}/{page_num:04d}_{panel_num:04d}.jpg
+
+    Returns partial metadata; fields are None if pattern doesn't match.
+    """
+    parts = image_path.parts
+    # Expect: ...raw_panel_images/<book_id>/<filename>
+    try:
+        panel_dir_idx = next(i for i, p in enumerate(parts) if p == "raw_panel_images")
+        book_id = parts[panel_dir_idx + 1] if panel_dir_idx + 1 < len(parts) else None
+    except StopIteration:
+        book_id = None
+
+    page_num, panel_num = None, None
+    m = re.match(r"^(\d+)_(\d+)\.(jpg|jpeg|png)$", image_path.name, re.IGNORECASE)
+    if m:
+        page_num = int(m.group(1))
+        panel_num = int(m.group(2))
+
+    comic_id = book_id
+    page_id = f"{book_id}_p{page_num:04d}" if book_id and page_num is not None else None
+    panel_id = (
+        f"{comic_id}:{page_num}:{panel_num}"
+        if comic_id and page_num is not None and panel_num is not None
+        else f"comics:{stable_hash(str(image_path))}"
+    )
+
+    return {
+        "panel_id": panel_id,
+        "comic_id": comic_id,
+        "book_id": book_id,
+        "page_id": page_id,
+        "page_num": page_num,
+        "panel_num": panel_num,
+    }
+
+
+def load_ad_pages(ad_pages_path: Path) -> set[str]:
+    """Load advertisement page identifiers from predadpages.txt."""
+    if not ad_pages_path.exists():
+        print(f"  Warning: {ad_pages_path} not found; no ad-page flags will be set")
+        return set()
+    lines = ad_pages_path.read_text().splitlines()
+    return {line.strip() for line in lines if line.strip()}
+
+
+def build_ocr_lookup(ocr_df: pd.DataFrame) -> dict[str, str]:
+    """
+    Build a mapping from panel_id keys to OCR text.
+
+    After inspecting the OCR CSV columns, this function maps panel image
+    paths to their OCR text. The COMICS OCR CSV format uses columns:
+      comic_no, page_no, panel_no, ocr  (or similar)
+
+    Prints column info on first run so you can verify the mapping is correct.
+    """
+    print("\nOCR CSV columns:", list(ocr_df.columns))
+    print("OCR CSV shape:", ocr_df.shape)
+    print("OCR CSV sample rows:")
+    print(ocr_df.head(3).to_string())
+    print()
+
+    # Best-effort mapping based on common COMICS dataset column names.
+    # Adjust the column names below after inspecting the output above.
+    lookup: dict[str, str] = {}
+
+    # Try to find comic/book, page, panel, and text columns
+    col_map = {c.lower(): c for c in ocr_df.columns}
+
+    book_col = next((col_map[k] for k in ["comic_no", "book_id", "comic_id", "book_no"] if k in col_map), None)
+    page_col = next((col_map[k] for k in ["page_no", "page_num", "page_id", "page"] if k in col_map), None)
+    panel_col = next((col_map[k] for k in ["panel_no", "panel_num", "panel_id", "panel"] if k in col_map), None)
+    text_col = next((col_map[k] for k in ["ocr", "text", "ocr_text", "dialogue"] if k in col_map), None)
+
+    if not all([book_col, page_col, panel_col, text_col]):
+        print(f"  Warning: Could not auto-map all OCR columns.")
+        print(f"  book_col={book_col}, page_col={page_col}, panel_col={panel_col}, text_col={text_col}")
+        print("  Update build_ocr_lookup() with the correct column names after inspection.")
+        return lookup
+
+    print(f"  Using columns: book={book_col}, page={page_col}, panel={panel_col}, text={text_col}")
+
+    for row in ocr_df.itertuples(index=False):
+        book = str(getattr(row, book_col)).strip()
+        try:
+            page = int(getattr(row, page_col))
+            panel = int(getattr(row, panel_col))
+        except (ValueError, TypeError):
+            continue
+        text = str(getattr(row, text_col)) if getattr(row, text_col, None) is not None else ""
+        key = f"{book}:{page}:{panel}"
+        lookup[key] = text
+
+    print(f"  OCR lookup entries: {len(lookup)}")
+    return lookup
+
+
+def main():
+    panel_dir = RAW_DIR / "raw_panel_images"
+    ocr_path = RAW_DIR / "COMICS_ocr_file.csv"
+    ad_pages_path = RAW_DIR / "predadpages.txt"
+
+    if not panel_dir.exists():
+        raise FileNotFoundError(
+            f"Panel image directory not found: {panel_dir}\n"
+            "Run the download step first:\n"
+            "  mkdir -p data/comics/raw && cd data/comics/raw\n"
+            "  wget -c https://obj.umiacs.umd.edu/comics/raw_panel_images.tar.gz\n"
+            "  tar -xzf raw_panel_images.tar.gz"
+        )
+
+    panel_paths = sorted(
+        p for p in panel_dir.rglob("*")
+        if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png"}
+    )
+    print(f"Found {len(panel_paths)} panel images")
+
+    ocr_lookup: dict[str, str] = {}
+    if ocr_path.exists():
+        ocr_df = pd.read_csv(ocr_path)
+        ocr_lookup = build_ocr_lookup(ocr_df)
+    else:
+        print(f"Warning: OCR file not found at {ocr_path}; all panels will have empty OCR")
+
+    ad_pages = load_ad_pages(ad_pages_path)
+
+    rows = []
+    unmatched_images = []
+
+    for image_path in panel_paths:
+        meta = parse_panel_path(image_path)
+        panel_id = meta["panel_id"]
+
+        ocr_text_raw = ocr_lookup.get(panel_id, "")
+        if not ocr_text_raw and meta["comic_id"] and meta["page_num"] is not None:
+            # Try zero-padded fallback keys
+            for fmt in [f"{meta['comic_id']}:{meta['page_num']:04d}:{meta['panel_num']:04d}"]:
+                if fmt in ocr_lookup:
+                    ocr_text_raw = ocr_lookup[fmt]
+                    break
+
+        if not ocr_text_raw:
+            unmatched_images.append(str(image_path))
+
+        ocr_text_clean = clean_ocr(ocr_text_raw)
+        search_text = build_search_text(ocr_text_clean)
+
+        # Check if this is an ad page
+        is_ad = (
+            meta["page_id"] in ad_pages
+            or str(image_path) in ad_pages
+            or image_path.name in ad_pages
+        )
+
+        rows.append({
+            "panel_id": panel_id,
+            "image_path": str(image_path),
+            "ocr_text_raw": ocr_text_raw,
+            "ocr_text_clean": ocr_text_clean,
+            "search_text": search_text,
+            "comic_id": meta["comic_id"],
+            "book_id": meta["book_id"],
+            "page_id": meta["page_id"],
+            "page_num": meta["page_num"],
+            "panel_num": meta["panel_num"],
+            "is_ad_page": is_ad,
+            "source": "COMICS",
+        })
+
+    manifest = pd.DataFrame(rows)
+
+    # Validate: all panel_ids must be unique
+    dupes = manifest["panel_id"].duplicated().sum()
+    if dupes:
+        print(f"Warning: {dupes} duplicate panel_ids found; using path-hash fallback for those rows")
+        dup_mask = manifest["panel_id"].duplicated(keep=False)
+        manifest.loc[dup_mask, "panel_id"] = manifest.loc[dup_mask, "image_path"].apply(
+            lambda p: f"comics:{stable_hash(p)}"
+        )
+
+    manifest.to_parquet(PROCESSED_DIR / "panels_manifest.parquet", index=False)
+    manifest.to_json(PROCESSED_DIR / "panels_manifest.jsonl", orient="records", lines=True)
+
+    print(f"\nManifest written: {len(manifest)} panels")
+    print(f"  With OCR text: {(manifest['ocr_text_clean'] != '').sum()}")
+    print(f"  Empty OCR:     {(manifest['ocr_text_clean'] == '').sum()}")
+    print(f"  Ad pages:      {manifest['is_ad_page'].sum()}")
+    print(f"  Unmatched OCR: {len(unmatched_images)}")
+
+    if unmatched_images[:5]:
+        print(f"  Sample unmatched: {unmatched_images[:5]}")
+
+    unmatched_ocr_keys = set(ocr_lookup) - {r["panel_id"] for r in rows}
+    if unmatched_ocr_keys:
+        print(f"  OCR rows with no matching image: {len(unmatched_ocr_keys)}")
+
+
+if __name__ == "__main__":
+    main()

src/pinecone/build_documents.py

@@ -0,0 +1,67 @@
+"""Join manifest, dense vectors, and sparse vectors into Pinecone documents."""
+
+from pathlib import Path
+import json
+import pandas as pd
+
+
+PROCESSED_DIR = Path("data/comics/processed")
+MANIFEST_PATH = PROCESSED_DIR / "panels_manifest.parquet"
+DENSE_PATH = PROCESSED_DIR / "image_dense_vectors.parquet"
+SPARSE_PATH = PROCESSED_DIR / "text_sparse_vectors.parquet"
+OUTPUT_PATH = PROCESSED_DIR / "pinecone_documents.jsonl"
+
+
+def valid_sparse(sparse) -> bool:
+    if not isinstance(sparse, dict):
+        return False
+    return bool(sparse.get("indices")) and bool(sparse.get("values"))
+
+
+def main():
+    manifest = pd.read_parquet(MANIFEST_PATH)
+    dense = pd.read_parquet(DENSE_PATH)
+    sparse = pd.read_parquet(SPARSE_PATH)
+
+    # Inner join on dense — only panels that were successfully embedded
+    df = manifest.merge(dense, on="panel_id", how="inner")
+    # Left join on sparse — panels without text keep their document slot
+    df = df.merge(sparse, on="panel_id", how="left")
+
+    dropped = len(manifest) - len(df)
+    if dropped:
+        print(f"Warning: {dropped} panels dropped (missing dense vector)")
+
+    count = 0
+    with open(OUTPUT_PATH, "w") as f:
+        for row in df.itertuples(index=False):
+            doc: dict = {
+                "_id": row.panel_id,
+                "image_dense": row.image_dense,
+                "ocr_text": row.ocr_text_clean or "",
+                "search_text": row.search_text or "",
+                "comic_id": row.comic_id,
+                "book_id": row.book_id,
+                "page_id": row.page_id,
+                "page_num": int(row.page_num) if row.page_num is not None and not pd.isna(row.page_num) else None,
+                "panel_num": int(row.panel_num) if row.panel_num is not None and not pd.isna(row.panel_num) else None,
+                "image_path": row.image_path,
+                "source": "COMICS",
+                "is_ad_page": bool(row.is_ad_page),
+            }
+
+            sparse_vec = getattr(row, "text_sparse", None)
+            if valid_sparse(sparse_vec):
+                doc["text_sparse"] = sparse_vec
+
+            f.write(json.dumps(doc) + "\n")
+            count += 1
+
+    print(f"Wrote {count} documents → {OUTPUT_PATH}")
+    with_sparse = sum(1 for row in df.itertuples(index=False) if valid_sparse(getattr(row, "text_sparse", None)))
+    print(f"  With sparse vector: {with_sparse}")
+    print(f"  Dense only:         {count - with_sparse}")
+
+
+if __name__ == "__main__":
+    main()

src/pinecone/create_index.py

@@ -0,0 +1,55 @@
+"""Create the Pinecone document-schema index for comic panel search."""
+
+import os
+import time
+from pinecone import Pinecone
+from pinecone.preview import SchemaBuilder
+
+
+INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "comic-panels")
+IMAGE_EMBED_DIM = int(os.environ.get("IMAGE_EMBED_DIM", "512"))
+IMAGE_EMBED_METRIC = os.environ.get("IMAGE_EMBED_METRIC", "cosine")
+
+
+def create_index(pc: Pinecone) -> None:
+    existing = [idx.name for idx in pc.list_indexes()]
+    if INDEX_NAME in existing:
+        print(f"Index '{INDEX_NAME}' already exists — skipping creation")
+        return
+
+    print(f"Creating index '{INDEX_NAME}' (dim={IMAGE_EMBED_DIM}, metric={IMAGE_EMBED_METRIC})")
+
+    schema = (
+        SchemaBuilder()
+        .add_dense_vector_field(
+            "image_dense",
+            dimension=IMAGE_EMBED_DIM,
+            metric=IMAGE_EMBED_METRIC,
+        )
+        .add_sparse_vector_field("text_sparse")
+        .add_string_field("ocr_text", full_text_search={"language": "en"})
+        .add_string_field("search_text", full_text_search={"language": "en"})
+        .build()
+    )
+
+    pc.preview.indexes.create(name=INDEX_NAME, schema=schema)
+
+    # Wait for the index to be ready
+    for _ in range(60):
+        status = pc.describe_index(INDEX_NAME).status
+        if status.get("ready"):
+            print(f"Index '{INDEX_NAME}' is ready")
+            return
+        print("  waiting for index to be ready...")
+        time.sleep(5)
+
+    raise TimeoutError(f"Index '{INDEX_NAME}' did not become ready within 5 minutes")
+
+
+def main():
+    pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+    create_index(pc)
+
+
+if __name__ == "__main__":
+    main()

src/pinecone/upsert_panels.py

@@ -0,0 +1,84 @@
+"""Upsert comic panel documents into Pinecone."""
+
+import json
+import os
+import time
+from pathlib import Path
+
+from pinecone import Pinecone
+
+
+INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "comic-panels")
+NAMESPACE = os.environ.get("PINECONE_NAMESPACE", "comics-v1")
+DOCS_PATH = Path("data/comics/processed/pinecone_documents.jsonl")
+BATCH_SIZE = 100
+
+
+def batched_jsonl(path: Path, batch_size: int = BATCH_SIZE):
+    batch = []
+    with open(path) as f:
+        for line in f:
+            batch.append(json.loads(line))
+            if len(batch) >= batch_size:
+                yield batch
+                batch = []
+    if batch:
+        yield batch
+
+
+def upsert_with_retry(index, batch: list, retries: int = 5):
+    for attempt in range(retries):
+        try:
+            return index.documents.upsert(namespace=NAMESPACE, documents=batch)
+        except Exception as exc:
+            if attempt == retries - 1:
+                raise
+            wait = 2 ** attempt
+            print(f"  Upsert attempt {attempt + 1} failed: {exc}; retrying in {wait}s")
+            time.sleep(wait)
+
+
+def validate_search(index) -> None:
+    """Quick sanity check: run a dense search after upsert."""
+    import random
+    dummy_vector = [random.gauss(0, 1) for _ in range(int(os.environ.get("IMAGE_EMBED_DIM", "512")))]
+    norm = sum(x ** 2 for x in dummy_vector) ** 0.5
+    dummy_vector = [x / norm for x in dummy_vector]
+
+    results = index.documents.search(
+        namespace=NAMESPACE,
+        top_k=3,
+        score_by=[{"type": "dense_vector", "field": "image_dense", "values": dummy_vector}],
+        include_fields=["panel_id", "comic_id", "page_num"],
+    )
+    hits = results.get("result", {}).get("hits", [])
+    print(f"Validation search returned {len(hits)} hits")
+    for h in hits:
+        print(f"  {h.get('_id')} score={h.get('_score'):.4f}")
+
+
+def main():
+    pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+    index = pc.preview.index(name=INDEX_NAME)
+
+    total = 0
+    failed_batches = 0
+
+    for batch_num, batch in enumerate(batched_jsonl(DOCS_PATH), start=1):
+        try:
+            upsert_with_retry(index, batch)
+            total += len(batch)
+            if batch_num % 10 == 0:
+                print(f"  Upserted {total} documents...")
+        except Exception as exc:
+            failed_batches += 1
+            print(f"  Batch {batch_num} failed permanently: {exc}")
+
+    print(f"\nUpsert complete: {total} documents, {failed_batches} failed batches")
+
+    if total > 0:
+        validate_search(index)
+
+
+if __name__ == "__main__":
+    main()

src/search/fusion.py

@@ -0,0 +1,42 @@
+"""Reciprocal Rank Fusion for merging results from multiple search signals."""
+
+
+def rrf_merge(result_groups: list[tuple[str, list]], k: int = 60) -> list[dict]:
+    """
+    Merge ranked result lists using Reciprocal Rank Fusion.
+
+    Args:
+        result_groups: list of (source_name, hits) tuples where each hit is a
+                       dict with at least '_id'.
+        k: RRF smoothing constant (default 60 per original paper).
+
+    Returns:
+        List of hit dicts sorted by descending rrf_score, with a 'sources' key
+        added to each hit indicating which signals returned it.
+    """
+    scores: dict[str, float] = {}
+    payloads: dict[str, dict] = {}
+
+    for source_name, hits in result_groups:
+        for rank, hit in enumerate(hits, start=1):
+            panel_id = hit.get("_id") or hit.get("panel_id")
+            if panel_id is None:
+                continue
+
+            scores[panel_id] = scores.get(panel_id, 0.0) + 1.0 / (k + rank)
+
+            if panel_id not in payloads:
+                payloads[panel_id] = dict(hit)
+                payloads[panel_id]["sources"] = []
+
+            payloads[panel_id]["sources"].append(source_name)
+
+    ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+
+    output = []
+    for panel_id, score in ranked:
+        item = payloads[panel_id]
+        item["rrf_score"] = score
+        output.append(item)
+
+    return output

src/search/router.py

@@ -0,0 +1,36 @@
+"""Route a query to the appropriate search signals."""
+
+
+_EXACT_INTENT_PHRASES = {
+    "says", "where they say", "exact", "phrase", "quote",
+    "dialogue", "sound effect", "contains",
+}
+
+
+def route_query(query: str) -> dict[str, bool]:
+    """
+    Decide which search signals to run for a given query string.
+
+    Returns a dict with keys: dense, sparse, fts — each True or False.
+
+    Routing logic:
+      - dense is always True (visual/semantic search runs for every query)
+      - sparse is True for any non-empty query (keyword overlap)
+      - fts is True when the query has quoted phrases, Boolean operators,
+        all-caps sound effects, or explicit exact-match intent language
+    """
+    q = query.strip()
+    if not q:
+        return {"dense": False, "sparse": False, "fts": False}
+
+    tokens = q.split()
+    has_quotes = '"' in q or "'" in q
+    has_boolean = any(t.upper() in {"AND", "OR", "NOT"} for t in tokens)
+    has_all_caps = any(t.isupper() and len(t) >= 3 for t in tokens)
+    has_exact_intent = any(phrase in q.lower() for phrase in _EXACT_INTENT_PHRASES)
+
+    return {
+        "dense": True,
+        "sparse": True,
+        "fts": has_quotes or has_boolean or has_all_caps or has_exact_intent,
+    }

src/search/search_combined.py

@@ -0,0 +1,82 @@
+"""Combined search: run dense, sparse, and FTS in parallel and merge with RRF."""
+
+import os
+from pinecone import Pinecone
+
+from src.search.search_dense import search_dense
+from src.search.search_sparse import search_sparse
+from src.search.search_fts import search_fts
+from src.search.fusion import rrf_merge
+
+
+def search_combined(
+    index,
+    namespace: str,
+    query_text: str,
+    dense_query_vector: list[float] | None = None,
+    sparse_query_vector: dict | None = None,
+    top_k: int = 20,
+    filters: dict | None = None,
+    run_fts: bool = True,
+) -> list[dict]:
+    result_groups = []
+
+    if dense_query_vector is not None:
+        dense_response = search_dense(index, namespace, dense_query_vector, top_k=top_k, filters=filters)
+        result_groups.append(("dense", dense_response.get("result", {}).get("hits", [])))
+
+    if sparse_query_vector is not None:
+        sparse_response = search_sparse(index, namespace, sparse_query_vector, top_k=top_k, filters=filters)
+        result_groups.append(("sparse", sparse_response.get("result", {}).get("hits", [])))
+
+    if run_fts and query_text.strip():
+        fts_response = search_fts(index, namespace, query_text, top_k=top_k, filters=filters)
+        result_groups.append(("fts", fts_response.get("result", {}).get("hits", [])))
+
+    if not result_groups:
+        return []
+
+    return rrf_merge(result_groups)[:top_k]
+
+
+def main():
+    import argparse
+    from src.search.router import route_query
+    from src.embeddings.embed_images import embed_text_query
+    from src.embeddings.embed_sparse_text import embed_sparse_query
+
+    parser = argparse.ArgumentParser(description="Combined dense+sparse+FTS search")
+    parser.add_argument("--query", required=True)
+    parser.add_argument("--top-k", type=int, default=10)
+    parser.add_argument("--no-ads", action="store_true", default=True)
+    args = parser.parse_args()
+
+    pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+    index = pc.preview.index(name=os.environ.get("PINECONE_INDEX_NAME", "comic-panels"))
+    namespace = os.environ.get("PINECONE_NAMESPACE", "comics-v1")
+
+    route = route_query(args.query)
+    dense_vector = embed_text_query(args.query) if route["dense"] else None
+    sparse_vector = embed_sparse_query(args.query) if route["sparse"] else None
+    filters = {"is_ad_page": False} if args.no_ads else {}
+
+    hits = search_combined(
+        index=index,
+        namespace=namespace,
+        query_text=args.query,
+        dense_query_vector=dense_vector,
+        sparse_query_vector=sparse_vector,
+        top_k=args.top_k,
+        filters=filters,
+        run_fts=route["fts"],
+    )
+
+    print(f"Query: {args.query!r}  route={route}  ({len(hits)} hits)")
+    for i, hit in enumerate(hits, 1):
+        fields = hit.get("fields", hit)
+        sources = hit.get("sources", [])
+        print(f"  {i}. [rrf={hit.get('rrf_score', 0):.4f} src={sources}] {hit.get('_id')}  |  {fields.get('ocr_text', '')[:80]}")
+
+
+if __name__ == "__main__":
+    main()

src/search/search_dense.py

@@ -0,0 +1,68 @@
+"""Dense vector search over panel image embeddings."""
+
+import os
+from pinecone import Pinecone
+
+
+_INCLUDE_FIELDS = [
+    "ocr_text", "search_text", "comic_id", "book_id",
+    "page_id", "page_num", "panel_num", "image_path", "source", "is_ad_page",
+]
+
+
+def search_dense(index, namespace: str, query_vector: list[float], top_k: int = 20, filters: dict | None = None) -> dict:
+    return index.documents.search(
+        namespace=namespace,
+        top_k=top_k,
+        score_by=[{"type": "dense_vector", "field": "image_dense", "values": query_vector}],
+        filter=filters or {},
+        include_fields=_INCLUDE_FIELDS,
+    )
+
+
+def search_dense_with_phrase_filter(
+    index,
+    namespace: str,
+    query_vector: list[float],
+    phrase: str,
+    top_k: int = 20,
+    filters: dict | None = None,
+) -> dict:
+    merged_filter = dict(filters) if filters else {}
+    merged_filter["search_text"] = {"$match_phrase": phrase}
+    return index.documents.search(
+        namespace=namespace,
+        top_k=top_k,
+        score_by=[{"type": "dense_vector", "field": "image_dense", "values": query_vector}],
+        filter=merged_filter,
+        include_fields=_INCLUDE_FIELDS,
+    )
+
+
+def main():
+    import argparse
+    from src.embeddings.embed_images import embed_text_query
+
+    parser = argparse.ArgumentParser(description="Dense image search")
+    parser.add_argument("--query", required=True)
+    parser.add_argument("--top-k", type=int, default=10)
+    parser.add_argument("--no-ads", action="store_true", default=True)
+    args = parser.parse_args()
+
+    pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+    index = pc.preview.index(name=os.environ.get("PINECONE_INDEX_NAME", "comic-panels"))
+    namespace = os.environ.get("PINECONE_NAMESPACE", "comics-v1")
+
+    query_vector = embed_text_query(args.query)
+    filters = {"is_ad_page": False} if args.no_ads else {}
+    response = search_dense(index, namespace, query_vector, top_k=args.top_k, filters=filters)
+
+    hits = response.get("result", {}).get("hits", [])
+    print(f"Query: {args.query!r}  ({len(hits)} hits)")
+    for i, hit in enumerate(hits, 1):
+        fields = hit.get("fields", {})
+        print(f"  {i}. [{hit.get('_score', 0):.4f}] {hit['_id']}  |  {fields.get('ocr_text', '')[:80]}")
+
+
+if __name__ == "__main__":
+    main()

src/search/search_fts.py

@@ -0,0 +1,47 @@
+"""Full-text search over panel OCR/dialogue text."""
+
+import os
+from pinecone import Pinecone
+
+
+_INCLUDE_FIELDS = [
+    "ocr_text", "search_text", "comic_id", "book_id",
+    "page_id", "page_num", "panel_num", "image_path", "source", "is_ad_page",
+]
+
+
+def search_fts(index, namespace: str, query: str, top_k: int = 20, filters: dict | None = None) -> dict:
+    return index.documents.search(
+        namespace=namespace,
+        top_k=top_k,
+        score_by=[{"type": "text", "field": "search_text", "query": query}],
+        filter=filters or {},
+        include_fields=_INCLUDE_FIELDS,
+    )
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Full-text search over comic dialogue")
+    parser.add_argument("--query", required=True, help='e.g. "secret formula" or BANG or formula AND detective')
+    parser.add_argument("--top-k", type=int, default=10)
+    parser.add_argument("--no-ads", action="store_true", default=True)
+    args = parser.parse_args()
+
+    pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+    index = pc.preview.index(name=os.environ.get("PINECONE_INDEX_NAME", "comic-panels"))
+    namespace = os.environ.get("PINECONE_NAMESPACE", "comics-v1")
+
+    filters = {"is_ad_page": False} if args.no_ads else {}
+    response = search_fts(index, namespace, args.query, top_k=args.top_k, filters=filters)
+
+    hits = response.get("result", {}).get("hits", [])
+    print(f"Query: {args.query!r}  ({len(hits)} hits)")
+    for i, hit in enumerate(hits, 1):
+        fields = hit.get("fields", {})
+        print(f"  {i}. [{hit.get('_score', 0):.4f}] {hit['_id']}  |  {fields.get('ocr_text', '')[:80]}")
+
+
+if __name__ == "__main__":
+    main()

src/search/search_sparse.py

@@ -0,0 +1,53 @@
+"""Sparse vector search over panel OCR/dialogue text."""
+
+import os
+from pinecone import Pinecone
+
+
+_INCLUDE_FIELDS = [
+    "ocr_text", "search_text", "comic_id", "book_id",
+    "page_id", "page_num", "panel_num", "image_path", "source", "is_ad_page",
+]
+
+
+def search_sparse(index, namespace: str, query_sparse: dict, top_k: int = 20, filters: dict | None = None) -> dict:
+    return index.documents.search(
+        namespace=namespace,
+        top_k=top_k,
+        score_by=[{"type": "sparse_vector", "field": "text_sparse", "sparse_values": query_sparse}],
+        filter=filters or {},
+        include_fields=_INCLUDE_FIELDS,
+    )
+
+
+def main():
+    import argparse
+    from src.embeddings.embed_sparse_text import embed_sparse_query
+
+    parser = argparse.ArgumentParser(description="Sparse text search")
+    parser.add_argument("--query", required=True)
+    parser.add_argument("--top-k", type=int, default=10)
+    parser.add_argument("--no-ads", action="store_true", default=True)
+    args = parser.parse_args()
+
+    pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+    index = pc.preview.index(name=os.environ.get("PINECONE_INDEX_NAME", "comic-panels"))
+    namespace = os.environ.get("PINECONE_NAMESPACE", "comics-v1")
+
+    query_sparse = embed_sparse_query(args.query)
+    if not query_sparse:
+        print("Empty query — no sparse vector produced")
+        return
+
+    filters = {"is_ad_page": False} if args.no_ads else {}
+    response = search_sparse(index, namespace, query_sparse, top_k=args.top_k, filters=filters)
+
+    hits = response.get("result", {}).get("hits", [])
+    print(f"Query: {args.query!r}  ({len(hits)} hits)")
+    for i, hit in enumerate(hits, 1):
+        fields = hit.get("fields", {})
+        print(f"  {i}. [{hit.get('_score', 0):.4f}] {hit['_id']}  |  {fields.get('ocr_text', '')[:80]}")
+
+
+if __name__ == "__main__":
+    main()

updated-build.md

@@ -0,0 +1,1610 @@
+# COMICS Panel Search: Pinecone Build Spec
+
+Last updated: 2026-05-27
+
+## Goal
+
+Build a Pinecone-backed panel search index over the COMICS dataset.
+
+Each individual comic panel should be one searchable document in Pinecone. The system should support:
+
+1. Dense vector search over panel images.
+2. Sparse vector search over panel OCR/search text.
+3. Full-text search over OCR/search text.
+4. Combined retrieval by running separate searches and merging results client-side.
+5. A Jupyter notebook for manually running and inspecting queries.
+
+Do not build the UI, Streamlit app, FastAPI app, story builder, sequencing logic, generation layer, or fine-tuning workflow yet.
+
+## Source notes
+
+Use these sources as grounding references while implementing:
+
+- COMICS GitHub repo: https://github.com/miyyer/comics
+- COMICS direct downloads: https://obj.umiacs.umd.edu/comics/index.html
+- Pinecone indexing overview: https://docs.pinecone.io/guides/index-data/indexing-overview
+- Pinecone data modeling: https://docs.pinecone.io/guides/index-data/data-modeling
+- Pinecone search overview: https://docs.pinecone.io/guides/search/search-overview
+
+Important implementation assumptions:
+
+- The COMICS repo provides `setup.sh`, which downloads raw panel images, OCR transcriptions, advertisement page IDs, cached features, and original page images.
+- For this project, use the raw dataset files directly instead of depending on the old Python 2/Theano training pipeline.
+- Pinecone document-schema indexes can hold a `dense_vector`, a `sparse_vector`, one or more FTS-enabled `string` fields, and metadata on the same JSON document.
+- Metadata fields do not need to be declared in the schema. Any undeclared fields upserted with a document are stored and indexed for filtering.
+- A document-schema index supports at most one dense vector field and at most one sparse vector field.
+- A single Pinecone document search request ranks by one scoring signal. To blend dense, sparse, and BM25/FTS rankings, run separate searches and merge results client-side.
+
+## Core data model
+
+One comic panel equals one Pinecone document.
+
+Do not index full pages as the primary retrieval unit. Store page-level metadata on each panel so the exact panel can be displayed and traced back to its source page later.
+
+Target document shape:
+
+```json
+{
+  "_id": "comic_id:page_num:panel_num",
+
+  "image_dense": [0.012, -0.034, 0.091],
+  "text_sparse": {
+    "indices": [123, 456, 789],
+    "values": [0.41, 0.29, 0.13]
+  },
+
+  "ocr_text": "Wait... I know that symbol.",
+  "search_text": "Wait... I know that symbol.",
+
+  "comic_id": "black_terror_001",
+  "book_id": "black_terror_001",
+  "page_id": "page_012",
+  "page_num": 12,
+  "panel_num": 4,
+  "image_path": "data/comics/raw/raw_panel_images/...",
+  "source": "COMICS",
+  "is_ad_page": false
+}
+```
+
+## Target repository structure
+
+```text
+comic-search/
+  data/
+    comics/
+      raw/
+        raw_panel_images.tar.gz
+        COMICS_ocr_file.csv
+        predadpages.txt
+        raw_panel_images/
+      processed/
+        panels_manifest.parquet
+        panels_manifest.jsonl
+        image_dense_vectors.parquet
+        text_sparse_vectors.parquet
+        pinecone_documents.jsonl
+  notebooks/
+    query_comics_index.ipynb
+  src/
+    ingest/
+      build_manifest.py
+      clean_text.py
+    embeddings/
+      embed_images.py
+      embed_sparse_text.py
+    pinecone/
+      create_index.py
+      build_documents.py
+      upsert_panels.py
+    search/
+      search_dense.py
+      search_sparse.py
+      search_fts.py
+      search_combined.py
+      fusion.py
+      router.py
+  README.md
+  requirements.txt
+  config.yaml
+```
+
+## Environment variables
+
+Required:
+
+```bash
+export PINECONE_API_KEY="..."
+export PINECONE_INDEX_NAME="comic-panels"
+export PINECONE_NAMESPACE="comics-v1"
+export COMICS_DATA_DIR="data/comics"
+```
+
+Optional:
+
+```bash
+export IMAGE_EMBED_MODEL="open_clip_ViT-B-32"
+export IMAGE_EMBED_DIM="512"
+export IMAGE_EMBED_METRIC="cosine"
+export SPARSE_EMBED_MODEL="pinecone-sparse-english-v0"
+```
+
+## Config file
+
+Create:
+
+```text
+config.yaml
+```
+
+Suggested contents:
+
+```yaml
+pinecone:
+  index_name: comic-panels
+  namespace: comics-v1
+
+dense_embedding:
+  model_name: open_clip_ViT-B-32
+  field_name: image_dense
+  dimension: 512
+  metric: cosine
+
+sparse_embedding:
+  model_name: pinecone-sparse-english-v0
+  field_name: text_sparse
+  source_text_field: search_text
+
+fields:
+  dense_vector: image_dense
+  sparse_vector: text_sparse
+  fts_text:
+    - ocr_text
+    - search_text
+
+data:
+  comics_data_dir: data/comics
+  manifest_path: data/comics/processed/panels_manifest.parquet
+  dense_vectors_path: data/comics/processed/image_dense_vectors.parquet
+  sparse_vectors_path: data/comics/processed/text_sparse_vectors.parquet
+  pinecone_documents_path: data/comics/processed/pinecone_documents.jsonl
+```
+
+If the selected dense image embedding model has a different dimension, update `dense_embedding.dimension` before creating the Pinecone index.
+
+## Step 1: Download COMICS files
+
+Create a download script or include this in the README:
+
+```bash
+mkdir -p data/comics/raw
+cd data/comics/raw
+
+wget -c https://obj.umiacs.umd.edu/comics/raw_panel_images.tar.gz
+wget -c https://obj.umiacs.umd.edu/comics/COMICS_ocr_file.csv
+wget -c https://obj.umiacs.umd.edu/comics/predadpages.txt
+
+tar -xzf raw_panel_images.tar.gz
+```
+
+Optional, only if page-level reconstruction is needed later:
+
+```bash
+wget -c https://obj.umiacs.umd.edu/comics/raw_page_images.tar.gz
+```
+
+Do not rely on the old COMICS training pipeline unless necessary. Use the downloaded raw panel images and OCR CSV directly.
+
+## Step 2: Clean OCR text
+
+Create:
+
+```text
+src/ingest/clean_text.py
+```
+
+Clean text conservatively. Comics contain sound effects, stylized spelling, names, unusual punctuation, and OCR noise.
+
+Requirements:
+
+- Normalize whitespace.
+- Remove obvious OCR junk.
+- Preserve punctuation where useful.
+- Preserve all-caps sound effects such as `BANG`, `WHAM`, `CRASH`.
+- Preserve names, unusual spellings, and comic-style phrasing.
+- Store both raw and cleaned text.
+- Do not aggressively lowercase the stored text fields. Pinecone FTS applies server-side tokenization and lowercasing for FTS-enabled fields.
+
+Example:
+
+```python
+import re
+
+
+def clean_ocr(text: str | None) -> str:
+    if not text:
+        return ""
+
+    text = str(text)
+    text = text.replace("\u0000", " ")
+    text = re.sub(r"\s+", " ", text).strip()
+
+    # Keep this conservative. Do not remove punctuation or all-caps words.
+    return text
+
+
+def build_search_text(ocr_text_clean: str, extra_terms: list[str] | None = None) -> str:
+    parts = [ocr_text_clean]
+    if extra_terms:
+        parts.extend(extra_terms)
+
+    return " ".join(p for p in parts if p).strip()
+```
+
+For v1:
+
+```python
+search_text = ocr_text_clean
+```
+
+## Step 3: Build the panel manifest
+
+Create:
+
+```text
+src/ingest/build_manifest.py
+```
+
+This script should read:
+
+```text
+data/comics/raw/raw_panel_images/
+data/comics/raw/COMICS_ocr_file.csv
+data/comics/raw/predadpages.txt
+```
+
+Do not hardcode assumptions about the CSV columns or image filename format before inspecting them. Build a deterministic mapping between panel image paths and OCR rows.
+
+Output one row per panel:
+
+```python
+{
+    "panel_id": str,
+    "image_path": str,
+    "ocr_text_raw": str,
+    "ocr_text_clean": str,
+    "search_text": str,
+    "comic_id": str | None,
+    "book_id": str | None,
+    "page_id": str | None,
+    "page_num": int | None,
+    "panel_num": int | None,
+    "is_ad_page": bool,
+    "source": "COMICS"
+}
+```
+
+ID requirements:
+
+```text
+Preferred: {comic_id}:{page_num}:{panel_num}
+Fallback:  comics:{sha1(image_path)}
+```
+
+Validation requirements:
+
+- Every row must have a stable `panel_id`.
+- Every row must point to an existing image file.
+- Empty OCR is allowed.
+- Empty OCR panels must still be indexed because they are visually searchable.
+- `search_text` must be non-null, but can be an empty string.
+- Ad pages should be flagged with `is_ad_page`, not automatically dropped.
+- Log unmatched panel images and unmatched OCR rows.
+- Do not silently drop rows unless the image is corrupt or unreadable.
+
+Write:
+
+```text
+data/comics/processed/panels_manifest.parquet
+data/comics/processed/panels_manifest.jsonl
+```
+
+Suggested skeleton:
+
+```python
+from pathlib import Path
+import hashlib
+import pandas as pd
+
+from src.ingest.clean_text import clean_ocr, build_search_text
+
+
+COMICS_DATA_DIR = Path("data/comics")
+RAW_DIR = COMICS_DATA_DIR / "raw"
+PROCESSED_DIR = COMICS_DATA_DIR / "processed"
+PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def stable_hash(value: str) -> str:
+    return hashlib.sha1(value.encode("utf-8")).hexdigest()[:16]
+
+
+def infer_panel_id(image_path: Path, row: dict | None = None) -> str:
+    # Prefer structured IDs when comic/page/panel metadata is available.
+    # Fallback to deterministic path hash.
+    return f"comics:{stable_hash(str(image_path))}"
+
+
+def main():
+    panel_dir = RAW_DIR / "raw_panel_images"
+    ocr_path = RAW_DIR / "COMICS_ocr_file.csv"
+    ad_pages_path = RAW_DIR / "predadpages.txt"
+
+    ocr_df = pd.read_csv(ocr_path)
+    panel_paths = sorted(panel_dir.rglob("*"))
+
+    panel_paths = [
+        p for p in panel_paths
+        if p.is_file() and p.suffix.lower() in {".jpg", ".jpeg", ".png"}
+    ]
+
+    # TODO: inspect ocr_df columns and build robust mapping to panel image paths.
+    print("OCR columns:", list(ocr_df.columns))
+    print("Panel count:", len(panel_paths))
+
+    rows = []
+    for image_path in panel_paths:
+        ocr_text_raw = ""
+        ocr_text_clean = clean_ocr(ocr_text_raw)
+        search_text = build_search_text(ocr_text_clean)
+
+        rows.append({
+            "panel_id": infer_panel_id(image_path),
+            "image_path": str(image_path),
+            "ocr_text_raw": ocr_text_raw,
+            "ocr_text_clean": ocr_text_clean,
+            "search_text": search_text,
+            "comic_id": None,
+            "book_id": None,
+            "page_id": None,
+            "page_num": None,
+            "panel_num": None,
+            "is_ad_page": False,
+            "source": "COMICS",
+        })
+
+    manifest = pd.DataFrame(rows)
+    manifest.to_parquet(PROCESSED_DIR / "panels_manifest.parquet", index=False)
+    manifest.to_json(PROCESSED_DIR / "panels_manifest.jsonl", orient="records", lines=True)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+The agent should replace the TODO with the real filename-to-OCR mapping after inspecting `COMICS_ocr_file.csv`.
+
+## Step 4: Create dense image embeddings
+
+Create:
+
+```text
+src/embeddings/embed_images.py
+```
+
+Use a multimodal image embedding model suitable for text-to-image and image-to-image search. CLIP or SigLIP-style models are acceptable for v1.
+
+Requirements:
+
+- Embed the panel image, not the page.
+- Use one dense vector per panel.
+- Use the same model family at query time.
+- Store the model name, dimension, and metric in `config.yaml`.
+- Save vectors before upsert so the process is restartable.
+- Log failed image reads.
+- Do not drop rows with empty OCR.
+
+Output:
+
+```text
+data/comics/processed/image_dense_vectors.parquet
+```
+
+Each row:
+
+```python
+{
+    "panel_id": str,
+    "image_dense": list[float]
+}
+```
+
+Suggested skeleton:
+
+```python
+from pathlib import Path
+import pandas as pd
+from PIL import Image
+from tqdm import tqdm
+
+
+MANIFEST_PATH = Path("data/comics/processed/panels_manifest.parquet")
+OUTPUT_PATH = Path("data/comics/processed/image_dense_vectors.parquet")
+
+
+def embed_image(image: Image.Image) -> list[float]:
+    """
+    Use the selected CLIP/SigLIP-style image encoder.
+    Must return a vector matching config.yaml dense_embedding.dimension.
+    """
+    raise NotImplementedError("Wire this to the selected image embedding model.")
+
+
+def main():
+    manifest = pd.read_parquet(MANIFEST_PATH)
+
+    rows = []
+    failures = []
+
+    for row in tqdm(manifest.itertuples(index=False), total=len(manifest)):
+        try:
+            image = Image.open(row.image_path).convert("RGB")
+            vector = embed_image(image)
+            rows.append({
+                "panel_id": row.panel_id,
+                "image_dense": vector,
+            })
+        except Exception as exc:
+            failures.append({
+                "panel_id": row.panel_id,
+                "image_path": row.image_path,
+                "error": str(exc),
+            })
+
+    pd.DataFrame(rows).to_parquet(OUTPUT_PATH, index=False)
+
+    if failures:
+        pd.DataFrame(failures).to_csv(
+            "data/comics/processed/image_embedding_failures.csv",
+            index=False,
+        )
+
+
+if __name__ == "__main__":
+    main()
+```
+
+## Step 5: Create sparse text embeddings
+
+Create:
+
+```text
+src/embeddings/embed_sparse_text.py
+```
+
+Use cleaned `search_text`.
+
+Preferred model:
+
+```text
+pinecone-sparse-english-v0
+```
+
+Requirements:
+
+- Use passage/document mode for indexing.
+- Use query mode for user queries.
+- Skip sparse embedding only when `search_text` is empty.
+- Store empty text rows with no sparse vector, but keep the panel document.
+- Save vectors before upsert.
+- Sparse vectors must use parallel `indices` and `values` arrays.
+- Respect sparse vector limits, including max non-zero values per vector.
+
+Output:
+
+```text
+data/comics/processed/text_sparse_vectors.parquet
+```
+
+Each row:
+
+```python
+{
+    "panel_id": str,
+    "text_sparse": {
+        "indices": [123, 456],
+        "values": [0.25, 0.19]
+    }
+}
+```
+
+Suggested skeleton:
+
+```python
+from pathlib import Path
+import pandas as pd
+from tqdm import tqdm
+
+
+MANIFEST_PATH = Path("data/comics/processed/panels_manifest.parquet")
+OUTPUT_PATH = Path("data/comics/processed/text_sparse_vectors.parquet")
+
+
+def embed_sparse_passage(text: str) -> dict:
+    """
+    Use pinecone-sparse-english-v0 or another sparse encoder in passage/document mode.
+    Must return:
+    {
+        "indices": [...],
+        "values": [...]
+    }
+    """
+    raise NotImplementedError("Wire this to the selected sparse embedding model.")
+
+
+def main():
+    manifest = pd.read_parquet(MANIFEST_PATH)
+
+    rows = []
+    for row in tqdm(manifest.itertuples(index=False), total=len(manifest)):
+        text = row.search_text or ""
+
+        if not text.strip():
+            sparse = None
+        else:
+            sparse = embed_sparse_passage(text)
+
+        rows.append({
+            "panel_id": row.panel_id,
+            "text_sparse": sparse,
+        })
+
+    pd.DataFrame(rows).to_parquet(OUTPUT_PATH, index=False)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+## Step 6: Create the Pinecone index
+
+Create:
+
+```text
+src/pinecone/create_index.py
+```
+
+Create one document-schema index with:
+
+```text
+image_dense   dense_vector
+text_sparse   sparse_vector
+ocr_text      string with full_text_search enabled
+search_text   string with full_text_search enabled
+```
+
+Metadata fields should not be declared in the schema. Include them at upsert time.
+
+Logical schema:
+
+```python
+from pinecone import Pinecone
+from pinecone.preview import SchemaBuilder
+import os
+
+pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+
+INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "comic-panels")
+IMAGE_EMBED_DIM = int(os.environ.get("IMAGE_EMBED_DIM", "512"))
+IMAGE_EMBED_METRIC = os.environ.get("IMAGE_EMBED_METRIC", "cosine")
+
+schema = (
+    SchemaBuilder()
+    .add_dense_vector_field(
+        "image_dense",
+        dimension=IMAGE_EMBED_DIM,
+        metric=IMAGE_EMBED_METRIC,
+    )
+    .add_sparse_vector_field("text_sparse")
+    .add_string_field(
+        "ocr_text",
+        full_text_search={"language": "en"},
+    )
+    .add_string_field(
+        "search_text",
+        full_text_search={"language": "en"},
+    )
+    .build()
+)
+
+pc.preview.indexes.create(
+    name=INDEX_NAME,
+    schema=schema,
+)
+```
+
+Important:
+
+- Do not create separate Pinecone indexes for image, sparse, and FTS search.
+- Use one document-schema index.
+- Do not declare metadata-only fields in the schema.
+- Plan fields carefully because schema migration may require recreating the index.
+- Use the dense vector dimension from the selected image embedding model.
+- Use cosine or dotproduct depending on the embedding model. If unsure, use cosine for normalized CLIP-style vectors.
+
+## Step 7: Build Pinecone documents
+
+Create:
+
+```text
+src/pinecone/build_documents.py
+```
+
+Join:
+
+```text
+panels_manifest.parquet
+image_dense_vectors.parquet
+text_sparse_vectors.parquet
+```
+
+Output:
+
+```text
+data/comics/processed/pinecone_documents.jsonl
+```
+
+Each document:
+
+```python
+{
+    "_id": panel_id,
+
+    "image_dense": image_dense,
+
+    # Only include this when available and non-empty.
+    "text_sparse": {
+        "indices": [...],
+        "values": [...]
+    },
+
+    "ocr_text": ocr_text_clean,
+    "search_text": search_text,
+
+    # Metadata fields:
+    "comic_id": comic_id,
+    "book_id": book_id,
+    "page_id": page_id,
+    "page_num": page_num,
+    "panel_num": panel_num,
+    "image_path": image_path,
+    "source": "COMICS",
+    "is_ad_page": is_ad_page
+}
+```
+
+Requirements:
+
+- Do not include raw image bytes.
+- Do not include huge blobs.
+- Store local image paths for now.
+- Later, image paths can become S3/GCS/R2 URLs.
+- Preserve panels with empty OCR.
+- Preserve panels with missing sparse vectors.
+- Every document must have `_id`, `image_dense`, `ocr_text`, `search_text`, and metadata.
+- Only include `text_sparse` when it has non-empty `indices` and `values`.
+
+Suggested skeleton:
+
+```python
+from pathlib import Path
+import json
+import pandas as pd
+
+
+PROCESSED_DIR = Path("data/comics/processed")
+MANIFEST_PATH = PROCESSED_DIR / "panels_manifest.parquet"
+DENSE_PATH = PROCESSED_DIR / "image_dense_vectors.parquet"
+SPARSE_PATH = PROCESSED_DIR / "text_sparse_vectors.parquet"
+OUTPUT_PATH = PROCESSED_DIR / "pinecone_documents.jsonl"
+
+
+def valid_sparse(sparse) -> bool:
+    if not isinstance(sparse, dict):
+        return False
+    return bool(sparse.get("indices")) and bool(sparse.get("values"))
+
+
+def main():
+    manifest = pd.read_parquet(MANIFEST_PATH)
+    dense = pd.read_parquet(DENSE_PATH)
+    sparse = pd.read_parquet(SPARSE_PATH)
+
+    df = manifest.merge(dense, on="panel_id", how="inner")
+    df = df.merge(sparse, on="panel_id", how="left")
+
+    with open(OUTPUT_PATH, "w") as f:
+        for row in df.itertuples(index=False):
+            doc = {
+                "_id": row.panel_id,
+                "image_dense": row.image_dense,
+                "ocr_text": row.ocr_text_clean or "",
+                "search_text": row.search_text or "",
+                "comic_id": row.comic_id,
+                "book_id": row.book_id,
+                "page_id": row.page_id,
+                "page_num": row.page_num,
+                "panel_num": row.panel_num,
+                "image_path": row.image_path,
+                "source": "COMICS",
+                "is_ad_page": bool(row.is_ad_page),
+            }
+
+            sparse_vec = getattr(row, "text_sparse", None)
+            if valid_sparse(sparse_vec):
+                doc["text_sparse"] = sparse_vec
+
+            f.write(json.dumps(doc) + "\n")
+
+
+if __name__ == "__main__":
+    main()
+```
+
+## Step 8: Upsert documents to Pinecone
+
+Create:
+
+```text
+src/pinecone/upsert_panels.py
+```
+
+Pseudo-code:
+
+```python
+import json
+import os
+import time
+from pathlib import Path
+
+from pinecone import Pinecone
+
+
+pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
+
+INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "comic-panels")
+NAMESPACE = os.environ.get("PINECONE_NAMESPACE", "comics-v1")
+DOCS_PATH = Path("data/comics/processed/pinecone_documents.jsonl")
+
+index = pc.preview.index(name=INDEX_NAME)
+
+
+def batched_jsonl(path: Path, batch_size: int = 100):
+    batch = []
+    with open(path) as f:
+        for line in f:
+            batch.append(json.loads(line))
+            if len(batch) >= batch_size:
+                yield batch
+                batch = []
+
+    if batch:
+        yield batch
+
+
+def upsert_with_retry(batch, retries=5):
+    for attempt in range(retries):
+        try:
+            return index.documents.upsert(
+                namespace=NAMESPACE,
+                documents=batch,
+            )
+        except Exception:
+            if attempt == retries - 1:
+                raise
+            time.sleep(2 ** attempt)
+
+
+def main():
+    for batch in batched_jsonl(DOCS_PATH, batch_size=100):
+        upsert_with_retry(batch)
+
+
+if __name__ == "__main__":
+    main()
+```
+
+Requirements:
+
+- Upserts must be idempotent.
+- Re-running the job should overwrite/update the same `panel_id`.
+- Use deterministic IDs.
+- Batch writes.
+- Log batch failures.
+- Retry transient failures with exponential backoff.
+- After upsert, run a quick validation search.
+
+## Step 9: Dense search primitive
+
+Create:
+
+```text
+src/search/search_dense.py
+```
+
+Use for visual/conceptual queries:
+
+```text
+a detective in a dark alley
+a superhero punching a robot
+a close-up of a frightened face
+```
+
+Process:
+
+1. Embed the user query using the text side of the same multimodal model used for panel image embeddings.
+2. Search the `image_dense` field.
+3. Return panel metadata and OCR fields.
+
+```python
+def search_dense(index, namespace, query_vector, top_k=20, filters=None):
+    return index.documents.search(
+        namespace=namespace,
+        top_k=top_k,
+        score_by=[
+            {
+                "type": "dense_vector",
+                "field": "image_dense",
+                "values": query_vector,
+            }
+        ],
+        filter=filters or {},
+        include_fields=[
+            "ocr_text",
+            "search_text",
+            "comic_id",
+            "book_id",
+            "page_id",
+            "page_num",
+            "panel_num",
+            "image_path",
+            "source",
+            "is_ad_page",
+        ],
+    )
+```
+
+## Step 10: Sparse search primitive
+
+Create:
+
+```text
+src/search/search_sparse.py
+```
+
+Use for text-heavy lexical/conceptual queries:
+
+```text
+masked villain laboratory
+secret treasure island
+police detective murder clue
+```
+
+Process:
+
+1. Sparse-embed the query using query mode.
+2. Search the `text_sparse` field.
+3. Return panel metadata and OCR fields.
+
+```python
+def search_sparse(index, namespace, query_sparse, top_k=20, filters=None):
+    return index.documents.search(
+        namespace=namespace,
+        top_k=top_k,
+        score_by=[
+            {
+                "type": "sparse_vector",
+                "field": "text_sparse",
+                "sparse_values": query_sparse,
+            }
+        ],
+        filter=filters or {},
+        include_fields=[
+            "ocr_text",
+            "search_text",
+            "comic_id",
+            "book_id",
+            "page_id",
+            "page_num",
+            "panel_num",
+            "image_path",
+            "source",
+            "is_ad_page",
+        ],
+    )
+```
+
+## Step 11: Full-text search primitive
+
+Create:
+
+```text
+src/search/search_fts.py
+```
+
+Use for:
+
+```text
+exact dialogue
+quoted phrases
+character names
+sound effects
+Boolean-style keyword search
+```
+
+Examples:
+
+```text
+"secret formula"
+BANG
+laboratory AND formula
+treasure NOT island
+```
+
+Basic FTS search:
+
+```python
+def search_fts(index, namespace, query, top_k=20, filters=None):
+    return index.documents.search(
+        namespace=namespace,
+        top_k=top_k,
+        score_by=[
+            {
+                "type": "text",
+                "field": "search_text",
+                "query": query,
+            }
+        ],
+        filter=filters or {},
+        include_fields=[
+            "ocr_text",
+            "search_text",
+            "comic_id",
+            "book_id",
+            "page_id",
+            "page_num",
+            "panel_num",
+            "image_path",
+            "source",
+            "is_ad_page",
+        ],
+    )
+```
+
+For exact phrase filters, use FTS-enabled string fields as filters:
+
+```python
+def search_dense_with_phrase_filter(
+    index,
+    namespace,
+    query_vector,
+    phrase,
+    top_k=20,
+    filters=None,
+):
+    merged_filter = filters.copy() if filters else {}
+    merged_filter["search_text"] = {"$match_phrase": phrase}
+
+    return index.documents.search(
+        namespace=namespace,
+        top_k=top_k,
+        score_by=[
+            {
+                "type": "dense_vector",
+                "field": "image_dense",
+                "values": query_vector,
+            }
+        ],
+        filter=merged_filter,
+        include_fields=[
+            "ocr_text",
+            "search_text",
+            "comic_id",
+            "book_id",
+            "page_id",
+            "page_num",
+            "panel_num",
+            "image_path",
+            "source",
+            "is_ad_page",
+        ],
+    )
+```
+
+## Step 12: Combined search primitive
+
+Create:
+
+```text
+src/search/search_combined.py
+src/search/fusion.py
+```
+
+A single Pinecone document search request ranks by one signal. For a true blend of dense, sparse, and FTS, run separate searches and merge client-side.
+
+Use Reciprocal Rank Fusion for v1.
+
+```python
+def rrf_merge(result_groups, k=60):
+    scores = {}
+    payloads = {}
+
+    for source_name, hits in result_groups:
+        for rank, hit in enumerate(hits, start=1):
+            panel_id = hit["_id"]
+
+            scores.setdefault(panel_id, 0.0)
+            scores[panel_id] += 1.0 / (k + rank)
+
+            if panel_id not in payloads:
+                payloads[panel_id] = dict(hit)
+                payloads[panel_id]["sources"] = []
+
+            payloads[panel_id]["sources"].append(source_name)
+
+    ranked = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+
+    output = []
+    for panel_id, score in ranked:
+        item = payloads[panel_id]
+        item["rrf_score"] = score
+        output.append(item)
+
+    return output
+```
+
+Combined search:
+
+```python
+from src.search.search_dense import search_dense
+from src.search.search_sparse import search_sparse
+from src.search.search_fts import search_fts
+from src.search.fusion import rrf_merge
+
+
+def search_combined(
+    index,
+    namespace,
+    query_text,
+    dense_query_vector,
+    sparse_query_vector=None,
+    top_k=20,
+    filters=None,
+    run_fts=True,
+):
+    result_groups = []
+
+    dense_response = search_dense(
+        index=index,
+        namespace=namespace,
+        query_vector=dense_query_vector,
+        top_k=top_k,
+        filters=filters,
+    )
+    result_groups.append(("dense", dense_response["result"]["hits"]))
+
+    if sparse_query_vector:
+        sparse_response = search_sparse(
+            index=index,
+            namespace=namespace,
+            query_sparse=sparse_query_vector,
+            top_k=top_k,
+            filters=filters,
+        )
+        result_groups.append(("sparse", sparse_response["result"]["hits"]))
+
+    if run_fts:
+        fts_response = search_fts(
+            index=index,
+            namespace=namespace,
+            query=query_text,
+            top_k=top_k,
+            filters=filters,
+        )
+        result_groups.append(("fts", fts_response["result"]["hits"]))
+
+    return rrf_merge(result_groups)[:top_k]
+```
+
+Do not directly compare raw dense, sparse, and BM25 scores. Merge by rank unless there is a later calibration step.
+
+## Step 13: Query router
+
+Create:
+
+```text
+src/search/router.py
+```
+
+The router should decide which signals to run.
+
+```python
+def route_query(query: str) -> dict:
+    q = query.strip()
+    tokens = q.split()
+
+    has_quotes = '"' in q or "'" in q
+    has_boolean = any(t.upper() in {"AND", "OR", "NOT"} for t in tokens)
+    has_all_caps = any(t.isupper() and len(t) >= 3 for t in tokens)
+    has_exact_intent = any(
+        phrase in q.lower()
+        for phrase in [
+            "says",
+            "where they say",
+            "exact",
+            "phrase",
+            "quote",
+            "dialogue",
+            "sound effect",
+            "contains",
+        ]
+    )
+
+    return {
+        "dense": True,
+        "sparse": bool(q),
+        "fts": has_quotes or has_boolean or has_all_caps or has_exact_intent,
+    }
+```
+
+Default behavior:
+
+```text
+Natural-language visual query: dense + sparse
+Quoted/exact/dialogue query: dense + sparse + FTS
+All-caps sound effect query: dense + sparse + FTS
+Boolean query: FTS + optional dense/sparse
+```
+
+## Step 14: Jupyter query notebook
+
+Create:
+
+```text
+notebooks/query_comics_index.ipynb
+```
+
+This notebook is not a UI. It is a developer-facing query playground for validating search quality, inspecting returned panels, and comparing dense, sparse, FTS, and combined retrieval.
+
+The notebook should make it easy to:
+
+1. Connect to the existing Pinecone index.
+2. Run dense image search from a text query.
+3. Run sparse text search from a text query.
+4. Run full-text search against OCR/search text.
+5. Run combined search with RRF.
+6. Display returned panel images inline.
+7. Compare results across search modes.
+8. Inspect metadata, OCR text, and source signals.
+
+The notebook should call the repo’s search primitives rather than duplicating search logic.
+
+### Notebook setup cell
+
+```python
+import os
+import sys
+from pathlib import Path
+
+from pinecone import Pinecone
+from PIL import Image
+import pandas as pd
+from IPython.display import display, Markdown
+
+PROJECT_ROOT = Path.cwd().parent if Path.cwd().name == "notebooks" else Path.cwd()
+sys.path.append(str(PROJECT_ROOT))
+
+PINECONE_API_KEY = os.environ["PINECONE_API_KEY"]
+PINECONE_INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "comic-panels")
+PINECONE_NAMESPACE = os.environ.get("PINECONE_NAMESPACE", "comics-v1")
+COMICS_DATA_DIR = Path(os.environ.get("COMICS_DATA_DIR", "data/comics"))
+
+pc = Pinecone(api_key=PINECONE_API_KEY)
+index = pc.preview.index(name=PINECONE_INDEX_NAME)
+
+print("Connected to:", PINECONE_INDEX_NAME)
+print("Namespace:", PINECONE_NAMESPACE)
+```
+
+### Notebook imports cell
+
+```python
+from src.search.search_dense import search_dense
+from src.search.search_sparse import search_sparse
+from src.search.search_fts import search_fts
+from src.search.search_combined import search_combined
+from src.search.router import route_query
+```
+
+### Notebook embedding helper cell
+
+The notebook needs helpers to embed queries. These should call the same embedding modules/config used during ingestion.
+
+```python
+def embed_query_dense(query: str):
+    """
+    Use the text side of the same multimodal model used for panel image embeddings.
+    Must return a dense vector with the same dimension as image_dense.
+    """
+    raise NotImplementedError("Wire this to the selected CLIP/SigLIP query embedding function.")
+
+
+def embed_query_sparse(query: str):
+    """
+    Use the sparse model in query mode.
+    Must return:
+    {
+        "indices": [...],
+        "values": [...]
+    }
+    """
+    raise NotImplementedError("Wire this to pinecone-sparse-english-v0 query embedding.")
+```
+
+Agents should replace these stubs with calls into the project’s embedding modules once those modules exist.
+
+### Notebook display helper cell
+
+```python
+def display_results(results, max_items=10, image_width=300):
+    for i, hit in enumerate(results[:max_items], start=1):
+        fields = hit.get("fields", hit)
+
+        panel_id = hit.get("_id") or hit.get("panel_id")
+        score = hit.get("_score") or hit.get("score") or hit.get("rrf_score")
+        sources = hit.get("sources", fields.get("sources", []))
+
+        image_path = fields.get("image_path") or hit.get("image_path")
+        ocr_text = fields.get("ocr_text") or hit.get("ocr_text", "")
+        comic_id = fields.get("comic_id") or hit.get("comic_id")
+        page_num = fields.get("page_num") or hit.get("page_num")
+        panel_num = fields.get("panel_num") or hit.get("panel_num")
+
+        display(Markdown(f"### {i}. `{panel_id}`"))
+        display(Markdown(
+            f"**Score:** `{score}`  \n"
+            f"**Sources:** `{sources}`  \n"
+            f"**Comic:** `{comic_id}`  \n"
+            f"**Page:** `{page_num}`  \n"
+            f"**Panel:** `{panel_num}`"
+        ))
+
+        if image_path and Path(image_path).exists():
+            img = Image.open(image_path)
+            display(img.resize((image_width, int(image_width * img.height / img.width))))
+        else:
+            display(Markdown(f"`Image not found: {image_path}`"))
+
+        if ocr_text:
+            display(Markdown(f"**OCR:** {ocr_text}"))
+
+        display(Markdown("---"))
+```
+
+### Dense search section
+
+```python
+query = "a detective in a dark alley"
+top_k = 10
+
+dense_vector = embed_query_dense(query)
+
+dense_response = search_dense(
+    index=index,
+    namespace=PINECONE_NAMESPACE,
+    query_vector=dense_vector,
+    top_k=top_k,
+    filters={"is_ad_page": False},
+)
+
+dense_hits = dense_response["result"]["hits"]
+display_results(dense_hits, max_items=top_k)
+```
+
+Use for searches like:
+
+```text
+a superhero flying through the sky
+a monster attacking a city
+a close-up of a frightened face
+a robot in a laboratory
+```
+
+### Sparse search section
+
+```python
+query = "masked villain laboratory"
+top_k = 10
+
+sparse_vector = embed_query_sparse(query)
+
+sparse_response = search_sparse(
+    index=index,
+    namespace=PINECONE_NAMESPACE,
+    query_sparse=sparse_vector,
+    top_k=top_k,
+    filters={"is_ad_page": False},
+)
+
+sparse_hits = sparse_response["result"]["hits"]
+display_results(sparse_hits, max_items=top_k)
+```
+
+Use for searches like:
+
+```text
+secret formula
+police detective clue
+treasure island
+masked villain laboratory
+```
+
+### Full-text search section
+
+```python
+query = '"secret formula"'
+top_k = 10
+
+fts_response = search_fts(
+    index=index,
+    namespace=PINECONE_NAMESPACE,
+    query=query,
+    top_k=top_k,
+    filters={"is_ad_page": False},
+)
+
+fts_hits = fts_response["result"]["hits"]
+display_results(fts_hits, max_items=top_k)
+```
+
+Use for searches like:
+
+```text
+"secret formula"
+BANG
+WHAM
+laboratory AND formula
+treasure NOT island
+```
+
+### Combined search section
+
+```python
+query = "a laboratory panel containing formula"
+top_k = 10
+
+route = route_query(query)
+
+dense_vector = embed_query_dense(query) if route["dense"] else None
+sparse_vector = embed_query_sparse(query) if route["sparse"] else None
+
+combined_hits = search_combined(
+    index=index,
+    namespace=PINECONE_NAMESPACE,
+    query_text=query,
+    dense_query_vector=dense_vector,
+    sparse_query_vector=sparse_vector,
+    top_k=top_k,
+    filters={"is_ad_page": False},
+    run_fts=route["fts"],
+)
+
+display_results(combined_hits, max_items=top_k)
+```
+
+### Compare modes section
+
+```python
+query = "secret formula laboratory"
+top_k = 10
+
+dense_vector = embed_query_dense(query)
+sparse_vector = embed_query_sparse(query)
+
+dense_hits = search_dense(
+    index=index,
+    namespace=PINECONE_NAMESPACE,
+    query_vector=dense_vector,
+    top_k=top_k,
+    filters={"is_ad_page": False},
+)["result"]["hits"]
+
+sparse_hits = search_sparse(
+    index=index,
+    namespace=PINECONE_NAMESPACE,
+    query_sparse=sparse_vector,
+    top_k=top_k,
+    filters={"is_ad_page": False},
+)["result"]["hits"]
+
+fts_hits = search_fts(
+    index=index,
+    namespace=PINECONE_NAMESPACE,
+    query=query,
+    top_k=top_k,
+    filters={"is_ad_page": False},
+)["result"]["hits"]
+
+
+def hits_to_rows(mode, hits):
+    rows = []
+    for rank, hit in enumerate(hits, start=1):
+        fields = hit.get("fields", {})
+        rows.append({
+            "mode": mode,
+            "rank": rank,
+            "panel_id": hit.get("_id"),
+            "score": hit.get("_score"),
+            "ocr_text": fields.get("ocr_text"),
+            "image_path": fields.get("image_path"),
+            "comic_id": fields.get("comic_id"),
+            "page_num": fields.get("page_num"),
+            "panel_num": fields.get("panel_num"),
+        })
+    return rows
+
+
+df = pd.DataFrame(
+    hits_to_rows("dense", dense_hits)
+    + hits_to_rows("sparse", sparse_hits)
+    + hits_to_rows("fts", fts_hits)
+)
+
+df
+```
+
+### Metadata filtering examples
+
+```python
+filters = {
+    "is_ad_page": False
+}
+```
+
+```python
+filters = {
+    "comic_id": "some_comic_id",
+    "is_ad_page": False
+}
+```
+
+```python
+filters = {
+    "page_num": {"$gte": 5, "$lte": 20},
+    "is_ad_page": False
+}
+```
+
+### Manual evaluation section
+
+```python
+test_queries = [
+    '"secret formula"',
+    "BANG",
+    "a detective in a dark alley",
+    "a superhero flying through the sky",
+    "masked villain laboratory",
+    "a laboratory panel containing formula",
+]
+
+for q in test_queries:
+    print("=" * 80)
+    print("Query:", q)
+
+    route = route_query(q)
+    dense_vector = embed_query_dense(q) if route["dense"] else None
+    sparse_vector = embed_query_sparse(q) if route["sparse"] else None
+
+    hits = search_combined(
+        index=index,
+        namespace=PINECONE_NAMESPACE,
+        query_text=q,
+        dense_query_vector=dense_vector,
+        sparse_query_vector=sparse_vector,
+        top_k=5,
+        filters={"is_ad_page": False},
+        run_fts=route["fts"],
+    )
+
+    display_results(hits, max_items=5)
+```
+
+## README command flow
+
+The README should let a new developer run:
+
+```bash
+python -m src.ingest.build_manifest
+python -m src.embeddings.embed_images
+python -m src.embeddings.embed_sparse_text
+python -m src.pinecone.create_index
+python -m src.pinecone.build_documents
+python -m src.pinecone.upsert_panels
+```
+
+Then launch the query notebook:
+
+```bash
+jupyter lab notebooks/query_comics_index.ipynb
+```
+
+or:
+
+```bash
+jupyter notebook notebooks/query_comics_index.ipynb
+```
+
+Optional search smoke tests:
+
+```bash
+python -m src.search.search_fts --query '"secret formula"'
+python -m src.search.search_dense --query "a detective in a dark alley"
+python -m src.search.search_combined --query "a laboratory panel containing formula"
+```
+
+## Non-goals
+
+Do not build:
+
+- UI
+- Streamlit app
+- FastAPI app
+- Story builder
+- Panel sequencing system
+- Generative story construction
+- Fine-tuning pipeline
+- ComicsPAP task evaluation
+- Page-level search as the primary retrieval unit
+
+## Final deliverables
+
+The agent should produce:
+
+1. COMICS download instructions.
+2. Manifest builder.
+3. OCR cleaning module.
+4. Dense image embedding script.
+5. Sparse text embedding script.
+6. Pinecone document-schema index creation script.
+7. Pinecone document builder.
+8. Pinecone upsert script.
+9. Dense search primitive.
+10. Sparse search primitive.
+11. FTS search primitive.
+12. Combined search primitive with RRF.
+13. Query router.
+14. Jupyter notebook for querying the Pinecone index.
+15. README command flow.
+16. `config.yaml`.
+17. `requirements.txt`.
+
+## Acceptance criteria
+
+The build is complete when:
+
+- The COMICS panel manifest exists.
+- Each panel has a deterministic `panel_id`.
+- Raw and cleaned OCR fields exist.
+- Panel image embeddings have been generated and saved.
+- Sparse text embeddings have been generated and saved where text exists.
+- A Pinecone document-schema index exists with:
+  - `image_dense`
+  - `text_sparse`
+  - `ocr_text`
+  - `search_text`
+- Documents have been upserted into the configured namespace.
+- Dense search returns relevant visual/conceptual panels.
+- Sparse search returns relevant text-heavy panels.
+- FTS search returns panels with matching dialogue, names, phrases, or sound effects.
+- Combined search merges dense, sparse, and FTS results with RRF.
+- The Jupyter notebook can run each search mode and display panel images inline.

utils/downloader.py

@@ -1,24 +0,0 @@
-"""Download comic archives from URLs to data/raw/."""
-import hashlib
-from pathlib import Path
-
-import requests
-
-
-def download_comic(url: str, raw_dir: Path, filename: str | None = None) -> Path:
-    """Download a CBZ/ZIP from url into raw_dir. Returns the local path."""
-    raw_dir.mkdir(parents=True, exist_ok=True)
-    if filename is None:
-        filename = url.split("/")[-1].split("?")[0] or hashlib.md5(url.encode()).hexdigest() + ".cbz"
-    dest = raw_dir / filename
-    if dest.exists():
-        print(f"  already downloaded: {dest.name}")
-        return dest
-    print(f"  downloading {filename}...")
-    resp = requests.get(url, stream=True, timeout=60)
-    resp.raise_for_status()
-    with open(dest, "wb") as f:
-        for chunk in resp.iter_content(chunk_size=65536):
-            f.write(chunk)
-    print(f"  saved → {dest}")
-    return dest

utils/embedders.py

@@ -1,48 +0,0 @@
-"""Shared embedding helpers for all notebooks."""
-import base64
-import os
-from io import BytesIO
-from pathlib import Path
-
-import voyageai
-from PIL import Image
-
-
-def _image_to_base64(image_path: str | Path) -> str:
-    img = Image.open(image_path).convert("RGB")
-    buf = BytesIO()
-    img.save(buf, format="JPEG", quality=85)
-    return base64.b64encode(buf.getvalue()).decode()
-
-
-def embed_images_voyage(
-    image_paths: list[str | Path],
-    batch_size: int = 10,
-) -> list[list[float]]:
-    """Embed comic page images with voyage-multimodal-3. Returns list of 1024-dim vectors."""
-    vo = voyageai.Client()
-    all_vectors = []
-    for i in range(0, len(image_paths), batch_size):
-        batch = image_paths[i : i + batch_size]
-        inputs = []
-        for p in batch:
-            img = Image.open(p).convert("RGB")
-            inputs.append([img])
-        result = vo.multimodal_embed(inputs, model="voyage-multimodal-3", input_type="document")
-        all_vectors.extend(result.embeddings)
-    return all_vectors
-
-
-def embed_query_voyage(query_text: str) -> list[float]:
-    """Embed a text query with voyage-multimodal-3 for cross-modal search."""
-    vo = voyageai.Client()
-    result = vo.multimodal_embed([[query_text]], model="voyage-multimodal-3", input_type="query")
-    return result.embeddings[0]
-
-
-def embed_query_image_voyage(image_path: str | Path) -> list[float]:
-    """Embed an image as a query with voyage-multimodal-3 for image-to-image search."""
-    vo = voyageai.Client()
-    img = Image.open(image_path).convert("RGB")
-    result = vo.multimodal_embed([[img]], model="voyage-multimodal-3", input_type="query")
-    return result.embeddings[0]

utils/extractor.py

@@ -1,36 +0,0 @@
-"""Extract pages from CBZ/ZIP comic archives into data/pages/<comic_slug>/."""
-import zipfile
-from pathlib import Path
-
-
-def slugify(name: str) -> str:
-    return name.lower().replace(" ", "_").replace("-", "_")
-
-
-def extract_comic(archive_path: Path, pages_dir: Path) -> list[dict]:
-    """Extract image pages from a CBZ/ZIP archive. Returns a list of page dicts."""
-    slug = slugify(archive_path.stem)
-    out_dir = pages_dir / slug
-    out_dir.mkdir(parents=True, exist_ok=True)
-
-    image_exts = {".jpg", ".jpeg", ".png", ".gif", ".webp"}
-    pages = []
-
-    with zipfile.ZipFile(archive_path) as zf:
-        image_names = sorted(
-            n for n in zf.namelist()
-            if Path(n).suffix.lower() in image_exts and not Path(n).name.startswith(".")
-        )
-        for i, name in enumerate(image_names, start=1):
-            suffix = Path(name).suffix.lower()
-            dest = out_dir / f"page_{i:03d}{suffix}"
-            if not dest.exists():
-                dest.write_bytes(zf.read(name))
-            pages.append({
-                "comic_slug": slug,
-                "comic_title": archive_path.stem,
-                "page_num": i,
-                "file_path": str(dest),
-            })
-
-    return pages