Merge pull request #1526 from feder-cr/details-keys

TUI: Space and Enter select, i opens the job with only its posting; install on Python 3.14

.github/workflows/counters.yml

@@ -0,0 +1,74 @@
+# counters.yml - the daily snapshot of the applications AIHawk has sent, and the README badge.
+#
+# GitHub keeps only the running total of each counter file's downloads (src/aihawk/counter.py), so the applications of a
+# day exist only if someone records the totals every day. This is that someone: no machine and no personal token, just
+# the scheduler and the built-in GITHUB_TOKEN. A run that fails sends GitHub's failure mail, and the run's log warns when
+# a day between two snapshots is missing. The way invisible_playwright records its browser launches.
+#
+# Everything it writes lives on the `badges` branch, which is not protected:
+#   counters/history.csv             date,key,value - every counter, once a day
+#   counters/applications_daily.csv  day_utc,sent,submitted,... - one row per measured day
+#   docs/badges/applications.svg
+#
+# 00:10 UTC, so the snapshot dated D closes calendar day D-1.
+name: counters
+
+on:
+  schedule:
+    - cron: "10 0 * * *"
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+
+concurrency:
+  group: counters
+  cancel-in-progress: false
+
+jobs:
+  snapshot:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Check out the script (main)
+        uses: actions/checkout@v7
+        with:
+          path: main
+
+      - name: Check out the badges branch
+        uses: actions/checkout@v7
+        with:
+          ref: badges
+          path: badges
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+
+      - name: Self-test
+        run: python main/scripts/usage_counters.py --selftest
+
+      - name: Record today's counters and redraw the badge
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >
+          python main/scripts/usage_counters.py
+          --history badges/counters/history.csv
+          --daily badges/counters/applications_daily.csv
+          --badges badges/docs/badges
+
+      - name: Commit to the badges branch
+        working-directory: badges
+        run: |
+          set -euo pipefail
+          git config user.name "feder-cr"
+          git config user.email "[email protected]"
+          git add counters docs/badges
+          if git diff --cached --quiet; then
+            echo "nothing changed"
+            exit 0
+          fi
+          sent=$(awk -F, '$2=="sent"{v=$3} END{print v}' counters/history.csv)
+          git commit -q -m "counters: $(date -u +%F), applications sent at ${sent}"
+          git push origin HEAD:badges

.github/workflows/jobs-database.yml

@@ -21,8 +21,8 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 200  # shorter than the five hours to the next run, so a hung build does not take its slot
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
         with:
           python-version: "3.12"
       - name: room on disk

.github/workflows/tests.yml

@@ -21,8 +21,8 @@ jobs:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python: ["3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python }}
       - run: pip install -e ".[dev]"
@@ -33,8 +33,8 @@ jobs:
     name: the real browser on a real form
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
         with:
           python-version: "3.12"
       - run: pip install -e ".[dev]"

.python-version

@@ -0,0 +1 @@
+3.14

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- Space and Enter both select a job, and `i` opens it: its role, company, place and the posting's own text, as paragraphs, with nothing else. `s`, the applications, is shown at the bottom; an application's details say how it went in plain words, and the models are no longer in the settings.
+- The one-command install runs on Python 3.14: on 3.15 PyYAML has no Windows build yet, and the install stopped asking for a C++ compiler.
+
 - Added [docs/sites.md](docs/sites.md): each site's status from real forms and what stops it; the README links to it.
 - Added [docs/data-sources.md](docs/data-sources.md), each source with its licence and how to ask for its removal, and [docs/configuration.md](docs/configuration.md).
 - A model error that a smaller batch cannot cure stops at once and says why, instead of ranking by title almost silently.

README.md

@@ -9,6 +9,8 @@
 </picture>
 <h3 align="center">An AI job application bot: automated job search, then it auto-applies to the jobs that match your resume, while you sleep.</h3>
 
+<a href="#the-count"><img alt="Applications sent with AIHawk, by all its users" src="https://raw.githubusercontent.com/feder-cr/Jobs_Applier_AI_Agent/badges/docs/badges/applications.svg"></a>
+
 <sub>FEATURED IN</sub><br>
 <a href="https://www.businessinsider.com/aihawk-applies-jobs-for-you-linkedin-risks-inaccuracies-mistakes-2024-11"><b>Business Insider</b></a> ·
 <a href="https://techcrunch.com/2024/10/10/a-reporter-used-ai-to-apply-to-2843-jobs/"><b>TechCrunch</b></a> ·
@@ -52,6 +54,10 @@ curl -LsSf https://raw.githubusercontent.com/feder-cr/Jobs_Applier_AI_Agent/main
 
 You need a key: [get one from OrcaRouter](https://www.orcarouter.ai/ref/ref_accd17afb78fc95ac4d7).
 
+## The count
+
+Each application AIHawk finishes downloads one small file from GitHub, named after how it went: sent and confirmed, sent without a confirmation, refused by the form, blocked, or failed. GitHub counts the downloads, and that is how we know how many applications AIHawk sends; the number is the badge above. Nothing about you or the job goes with it: the file is the same for everyone. The count is always on.
+
 
 ---
 

docs/configuration.md

@@ -10,7 +10,7 @@ With the environment variable `AIHAWK_DRY_RUN` set to anything, an application s
 
 ## What it costs
 
-AIHawk counts the tokens of every model call and prices them with the gateway's own list, read once per start. A sent application shows its cost in its details, and the log says what each application and each scoring cost. Jev's calls, which score the jobs, are not counted: they go through another endpoint and cost under a cent per hundred postings.
+AIHawk counts the tokens of every model call and prices them with the gateway's own list, read once per start. The log says what each application and each scoring cost. Jev's calls, which score the jobs, are not counted: they go through another endpoint and cost under a cent per hundred postings.
 
 ## The accounts it made
 
@@ -24,8 +24,8 @@ What the forms are told about you beyond the CV (email, phone, location, links,
 |---|---|---|
 | `name` | empty | Your name for this search, shown at the top of the list. |
 | `goals` | empty | What you are looking for, in your own words. Jev reads it. |
-| `database` | `https://huggingface.co/datasets/feder-cr/jobs/resolve/main/manifest.json` | Where the job database is: the URL of its `manifest.json`, or the path of a local one. The files it names sit beside it, unless its `base_url` says otherwise. |
-| `max_age_days` | `60` | Postings posted more than this many days ago are left out. A posting with no date is kept. |
+| `database` | `https://huggingface.co/datasets/feder-cr/jobs/resolve/main/library/manifest.json` | Where the job database is: the URL of its `manifest.json`, or the path of a local one. AIHawk downloads into `data/library` the chunks the search's days need, the newest jobs first; after that, only what changed. |
+| `max_age_days` | `15` | How many days back a search looks; "Posted within (days)" in Settings. AIHawk downloads only the part of the database these days need, all of it before the first search. The database holds the last 60 days, so a larger number finds nothing more. A posting its source does not date is dated by the day the database first saw it. |
 | `model` | `deepseek/deepseek-v4.1-flash` | Fills the application forms. |
 | `quick_model` | `google/gemini-3.5-flash` | Writes the search plan from your CV, and the other short calls of a search. |
 | `decision_model` | `typesafe/jev-1.14` | Jev, which answers for each posting whether it is in line with you; how sure it is makes the score, from 1 to 5. |

pipeline/build.py

@@ -6,9 +6,9 @@
 Scheduled every five hours in .github/workflows/jobs-database.yml, which publishes work/out to the Hugging Face dataset feder-cr/jobs. Each source is read into
 work/sources/<name>.parquet in the columns of COMMON. A source that fails is left out of that build's database and named in
 the manifest; publish.py refuses a database that lost too much of itself. Postings are the same posting when their links
-are the same after `posting_key` (src/aihawk/key.sql), and of those the row with the longest description is kept, the other rows adding their source's
-name. The output is cut in files of ROWS_PER_FILE postings, with index.parquet beside them: every posting without its
-description, and where to find that description. A search reads the index; only the jobs being judged read descriptions.
+are the same after `posting_key` (src/aihawk/key.sql), and of those the row with the longest description is kept. The
+output is cut in chunks of CHUNK_ROWS complete postings, the newest first, which AIHawk downloads one after the other:
+every column it reads and nothing else, the posting's address being its key.
 """
 
 from __future__ import annotations
@@ -22,6 +22,7 @@ import subprocess
 import sys
 import tempfile
 import time
+import urllib.error
 import urllib.request
 import zipfile
 from collections.abc import Callable
@@ -38,20 +39,43 @@ RAW = "https://raw.githubusercontent.com"
 JOBHIVE = "https://storage.stapply.ai/jobhive/v1/all.parquet"
 # The private dataset that keeps each source as the last build read it, with the version it was read at.
 SOURCES_DATASET = "feder-cr/jobs-sources"
-# Files of about 128 MB and row groups of 2048 rows: small row groups let a search read only the rows it needs over HTTP.
-# A group is a whole description column chunk, which DuckDB cannot write below 2048 rows.
-# Postings to a description file: about 120 MB at the measured 590 bytes a posting, under GitHub's 2 GiB a file, and small
-# enough that one file is sorted by key in memory.
-ROWS_PER_FILE = 200_000
+# The database is a list of chunks, each CHUNK_ROWS complete postings, the newest first: AIHawk downloads them one after the
+# other and searches what it has, so a person who has just come sees jobs arrive instead of waiting for all of them. A
+# posting with a description of at least DESCRIBED characters is in a `jobs_N` chunk; the others, whose text says nothing a
+# title does not, are in `titles_N` chunks, which come last and which a search puts below the rest.
+CHUNK_ROWS = 50_000
+# DuckDB cannot write row groups under 2048 rows: the parts it writes are rewritten by pyarrow into the chunks.
 ROW_GROUP = 2048
-# A description file's row group: a reader asking for one posting downloads its whole group. At 2048 rows that was about
-# 700 KB a posting; at 256, with statistics on the key alone (the other columns' made the footer ten times larger), a batch
-# of postings downloads about a quarter of the bytes. DuckDB cannot write groups this small, so pyarrow writes these files.
-DESCRIPTION_GROUP = 256
-DESCRIPTION_COLUMNS = ("key", "description", "sources")
-INDEX = "index.parquet"
-# A description shorter than this says nothing a title does not: the index marks the posting as having none, and a search
-# puts it below one that has.
+# A chunk's row group. A chunk is downloaded whole, so nothing reads a single group over the network: groups of 10,000
+# rows compress like one of 100,000 (1.85 GB for the whole database at zstd 12 against 2.20 GB in groups of 256, measured
+# on 2026-11-09), and a search on this computer still skips the groups of postings older than it asks for.
+CHUNK_GROUP = 10_000
+# zstd 12: 1.85 GB against 2.04 at level 6 and 1.62 at 19, which took ten times as long to write. Measured on 2026-11-09.
+CHUNK_LEVEL = 12
+CHUNK_THREADS = 4
+# The part number of the first `titles_N` chunk: the `jobs_N` ones count from 0.
+TITLES_PART = 1_000_000
+# A posting older than this is not published: a search is for the last 60 days by default, and the owner chose on
+# 2026-11-09 to keep no more. Measured then: 763,000 postings with a description, about 0.42 GB of the database's 2.62.
+MAX_AGE_DAYS = 60
+# A posting its source does not date is dated by the database: the day a build first saw it, kept from build to build,
+# so that it ages and goes like any other. The owner's decision of 2026-11-09: 1.48 million postings with a description,
+# half the database, had no date, and a search kept them whatever its days. The first build to date them had no day
+# seen for them: it dated them between BACKDATED days ago, so the first download is light, by their site, not one by
+# one: a site's postings, alike in their texts, stay together in a chunk and compress together. Dated one by one, they
+# took the chunks with text from 1.91 GB to 2.46 GB (measured on 2026-11-09). A second run of that build gives the
+# same dates.
+BACKDATED = (15, 60)
+# The manifest's mark that its builds date every posting, by the rule above: a manifest without it, or with another,
+# has its undated postings backdated again.
+FIRST_SEEN = 2
+# The manifest of the database now published, whose chunks say the day each posting was dated.
+LIBRARY = "https://huggingface.co/datasets/feder-cr/jobs/resolve/main/library/manifest.json"
+# What a remote posting's location says when it does not narrow the job to a place; the database says "Remote" instead.
+NO_PLACE = ("", "remote", "anywhere", "worldwide", "global")
+# What a chunk holds: nothing AIHawk does not read. The posting's address is its key.
+CHUNK_COLUMNS = ("url", "title", "company", "location", "country", "posted_at", "description")
+# A description shorter than this says nothing a title does not: it is not published, and the posting is a `titles_N` one.
 DESCRIBED = 200
 
 COMMON = (
@@ -64,10 +88,16 @@ COMMON = (
 KEY_SQL = Path(__file__).resolve().parents[1] / "src" / "aihawk" / "key.sql"
 KEY = "posting_key(url)"
 
-# One name for each vendor, whatever a source calls it.
+# The country of a posting as ISO 3166's two letters, whatever a source writes: its name, in English or in its own language,
+# or its three letters. Anything else, a city or "Remote", says no country. Measured on 2026-11-09: 1,725 different values,
+# 742,569 postings saying "UNITED STATES".
+COUNTRIES = {"UNITED STATES": "US", "UNITED STATES OF AMERICA": "US", "USA": "US", "U.S.": "US", "U.S.A.": "US", "AMERICA": "US", "UNITED KINGDOM": "GB", "GREAT BRITAIN": "GB", "ENGLAND": "GB", "SCOTLAND": "GB", "WALES": "GB", "NORTHERN IRELAND": "GB", "UK": "GB", "CANADA": "CA", "AUSTRALIA": "AU", "INDIA": "IN", "FRANCE": "FR", "KENYA": "KE", "NETHERLANDS": "NL", "THE NETHERLANDS": "NL", "NEDERLAND": "NL", "MEXICO": "MX", "MÉXICO": "MX", "UNITED ARAB EMIRATES": "AE", "UAE": "AE", "GERMANY": "DE", "DEUTSCHLAND": "DE", "SPAIN": "ES", "ESPAÑA": "ES", "MALAYSIA": "MY", "CHINA": "CN", "ITALY": "IT", "ITALIA": "IT", "AUSTRIA": "AT", "ÖSTERREICH": "AT", "PHILIPPINES": "PH", "BRAZIL": "BR", "BRASIL": "BR", "BELGIUM": "BE", "BELGIË": "BE", "BELGIQUE": "BE", "SOUTH AFRICA": "ZA", "NEW ZEALAND": "NZ", "POLAND": "PL", "POLSKA": "PL", "SINGAPORE": "SG", "IRELAND": "IE", "SWITZERLAND": "CH", "SCHWEIZ": "CH", "SUISSE": "CH", "SAUDI ARABIA": "SA", "TAIWAN": "TW", "PORTUGAL": "PT", "THAILAND": "TH", "JAPAN": "JP", "日本": "JP", "EGYPT": "EG", "HONG KONG": "HK", "NIGERIA": "NG", "VIETNAM": "VN", "VIET NAM": "VN", "INDONESIA": "ID", "SWEDEN": "SE", "SVERIGE": "SE", "CHILE": "CL", "PAKISTAN": "PK", "DENMARK": "DK", "DANMARK": "DK", "PERU": "PE", "PERÚ": "PE", "LUXEMBOURG": "LU", "COSTA RICA": "CR", "COLOMBIA": "CO", "CZECHIA": "CZ", "CZECH REPUBLIC": "CZ", "FINLAND": "FI", "SUOMI": "FI", "PUERTO RICO": "PR", "NORWAY": "NO", "NORGE": "NO", "ROMANIA": "RO", "UKRAINE": "UA", "УКРАЇНА": "UA", "ARGENTINA": "AR", "MALTA": "MT", "SERBIA": "RS", "CYPRUS": "CY", "ISRAEL": "IL", "KOREA, REPUBLIC OF": "KR", "SOUTH KOREA": "KR", "KOREA (SOUTH)": "KR", "KOREA": "KR", "REPUBLIC OF KOREA": "KR", "BULGARIA": "BG", "GREECE": "GR", "HUNGARY": "HU", "LEBANON": "LB", "GHANA": "GH", "QATAR": "QA", "TURKEY": "TR", "TÜRKIYE": "TR", "TURKIYE": "TR", "ARMENIA": "AM", "SRI LANKA": "LK", "JAMAICA": "JM", "LIBYA": "LY", "MOROCCO": "MA", "MAROC": "MA", "KUWAIT": "KW", "LATVIA": "LV", "JORDAN": "JO", "PANAMA": "PA", "PANAMÁ": "PA", "DOMINICAN REPUBLIC": "DO", "GUYANA": "GY", "LITHUANIA": "LT", "TANZANIA": "TZ", "TANZANIA, UNITED REPUBLIC OF": "TZ", "CAYMAN ISLANDS": "KY", "GUATEMALA": "GT", "BARBADOS": "BB", "BAHAMAS": "BS", "THE BAHAMAS": "BS", "BOLIVIA": "BO", "BERMUDA": "BM", "BELIZE": "BZ", "SLOVAKIA": "SK", "PAPUA NEW GUINEA": "PG", "MAURITIUS": "MU", "UGANDA": "UG", "CÔTE D'IVOIRE": "CI", "COTE D'IVOIRE": "CI", "IVORY COAST": "CI", "TUNISIA": "TN", "ECUADOR": "EC", "MACAO": "MO", "MACAU": "MO", "HONDURAS": "HN", "SLOVENIA": "SI", "ESTONIA": "EE", "NAMIBIA": "NA", "OMAN": "OM", "SULTANATE OF OMAN": "OM", "BOSNIA AND HERZEGOVINA": "BA", "MADAGASCAR": "MG", "ZAMBIA": "ZM", "BANGLADESH": "BD", "ZIMBABWE": "ZW", "ALBANIA": "AL", "TRINIDAD AND TOBAGO": "TT", "URUGUAY": "UY", "MOLDOVA": "MD", "MALDIVES": "MV", "SYRIA": "SY", "EL SALVADOR": "SV", "SENEGAL": "SN", "IRAQ": "IQ", "KOSOVO": "XK", "ICELAND": "IS", "TIMOR-LESTE": "TL", "GUINEA": "GN", "GUAM": "GU", "CROATIA": "HR", "HRVATSKA": "HR", "RUSSIA": "RU", "RUSSIAN FEDERATION": "RU", "IRAN": "IR", "ALGERIA": "DZ", "ETHIOPIA": "ET", "RWANDA": "RW", "BOTSWANA": "BW", "MOZAMBIQUE": "MZ", "ANGOLA": "AO", "CAMEROON": "CM", "NEPAL": "NP", "MONGOLIA": "MN", "KAZAKHSTAN": "KZ", "UZBEKISTAN": "UZ", "GEORGIA": "GE", "AZERBAIJAN": "AZ", "BELARUS": "BY", "NORTH MACEDONIA": "MK", "MONTENEGRO": "ME", "VENEZUELA": "VE", "PARAGUAY": "PY", "NICARAGUA": "NI", "CUBA": "CU", "HAITI": "HT", "BAHRAIN": "BH", "YEMEN": "YE", "AFGHANISTAN": "AF", "MYANMAR": "MM", "CAMBODIA": "KH", "LAOS": "LA", "BRUNEI": "BN", "FIJI": "FJ", "LIECHTENSTEIN": "LI", "MONACO": "MC", "ANDORRA": "AD", "GIBRALTAR": "GI", "ISLE OF MAN": "IM", "JERSEY": "JE", "GUERNSEY": "GG", "GREENLAND": "GL", "ARUBA": "AW", "CURAÇAO": "CW", "CURACAO": "CW", "MALAWI": "MW", "SUDAN": "SD", "SOMALIA": "SO", "DJIBOUTI": "DJ", "ERITREA": "ER", "MALI": "ML", "NIGER": "NE", "CHAD": "TD", "TOGO": "TG", "BENIN": "BJ", "BURKINA FASO": "BF", "SIERRA LEONE": "SL", "LIBERIA": "LR", "GAMBIA": "GM", "GABON": "GA", "CONGO": "CG", "DEMOCRATIC REPUBLIC OF THE CONGO": "CD", "SOUTH SUDAN": "SS", "LESOTHO": "LS", "ESWATINI": "SZ", "SEYCHELLES": "SC", "CAPE VERDE": "CV", "TURKS AND CAICOS ISLANDS": "TC", "VIRGIN ISLANDS": "VI", "SAINT LUCIA": "LC", "GRENADA": "GD", "DOMINICA": "DM", "ANTIGUA AND BARBUDA": "AG", "SAINT KITTS AND NEVIS": "KN", "SURINAME": "SR", "NEW CALEDONIA": "NC", "FRENCH POLYNESIA": "PF", "RÉUNION": "RE", "REUNION": "RE", "DEU": "DE", "SGP": "SG", "PHL": "PH", "MYS": "MY", "AUS": "AU", "CHN": "CN", "BRA": "BR", "JPN": "JP", "NLD": "NL", "IRL": "IE", "IRQ": "IQ", "KOR": "KR", "EGY": "EG", "VNM": "VN", "POL": "PL", "DJI": "DJ", "SYR": "SY", "TCA": "TC", "ITA": "IT", "KWT": "KW", "ISR": "IL", "FRA": "FR", "SWE": "SE", "PAK": "PK", "KEN": "KE", "ARE": "AE", "DOM": "DO", "SAU": "SA", "GBR": "GB", "CAN": "CA", "IND": "IN", "ESP": "ES", "MEX": "MX", "CHE": "CH", "AUT": "AT", "BEL": "BE", "PRT": "PT", "NZL": "NZ", "ZAF": "ZA", "ARG": "AR", "COL": "CO", "CHL":Line truncated
+NOT_COUNTRIES = ("XX", "EU", "UN", "ZZ")
+# One name for each vendor, whatever a source calls it. None: a label that names no vendor. One source labels postings of
+# every site "datastax", 34,477 of them LinkedIn's (measured on 2026-11-09).
 VENDOR_ALIASES = {
     "ashbyhq": "ashby", "join_com": "join", "oracle_hcm": "oracle", "wttj": "welcometothejungle", "csod": "cornerstone",
-    "ultipro": "ukg", "beisen_legacy": "beisen",
+    "ultipro": "ukg", "beisen_legacy": "beisen", "datastax": None,
 }  # fmt: skip
 # The vendor a link's host names, for the postings whose source does not say. The first that matches.
 VENDOR_HOSTS = (
@@ -86,6 +116,16 @@ VENDOR_HOSTS = (
     ("recruiterbox.com", "recruiterbox"), ("betterteam.com", "betterteam"), ("linkedin.com", "linkedin"),
     ("indeed.", "indeed"), ("glassdoor.", "glassdoor"), ("europa.eu", "eures"), ("arbeitsagentur.de", "bundesagentur"),
     ("jobbank.gc.ca", "jobbankca"), ("jobicy.com", "jobicy"), ("welcometothejungle.com", "welcometothejungle"),
+    # Read from the hosts of the postings no vendor was known for, on 2026-11-09: forms first, then boards.
+    ("brassring.com", "brassring"), ("governmentjobs.com", "neogov"), ("schooljobs.com", "neogov"),
+    ("recruitingbypaycor.com", "paycor"), ("applitrack.com", "applitrack"), ("hrmdirect.com", "hrmdirect"),
+    ("cadienttalent.com", "cadient"), ("hireology.com", "hireology"), ("workstream.us", "workstream"),
+    ("paradox.ai", "paradox"), ("apploi.com", "apploi"), ("pageuppeople.com", "pageup"), ("comeet.com", "comeet"),
+    ("dover.com", "dover"), ("silkroad.com", "silkroad"), ("hirebridge.com", "hirebridge"), ("careers-page.com", "manatal"),
+    ("jobappnetwork.com", "jobappnetwork"), ("arbeitnow.com", "arbeitnow"), ("builtin.com", "builtin"),
+    ("trudvsem.ru", "trudvsem"), ("jobsyn.org", "directemployers"), ("usajobs.gov", "usajobs"), ("glints.com", "glints"),
+    ("jobvision.ir", "jobvision"), ("jobinja.ir", "jobinja"), ("irantalent.com", "irantalent"), ("51job.com", "51job"),
+    ("myjob.mu", "myjobmu"), ("hr.ge", "hrge"), ("jobmaster.co.il", "jobmaster"), ("reed.co.uk", "reed"),
 )  # fmt: skip
 # Vendors that list other companies' postings rather than take the application: a copy on one is never the one kept when
 # the company's own form carries the same job.
@@ -93,6 +133,8 @@ AGGREGATORS = (
     "eures", "bundesagentur", "jobbankca", "linkedin", "indeed", "glassdoor", "jobicy", "arbeitnow", "weworkremotely",
     "remoteok", "remotive", "himalayas", "workingnomads", "builtin", "themuse", "wellfound", "ycombinator", "getonbrd",
     "cryptojobs", "fashionjobs", "jobsch", "programathor", "jobtech", "arbetsformedlingen", "manfred", "welcometothejungle",
+    "trudvsem", "directemployers", "usajobs", "glints", "jobvision", "jobinja", "irantalent", "51job", "myjobmu", "hrge",
+    "jobmaster", "reed",
 )  # fmt: skip
 # Boards whose posting pages, opened on 2026-11-08, sent the applicant on to one of the companies' own forms AIHawk applies
 # on: such a posting goes, as the job is the form's. EURES, the Bundesagentur, Welcome to the Jungle and Job Bank showed no
@@ -113,13 +155,22 @@ LEGAL_FORMS = (
 
 def _macros() -> str:
     """The vendor a posting belongs to, and the print that knows one job under links of different sites."""
-    aliases = " ".join(f"WHEN '{name}' THEN '{to}'" for name, to in VENDOR_ALIASES.items())
+    aliases = " ".join(f"WHEN '{name}' THEN {f'{chr(39)}{to}{chr(39)}' if to else 'NULL'}" for name, to in VENDOR_ALIASES.items())
     hosts = " ".join(f"WHEN contains(regexp_extract(lower(url), '^https?://([^/]+)', 1), '{host}') THEN '{name}'" for host, name in VENDOR_HOSTS)
     forms = "|".join(LEGAL_FORMS)
+    # Quoted for SQL: a name may hold an apostrophe, as Côte d'Ivoire's does.
+    countries = ", ".join("'" + name.replace("'", "''") + "'" for name in COUNTRIES)
+    codes = " ".join("WHEN '" + name.replace("'", "''") + f"' THEN '{code}'" for name, code in COUNTRIES.items())
     return f"""
+        -- The link's host first: it is where the application goes, and a source's label can be wrong.
         CREATE OR REPLACE MACRO vendor(ats, url) AS coalesce(
-            CASE lower(nullif(trim(ats), '')) {aliases} ELSE lower(nullif(trim(ats), '')) END,
-            CASE {hosts} END);
+            CASE {hosts} END,
+            CASE lower(nullif(trim(ats), '')) {aliases} ELSE lower(nullif(trim(ats), '')) END);
+        CREATE OR REPLACE MACRO country_code(country) AS CASE
+            WHEN upper(trim(country)) IN {NOT_COUNTRIES!r} THEN NULL
+            WHEN upper(trim(country)) IN ({countries}) THEN CASE upper(trim(country)) {codes} END
+            WHEN regexp_matches(upper(trim(country)), '^[A-Z]{{2}}$') THEN upper(trim(country))
+            END;
         CREATE OR REPLACE MACRO plain(text) AS trim(regexp_replace(regexp_replace(lower(coalesce(text, '')), '[^\\pL\\pN]+', ' ', 'g'), '\\s+', ' ', 'g'));
         CREATE OR REPLACE MACRO employer(company) AS trim(regexp_replace(regexp_replace(plain(company), '\\b({forms})\\b', ' ', 'g'), '\\s+', ' ', 'g'));
         CREATE OR REPLACE MACRO city(location) AS plain(split_part(coalesce(location, ''), ',', 1));
@@ -609,7 +660,9 @@ def read_source(connection: duckdb.DuckDBPyConnection, source: Source, work: Pat
     return report
 
 
-def merge(connection: duckdb.DuckDBPyConnection, work: Path, sent_on: Callable[[list[tuple[str, str]]], set[str]] = lambda _: set()) -> dict[str, int]:
+def merge(
+    connection: duckdb.DuckDBPyConnection, work: Path, sent_on: Callable[[list[tuple[str, str]]], set[str]] = lambda _: set(), backdate: bool = False
+) -> dict[str, int]:
     """One row per posting, cut in files. Returns how many rows went in and came out.
 
     Which row wins is decided on the keys alone, and only then are the winners' rows, descriptions included, read and
@@ -628,13 +681,17 @@ def merge(connection: duckdb.DuckDBPyConnection, work: Path, sent_on: Callable[[
             -- came out changed in a delta though nothing about them had.
             arg_max(
                 struct_pack(filename := filename, row := file_row_number, length := length, source := source, print := print,
-                    employer := vendor IS NOT NULL AND vendor NOT IN {AGGREGATORS!r}, host := host, vendor := vendor, url := url),
+                    employer := vendor IS NOT NULL AND vendor NOT IN {AGGREGATORS!r}, host := host, vendor := vendor, url := url,
+                    posted := posted),
                 struct_pack(length := length, source := source, row := -file_row_number)
             ) AS winner
         FROM (
             SELECT {KEY} AS key, regexp_extract(filename, '([^/\\\\]+)\\.parquet$', 1) AS source, filename, file_row_number,
                 length(coalesce(description, '')) AS length, vendor(ats, url) AS vendor, job_print(company, title, location) AS print,
-                regexp_extract(posting_link(url), '^[^/]+') AS host, url
+                regexp_extract(posting_link(url), '^[^/]+') AS host, url,
+                -- A date after the build is a source's mistake: the posting is as new as one posted now, and no newer. Not
+                -- least(): it skips a NULL, and dated every posting with no date as now.
+                CASE WHEN TRY_CAST(posted_at AS TIMESTAMP) > now()::TIMESTAMP THEN now()::TIMESTAMP ELSE TRY_CAST(posted_at AS TIMESTAMP) END AS posted
             FROM {every_row}
             -- No LinkedIn posting is kept, whatever source carried it: the owner's decision of 2026-11-08.
             WHERE url LIKE 'http%' AND coalesce(title, '') <> '' AND vendor(ats, url) IS DISTINCT FROM 'linkedin'
@@ -694,67 +751,108 @@ def merge(connection: duckdb.DuckDBPyConnection, work: Path, sent_on: Callable[[
         connection.execute("CREATE OR REPLACE TEMP TABLE sent AS SELECT unnest(?) AS key", [sorted(sent)])
         connection.execute("DELETE FROM best WHERE key IN (SELECT key FROM sent)")
     print(f"boards: {len(sent)} of {len(on_boards)} postings checked send the applicant on to a company's form, and go", flush=True)
-    rows_in, rows_out_expected = connection.execute("SELECT sum(copies), count(*) FROM best").fetchone()
+    rows_in = connection.execute("SELECT sum(copies) FROM best").fetchone()[0]
     out = work / "out"
     shutil.rmtree(out, ignore_errors=True)
     out.mkdir(parents=True)
-    columns = ", ".join("vendor(row.ats, row.url) AS ats" if column == "ats" else f"row.{column}" for column in COMMON)
-    # The files are in key order, so a reader that asks for a posting by its key reads the one row group that can hold it,
-    # whatever build it came from. Sorting ten million rows with their descriptions at once ran a runner out of memory,
-    # so only the keys are sorted, into ranges of ROWS_PER_FILE; the rows go to their range's part in one pass; and each
-    # part, a file's worth, is sorted alone.
-    connection.execute(f"CREATE OR REPLACE TEMP TABLE ranges AS SELECT key, (row_number() OVER (ORDER BY key) - 1) // {ROWS_PER_FILE} AS part FROM best")
+    # A remote posting that names no place says "Remote": the database has no column for it, and a search takes it for
+    # one open to anyone, as it took one flagged remote with no place.
+    unnamed = ", ".join(repr(word) for word in NO_PLACE)
+    location = f"CASE WHEN row.remote AND coalesce(row.country, '') = '' AND lower(trim(coalesce(row.location, ''))) IN ({unnamed}) THEN 'Remote' ELSE row.location END"
+    # Sorting ten million rows with their descriptions at once ran a runner out of memory, so only the keys are sorted: the
+    # newest first, into chunks of CHUNK_ROWS, the described ones apart from the others; the rows go to their chunk's part in
+    # one pass; and each part is sorted alone. A posting older than MAX_AGE_DAYS is not kept.
+    # Each posting's date: its source's; else the day the database first saw it, from `seen`; else today, or, for the first
+    # build that dates them, a day between BACKDATED days ago.
+    if not connection.execute("SELECT count(*) FROM duckdb_tables() WHERE table_name = 'seen'").fetchone()[0]:
+        connection.execute("CREATE TEMP TABLE seen (url VARCHAR, posted_at TIMESTAMP)")
+    low, high = BACKDATED
+    first = f"now()::TIMESTAMP - to_days(CAST({low} + hash(best.winner.host) % {high - low + 1} AS INTEGER))" if backdate else "now()::TIMESTAMP"
+    connection.execute(f"""
+        CREATE OR REPLACE TEMP TABLE dated AS
+        SELECT best.key, coalesce(best.winner.posted, seen.posted_at, {first}) AS posted, best.winner.length >= {DESCRIBED} AS described
+        FROM best LEFT JOIN (SELECT url, max(posted_at) AS posted_at FROM seen GROUP BY url) AS seen ON seen.url = best.winner.url
+    """)
+    connection.execute(f"""
+        CREATE OR REPLACE TEMP TABLE ranges AS
+        SELECT key, posted, CASE WHEN described THEN 0 ELSE {TITLES_PART} END
+            + (row_number() OVER (PARTITION BY described ORDER BY posted DESC, key) - 1) // {CHUNK_ROWS} AS part
+        FROM dated
+        WHERE posted >= now()::TIMESTAMP - INTERVAL {MAX_AGE_DAYS} DAY
+    """)
     parts = work / "tmp" / "parts"
     shutil.rmtree(parts, ignore_errors=True)
     connection.execute(f"""
         COPY (
-            SELECT best.key, {columns}, best.sources, ranges.part
+            SELECT row.url, row.title, row.company, {location} AS location, country_code(row.country) AS country, ranges.posted AS posted_at,
+                CASE WHEN best.winner.length >= {DESCRIBED} THEN row.description END AS description, ranges.part
             FROM {every_row} AS row
             JOIN best ON row.filename = best.winner.filename AND row.file_row_number = best.winner.row
             JOIN ranges ON ranges.key = best.key
         ) TO '{parts.as_posix()}' (
-            -- One open file a range, each holding a row group in memory: small row groups keep fifty of them light.
+            -- One open file a part, each holding a row group in memory: small row groups keep a hundred of them light.
             FORMAT parquet, COMPRESSION zstd, PARTITION_BY (part), ROW_GROUP_SIZE {ROW_GROUP}
         )
     """)
-    ordered = work / "tmp" / "ordered"
-    shutil.rmtree(ordered, ignore_errors=True)
-    ordered.mkdir(parents=True)
-    for part in sorted(parts.glob("part=*"), key=lambda folder: int(folder.name.split("=")[1])):
-        number = part.name.split("=")[1]
-        connection.execute(f"""
-            COPY (SELECT * FROM read_parquet('{part.as_posix()}/*.parquet', hive_partitioning = false) ORDER BY key)
-            TO '{(ordered / f"{number}.parquet").as_posix()}' (FORMAT parquet, COMPRESSION zstd, ROW_GROUP_SIZE {ROW_GROUP})
-        """)
-        shutil.rmtree(part)
-        description_file(ordered / f"{number}.parquet", out / f"jobs_{number}.parquet")
-    rows_out = connection.execute(f"SELECT count(*) FROM read_parquet('{out.as_posix()}/jobs_*.parquet')").fetchone()[0]
-    if rows_out != rows_out_expected:
-        raise SystemExit(f"{rows_out_expected} postings chosen and {rows_out} written")
-    # Every posting without its description, which a reader finds by the key in the file whose range holds it.
-    connection.execute(f"""
-        COPY (
-            SELECT key, url, title, company, ats, location, country, remote, salary_min, salary_max, salary_currency, posted_at,
-                length(coalesce(description, '')) >= {DESCRIBED} AS described
-            FROM read_parquet('{ordered.as_posix()}/*.parquet')
-        ) TO '{(out / INDEX).as_posix()}' (FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 9)
-    """)
-    shutil.rmtree(ordered)
+    folders = sorted(parts.glob("part=*"), key=lambda folder: int(folder.name.split("=")[1]))
+    with ThreadPoolExecutor(CHUNK_THREADS) as pool:
+        list(pool.map(lambda folder: write_chunk(folder, out), folders))
+    rows_out = connection.execute(f"SELECT count(*) FROM read_parquet('{out.as_posix()}/*.parquet')").fetchone()[0]
+    expected = connection.execute("SELECT count(*) FROM ranges").fetchone()[0]
+    if rows_out != expected:
+        raise SystemExit(f"{expected} postings chosen and {rows_out} written")
     return {"rows_in": int(rows_in), "rows_out": rows_out}
 
 
-def description_file(ordered: Path, out: Path) -> None:
-    """A part's descriptions, as a reader looks them up: by key alone, so the other columns stay in the index."""
+def seen_dates(connection: duckdb.DuckDBPyConnection) -> bool:
+    """`seen`, the date of each posting in the database now published, when its builds date every posting; False when they
+    do not yet, and the postings no source dates are to be backdated."""
+    try:
+        with urllib.request.urlopen(urllib.request.Request(LIBRARY, headers={"User-Agent": "AIHawk build"}), timeout=60) as response:
+            manifest = json.load(response)
+    except urllib.error.HTTPError as error:
+        # The first build of all has no manifest before it; any other failure stops the build, rather than date anew,
+        # and so change, every posting no source dates.
+        if error.code not in (401, 404):
+            raise
+        print(f"seen dates: no database published yet ({error.code})", flush=True)
+        return False
+    if manifest.get("first_seen") != FIRST_SEEN:
+        return False
+    files = [f"{manifest['base_url']}/{chunk['name']}" for chunk in manifest["chunks"]]
+    connection.execute("CREATE OR REPLACE TEMP TABLE seen AS SELECT url, posted_at FROM read_parquet(?)", [files])
+    print(f"seen dates: {connection.execute('SELECT count(*) FROM seen').fetchone()[0]} postings dated by {manifest['build']}", flush=True)
+    return True
+
+
+def write_chunk(folder: Path, out: Path) -> None:
+    """A part as a chunk: its postings newest first, in row groups of CHUNK_GROUP, and the part's folder gone."""
     import pyarrow.parquet as pq  # imported here: only the build writes these files
 
-    table = pq.read_table(ordered, columns=list(DESCRIPTION_COLUMNS))
-    pq.write_table(table, out, row_group_size=DESCRIPTION_GROUP, compression="zstd", write_statistics=["key"])
+    part = int(folder.name.split("=")[1])
+    name = f"titles_{part - TITLES_PART}.parquet" if part >= TITLES_PART else f"jobs_{part}.parquet"
+    table = pq.read_table([path.as_posix() for path in sorted(folder.glob("*.parquet"))], columns=list(CHUNK_COLUMNS))
+    # By address within the chunk: a site's postings side by side compress together, 4 to 8% smaller than by date.
+    table = table.sort_by([("url", "ascending")])
+    # Statistics on the date alone: a search for the last 60 days skips the groups of older postings. Those of the other
+    # columns made the footer ten times larger.
+    pq.write_table(table, out / name, row_group_size=CHUNK_GROUP, compression="zstd", compression_level=CHUNK_LEVEL, write_statistics=["posted_at"])
+    shutil.rmtree(folder)
 
 
-def file_entry(connection: duckdb.DuckDBPyConnection, path: Path) -> dict[str, object]:
-    """A description file as the manifest names it: its size, and the first and last key it holds."""
-    first, last = connection.execute("SELECT min(key), max(key) FROM read_parquet(?)", [path.as_posix()]).fetchone()
-    return {"name": path.name, "bytes": path.stat().st_size, "first_key": first, "last_key": last}
+def chunk_entry(connection: duckdb.DuckDBPyConnection, path: Path) -> dict[str, object]:
+    """A chunk as the manifest names it: its size, its postings, the dates it spans, and whether its postings have no text."""
+    rows, newest, oldest = connection.execute("SELECT count(*), max(posted_at), min(posted_at) FROM read_parquet(?)", [path.as_posix()]).fetchone()
+    return {
+        "name": path.name, "bytes": path.stat().st_size, "rows": rows, "titles_only": path.name.startswith("titles_"),
+        "newest": newest.isoformat() if newest else None, "oldest": oldest.isoformat() if oldest else None,
+    }
+
+
+def chunk_order(path: Path) -> tuple[int, int]:
+    """The order AIHawk downloads them in: the `jobs_N` chunks, then the `titles_N` ones, each the newest first."""
+    kind, _, number = path.stem.partition("_")
+    return (kind == "titles", int(number))
 
 
 def main() -> None:
@@ -778,22 +876,24 @@ def main() -> None:
     if not arguments.reuse:
         reports = read_all(connection, chosen, work, cache)
     _clean_up(work)
-    counts = merge(connection, work, board_check(cache, work))
+    dated = seen_dates(connection)
+    counts = merge(connection, work, board_check(cache, work), backdate=not dated)
     out = work / "out"
     manifest = {
-        # 2: the description files are in key order and say their first and last key, and the dataset carries a
-        # delta of the index for each build. Everything schema 1 had is still here, for an AIHawk not yet updated.
-        "schema": 2,
+        # 3: the database is a list of chunks of complete postings, the newest first, which AIHawk downloads one after the
+        # other, and a delta of the postings added, changed or gone for each build.
+        "schema": 3,
         "built_at": datetime.now(UTC).isoformat(timespec="seconds"),
         **counts,
         "seconds": round(time.monotonic() - started),
-        "columns": list(DESCRIPTION_COLUMNS),
-        "index": {"name": INDEX, "bytes": (out / INDEX).stat().st_size},
-        "files": [file_entry(connection, path) for path in sorted(out.glob("jobs_*.parquet"))],
+        "columns": list(CHUNK_COLUMNS),
+        "chunks": [chunk_entry(connection, path) for path in sorted(out.glob("*.parquet"), key=chunk_order)],
+        # Every posting is dated: by its source, or by the day a build first saw it.
+        "first_seen": FIRST_SEEN,
         "sources": reports,
     }
     (out / "manifest.json").write_text(json.dumps(manifest, indent=1), encoding="utf-8")
-    print(f"{counts['rows_in']} rows in, {counts['rows_out']} postings out, {len(manifest['files'])} files, {manifest['seconds']} s")
+    print(f"{counts['rows_in']} rows in, {counts['rows_out']} postings out, {len(manifest['chunks'])} chunks, {manifest['seconds']} s")
 
 
 if __name__ == "__main__":

pipeline/publish.py

@@ -1,12 +1,16 @@
-"""Publishes work/out to the Hugging Face dataset AIHawk reads, in one commit: the build's files, its delta and the
+"""Publishes work/out to the Hugging Face dataset AIHawk reads, in one commit: the build's chunks, its delta and the
 manifest that names them, so a reader never sees a half-replaced database.
 
+Everything is under `library/`: the chunks of a build in `library/builds/<tag>/`, the deltas in `library/deltas/`, the
+manifest at `library/manifest.json`. The layout before it, an index and description files read in place at the dataset's
+root, is removed by the first commit that finds it: AIHawk brings itself up to date at every start, and nothing reads it.
+
 A database much smaller than the one before it is refused: losing a big source must not silently empty everyone's search.
-The last KEEP builds stay, so a reader mid-search does not lose its files. Each build also publishes the delta of the
-index against the build before it: the postings added, the ones whose fields changed, and the keys gone. A reader that
-has the index applies them in order and never downloads the whole of it again; they are kept for ever, because a reader
-may come back after any time. After the commit the history is squashed: the files old commits hold would otherwise grow
-the dataset by three gigabytes a build.
+The last KEEP builds stay, so a reader still downloading one does not lose its chunks. Each build also publishes the delta
+against the build before it: every posting added or changed, whole, with its description, and the addresses gone. A
+reader that has a build applies the deltas after it in order and never downloads the whole database again; the last
+KEEP_DELTAS are kept, and a reader further behind downloads the newest build instead. After the commit the history is
+squashed: the files old commits hold would otherwise grow the dataset by two gigabytes a build.
 
 The database moved from GitHub's releases on 2026-11-08, when github.com answered 504 for its downloads for hours and its
 API allows sixty calls an hour with no account; Hugging Face allows three thousand every five minutes.
@@ -24,10 +28,14 @@ from pathlib import Path
 import duckdb
 
 DATASET = "feder-cr/jobs"
-RESOLVE = f"https://huggingface.co/datasets/{DATASET}/resolve/main"
-# The columns a delta carries: the index's own, without the file and row that point into one build's files.
-INDEX_COLUMNS = ("url", "title", "company", "ats", "location", "country", "remote", "salary_min", "salary_max", "salary_currency", "posted_at", "described")
-KEEP = 6  # five-hourly builds: about 30 hours of builds, so a reader mid-search does not lose its files
+FOLDER = "library"
+RESOLVE = f"https://huggingface.co/datasets/{DATASET}/resolve/main/{FOLDER}"
+# The layout before 2026-11-09, at the dataset's root: removed. The owner's decision of 2026-11-09.
+OLD_LAYOUT = ("builds", "deltas", "manifest.json")
+# The columns of a chunk, which a delta carries too, after its `op`: the address is the key.
+COLUMNS = ("url", "title", "company", "location", "country", "posted_at", "description")
+KEEP = 6  # five-hourly builds: about 30 hours, more than a slow line takes to download one
+KEEP_DELTAS = 60  # about twelve days of builds
 MIN_SHARE_OF_PREVIOUS = 0.7
 
 
@@ -42,41 +50,33 @@ def previous_manifest() -> dict | None:
         raise
 
 
-def build_of(manifest: dict) -> str:
-    """A build's tag: written in the manifest since schema 2, the end of its release's address before."""
-    return manifest.get("build") or manifest["base_url"].rstrip("/").rsplit("/", 1)[-1]
-
-
-def write_delta(previous_index: str, index: Path, out: Path) -> tuple[int, int]:
-    """The delta from the previous index to this one: an `add` row for each posting new or changed, a `drop` row for each key
-    gone. A changed posting is an add: a reader replaces whatever row it holds under that key. How many adds and drops."""
-    columns = ", ".join(INDEX_COLUMNS)
+def write_delta(previous: list[str], chunks: list[Path], out: Path) -> tuple[int, int]:
+    """The delta from the previous build's chunks to these: an `add` row for each posting new or changed, whole, and a
+    `drop` row for each address gone. A changed posting is an add: a reader replaces whatever row it holds under that
+    address. How many adds and drops."""
+    fields = ", ".join(column for column in COLUMNS if column != "url")
     connection = duckdb.connect()
     try:
         connection.execute("INSTALL httpfs; LOAD httpfs;")
-        # A column added since the previous build is NULL there: every posting is then changed once, and the delta carries it.
-        held = {row[0] for row in connection.execute("DESCRIBE SELECT * FROM read_parquet(?)", [previous_index]).fetchall()}
-        before = ", ".join(column if column in held else "NULL" for column in INDEX_COLUMNS)
-        connection.execute(f"CREATE TABLE before AS SELECT key, hash({before}) AS h FROM read_parquet(?)", [previous_index])
-        connection.execute(f"CREATE TABLE after AS SELECT key, hash({columns}) AS h FROM read_parquet(?)", [index.as_posix()])
+        connection.execute(f"CREATE TABLE before AS SELECT url, hash({fields}) AS h FROM read_parquet(?)", [previous])
+        now = [path.as_posix() for path in chunks]
+        connection.execute(f"CREATE TABLE after AS SELECT url, hash({fields}) AS h FROM read_parquet(?)", [now])
         connection.execute("""
             CREATE TABLE changed AS
-            SELECT coalesce(a.key, b.key) AS key, a.key IS NULL AS gone
-            FROM after AS a FULL OUTER JOIN before AS b ON a.key = b.key
-            WHERE a.key IS NULL OR b.key IS NULL OR a.h <> b.h
+            SELECT coalesce(a.url, b.url) AS url, a.url IS NULL AS gone
+            FROM after AS a FULL OUTER JOIN before AS b ON a.url = b.url
+            WHERE a.url IS NULL OR b.url IS NULL OR a.h <> b.h
         """)
         connection.execute(f"""
             COPY (
-                SELECT 'add' AS op, i.key, {", ".join(f"i.{column}" for column in INDEX_COLUMNS)}
-                FROM read_parquet(?) AS i SEMI JOIN changed AS c ON i.key = c.key AND NOT c.gone
+                SELECT 'add' AS op, {", ".join(f"c.{column}" for column in COLUMNS)}
+                FROM read_parquet(?) AS c SEMI JOIN changed AS d ON c.url = d.url AND NOT d.gone
                 UNION ALL
-                SELECT 'drop' AS op, c.key, {", ".join(f"NULL AS {column}" for column in INDEX_COLUMNS)}
-                FROM changed AS c WHERE c.gone
-            ) TO '{out.as_posix()}' (FORMAT parquet, COMPRESSION zstd)
-        """, [index.as_posix()])
-        adds, drops = connection.execute(
-            "SELECT count(*) FILTER (WHERE NOT gone), count(*) FILTER (WHERE gone) FROM changed"
-        ).fetchone()
+                SELECT 'drop' AS op, d.url, {", ".join(f"NULL AS {column}" for column in COLUMNS if column != "url")}
+                FROM changed AS d WHERE d.gone
+            ) TO '{out.as_posix()}' (FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 12)
+        """, [now])
+        adds, drops = connection.execute("SELECT count(*) FILTER (WHERE NOT gone), count(*) FILTER (WHERE gone) FROM changed").fetchone()
         return int(adds), int(drops)
     finally:
         connection.close()
@@ -89,16 +89,17 @@ def check(rows: int, previous: int | None) -> None:
         sys.exit(f"{rows} postings against {previous} in the previous build: nothing published. See the manifest's failed sources.")
 
 
-def commit_plan(out: Path, manifest: dict, delta: Path | None, built: list[str]) -> tuple[list[tuple[str, Path]], list[str]]:
-    """What the commit adds, as (path in the dataset, file here), and the build folders it removes: all but the last KEEP."""
+def commit_plan(out: Path, manifest: dict, delta: Path | None, built: list[str], stored_deltas: list[str]) -> tuple[list[tuple[str, Path]], list[str], list[str]]:
+    """What the commit adds, as (path in the dataset, file here), the build folders it removes, all but the last KEEP,
+    and the delta files it removes, those the manifest no longer names."""
     tag = manifest["build"]
-    names = [manifest["index"]["name"], *(item["name"] for item in manifest["files"])]
-    adds = [(f"builds/{tag}/{name}", out / name) for name in names]
+    adds = [(f"{FOLDER}/builds/{tag}/{chunk['name']}", out / chunk["name"]) for chunk in manifest["chunks"]]
     if delta is not None:
-        adds.append((f"deltas/{delta.name}", delta))
-    adds.append(("manifest.json", out / "manifest.json"))
+        adds.append((f"{FOLDER}/deltas/{delta.name}", delta))
+    adds.append((f"{FOLDER}/manifest.json", out / "manifest.json"))
     kept = sorted({*built, tag})[-KEEP:]
-    return adds, [f"builds/{old}" for old in sorted(built) if old not in kept]
+    named = {entry["url"].rsplit("/", 1)[-1] for entry in manifest["deltas"]}
+    return adds, [f"{FOLDER}/builds/{old}" for old in sorted(built) if old not in kept], [f"{FOLDER}/deltas/{name}" for name in sorted(stored_deltas) if name not in named]
 
 
 def main() -> None:
@@ -123,21 +124,28 @@ def main() -> None:
     delta = None
     if previous is not None:
         delta = arguments.out / f"delta-{tag}.parquet"
-        adds, drops = write_delta(f"{previous['base_url']}/{previous['index']['name']}", arguments.out / manifest["index"]["name"], delta)
-        manifest["deltas"] = [*previous.get("deltas", []), {"build": tag, "after": build_of(previous), "url": f"{RESOLVE}/deltas/{delta.name}", "adds": adds, "drops": drops}]
-        print(f"delta against {build_of(previous)}: {adds} added or changed, {drops} gone")
+        chunks = [arguments.out / chunk["name"] for chunk in manifest["chunks"]]
+        adds, drops = write_delta([f"{previous['base_url']}/{chunk['name']}" for chunk in previous["chunks"]], chunks, delta)
+        entry = {"build": tag, "after": previous["build"], "url": f"{RESOLVE}/deltas/{delta.name}", "bytes": delta.stat().st_size, "adds": adds, "drops": drops}
+        manifest["deltas"] = [*previous.get("deltas", []), entry][-KEEP_DELTAS:]
+        print(f"delta against {previous['build']}: {adds} added or changed, {drops} gone, {delta.stat().st_size // 1_000_000} MB")
     manifest_path.write_text(json.dumps(manifest, indent=1), encoding="utf-8")
 
-    try:
-        built = [item.path.removeprefix("builds/") for item in api.list_repo_tree(DATASET, path_in_repo="builds", repo_type="dataset")]
-    except Exception:  # noqa: BLE001  The first build: there is no builds folder yet.
-        built = []
-    adds, removed = commit_plan(arguments.out, manifest, delta, built)
+    def listed(folder: str) -> list[str]:
+        try:
+            return [item.path.rsplit("/", 1)[-1] for item in api.list_repo_tree(DATASET, path_in_repo=f"{FOLDER}/{folder}", repo_type="dataset")]
+        except Exception:  # noqa: BLE001  The first build: there is no such folder yet.
+            return []
+
+    adds, removed, old_deltas = commit_plan(arguments.out, manifest, delta, listed("builds"), listed("deltas"))
     operations = [CommitOperationAdd(path_in_repo=path, path_or_fileobj=str(local)) for path, local in adds]
     operations += [CommitOperationDelete(path_in_repo=f"{folder}/", is_folder=True) for folder in removed]
+    operations += [CommitOperationDelete(path_in_repo=path) for path in old_deltas]
+    root = {item.path for item in api.list_repo_tree(DATASET, repo_type="dataset")}
+    operations += [CommitOperationDelete(path_in_repo=f"{path}/" if "." not in path else path, is_folder="." not in path) for path in OLD_LAYOUT if path in root]
     api.create_commit(DATASET, repo_type="dataset", operations=operations, commit_message=f"{tag}: {manifest['rows_out']} postings")
     api.super_squash_history(DATASET, repo_type="dataset", commit_message=f"The job database as of {tag}")
-    print(f"published {tag}: {manifest['rows_out']} postings, {len(adds)} files; removed {len(removed)} old builds")
+    print(f"published {tag}: {manifest['rows_out']} postings, {len(adds)} files; removed {len(removed)} old builds and {len(old_deltas)} old deltas")
 
 
 if __name__ == "__main__":

run.ps1

@@ -3,4 +3,5 @@ if (-not (Get-Command uvx -ErrorAction SilentlyContinue)) {
     Invoke-RestMethod https://astral.sh/uv/install.ps1 | Invoke-Expression
     $env:Path = "$env:USERPROFILE\.local\bin;$env:Path"
 }
-uvx --refresh-package aihawk --from git+https://github.com/feder-cr/Jobs_Applier_AI_Agent aihawk
+# Python 3.14: on 3.15, the newest, PyYAML has no Windows build yet, and the install stopped asking for a C++ compiler.
+uvx --python 3.14 --refresh-package aihawk --from git+https://github.com/feder-cr/Jobs_Applier_AI_Agent aihawk

run.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # AIHawk in one command: installs uv if it is missing, brings AIHawk up to date, and starts it.
 command -v uvx >/dev/null 2>&1 || { curl -LsSf https://astral.sh/uv/install.sh | sh && . "$HOME/.local/bin/env"; }
-# This script arrives through a pipe, and the screen needs the terminal.
-exec uvx --refresh-package aihawk --from git+https://github.com/feder-cr/Jobs_Applier_AI_Agent aihawk "$@" </dev/tty
+# This script arrives through a pipe, and the screen needs the terminal. Python 3.14: on 3.15, the newest, some of
+# AIHawk's libraries have no ready build yet, and the install stopped to compile them.
+exec uvx --python 3.14 --refresh-package aihawk --from git+https://github.com/feder-cr/Jobs_Applier_AI_Agent aihawk "$@" </dev/tty

scripts/usage_counters.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""Daily snapshot of the applications AIHawk has sent, for all its users together, and the README badge drawn from it.
+
+    python scripts/usage_counters.py --history H.csv --daily D.csv --badges DIR
+    python scripts/usage_counters.py --selftest
+
+Run once a day by .github/workflows/counters.yml, which keeps the files on the unprotected `badges` branch: no person and
+no machine of ours, every input a public GitHub API answer, every output a file on that branch. The way
+invisible_playwright records its browser launches.
+
+WHAT IS COUNTED. Every application that ends makes one download of the `usage-counter` release's file named after how it
+ended (src/aihawk/counter.py), and GitHub counts each file's downloads as its `download_count`. GitHub keeps only the
+running totals, never a history: a day's applications are the difference between two daily totals, and a day nobody
+recorded is lost for good.
+
+`sent` is what the badge shows: the applications whose send was clicked, confirmed by the page or not.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import datetime as dt
+import json
+import os
+import re
+import sys
+import urllib.request
+from collections.abc import Callable
+from dataclasses import dataclass
+from itertools import pairwise
+from pathlib import Path
+
+REPO = "feder-cr/Jobs_Applier_AI_Agent"
+COUNTER_TAG = "usage-counter"
+OUTCOMES = ("submitted", "unconfirmed", "refused", "blocked", "failed")
+SENT = ("submitted", "unconfirmed")
+
+Api = Callable[[str], object]
+
+
+def github_api(token: str | None) -> Api:
+    def get(path: str) -> object:
+        request = urllib.request.Request("https://api.github.com" + path)
+        request.add_header("Accept", "application/vnd.github+json")
+        if token:
+            request.add_header("Authorization", "Bearer " + token)
+        with urllib.request.urlopen(request, timeout=60) as response:
+            return json.load(response)
+
+    return get
+
+
+def collect(api: Api) -> dict[str, int]:
+    """Each outcome's total, the ones sent, all of them, and the stars."""
+    release = api(f"/repos/{REPO}/releases/tags/{COUNTER_TAG}")
+    downloads = {asset["name"].removesuffix(".txt"): asset["download_count"] for asset in release["assets"]}  # type: ignore[index]
+    out = {outcome: int(downloads.get(outcome, 0)) for outcome in OUTCOMES}
+    out["sent"] = sum(out[outcome] for outcome in SENT)
+    out["applications"] = sum(out[outcome] for outcome in OUTCOMES)
+    out["stars"] = int(api(f"/repos/{REPO}")["stargazers_count"])  # type: ignore[index]
+    return out
+
+
+# ── history ───────────────────────────────────────────────────────────────────────────────────────────────────────────
+
+
+def read_history(path: Path) -> dict[str, dict[str, int]]:
+    out: dict[str, dict[str, int]] = {}
+    if path.exists():
+        with path.open(encoding="utf-8", newline="") as file:
+            for row in csv.DictReader(file):
+                out.setdefault(row["date"], {})[row["key"]] = int(float(row["value"]))
+    return out
+
+
+def append_snapshot(path: Path, day: str, data: dict[str, int]) -> list[str]:
+    """Append today's rows. Idempotent per (date, key): a rerun the same day writes only keys that are not there yet, and
+    never a second value."""
+    seen = set(read_history(path).get(day, {}))
+    fresh = {key: value for key, value in data.items() if key not in seen}
+    if not fresh:
+        return []
+    new = not path.exists()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8", newline="") as file:
+        writer = csv.writer(file, lineterminator="\n")
+        if new:
+            writer.writerow(["date", "key", "value"])
+        for key in sorted(fresh):
+            writer.writerow([day, key, fresh[key]])
+    return list(fresh)
+
+
+def per_day(history: dict[str, dict[str, int]], keys: tuple[str, ...]) -> list[tuple[str, dict[str, int]]]:
+    """(day counted, each key's count that day) for each pair of snapshots one day apart. The snapshot dated D is taken just
+    after midnight UTC, so its difference from D-1's is calendar day D-1. A gap between snapshots is not a day, and is never
+    spread over the missing days."""
+    dates = sorted(day for day in history if all(key in history[day] for key in keys))
+    out = []
+    for first, second in pairwise(dates):
+        if (dt.date.fromisoformat(second) - dt.date.fromisoformat(first)).days != 1:
+            continue
+        out.append((first, {key: max(0, history[second][key] - history[first][key]) for key in keys}))
+    return out
+
+
+def write_daily(path: Path, history: dict[str, dict[str, int]]) -> None:
+    keys = ("sent", *OUTCOMES)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8", newline="") as file:
+        writer = csv.writer(file, lineterminator="\n")
+        writer.writerow(["day_utc", *keys])
+        writer.writerows([day, *(counts[key] for key in keys)] for day, counts in per_day(history, keys))
+
+
+# ── badge ─────────────────────────────────────────────────────────────────────────────────────────────────────────────
+
+VALUE_PAD = 6
+COLOUR_LABEL = "#555"
+CHAR_W = {".": 4.0, "M": 10.0, "k": 6.0}  # Verdana 11px advances
+DIGIT_W = 7.0
+SAFE_VALUE = re.compile(r"^[0-9]+(\.[0-9]+)?[kM]?$")
+
+
+@dataclass(frozen=True)
+class Badge:
+    label: str
+    label_box: int  # the label's width in Verdana 11px, with its padding
+    colour: str
+    file_name: str
+    key: str
+
+
+BADGES = [Badge("applications sent", 113, "#2ea44f", "applications.svg", "sent")]
+
+
+def format_count(number: int) -> str:
+    """Two decimals, truncated: an adoption figure is never rounded up."""
+    if number < 1000:
+        return str(number)
+    scaled, suffix = (number / 1000, "k") if number < 1_000_000 else (number / 1_000_000, "M")
+    return f"{int(scaled * 100) / 100:.2f}".rstrip("0").rstrip(".") + suffix
+
+
+def render(value: str, badge: Badge) -> bytes:
+    if not SAFE_VALUE.match(value):
+        raise ValueError(f"refusing to draw an unexpected value: {value!r}")
+    box = int(sum(CHAR_W.get(char, DIGIT_W) for char in value) + VALUE_PAD)
+    total = badge.label_box + box
+    label_x, value_x = badge.label_box / 2, badge.label_box + box / 2
+    aria = f"{badge.label}: {value}"
+    lines = [
+        f'<svg xmlns="http://www.w3.org/2000/svg" width="{total}" height="20" role="img" aria-label="{aria}">',
+        f"<title>{aria}</title>",
+        '<linearGradient id="s" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient>',
+        f'<clipPath id="r"><rect width="{total}" height="20" rx="3" fill="#fff"/></clipPath>',
+        '<g clip-path="url(#r)">',
+        f'<rect width="{badge.label_box}" height="20" fill="{COLOUR_LABEL}"/>',
+        f'<rect x="{badge.label_box}" width="{box}" height="20" fill="{badge.colour}"/>',
+        f'<rect width="{total}" height="20" fill="url(#s)"/>',
+        "</g>",
+        '<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11">',
+        f'<text x="{label_x:.1f}" y="15" fill="#010101" fill-opacity=".3">{badge.label}</text>',
+        f'<text x="{label_x:.1f}" y="14">{badge.label}</text>',
+        f'<text x="{value_x:.1f}" y="15" fill="#010101" fill-opacity=".3">{value}</text>',
+        f'<text x="{value_x:.1f}" y="14">{value}</text>',
+        "</g>",
+        "</svg>",
+    ]
+    return "\n".join(lines).encode("utf-8")
+
+
+def write_badges(directory: Path, history: dict[str, dict[str, int]]) -> None:
+    directory.mkdir(parents=True, exist_ok=True)
+    for badge in BADGES:
+        days = [day for day in history if badge.key in history[day]]
+        if not days:
+            raise SystemExit(f"no {badge.key} in the history")
+        (directory / badge.file_name).write_bytes(render(format_count(history[max(days)][badge.key]), badge))
+
+
+# ── entry point ───────────────────────────────────────────────────────────────────────────────────────────────────────
+
+
+def run(api: Api, history: Path, daily: Path, badges: Path, today: dt.date) -> int:
+    day = today.isoformat()
+    before = read_history(history)
+    written = append_snapshot(history, day, collect(api))
+    after = read_history(history)
+    write_daily(daily, after)
+    write_badges(badges, after)
+    print(f"[counters] {day}: wrote {len(written)} keys, sent={after[day]['sent']}, applications={after[day]['applications']}")
+    previous = sorted(date for date in before if date < day)
+    if previous and (today - dt.date.fromisoformat(previous[-1])).days > 1:
+        # Recorded anyway, but a hole is a lost measurement: said in the run.
+        print(f"::warning::no snapshot between {previous[-1]} and {day}: those days cannot be split")
+    return 0
+
+
+def selftest() -> int:
+    import tempfile
+
+    fails = []
+    release = {"assets": [{"name": "submitted.txt", "download_count": 7}, {"name": "unconfirmed.txt", "download_count": 2}, {"name": "failed.txt", "download_count": 1}]}
+    answers = {f"/repos/{REPO}/releases/tags/{COUNTER_TAG}": release, f"/repos/{REPO}": {"stargazers_count": 5}}
+    if collect(answers.__getitem__) != {"submitted": 7, "unconfirmed": 2, "refused": 0, "blocked": 0, "failed": 1, "sent": 9, "applications": 10, "stars": 5}:
+        fails.append("collect")
+    if format_count(4_616_723) != "4.61M" or format_count(2554) != "2.55k" or format_count(999) != "999":
+        fails.append("format")
+    history = {"2026-11-01": {"sent": 100}, "2026-11-02": {"sent": 160}, "2026-11-04": {"sent": 300}}
+    if per_day(history, ("sent",)) != [("2026-11-01", {"sent": 60})]:
+        fails.append("gap")
+    with tempfile.TemporaryDirectory() as folder:
+        path = Path(folder) / "h.csv"
+        append_snapshot(path, "2026-11-08", {"stars": 1})
+        if append_snapshot(path, "2026-11-08", {"stars": 2}) or read_history(path)["2026-11-08"]["stars"] != 1:
+            fails.append("idempotent")
+    print("selftest:", "OK" if not fails else "FAIL " + ", ".join(fails))
+    return 1 if fails else 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--selftest", action="store_true")
+    parser.add_argument("--history", type=Path)
+    parser.add_argument("--daily", type=Path)
+    parser.add_argument("--badges", type=Path)
+    arguments = parser.parse_args()
+    if arguments.selftest:
+        return selftest()
+    if not (arguments.history and arguments.daily and arguments.badges):
+        parser.error("--history, --daily and --badges are required")
+    api = github_api(os.environ.get("GITHUB_TOKEN"))
+    return run(api, arguments.history, arguments.daily, arguments.badges, dt.datetime.now(dt.UTC).date())
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/aihawk/config.py

@@ -24,8 +24,11 @@ DEFAULT_QUICK_MODEL = "google/gemini-3.5-flash"
 # Answers, for each posting, whether it is in line with the candidate, and how sure it is: that is the posting's score.
 DEFAULT_DECISION_MODEL = "typesafe/jev-1.14"
 # The Hugging Face dataset the pipeline publishes to. It was GitHub's releases until 2026-11-08: see pipeline/publish.py.
-DEFAULT_DATABASE = "https://huggingface.co/datasets/feder-cr/jobs/resolve/main/manifest.json"
-DEFAULT_MAX_AGE_DAYS = 60
+DEFAULT_DATABASE = "https://huggingface.co/datasets/feder-cr/jobs/resolve/main/library/manifest.json"
+# The manifest of the layout before 2026-11-09, which a settings file may still name: it means the default now.
+OLD_DATABASE = "https://huggingface.co/datasets/feder-cr/jobs/resolve/main/manifest.json"
+# A search looks back this many days by default: the owner's decision of 2026-11-09. The database holds 60.
+DEFAULT_MAX_AGE_DAYS = 15
 
 
 class ConfigError(Exception):
@@ -240,7 +243,7 @@ def load_config(workspace: Path) -> Config:
         workspace=workspace,
         name=str(raw.get("name") or ""),
         goals=str(raw.get("goals") or ""),
-        database=str(raw.get("database") or DEFAULT_DATABASE),
+        database=DEFAULT_DATABASE if str(raw.get("database") or OLD_DATABASE) == OLD_DATABASE else str(raw["database"]),
         max_age_days=_days(raw.get("max_age_days", DEFAULT_MAX_AGE_DAYS), "max_age_days"),
         model=str(raw.get("model") or DEFAULT_MODEL),
         quick_model=str(raw.get("quick_model") or DEFAULT_QUICK_MODEL),

src/aihawk/counter.py

@@ -0,0 +1,37 @@
+"""The count of the applications AIHawk sends, for all its users together.
+
+Every application that ends makes one download of a small file from the `usage-counter` release of AIHawk's repository,
+the file named after how it ended: submitted, unconfirmed, refused, blocked or failed. GitHub counts each file's
+downloads, and those counts are how many applications AIHawk has sent, and how they went. Nothing about the person or
+the job is sent: the download is the same for everyone. The way invisible_playwright counts its browser launches.
+
+The owner's decision of 2026-11-09: the count cannot be turned off, and the README says so. A dry run is not counted.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import urllib.request
+
+from aihawk.apply import DRY_RUN_ENV
+
+COUNTER = "https://github.com/feder-cr/Jobs_Applier_AI_Agent/releases/download/usage-counter/{outcome}.txt"
+OUTCOMES = ("submitted", "unconfirmed", "refused", "blocked", "failed")
+
+
+def count(outcome: str) -> None:
+    """One download of the outcome's file, in the background: the next application does not wait for it. Not a daemon
+    thread, so quitting right after a send still counts it, within the download's timeout."""
+    if outcome not in OUTCOMES or os.environ.get(DRY_RUN_ENV):
+        return
+    threading.Thread(target=fetch, args=(COUNTER.format(outcome=outcome),), name="usage-counter").start()
+
+
+def fetch(url: str) -> None:
+    """The download, read and dropped. A count that cannot be made is lost, and nothing else happens."""
+    try:
+        with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "AIHawk"}), timeout=15) as response:
+            response.read()
+    except Exception:  # noqa: BLE001, S110  The count must never stop or slow an application.
+        pass

src/aihawk/database.py

@@ -1,41 +1,34 @@
-"""The job database AIHawk reads, as published on Hugging Face: read in place, so a search downloads only what it needs."""
+"""The search over the job database on this computer (library.py): the postings that best fit, by their titles."""
 
 from __future__ import annotations
 
-import errno
 import html
 import json
 import math
 import re
 import urllib.request
-from bisect import bisect_right
-from collections.abc import Callable, Iterable
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections.abc import Iterable
 from dataclasses import dataclass
-from datetime import UTC, datetime
+from datetime import datetime
 from importlib import resources
-from itertools import pairwise
 from pathlib import Path
 from typing import Any, NamedTuple
 
 import duckdb
 
 from aihawk.config import Arc, Filters, Form, Node, Plan
+from aihawk.library import Library
 
 # How many postings a search brings back to be judged. Past these, a title fits the search too little to be worth a look.
 RECALL = 600
-# Description files read at once: each read waits on the network, not on this computer.
-FILES_AT_ONCE = 8
 # What being in the person's own city is worth, against a node of the plan: about as much as a named skill.
 CITY_WEIGHT = 1.5
 # What a posting with no description keeps of its score: judged on its title alone, it needs a title twice as fitting to
-# come before one that has a description. The pipeline marks which have one.
+# come before one that has a description.
 UNDESCRIBED = 0.5
-# The columns of the index that a delta carries and an index kept on this computer is read by. A description is found by
-# the posting's key, in the description file whose range holds it.
-INDEX_COLUMNS = ("key", "url", "title", "company", "ats", "location", "country", "remote", "salary_min", "salary_max", "salary_currency", "posted_at", "described")
-# What a remote posting's location says when it does not narrow the job to a place.
-NO_PLACE = ("", "remote", "anywhere", "worldwide", "global")
+# What a remote posting's location says when it does not narrow the job to a place: the database writes "Remote" for one
+# flagged remote that names no place.
+NO_PLACE = ("remote", "anywhere", "worldwide", "global")
 
 
 class DatabaseError(Exception):
@@ -67,69 +60,6 @@ def posting_keys(urls: list[str]) -> list[str]:
         connection.close()
 
 
-def description_files(manifest: dict[str, Any], location: str) -> tuple[tuple[str, str, str], ...]:
-    """Where a description is found by its posting's key: each file's first key, last key and address, in key order."""
-    base = _base(manifest, location)
-    found = [(item["first_key"], item["last_key"], f"{base}/{item['name']}") for item in manifest.get("files", []) if "first_key" in item]
-    return tuple(sorted(found))
-
-
-def _holder(key: str, files: tuple[tuple[str, str, str], ...]) -> str | None:
-    """The file whose range of keys holds this one, or None."""
-    at = bisect_right([first for first, _, _ in files], key) - 1
-    return files[at][2] if at >= 0 and key <= files[at][1] else None
-
-
-def stored_descriptions(
-    keys: list[str],
-    files: tuple[tuple[str, str, str], ...],
-    done: Callable[[int], None] = lambda _: None,
-    problem: Callable[[str], None] = lambda _: None,
-) -> list[str]:
-    """The descriptions the database holds for these postings, by their keys, in order; empty where its sources published
-    none. One query per file, FILES_AT_ONCE files at a time, each on its own connection: a key read in a file in key order
-    reads only the row group that can hold it, and every read waits on the network. On Hugging Face's servers one file
-    took 3 to 4 seconds, and 25 of them one after the other 249.
-    `done` is told how many have been read, as each file is. A file that cannot be read is told once to `problem`, and
-    the others are still read: the search goes on with what it has.
-    """
-    texts = [""] * len(keys)
-    by_file: dict[str, list[tuple[int, str]]] = {}
-    count = 0
-    for position, key in enumerate(keys):
-        if key and (file := _holder(key, files)):
-            by_file.setdefault(file, []).append((position, key))
-        else:
-            count += 1
-            done(count)
-
-    def read(file: str, held: list[tuple[int, str]]) -> dict[str, str] | None:
-        connection = duckdb.connect()
-        try:
-            connection.execute("INSTALL httpfs; LOAD httpfs;")
-            # Constants, not a subquery: that is what lets the reader skip every row group whose range cannot hold them.
-            marks = ", ".join("?" for _ in held)
-            return dict(connection.execute(f"SELECT key, description FROM read_parquet(?) WHERE key IN ({marks})", [file, *(key for _, key in held)]).fetchall())
-        except duckdb.Error:
-            return None
-        finally:
-            connection.close()
-
-    failed = False
-    with ThreadPoolExecutor(FILES_AT_ONCE) as pool:
-        reads = {pool.submit(read, file, held): held for file, held in by_file.items()}
-        for finished in as_completed(reads):
-            found, held = finished.result(), reads[finished]
-            failed = failed or found is None
-            for position, key in held:
-                texts[position] = (found or {}).get(key) or ""
-            count += len(held)
-            done(count)
-    if failed:
-        problem("Some job descriptions could not be read from the job database, so those postings are judged without them.")
-    return texts
-
-
 def unreachable(error: BaseException) -> str:
     """What a database that cannot be downloaded says: in plain words, the detail kept after them."""
     return f"The job database cannot be reached: check your internet connection, then start AIHawk again. ({error})"
@@ -147,111 +77,6 @@ def load_manifest(location: str) -> dict[str, Any]:
         raise DatabaseError(f"the job database's manifest at {location} cannot be read: {error}") from error
 
 
-def built_at(manifest: dict[str, Any]) -> datetime:
-    """When the database was built, as the manifest says: a time with no zone is UTC, as the pipeline's is."""
-    built = datetime.fromisoformat(manifest["built_at"])
-    return built if built.tzinfo else built.replace(tzinfo=UTC)
-
-
-def _base(manifest: dict[str, Any], location: str) -> str:
-    """Where the database's files are: the release the manifest names, or beside the manifest."""
-    return manifest.get("base_url") or location.rsplit("/", 1)[0].rsplit("\\", 1)[0]
-
-
-def local_index(manifest: dict[str, Any], location: str, cache: Path, log: Callable[[str], None] = lambda _: None) -> Path:
-    """The index on this computer, as of the manifest's build: downloaded whole once, then kept up to date by applying the
-    deltas it has not seen, in order. A database on this computer is read where it is."""
-    source = f"{_base(manifest, location)}/{manifest['index']['name']}"
-    if not source.startswith(("http://", "https://")):
-        return Path(source)
-    cache.mkdir(parents=True, exist_ok=True)
-    path = cache / "index.parquet"
-    state = cache / "index.build"
-    build = manifest.get("build") or manifest["built_at"]
-    have = state.read_text(encoding="utf-8").strip() if path.is_file() and state.is_file() else ""
-    if have == build:
-        return path
-    chain = _chain(manifest.get("deltas", []), have) if have else None
-    if chain is not None:
-        log("Getting the latest jobs.")
-        _apply(path, chain)
-    else:
-        # The first time, or a gap no delta covers: the whole index, once.
-        _download(manifest, source, path, log)
-    state.write_text(build, encoding="utf-8")
-    return path
-
-
-def _chain(deltas: list[dict[str, Any]], have: str) -> list[dict[str, Any]] | None:
-    """The deltas that lead from the build this computer has to the newest, in order: none when it has the newest already,
-    None when they cannot be followed."""
-    if deltas and deltas[-1]["build"] == have:
-        return []
-    start = next((at for at, delta in enumerate(deltas) if delta["after"] == have), None)
-    if start is None:
-        return None
-    chain = deltas[start:]
-    if any(later["after"] != earlier["build"] for earlier, later in pairwise(chain)):
-        return None
-    return chain
-
-
-def _apply(path: Path, chain: list[dict[str, Any]]) -> None:
-    """Apply the deltas to the index in one rewrite. A key in several deltas takes its newest row, as Delta Lake's log does:
-    the last action on a key is its state, so a posting added, then changed, then gone, ends gone."""
-    columns = ", ".join(INDEX_COLUMNS)
-    partial = path.with_suffix(".applying")
-    connection = duckdb.connect()
-    try:
-        connection.execute("INSTALL httpfs; LOAD httpfs;")
-        # A column added since this index was kept is NULL in it, until a delta brings the posting with it.
-        held = {row[0] for row in connection.execute("DESCRIBE SELECT * FROM read_parquet(?)", [path.as_posix()]).fetchall()}
-        kept = ", ".join(column if column in held else f"NULL AS {column}" for column in INDEX_COLUMNS)
-        connection.execute(
-            """CREATE TABLE changes AS
-               SELECT * EXCLUDE (filename, newest) FROM (
-                   SELECT *, row_number() OVER (PARTITION BY key ORDER BY filename DESC) AS newest
-                   FROM read_parquet(?, filename = true, union_by_name = true)
-               ) WHERE newest = 1""",
-            [[delta["url"] for delta in chain]],
-        )
-        connection.execute(f"""
-            COPY (
-                SELECT {kept} FROM read_parquet(?) WHERE key NOT IN (SELECT key FROM changes)
-                UNION ALL
-                SELECT {columns} FROM changes WHERE op = 'add'
-            ) TO '{partial.as_posix()}' (FORMAT parquet, COMPRESSION zstd)
-        """, [path.as_posix()])
-    except duckdb.Error as error:
-        partial.unlink(missing_ok=True)
-        raise DatabaseError(f"the job index could not be brought up to date: {error}") from error
-    finally:
-        connection.close()
-    partial.replace(path)
-
-
-def _download(manifest: dict[str, Any], source: str, path: Path, log: Callable[[str], None]) -> None:
-    """The release's whole index into `path`, in one rename at the end, saying how far it has come."""
-    log(f"Getting the jobs ready, only the first time ({manifest['index']['bytes'] // 1_000_000} MB).")
-    partial = path.with_suffix(".part")
-    try:
-        with urllib.request.urlopen(source, timeout=120) as response, partial.open("wb") as out:
-            copied = told = 0
-            while chunk := response.read(1 << 20):
-                out.write(chunk)
-                copied += len(chunk)
-                # A quarter at a time: this takes minutes on a slow line, and must not look stuck.
-                if copied * 4 // manifest["index"]["bytes"] > told and copied < manifest["index"]["bytes"]:
-                    told = copied * 4 // manifest["index"]["bytes"]
-                    log(f"  {told * 25}%")
-    except OSError as error:
-        partial.unlink(missing_ok=True)
-        if error.errno == errno.ENOSPC:
-            raise DatabaseError(f"The disk is full: the job index needs {manifest['index']['bytes'] // 1_000_000} MB free in {path.parent}. Free some space, then start AIHawk again.") from error
-        raise DatabaseError(unreachable(error)) from error
-    partial.replace(path)
-
-
 def _word_pattern(words: tuple[str, ...]) -> str:
     """Whole words, in any case: `java` is not `javascript`, and `roma` is not `Romania`."""
     # Not \\b: it finds no end after a symbol, as in `c++`, and takes accented letters for edges.
@@ -274,7 +99,6 @@ def rarity(postings: int, holding: int) -> float:
 
 
 class _Posting(NamedTuple):
-    key: str
     company: str
     title: str
     location: str
@@ -318,10 +142,10 @@ def _place_and_age(filters: Filters, max_age_days: int) -> tuple[list[str], list
     if places:
         # A remote posting that names no place could be open to anyone, and its description will say.
         unnamed = ", ".join(repr(word) for word in NO_PLACE)
-        places.append(f"(remote AND coalesce(country, '') = '' AND lower(trim(coalesce(location, ''))) IN ({unnamed}))")
+        places.append(f"(coalesce(country, '') = '' AND lower(trim(coalesce(location, ''))) IN ({unnamed}))")
         clauses.append("(" + " OR ".join(places) + ")")
     # A posting with no date is kept: many sources have none.
-    clauses.append(f"(TRY_CAST(posted_at AS TIMESTAMP) IS NULL OR TRY_CAST(posted_at AS TIMESTAMP) >= now()::TIMESTAMP - INTERVAL {int(max_age_days)} DAY)")
+    clauses.append(f"(posted_at IS NULL OR posted_at >= now()::TIMESTAMP - INTERVAL {int(max_age_days)} DAY)")
     return clauses, parameters
 
 
@@ -376,15 +200,12 @@ def _rank(score: float, posting: _Posting) -> tuple[float, bool, datetime]:
     return score, posting.posted is not None, posting.posted or datetime.min
 
 
-def search(
-    location: str, filters: Filters, max_age_days: int, cache: Path, log: Callable[[str], None] = lambda _: None, limit: int = RECALL
-) -> tuple[dict[str, Any], list[Job]]:
-    """The postings that best fit the search, by their titles, the best first, with the manifest they came from.
+def search(library: Library, filters: Filters, max_age_days: int, limit: int = RECALL) -> list[Job]:
+    """The postings here that best fit the search, by their titles, the best first: none while nothing has come.
 
     The plan's nodes score a title by their best wording that it holds, weighted by how rare the wording is among the
     postings of this place: "Python" tells more than "Engineer". Arcs then scale the title, and the person's own city adds
-    to it. The same title at the same company, posted once per place, is one job, in its best place. The description
-    stays in the database until a job is judged.
+    to it. A posting is known by its address. Its description is read when a job is judged, not here.
     """
     plan = _plan_of(filters)
     nodes = tuple(node for node in plan.nodes if node.forms)
@@ -393,8 +214,6 @@ def search(
     mean = sum(node.weight for node in nodes) / len(nodes)
     if mean <= 0:
         raise DatabaseError("the plan weighs nothing: give a node a weight above 0")
-    manifest = load_manifest(location)
-    index = str(local_index(manifest, location, cache, log))
     texts = tuple(dict.fromkeys(form.text for node in nodes for form in node.forms))
     patterns = {text: _whole_words((text,)) for text in texts}
     cities = _whole_words(filters.cities) if filters.cities else None
@@ -411,20 +230,19 @@ def search(
 
     connection = duckdb.connect()
     try:
-        (postings,) = connection.execute(
-            f"SELECT count(*) FROM read_parquet(?) WHERE {' AND '.join(place_clauses)}", [index, *place_parameters]
-        ).fetchone()
+        if not library.view(connection):
+            return []
+        (postings,) = connection.execute(f"SELECT count(*) FROM postings WHERE {' AND '.join(place_clauses)}", place_parameters).fetchone()
         rows = [
             _Posting(*row)
             for row in connection.execute(
-                f"""SELECT key, coalesce(company, ''), title, coalesce(location, ''), url,
-                        TRY_CAST(posted_at AS TIMESTAMP), coalesce(described, true)
-                    FROM read_parquet(?) WHERE {' AND '.join(clauses)}""",
-                [index, *parameters],
+                f"""SELECT coalesce(company, ''), title, coalesce(location, ''), url, posted_at, described
+                    FROM postings WHERE {' AND '.join(clauses)}""",
+                parameters,
             ).fetchall()
         ]
     except duckdb.Error as error:
-        raise DatabaseError(f"the job index cannot be read: {error}") from error
+        raise DatabaseError(f"the jobs on this computer cannot be read: {error}") from error
     finally:
         connection.close()
 
@@ -436,21 +254,10 @@ def search(
         in_city = cities is not None and cities.search(row.location) is not None
         scored.append((_score(nodes, evidence, document, in_city, mean) * (1.0 if row.described else UNDESCRIBED), row))
 
-    # Every posting stands: the database already made one of a job carried by several sites.
     ranked = sorted(scored, key=lambda item: _rank(*item), reverse=True)[:limit]
-
     best = max((score for score, _ in ranked), default=1.0) or 1.0
     # Some sources publish their names with HTML entities still in them.
-    return manifest, [
-        Job(
-            row.key,
-            html.unescape(row.company),
-            html.unescape(row.title),
-            html.unescape(row.location),
-            row.url,
-            row.url,
-            "",
-            score / best,
-        )
+    return [
+        Job(row.url, html.unescape(row.company), html.unescape(row.title), html.unescape(row.location), row.url, row.url, "", score / best)
         for score, row in ranked
     ]

src/aihawk/library.py

@@ -0,0 +1,240 @@
+"""The job database on this computer: the chunks of one build, downloaded one after the other, the newest postings first,
+and the deltas of the builds after it: of a build, only the chunks a search over the person's days reads, all of them before
+it runs. Nothing is read over the network once a chunk is here.
+
+A posting is its newest version: the one in the latest delta that names its address, else the one in the chunks; a
+delta's `drop` takes it away. The chunks can come before or after the deltas that change them, and the answer is the same,
+so the deltas are fetched first, being small, and the chunks after, one by one.
+"""
+
+from __future__ import annotations
+
+import errno
+import json
+import shutil
+import urllib.error
+import urllib.request
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+from itertools import pairwise
+from pathlib import Path
+from typing import Any
+
+import duckdb
+
+# What each posting row holds, in the chunks and in the deltas, after a delta's `op`.
+COLUMNS = ("url", "title", "company", "location", "country", "posted_at", "description")
+STATE = "state.json"
+# Deltas kept apart before they are merged into one: a search reads them all, and a computer used every day gets five.
+DELTAS_APART = 8
+
+
+class LibraryError(Exception):
+    """The database cannot be brought to this computer."""
+
+
+def _literal(paths: list[Path]) -> str:
+    """A list of files as SQL: a view holds no parameters."""
+    return "[" + ", ".join("'" + path.as_posix().replace("'", "''") + "'" for path in paths) + "]"
+
+
+class Library:
+    def __init__(self, folder: Path) -> None:
+        self.folder = folder
+        self.chunks = folder / "chunks"
+        self.deltas = folder / "deltas"
+        self.state: dict[str, Any] = {}
+        # The days a search looks back: only the chunks holding postings that new are downloaded, all of them before a
+        # search runs. None: every chunk. The owner's decision of 2026-11-09, with 15 days by default.
+        self.days: int | None = None
+        if (folder / STATE).is_file():
+            try:
+                self.state = json.loads((folder / STATE).read_text(encoding="utf-8"))
+            except ValueError:
+                self.state = {}
+
+    # What is here.
+
+    def chunk_files(self) -> list[Path]:
+        """The chunks downloaded so far, in the order they came."""
+        have = set(self.state.get("have", []))
+        return [self.chunks / chunk["name"] for chunk in self.state.get("chunks", []) if chunk["name"] in have and (self.chunks / chunk["name"]).is_file()]
+
+    def delta_files(self) -> list[Path]:
+        """The deltas downloaded, the oldest first: their names follow their builds' times."""
+        return sorted(self.deltas.glob("delta-*.parquet")) if self.deltas.is_dir() else []
+
+    def postings_here(self) -> int:
+        """How many postings the chunks downloaded hold."""
+        have = set(self.state.get("have", []))
+        return sum(int(chunk.get("rows", 0)) for chunk in self.state.get("chunks", []) if chunk["name"] in have)
+
+    def needed(self) -> list[dict[str, Any]]:
+        """The chunks a search over `days` reads: those holding a posting that new. The chunks go newest first, so these
+        are the first ones of each kind."""
+        chunks = self.state.get("chunks", [])
+        if self.days is None:
+            return chunks
+        since = (datetime.now(UTC) - timedelta(days=self.days)).replace(tzinfo=None).isoformat()
+        return [chunk for chunk in chunks if not chunk.get("newest") or chunk["newest"] >= since]
+
+    def complete(self) -> bool:
+        """Whether every chunk a search over `days` reads is here."""
+        have = set(self.state.get("have", []))
+        return bool(self.state.get("chunks")) and all(chunk["name"] in have for chunk in self.needed())
+
+    def readiness(self) -> float:
+        """How much of what a search over `days` reads is here, from 0 to 1: a search runs at 1."""
+        if not self.state.get("chunks"):
+            return 0.0
+        have = set(self.state.get("have", []))
+        needed = self.needed()
+        total = sum(int(chunk.get("rows", 0)) or 1 for chunk in needed)
+        return sum(int(chunk.get("rows", 0)) or 1 for chunk in needed if chunk["name"] in have) / total if total else 1.0
+
+    def ready(self) -> bool:
+        return self.complete()
+
+    def view(self, connection: duckdb.DuckDBPyConnection) -> bool:
+        """`postings`, in this connection: every posting here at its newest, with `described` saying whether it has a text.
+        False when nothing has come yet."""
+        chunks, deltas = self.chunk_files(), self.delta_files()
+        if not chunks and not deltas:
+            return False
+        columns = ", ".join(COLUMNS)
+        parts = []
+        if deltas:
+            connection.execute(f"""
+                CREATE OR REPLACE TEMP TABLE changes AS
+                SELECT * EXCLUDE (filename, newest) FROM (
+                    SELECT *, row_number() OVER (PARTITION BY url ORDER BY filename DESC) AS newest
+                    FROM read_parquet({_literal(deltas)}, filename = true, union_by_name = true)
+                ) WHERE newest = 1
+            """)
+        if chunks:
+            # A chunk of postings with no text says so by its name: the description column is not read to know it.
+            kept = " WHERE url NOT IN (SELECT url FROM changes)" if deltas else ""
+            parts.append(f"SELECT {columns}, NOT contains(filename, 'titles_') AS described FROM read_parquet({_literal(chunks)}, filename = true){kept}")
+        if deltas:
+            parts.append(f"SELECT {columns}, description IS NOT NULL AS described FROM changes WHERE op = 'add'")
+        connection.execute("CREATE OR REPLACE TEMP VIEW postings AS " + " UNION ALL ".join(parts))
+        return True
+
+    def descriptions(self, urls: list[str]) -> list[str]:
+        """The descriptions of these postings, in order; empty where there is none, or the posting is not here."""
+        connection = duckdb.connect()
+        try:
+            if not urls or not self.view(connection):
+                return [""] * len(urls)
+            marks = ", ".join("?" for _ in urls)
+            found = dict(connection.execute(f"SELECT url, description FROM postings WHERE url IN ({marks})", urls).fetchall())
+        finally:
+            connection.close()
+        return [found.get(url) or "" for url in urls]
+
+    # Bringing it here.
+
+    def sync(self, manifest: dict[str, Any], location: str, progress: Callable[[int, int], None] = lambda done, size: None, arrived: Callable[[], None] = lambda: None) -> None:
+        """This computer's database brought to the manifest's build: the deltas it has not seen, then the chunks it does not
+        have, one at a time, `arrived` told after each that changes what a search finds, and `progress` the chunks here of
+        how many. A delta chain that cannot be followed, or a build the dataset no longer has, starts over from the newest."""
+        base = manifest.get("base_url") or location.replace("\\", "/").rsplit("/", 1)[0]
+        if not self.state or self._chain(manifest) is None:
+            self._start(manifest, base)
+        for delta in self._chain(manifest) or []:
+            self._fetch(delta["url"], self.deltas / delta["url"].rsplit("/", 1)[-1])
+            self.state["at"] = delta["build"]
+            self._save()
+            arrived()
+        self.state["at"] = manifest["build"]
+        self._save()
+        self._merge_deltas()
+        needed = self.needed()
+        total = len(needed)
+        progress(sum(1 for chunk in needed if chunk["name"] in self.state["have"]), total)
+        for chunk in needed:
+            if chunk["name"] in self.state["have"]:
+                continue
+            try:
+                self._fetch(f"{self.state['base_url']}/{chunk['name']}", self.chunks / chunk["name"])
+            except LibraryError as error:
+                gone = "(404)" in str(error) or "(401)" in str(error)
+                if not gone or self.state["build"] == manifest["build"]:
+                    raise
+                # The build this computer started from is gone from the dataset: the newest one, from the start.
+                self._start(manifest, base)
+                self.sync(manifest, location, progress, arrived)
+                return
+            self.state["have"].append(chunk["name"])
+            self._save()
+            progress(sum(1 for each in needed if each["name"] in self.state["have"]), total)
+            arrived()
+
+    def _chain(self, manifest: dict[str, Any]) -> list[dict[str, Any]] | None:
+        """The deltas from the build this computer is at to the manifest's, in order; None when they cannot be followed."""
+        at = self.state.get("at")
+        if at == manifest["build"]:
+            return []
+        deltas = manifest.get("deltas", [])
+        start = next((index for index, delta in enumerate(deltas) if delta["after"] == at), None)
+        if start is None:
+            return None
+        chain = deltas[start:]
+        if chain[-1]["build"] != manifest["build"] or any(later["after"] != earlier["build"] for earlier, later in pairwise(chain)):
+            return None
+        return chain
+
+    def _merge_deltas(self) -> None:
+        """Past DELTAS_APART, the deltas become one, each address at its newest action, under the newest one's name: what a
+        search reads is the same, and it reads one file."""
+        deltas = self.delta_files()
+        if len(deltas) <= DELTAS_APART:
+            return
+        merged = deltas[-1].with_suffix(".merging")
+        connection = duckdb.connect()
+        try:
+            connection.execute(f"""
+                COPY (
+                    SELECT * EXCLUDE (filename, newest) FROM (
+                        SELECT *, row_number() OVER (PARTITION BY url ORDER BY filename DESC) AS newest
+                        FROM read_parquet({_literal(deltas)}, filename = true, union_by_name = true)
+                    ) WHERE newest = 1
+                ) TO '{merged.as_posix()}' (FORMAT parquet, COMPRESSION zstd)
+            """)
+        finally:
+            connection.close()
+        for delta in deltas:
+            delta.unlink()
+        merged.replace(deltas[-1])
+
+    def _start(self, manifest: dict[str, Any], base: str) -> None:
+        """Everything here forgotten, and the manifest's build the one to download."""
+        shutil.rmtree(self.chunks, ignore_errors=True)
+        shutil.rmtree(self.deltas, ignore_errors=True)
+        self.state = {"build": manifest["build"], "at": manifest["build"], "base_url": base, "chunks": manifest["chunks"], "have": []}
+        self._save()
+
+    def _save(self) -> None:
+        self.folder.mkdir(parents=True, exist_ok=True)
+        partial = self.folder / (STATE + ".part")
+        partial.write_text(json.dumps(self.state), encoding="utf-8")
+        partial.replace(self.folder / STATE)
+
+    @staticmethod
+    def _fetch(source: str, path: Path) -> None:
+        """A file into `path`, in one rename at the end: a search never reads half of one."""
+        path.parent.mkdir(parents=True, exist_ok=True)
+        partial = path.with_suffix(".part")
+        try:
+            if source.startswith(("http://", "https://")):
+                with urllib.request.urlopen(urllib.request.Request(source, headers={"User-Agent": "AIHawk"}), timeout=120) as response, partial.open("wb") as out:
+                    shutil.copyfileobj(response, out, 1 << 20)
+            else:
+                shutil.copyfile(source, partial)
+        except OSError as error:
+            partial.unlink(missing_ok=True)
+            if error.errno == errno.ENOSPC:
+                raise LibraryError(f"The disk is full: AIHawk's jobs need about 2 GB free in {path.parent}. Free some space, then start AIHawk again.") from error
+            code = f" ({error.code})" if isinstance(error, urllib.error.HTTPError) else ""
+            raise LibraryError(f"The jobs cannot be downloaded{code}: check your internet connection, then start AIHawk again. ({error})") from error
+        partial.replace(path)

src/aihawk/screens.py

@@ -3,6 +3,8 @@ browser left open for a sign-in. Each is self-contained and reports back by what
 
 from __future__ import annotations
 
+import html
+import re
 import webbrowser
 from typing import Any, ClassVar
 
@@ -14,8 +16,7 @@ from textual.containers import Vertical, VerticalScroll
 from textual.screen import ModalScreen
 from textual.widgets import DataTable, Footer, Input, RichLog, Static
 
-from aihawk.llm import money
-from aihawk.store import Sent, Stored, said, sent_word
+from aihawk.store import SUBMITTED, Sent, Stored, sent_word
 
 
 class Said(RichLog):
@@ -138,14 +139,15 @@ class SentList(ModalScreen[None]):
 
     def on_mount(self) -> None:
         table = self.query_one(DataTable)
-        # What is left beside the time and the status, inside this screen's 95% and its padding, goes to the words.
+        # What is left beside the time and the status, inside this screen's 95% and its padding, goes to the words. What
+        # the page said is in each one's details: cut to a column, it was half a sentence.
         wide = self.app.size.width >= WIDE_ENOUGH
-        left = int(self.app.size.width * 0.95) - 8 - 2 * (5 if wide else 3) - 16 - 19
-        words = [("Company", left // 5), ("Role", left * 3 // 10)] if wide else [("Company", left)]
-        for label, width in (("When", 16), *words, ("Status", 19), *([("What the page said", left // 2)] if wide else [])):
+        left = int(self.app.size.width * 0.95) - 8 - 2 * (4 if wide else 3) - 16 - 19
+        words = [("Company", left * 2 // 5), ("Role", left - left * 2 // 5)] if wide else [("Company", left)]
+        for label, width in (("When", 16), *words, ("Status", 19)):
             table.add_column(label, key=label, width=width)
         for item in self.items:
-            cells = (when_short(item.when), item.job.company, *([item.job.title] if wide else []), sent_word(item), *([short(said(item))] if wide else []))
+            cells = (when_short(item.when), item.job.company, *([item.job.title] if wide else []), sent_word(item))
             table.add_row(*(Text(cell) for cell in cells), key=item.job.id)
         table.focus()
 
@@ -168,52 +170,71 @@ class SentDetails(ModalScreen[None]):
         self.item = item
 
     def compose(self) -> ComposeResult:
-        item, proof = self.item, self.item.proof
-        lines = [
-            item.job.title,
-            f"{item.job.company} · {item.job.location}",
-            "",
-            f"{sent_word(item)} · {item.when}" + (f" · {money(item.cost)}" if item.cost is not None else ""),
-            f"Posting: {item.job.url}",
-            f"Address after the send: {proof.url or '-'}",
-            f"Screenshot: {proof.screenshot or '-'}",
-            "",
-            f"Filled in:\n{item.note or '-'}",
-            "",
-            f"What the page said: {proof.evidence or '-'}",
-            "",
-            proof.text or "The page's text was not read.",
-        ]
+        # How it went, in a person's words: what the form was filled with and the page's files stay in the store.
+        item = self.item
+        lines = [f"{sent_word(item).capitalize()} · {when_short(item.when)}"]
+        if item.proof.evidence:
+            lines.append(f"The page said: {item.proof.evidence}")
+        elif item.status not in SUBMITTED and item.note:
+            # Why it was not sent; a sent one's note is what its form was filled with.
+            lines.append(item.note)
         with Vertical(id="dialog"):
+            yield Static(Text(item.job.title), id="question")
+            yield Static(Text(" · ".join(part for part in (item.job.company, item.job.location) if part)), classes="hint")
             with VerticalScroll(id="summary"):
-                yield Static("\n".join(lines), markup=False)
+                yield Static(Text("\n".join(lines)))
             yield Footer()
 
     def action_close(self) -> None:
         self.dismiss(None)
 
 
+def readable(text: str) -> str:
+    """A posting's text as a person reads it. Sources write HTML, some of it escaped once or twice over: here it is
+    paragraphs and bullets, with no tag left."""
+    for _ in range(3):
+        if not re.search(r"&(amp;)*lt;/?[a-zA-Z]", text):
+            break
+        text = html.unescape(text)
+    text = re.sub(r"(?is)<(script|style)\b.*?</\1\s*>", "", text)
+    text = re.sub(r"(?i)<li\b[^>]*>", "\n• ", text)
+    text = re.sub(r"(?i)<br\b[^>]*>", "\n", text)
+    text = re.sub(r"(?i)</?(p|div|h[1-6]|ul|ol|table|tr|section|article|header|footer|blockquote)\b[^>]*>", "\n\n", text)
+    text = html.unescape(re.sub(r"<[^>]+>", "", text))
+    lines: list[str] = []
+    for line in (" ".join(line.split()) for line in text.splitlines()):
+        # One empty line between paragraphs, never two.
+        if line or (lines and lines[-1]):
+            lines.append(line)
+    return "\n".join(lines).strip()
+
+
 class JobDetails(ModalScreen[None]):
-    BINDINGS: ClassVar = [Binding("space", "select", "Select"), Binding("o", "open_link", "Open"), Binding("escape,enter,q", "close", "Back")]
+    """The job as its posting says it: the role, the company, where, and the posting's text. Nothing else."""
+
+    BINDINGS: ClassVar = [
+        Binding("space,enter", "select", "Select", key_display="space"),
+        Binding("escape,i,q", "close", "Back", key_display="esc"),
+        Binding("o", "open_link", show=False),
+    ]
 
     def __init__(self, item: Stored) -> None:
         super().__init__()
         self.item = item
 
     def compose(self) -> ComposeResult:
-        job, score = self.item.job, self.item.score
-        lines = [f"{job.title}", f"{job.company} · {job.location}", job.url]
-        if score is not None:
-            lines += ["", f"Score {score:.1f}/5"]
-            if not job.description.strip():
-                lines += ["Scored from the title alone: the posting had no text to read."]
-        if self.item.note:
-            lines += ["", "Last attempt:", self.item.note]
-        with Vertical(id="dialog"):
-            with VerticalScroll(id="summary"):
-                yield Static("\n".join(lines), markup=False)
+        job = self.item.job
+        with Vertical(id="reader"):
+            yield Static(Text(job.title), id="question")
+            yield Static(Text(" · ".join(part for part in (job.company, job.location) if part)), classes="hint")
+            with VerticalScroll(id="text"):
+                yield Static(Text(readable(job.description) or "This posting has no description."))
             yield Footer()
 
+    def on_mount(self) -> None:
+        # The arrows and Page Down read the text.
+        self.query_one("#text").focus()
+
     def action_close(self) -> None:
         self.dismiss(None)
 

src/aihawk/session.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import asyncio
+import shutil
 from collections.abc import Awaitable, Callable
 from dataclasses import replace
 from pathlib import Path
@@ -12,7 +13,9 @@ from openai import AsyncOpenAI
 from aihawk.apply import MAILBOXES, Outcome, apply_to_job, signed_in, site_of
 from aihawk.apply import sign_in as sign_in_browser
 from aihawk.config import KEY_ENV, KEY_URL, Config, ConfigError, load_config
-from aihawk.database import description_files, search, stored_descriptions
+from aihawk.counter import count
+from aihawk.database import load_manifest, search
+from aihawk.library import Library
 from aihawk.llm import SPEND, explain, make_client, money, refused
 from aihawk.plugins import Plugin, always_plugins, load_plugins, plugin_for
 from aihawk.profile import ensure_account_password, import_answers_file, install_resume, save_setting
@@ -37,9 +40,10 @@ class Session:
         import_answers_file(self.config, self.store)
         ensure_account_password(self.store)
         self._client: AsyncOpenAI | None = None
-        # Where descriptions are found by key, from the database the last search read.
-        self.description_files: tuple[tuple[str, str, str], ...] = ()
-        # How many jobs the whole database holds, once a search has read it.
+        # The job database on this computer, which a search reads and the chunks still coming add to.
+        self.library = Library(self.config.data_dir / "library")
+        self.library.days = self.config.max_age_days
+        # How many jobs the whole database holds, once its manifest has been read.
         self.database_jobs = 0
 
     def close(self) -> None:
@@ -102,28 +106,36 @@ class Session:
         await SPEND.settle(self.client)
         return SPEND.total
 
-    async def find(self, log: Log) -> int:
-        """Bring the postings for this search from the job database into the store; how many are new."""
+    async def sync(self, progress: Callable[[int, int], None], arrived: Callable[[], None]) -> None:
+        """The job database on this computer brought to the newest build: what is new since the last time, then the chunks
+        still missing, one by one. `arrived` is told after each that changes what a search finds, `progress` how many chunks
+        are here of how many. Both are told on the loop, which alone touches the screen."""
         loop = asyncio.get_running_loop()
-        index = self.config.data_dir / "index"
-        # search runs in a thread, and the screen may be touched only from the loop: its lines are handed over to it.
-        manifest, jobs = await asyncio.to_thread(
-            search, self.config.database, self.config.filters, self.config.max_age_days, index, lambda line: loop.call_soon_threadsafe(log, line)
-        )
-        self.description_files = description_files(manifest, self.config.database)
+        # The days of this search, which may have changed in the settings since the last time.
+        self.library.days = self.config.max_age_days
+        manifest = await asyncio.to_thread(load_manifest, self.config.database)
         self.database_jobs = int(manifest.get("rows_out") or 0)
-        new = sum(self.store.add(job) for job in jobs)
-        return new
+        # The index and the description files read in place, before 2026-11-09: nothing reads them any more.
+        await asyncio.to_thread(shutil.rmtree, self.config.data_dir / "index", True)
+        await asyncio.to_thread(
+            self.library.sync, manifest, self.config.database,
+            lambda done, size: loop.call_soon_threadsafe(progress, done, size), lambda: loop.call_soon_threadsafe(arrived),
+        )
 
-    async def read(self, log: Log, status: Callable[[int, int], None]) -> int:
+    async def find(self, log: Log) -> int:
+        """Bring the postings for this search, of those on this computer, into the store; how many are new."""
+        jobs = await asyncio.to_thread(search, self.library, self.config.filters, self.config.max_age_days)
+        return self.store.add_all(jobs)
+
+    async def read(self, log: Log) -> int:
         """Jev reads the next WIDE postings the search ranked best and scores each one by how sure it is that the job is in
-        line with the candidate. `status` is told the descriptions read so far, of how many. A posting Jev gave no answer for
-        is set aside, not asked again and again. Returns how many were scored."""
+        line with the candidate. A posting Jev gave no answer for is set aside, not asked again and again. Returns how
+        many were scored."""
         batch = self.store.with_status("new")[:WIDE]
         if not batch:
             return 0
-        # By key: that reaches jobs found in any earlier build too.
-        texts = await _read_descriptions([item.job.id for item in batch], self.description_files, lambda done: status(done, len(batch)), log)
+        # From this computer, by the posting's address: a job kept from an earlier build has its text there too.
+        texts = await asyncio.to_thread(self.library.descriptions, [item.job.url for item in batch])
         for item, text in zip(batch, texts, strict=True):
             self.store.set_description(item.job.id, text)
         jobs = [replace(item.job, description=text) for item, text in zip(batch, texts, strict=True)]
@@ -200,6 +212,8 @@ class Session:
             self.store.set_account(host, email)
         self.store.set_status(item.job.id, outcome.status, outcome.summary)
         self.store.set_proof(item.job.id, outcome.proof)
+        # One more application sent, in the count of all AIHawk's users: see counter.py.
+        count(outcome.status)
         await SPEND.settle(self.client)
         self.store.set_cost(item.job.id, SPEND.total - before)
         # The file for the person outside the app. One that another program holds open must not stop the next send.
@@ -210,20 +224,6 @@ class Session:
         return outcome
 
 
-async def _read_descriptions(keys: list[str], files: tuple[tuple[str, str, str], ...], status: Callable[[int], None], log: Log) -> list[str]:
-    """The descriptions the database holds; the count of those read is told as each file is read, and a failure is logged."""
-    loop = asyncio.get_running_loop()
-
-    def shown(done: int) -> None:
-        # The read runs in a thread: its count is handed to the loop, which alone touches the screen.
-        loop.call_soon_threadsafe(status, done)
-
-    def problem(text: str) -> None:
-        loop.call_soon_threadsafe(log, text)
-
-    return await asyncio.to_thread(stored_descriptions, keys, files, shown, problem)
-
-
 def _places(config: Config) -> str:
     """The places a posting would name for this search: its cities, then its country, each one only when there is one."""
     parts = [" or ".join(config.filters.cities[:3]), (config.filters.locations or [""])[-1]]

src/aihawk/store.py

@@ -73,6 +73,9 @@ PROGRESS = ("new", "passed_over", "shortlisted", "scored", "failed", "blocked",
 KEY_VERSION = 2
 # 3: Jev alone scores; the jobs an earlier version left to its judge are new again.
 JEV_ONLY = 3
+# 4: a job's id is its address, as the database's since 2026-11-09, and its key is kept beside it: the same posting under
+# another of its links is known by the key, and never comes back as new, to be sent twice.
+URL_IDS = 4
 # The jobs the person sees as tried, and what each status says in plain words.
 SENT_WORDS = {
     "submitted": "sent and confirmed",
@@ -165,9 +168,14 @@ class Store:
         self._db.execute(ANSWERS_SCHEMA)
         self._db.execute(ACCOUNTS_SCHEMA)
         # A store from before the cost was kept gets the column; the rows keep their nulls.
-        if "cost" not in {row[1] for row in self._db.execute("PRAGMA table_info(jobs)")}:
+        columns = {row[1] for row in self._db.execute("PRAGMA table_info(jobs)")}
+        if "cost" not in columns:
             self._db.execute("ALTER TABLE jobs ADD COLUMN cost REAL")
             self._db.commit()
+        if "key" not in columns:
+            self._db.execute("ALTER TABLE jobs ADD COLUMN key TEXT")
+            self._db.commit()
+        self._db.execute("CREATE INDEX IF NOT EXISTS jobs_key ON jobs (key)")
         if self._db.execute("PRAGMA user_version").fetchone()[0] < KEY_VERSION:
             self._rekey()
         if self._db.execute("PRAGMA user_version").fetchone()[0] < JEV_ONLY:
@@ -175,6 +183,25 @@ class Store:
             self._db.execute("UPDATE jobs SET status = 'new' WHERE status IN ('shortlisted', 'passed_over') AND score IS NULL")
             self._db.execute(f"PRAGMA user_version = {JEV_ONLY}")
             self._db.commit()
+        if self._db.execute("PRAGMA user_version").fetchone()[0] < URL_IDS:
+            self._by_address()
+
+    def _by_address(self) -> None:
+        """Each job under its address, its key kept beside it. Two jobs with one address are one row, the one furthest along."""
+        rows = self._db.execute("SELECT id, url, status, score FROM jobs").fetchall()
+        rows.sort(key=lambda row: (PROGRESS.index(row[2]) if row[2] in PROGRESS else 0, row[3] or 0.0), reverse=True)
+        self._db.execute("UPDATE jobs SET key = id WHERE key IS NULL")
+        taken: set[str] = set()
+        for job_id, url, _, _ in rows:
+            if url in taken:
+                self._db.execute("DELETE FROM jobs WHERE id = ?", (job_id,))
+                continue
+            taken.add(url)
+            if job_id != url:
+                self._db.execute("DELETE FROM jobs WHERE id = ?", (url,))
+                self._db.execute("UPDATE jobs SET id = ? WHERE id = ?", (url, job_id))
+        self._db.execute(f"PRAGMA user_version = {URL_IDS}")
+        self._db.commit()
 
     def _rekey(self) -> None:
         """The jobs stored before a key changed, under the key their link has now. Two that turn out to be one posting are
@@ -200,14 +227,26 @@ class Store:
 
     def add(self, job: Job) -> bool:
         """Record a job seen in a search; True when it is new. A known job keeps its status and score."""
+        return self.add_all([job]) == 1
+
+    def add_all(self, jobs: list[Job]) -> int:
+        """Record the jobs a search found; how many are new. A known job keeps its status and score, and so does one
+        known under another of its links: the same key is the same posting, and is not added again."""
         now = _now()
-        cursor = self._db.execute(
-            "INSERT OR IGNORE INTO jobs (id, company, title, location, url, apply_url, description, lexical, first_seen, updated)"
-            " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
-            (job.id, job.company, job.title, job.location, job.url, job.apply_url, job.description, job.lexical, now, now),
-        )
+        new = 0
+        for job, key in zip(jobs, posting_keys([job.url for job in jobs]), strict=True):
+            # Only a link has a key worth comparing: another address entirely is another job.
+            known = "SELECT 1 FROM jobs WHERE id = ? OR (key = ? AND url LIKE 'http%')" if job.url.startswith("http") else "SELECT 1 FROM jobs WHERE id = ?"
+            if self._db.execute(known, (job.id, key) if job.url.startswith("http") else (job.id,)).fetchone():
+                continue
+            self._db.execute(
+                "INSERT INTO jobs (id, key, company, title, location, url, apply_url, description, lexical, first_seen, updated)"
+                " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (job.id, key, job.company, job.title, job.location, job.url, job.apply_url, job.description, job.lexical, now, now),
+            )
+            new += 1
         self._db.commit()
-        return cursor.rowcount == 1
+        return new
 
     @staticmethod
     def _job(row: sqlite3.Row) -> Job:

src/aihawk/tui.py

@@ -16,8 +16,9 @@ from textual.widgets import DataTable, Footer, Static
 from aihawk.apply import MAILBOXES, Outcome, mail_provider
 from aihawk.config import KEY_ENV, KEY_URL, ConfigError
 from aihawk.database import DatabaseError
+from aihawk.library import LibraryError
 from aihawk.llm import explain, refused
-from aihawk.screens import WIDE_ENOUGH, JobDetails, Rows, Said, SendBatch, SentDetails, SentList, Settings, SignIn, TextPrompt
+from aihawk.screens import WIDE_ENOUGH, JobDetails, Rows, Said, SendBatch, SentList, Settings, SignIn, TextPrompt
 from aihawk.session import Session
 from aihawk.store import SUBMITTED, Stored
 
@@ -61,13 +62,17 @@ class AIHawk(App[None]):
     #sent, #settings { height: auto; max-height: 24; }
     #question { text-style: bold; margin-bottom: 1; }
     #summary { height: auto; max-height: 24; margin: 1 0; }
+    #reader { width: 90%; max-width: 110; height: 90%; padding: 1 2; background: $surface; border: thick $primary; }
+    #reader #question { margin-bottom: 0; }
+    #text { height: 1fr; margin: 1 0; }
     .hint { color: $text-muted; }
     """
     BINDINGS: ClassVar = [
+        # Space and Enter both select: the table's own Enter comes as RowSelected, below. i reads the job.
         Binding("space", "toggle", "Select"),
-        Binding("enter", "details", show=False),
+        Binding("i", "details", "Details"),
         Binding("a", "apply", "Apply"),
-        Binding("s", "sent", show=False),
+        Binding("s", "sent", "Sent"),
         Binding("o", "open_link", show=False),
         Binding("e", "settings", "Settings"),
         Binding("r", "new_resume", show=False),
@@ -85,8 +90,12 @@ class AIHawk(App[None]):
         self.pages: list[list[str]] = []
         self.page = 0
         self.wanted: int | None = None
-        # The descriptions read so far of the round under way, and of how many.
+        # The chunks of the database on this computer, and of how many, while they come; and whether a search has to run
+        # again over the chunks come since the last one.
         self.progress: tuple[int, int] | None = None
+        self.more_found = False
+        # The chunks this computer had when it was last short of the search's days, plus one.
+        self.ready_at = 0
         self.signing_in = False
         self.columns_for = 0
         self.with_location = True
@@ -155,7 +164,7 @@ class AIHawk(App[None]):
         self.query_one("#heading", Static).update(" · ".join(part for part in ("AIHawk", wanted, place) if part))
 
     def row(self, item: Stored) -> tuple[Text | str, ...]:
-        # One sign, or none: selected, or sent. What else happened to a job is in its details, on Enter.
+        # One sign, or none: selected, or sent. How a send went is in the applications, on s.
         mark = "●" if item.job.id in self.selected else "✓" if item.status in SUBMITTED else ""
         score = f"{item.score:.1f}" if item.score is not None else "-"
         # Text, because a string cell is read as markup, and a posting's words are a source's: "[/x] Engineer" would stop the screen.
@@ -223,11 +232,17 @@ class AIHawk(App[None]):
                 self.show_heading()
             if not session.mail()[1]:
                 await self.set_up_mail()
-            await session.find(self.say)
             # The jobs scored before, by an earlier search: the first pages, sorted as they are.
             scored = [item.job.id for item in session.store.all() if item.score is not None]
             self.pages = [scored[start : start + PAGE] for start in range(0, len(scored), PAGE)]
             self.refresh_jobs()
+            # The jobs come in the background, a chunk at a time: those of the days the search looks back, all of them
+            # before anything is searched, so the first matches are the best ones.
+            self.bring_jobs()
+            if not session.library.ready():
+                self.show_progress()
+                return
+            await session.find(self.say)
             await self.score()
         except (ConfigError, DatabaseError) as error:
             self.say(str(error))
@@ -238,24 +253,65 @@ class AIHawk(App[None]):
 
     async def score(self) -> None:
         """Pages, PAGE jobs each, up to the one after the page shown, and a round of scored jobs no page holds yet in
-        reserve: Jev reads the next round while the person reads, so a page they turn to is there. A round's wait is its
-        descriptions' download, half a minute; it is counted on the line below only while the person waits for a page."""
+        reserve: Jev reads the next round while the person reads, so a page they turn to is there. A round takes Jev a few
+        seconds; until the first jobs are on this computer, the line below counts the chunks coming."""
         session = self.session
         store = session.store
         while True:
             self.make_pages()
             if not store.with_status("new") or (len(self.pages) >= self.page + 2 and len(self.spare()) >= AHEAD):
                 break
+            self.show_progress()
+            await session.read(self.say)
+        self.make_pages()
+
+    @work(exclusive=True, group="library")
+    async def bring_jobs(self) -> None:
+        """The database brought to this computer, the newest jobs first; each chunk that comes is searched."""
+        try:
+            await self.session.sync(self.chunks_came, self.found_more)
+        except (LibraryError, DatabaseError) as error:
+            self.say(str(error))
+        except Exception as error:  # noqa: BLE001  Said on the screen, as in start.
+            self.failed(error)
+        finally:
             self.progress = None
+            if self.is_running:
+                self.refresh_jobs()
+
+    def chunks_came(self, done: int, size: int) -> None:
+        # Whether the person was waiting for the jobs: told by the count before this chunk, not by this one's.
+        waiting = self.progress is not None and self.progress[0] < self.ready_at
+        self.progress = (done, size)
+        if not self.session.library.ready():
+            self.ready_at = done + 1
+        if not self.is_running:
+            return
+        self.refresh_jobs()
+        if waiting and self.session.library.ready():
+            # The count of the wait gives way: the person sees it is over.
+            self.query_one(Said).say("Ready.", again=True)
+        else:
             self.show_progress()
 
-            def counted(done: int, size: int) -> None:
-                self.progress = (done, size)
-                self.show_progress()
+    def found_more(self) -> None:
+        """New jobs on this computer: the search runs again over them, once for however many came while it ran. Not
+        before every chunk of the search's days is here."""
+        if not self.session.library.ready():
+            return
+        self.more_found = True
+        if self.is_running and not any(worker.group == "find" and worker.is_running for worker in self.workers):
+            self.find_again()
 
-            await session.read(self.say, counted)
-            self.progress = None
-        self.make_pages()
+    @work(group="find")
+    async def find_again(self) -> None:
+        try:
+            while self.more_found and self.is_running:
+                self.more_found = False
+                await self.session.find(self.say)
+                self.keep_ahead()
+        except DatabaseError as error:
+            self.say(str(error))
 
     def spare(self) -> list[str]:
         """The scored jobs no page holds yet, the best first."""
@@ -281,12 +337,15 @@ class AIHawk(App[None]):
             self.refresh_jobs()
 
     def show_progress(self) -> None:
-        """How far the page the person waits for is, on one line: only while they wait, the first page or one turned to."""
-        if self.wanted is None and len(self.pages) > self.page:
+        """How far the page the person waits for is, on one line: only while they wait, the first page or one turned to.
+        Until the jobs of the search's days are on this computer, how far that is."""
+        if self.wanted is None and len(self.pages) > self.page and self.session.library.ready():
             return
-        number = (self.wanted if self.wanted is not None else self.page) + 1
-        counted = f" {self.progress[0]} of {self.progress[1]}" if self.progress else ""
-        self.query_one(Said).say(f"Preparing page {number}…{counted}", again=True)
+        said = self.query_one(Said)
+        if not self.session.library.ready():
+            said.say(f"Getting the jobs ready… {int(self.session.library.readiness() * 100)}%", again=True)
+            return
+        said.say(f"Preparing page {(self.wanted if self.wanted is not None else self.page) + 1}…", again=True)
 
     def turn(self, step: int) -> None:
         """To the page before or after, on its first row. A page not ready yet opens once it is; the line below says how
@@ -304,7 +363,9 @@ class AIHawk(App[None]):
             return
         store = self.session.store
         paged = {job_id for page in self.pages for job_id in page}
-        if not store.with_status("new") and all(item.job.id in paged for item in store.all() if item.score is not None):
+        nothing_left = not store.with_status("new") and all(item.job.id in paged for item in store.all() if item.score is not None)
+        # Unless more are on their way: the page opens when the chunks still coming bring its jobs.
+        if nothing_left and self.session.library.complete():
             self.say("No more jobs for this search.")
             return
         self.wanted = target
@@ -328,15 +389,20 @@ class AIHawk(App[None]):
 
     @on(DataTable.RowSelected)
     def row_chosen(self) -> None:
-        self.action_details()
+        self.action_toggle()
 
     def action_details(self) -> None:
-        """The job under the cursor: what was sent and what the page said when it was tried, otherwise why it is listed."""
+        """The job under the cursor, as its posting says it."""
         item = self.current()
         if not item:
             return
-        tried = next((sent for sent in self.session.store.sent() if sent.job.id == item.job.id), None)
-        self.push_screen(SentDetails(tried) if tried else JobDetails(item))
+        if not item.job.description.strip():
+            # A job listed before its text was kept with it: the text is in the database on this computer.
+            text = self.session.library.descriptions([item.job.url])[0]
+            if text:
+                self.session.store.set_description(item.job.id, text)
+                item = self.session.store.get(item.job.id) or item
+        self.push_screen(JobDetails(item))
 
     def action_open_link(self) -> None:
         item = self.current()
@@ -378,9 +444,7 @@ class AIHawk(App[None]):
             ("answers", "Your answers", f"{len(self.session.store.answers())}, given with the CV to every form"),
             ("mail", "Your mail", self.mail_words()),
             ("key", "OrcaRouter key", f"from {KEY_ENV}" if config.key_from_environment() else "saved" if config.api_key() else "none yet"),
-            ("model", "Forms model", config.model),
-            ("quick_model", "Search plan model", config.quick_model),
-            ("decision_model", "Scoring model", config.decision_model),
+            # The models are not here: a person does not choose one, and aihawk.yaml still changes them.
             ("folder", "Your files", str(config.workspace)),
         ]
 

tests/conftest.py

@@ -0,0 +1,15 @@
+"""What every test shares."""
+
+from __future__ import annotations
+
+import pytest
+
+from aihawk import counter
+
+
+@pytest.fixture(autouse=True)
+def counted(monkeypatch: pytest.MonkeyPatch) -> list[str]:
+    """The applications' count goes to GitHub: in a test it goes here, and a test that asks for it reads what was counted."""
+    downloads: list[str] = []
+    monkeypatch.setattr(counter, "fetch", downloads.append)
+    return downloads

tests/fakes.py

@@ -65,14 +65,27 @@ class ScriptedBrowser:
 
 
 def publish(connection: Any, folder: Path, rows_out: int) -> str:
-    """Table t as the pipeline publishes it: a description file in key order whose range the manifest names, the index
-    beside it, the manifest. Returns the manifest's path."""
+    """Table t as the pipeline publishes it: a chunk of the postings with a description of 200 characters or more, one of
+    the others with none, and the manifest naming them. A posting is known by its address: rows that share one get their
+    key added to it. `keys.json` beside the manifest says each address's key, for the tests that name postings by it.
+    Returns the manifest's path."""
     connection.execute("ALTER TABLE t ADD COLUMN IF NOT EXISTS country VARCHAR; ALTER TABLE t ADD COLUMN IF NOT EXISTS remote BOOLEAN")
-    connection.execute(f"COPY (SELECT * FROM t ORDER BY key) TO '{(folder / 'jobs_0.parquet').as_posix()}' (FORMAT parquet)")
-    connection.execute(f"""COPY (SELECT * EXCLUDE (description), length(coalesce(description, '')) >= 200 AS described FROM t)
-        TO '{(folder / 'index.parquet').as_posix()}' (FORMAT parquet)""")
-    first, last = connection.execute("SELECT min(key), max(key) FROM t").fetchone()
-    manifest = {"schema": 2, "rows_out": rows_out, "built_at": "2026-11-04T00:00:00", "index": {"name": "index.parquet", "bytes": 1},
-                "files": [{"name": "jobs_0.parquet", "first_key": first, "last_key": last}]}
+    connection.execute("""CREATE OR REPLACE TABLE posted AS SELECT key,
+        CASE WHEN count(*) OVER (PARTITION BY url) > 1 THEN url || key ELSE url END AS url, title, company,
+        CASE WHEN remote AND coalesce(country, '') = '' AND lower(trim(coalesce(location, ''))) IN ('', 'remote', 'anywhere', 'worldwide', 'global')
+            THEN 'Remote' ELSE location END AS location,
+        country, TRY_CAST(posted_at AS TIMESTAMP) AS posted_at, description FROM t""")
+    chunks = []
+    for name, described in (("jobs_0.parquet", True), ("titles_0.parquet", False)):
+        test = "length(coalesce(description, '')) >= 200"
+        rows = connection.execute(f"SELECT count(*) FROM posted WHERE {'' if described else 'NOT '}{test}").fetchone()[0]
+        if not rows:
+            continue
+        connection.execute(f"""COPY (SELECT url, title, company, location, country, posted_at, {'description' if described else 'NULL::VARCHAR AS description'}
+            FROM posted WHERE {'' if described else 'NOT '}{test}) TO '{(folder / name).as_posix()}' (FORMAT parquet)""")
+        chunks.append({"name": name, "bytes": (folder / name).stat().st_size, "rows": rows, "titles_only": not described})
+    keys = dict(connection.execute("SELECT url, key FROM posted").fetchall())
+    (folder / "keys.json").write_text(json.dumps(keys), encoding="utf-8")
+    manifest = {"schema": 3, "build": "jobs-20261104-000000", "rows_out": rows_out, "built_at": "2026-11-04T00:00:00", "chunks": chunks, "deltas": []}
     (folder / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
     return str(folder / "manifest.json")

tests/test_funnel.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 
 import asyncio
 import json
-import threading
 from pathlib import Path
 from types import SimpleNamespace
 from typing import Any
@@ -51,7 +50,7 @@ def ids(numbers: range) -> set[str]:
 
 
 def read(session: Session) -> int:
-    return asyncio.run(session.read(lambda line: None, lambda done, size: None))
+    return asyncio.run(session.read(lambda line: None))
 
 
 def scored(session: Session) -> dict[str, float | None]:
@@ -79,20 +78,13 @@ def test_a_posting_jev_gave_no_answer_for_is_set_aside_and_said_once(tmp_path: P
     values.update({"p001": None})
     session = session_with(tmp_path, 3)
     said: list[str] = []
-    assert asyncio.run(session.read(said.append, lambda done, size: None)) == 2
+    assert asyncio.run(session.read(said.append)) == 2
     assert [item.job.id for item in session.store.with_status("passed_over")] == ["p001"]
     assert said == ["1 of 3 jobs could not be read."]
     # Not asked again: the next round has nothing left.
     assert read(session) == 0
 
 
-def test_the_round_says_how_many_postings_jev_has_read(tmp_path: Path) -> None:
-    session = session_with(tmp_path, 30)
-    heard: list[tuple[int, int]] = []
-    asyncio.run(session.read(lambda line: None, lambda done, size: heard.append((done, size))))
-    assert heard[-1] == (30, 30)
-
-
 def test_a_failed_request_is_said_with_its_reason_and_the_others_are_scored(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     def scripted_endpoint(request: httpx.Request) -> httpx.Response:
         if b"Engineer 7 at" in request.content:
@@ -105,7 +97,7 @@ def test_a_failed_request_is_said_with_its_reason_and_the_others_are_scored(tmp_
     session = session_with(tmp_path, 20)
     session._client = AsyncOpenAI(base_url="https://api.orcarouter.ai/v1", api_key="sk-test", max_retries=0, http_client=httpx.AsyncClient(transport=httpx.MockTransport(scripted_endpoint)))
     said: list[str] = []
-    assert asyncio.run(session.read(said.append, lambda done, size: None)) == 19
+    assert asyncio.run(session.read(said.append)) == 19
     assert scored(session)["p008"] == 4.0 and "p007" not in scored(session)
     assert len(said) == 1 and said[0].startswith("1 of 20 jobs could not be read (OrcaRouter rejected the request.")
 
@@ -113,7 +105,7 @@ def test_a_failed_request_is_said_with_its_reason_and_the_others_are_scored(tmp_
 def test_the_jobs_an_earlier_version_left_to_its_judge_are_read_again(tmp_path: Path) -> None:
     store = Store(tmp_path / "jobs.db")
     for name in ("waiting", "passed", "done"):
-        store.add(Job(name, "Firm", "Engineer", "Rome", "u", "a", ""))
+        store.add(Job(name, "Firm", "Engineer", "Rome", name, "a", ""))
     store.set_statuses(["waiting"], "shortlisted")
     store.set_statuses(["passed"], "passed_over")
     store.set_score("done", 4.0, {})
@@ -125,21 +117,6 @@ def test_the_jobs_an_earlier_version_left_to_its_judge_are_read_again(tmp_path:
     assert [item.job.id for item in store.with_status("scored")] == ["done"]
 
 
-def test_a_scan_that_logs_from_its_thread_says_it_on_the_loop(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    seen: list[tuple[str, int]] = []
-
-    def searching(*args: Any) -> tuple[dict[str, Any], list[Job]]:
-        if len(args) == 5:
-            args[4]("from the thread")
-        return {"rows_out": 0, "built_at": "2026-11-04T00:00:00"}, []
-
-    monkeypatch.setattr(session_module, "search", searching)
-    session = Session(tmp_path)
-    loop_thread = threading.get_ident()
-    asyncio.run(session.find(lambda line: seen.append((line, threading.get_ident()))))
-    assert seen == [("from the thread", loop_thread)]
-
-
 def test_with_no_postings_left_jev_reads_nothing(tmp_path: Path) -> None:
     session = session_with(tmp_path, 0)
     assert read(session) == 0
@@ -151,15 +128,3 @@ def test_with_no_postings_left_jev_reads_nothing(tmp_path: Path) -> None:
 )
 def test_the_places_a_posting_names_have_no_empty_part(cities: list[str], locations: list[str], places: str) -> None:
     assert _places(Config(workspace=Path("."), filters=Filters(cities=tuple(cities), locations=tuple(locations)))) == places
-
-
-def test_a_description_database_that_cannot_be_read_is_said_once_and_jev_goes_on(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    def unreadable(refs: list[str], files: Any, done: Any, problem: Any) -> list[str]:
-        problem("Some job descriptions could not be read from the job database, so those postings are judged without them.")
-        return [""] * len(refs)
-
-    monkeypatch.setattr(session_module, "stored_descriptions", unreadable)
-    session = session_with(tmp_path, 30)
-    said: list[str] = []
-    assert asyncio.run(session.read(said.append, lambda done, size: None)) == 30
-    assert len(said) == 1 and "could not be read" in said[0]

tests/test_jobs.py

@@ -9,31 +9,29 @@ import json
 import socket
 import textwrap
 import urllib.error
+from dataclasses import replace
 from pathlib import Path
 
 import duckdb
 import pytest
 
 from aihawk.config import Arc, ConfigError, Filters, Form, Node, Plan, load_config
-from aihawk.database import (
-    INDEX_COLUMNS,
-    UNDESCRIBED,
-    DatabaseError,
-    Job,
-    _apply,
-    _chain,
-    description_files,
-    local_index,
-    rarity,
-    search,
-    stored_descriptions,
-)
+from aihawk.database import RECALL, UNDESCRIBED, DatabaseError, Job, load_manifest, rarity, search
+from aihawk.library import Library, LibraryError
 from aihawk.store import COLUMNS, Proof, Store, said, sent_word, write_applied
 from fakes import publish
 
 JOB = Job("g:a:1", "Acme", "Senior Python Engineer", "Rome, Italy", "u", "a", "d")
 
 
+def found(location: str, filters: Filters, max_age_days: int, cache: Path, limit: int = RECALL) -> list[Job]:
+    """The search over the database at `location`, brought to this computer first, each job named by its row's key."""
+    library = Library(cache / "library")
+    library.sync(load_manifest(location), location)
+    keys = json.loads(Path(location).with_name("keys.json").read_text(encoding="utf-8"))
+    return [replace(job, id=keys.get(job.url, job.id)) for job in search(library, filters, max_age_days, limit)]
+
+
 def database(tmp_path: Path) -> str:
     rows = [
         ("k1", "https://jobs.lever.co/a/1", "Senior Python Engineer", "Acme", "Rome, Italy", "Build", "2099-01-01"),
@@ -57,7 +55,7 @@ def database(tmp_path: Path) -> str:
     ],
 )
 def test_search_keeps_what_the_filters_say_and_leaves_out_what_is_old(tmp_path: Path, filters: Filters, titles: list[str]) -> None:
-    _, jobs = search(database(tmp_path), filters, 60, tmp_path)
+    jobs = found(database(tmp_path), filters, 60, tmp_path)
     assert [job.title for job in jobs] == titles
 
 
@@ -76,7 +74,7 @@ def test_a_title_holding_the_rare_node_comes_before_one_holding_only_the_common_
     connection.executemany("INSERT INTO t VALUES (?, 'https://a.example/', ?, 'c', 'Rome', 'd', NULL)", rows)
     path = publish(connection, tmp_path, 4)
     plan = Plan((Node("role", 1.0, (Form("Engineer"),)), Node("skill", 1.5, (Form("Fortran"),))))
-    _, jobs = search(path, Filters(plan=plan), 60, tmp_path, limit=3)
+    jobs = found(path, Filters(plan=plan), 60, tmp_path, limit=3)
     assert jobs[0].id in {"rare", "both"} and jobs[1].id in {"rare", "both"}
     # A node counts once, however many times a title says it: the two score the same.
     assert jobs[0].lexical == jobs[1].lexical == 1.0
@@ -86,7 +84,7 @@ def test_a_title_holding_the_rare_node_comes_before_one_holding_only_the_common_
 def test_a_node_holds_its_best_wording_and_wordings_never_add_up(tmp_path: Path) -> None:
     path = catalogue(tmp_path, [("both", "Engineer Developer", "A", "Rome"), ("dev", "Developer", "B", "Rome"), ("eng", "Engineer", "C", "Rome")])
     plan = Plan((Node("role", 1.0, (Form("Engineer"), Form("Developer"))),))
-    _, jobs = search(path, Filters(plan=plan), 60, tmp_path)
+    jobs = found(path, Filters(plan=plan), 60, tmp_path)
     # Each title holds one wording of the node as rare as any other: all three score the same, and "both" is no higher.
     assert {job.id: job.lexical for job in jobs} == {"both": 1.0, "dev": 1.0, "eng": 1.0}
 
@@ -95,7 +93,7 @@ def test_a_node_holds_its_best_wording_and_wordings_never_add_up(tmp_path: Path)
 def test_a_wording_weight_changes_the_order(tmp_path: Path, weights: tuple[float, float], order: list[str]) -> None:
     path = catalogue(tmp_path, [("e", "Engineer", "A", "Rome"), ("d", "Developer", "B", "Rome")])
     plan = Plan((Node("role", 1.0, (Form("Engineer", weights[0]), Form("Developer", weights[1]))),))
-    _, jobs = search(path, Filters(plan=plan), 60, tmp_path)
+    jobs = found(path, Filters(plan=plan), 60, tmp_path)
     assert [job.title for job in jobs] == order
 
 
@@ -103,14 +101,14 @@ def test_a_rare_wording_beats_a_common_one(tmp_path: Path) -> None:
     rows = [(f"common-{number}", "Engineer", f"Firm {number}", "Rome") for number in range(50)] + [("rare", "Fortran", "Lab", "Rome")]
     path = catalogue(tmp_path, rows)
     plan = Plan((Node("role", 1.0, (Form("Engineer"),)), Node("skill", 1.0, (Form("Fortran"),))))
-    _, jobs = search(path, Filters(plan=plan), 60, tmp_path, limit=1)
+    jobs = found(path, Filters(plan=plan), 60, tmp_path, limit=1)
     assert jobs[0].id == "rare"
 
 
 def test_a_when_arc_on_a_phrase_scales_the_whole_title_down(tmp_path: Path) -> None:
     path = catalogue(tmp_path, [("human", "Nurse", "Clinic", "Rome"), ("vet", "Veterinary Nurse", "Vets", "Rome")])
     plan = Plan((Node("role", 1.0, (Form("Nurse"),)),), (Arc((("*", 0.2),), when="Veterinary"),))
-    _, jobs = search(path, Filters(plan=plan), 60, tmp_path)
+    jobs = found(path, Filters(plan=plan), 60, tmp_path)
     assert [job.id for job in jobs] == ["human", "vet"]
     assert jobs[1].lexical == pytest.approx(0.2)
 
@@ -118,7 +116,7 @@ def test_a_when_arc_on_a_phrase_scales_the_whole_title_down(tmp_path: Path) -> N
 def test_an_unless_arc_scales_the_titles_that_lack_the_phrase(tmp_path: Path) -> None:
     path = catalogue(tmp_path, [("plain", "Engineer", "A", "Rome"), ("senior", "Senior Engineer", "B", "Rome")])
     plan = Plan((Node("role", 1.0, (Form("Engineer"),)),), (Arc((("*", 0.3),), unless="Senior"),))
-    _, jobs = search(path, Filters(plan=plan), 60, tmp_path)
+    jobs = found(path, Filters(plan=plan), 60, tmp_path)
     assert [job.id for job in jobs] == ["senior", "plain"]
     assert jobs[1].lexical == pytest.approx(0.3)
 
@@ -129,7 +127,7 @@ def test_a_node_of_weight_zero_scores_nothing_but_still_fires_an_arc(tmp_path: P
         (Node("role", 1.0, (Form("Engineer"),)), Node("remote", 0.0, (Form("Remote"),))),
         (Arc((("role", 2.0),), when="remote"),),
     )
-    _, jobs = search(path, Filters(plan=plan), 60, tmp_path)
+    jobs = found(path, Filters(plan=plan), 60, tmp_path)
     # The remote title's engineer is boosted by the arc; a title holding only the weight-0 node is not brought in.
     assert [job.id for job in jobs] == ["remote", "plain"]
 
@@ -137,9 +135,9 @@ def test_a_node_of_weight_zero_scores_nothing_but_still_fires_an_arc(tmp_path: P
 def test_an_arc_multiplier_is_clamped_to_three_at_most_and_a_twentieth_at_least(tmp_path: Path) -> None:
     path = catalogue(tmp_path, [("senior", "Senior Nurse", "A", "Rome"), ("plain", "Nurse", "B", "Rome")])
     role = Node("role", 1.0, (Form("Nurse"),))
-    _, boosted = search(path, Filters(plan=Plan((role,), (Arc((("*", 100.0),), when="Senior"),))), 60, tmp_path)
+    boosted = found(path, Filters(plan=Plan((role,), (Arc((("*", 100.0),), when="Senior"),))), 60, tmp_path)
     assert {job.id: job.lexical for job in boosted}["plain"] == pytest.approx(1 / 3)
-    _, damped = search(path, Filters(plan=Plan((role,), (Arc((("*", 0.0),), when="Senior"),))), 60, tmp_path)
+    damped = found(path, Filters(plan=Plan((role,), (Arc((("*", 0.0),), when="Senior"),))), 60, tmp_path)
     assert {job.id: job.lexical for job in damped}["senior"] == pytest.approx(0.05)
 
 
@@ -196,7 +194,7 @@ def test_a_place_is_found_by_its_word_by_its_country_code_or_by_a_remote_posting
     ]
     connection.executemany("INSERT INTO t VALUES (?, 'https://a.example/', 'Engineer', ?, ?, ?, ?, 'd', NULL)", [(row[0], *row) for row in rows])
     path = publish(connection, tmp_path, 4)
-    _, jobs = search(path, Filters(include=("Engineer",), locations=("Milano", "Italy"), countries=("IT",)), 60, tmp_path)
+    jobs = found(path, Filters(include=("Engineer",), locations=("Milano", "Italy"), countries=("IT",)), 60, tmp_path)
     assert {job.id for job in jobs} == {"word", "code", "spelled", "open-remote"}
 
 
@@ -210,7 +208,7 @@ def test_the_titles_a_person_reads_are_the_first_wordings_not_the_ids() -> None:
 def test_a_place_word_is_a_whole_part_of_the_location_not_a_word_inside_a_longer_name(tmp_path: Path) -> None:
     # The Australian posting has no country, so only its location can keep it out of a search for Wales.
     path = catalogue(tmp_path, [("au", "Engineer", "Firm", "Moorebank, New South Wales, Australia"), ("uk", "Engineer", "Firm", "Cardiff, Wales, GB")])
-    _, jobs = search(path, Filters(include=("Engineer",), locations=("Wales",)), 60, tmp_path)
+    jobs = found(path, Filters(include=("Engineer",), locations=("Wales",)), 60, tmp_path)
     assert [job.id for job in jobs] == ["uk"]
 
 
@@ -220,7 +218,7 @@ def test_a_place_starts_the_location_or_follows_a_separator_a_dash_or_a_number_n
     # Each posting has its own company: one job per title and company would keep only one of them.
     path = catalogue(tmp_path, [(location, "Engineer", location, location) for location in kept + refused])
     places = ("London", "Milan", "Milano", "Dallas", "Berlin", "Rome", "Italy", "Wales")
-    _, jobs = search(path, Filters(include=("Engineer",), locations=places), 60, tmp_path)
+    jobs = found(path, Filters(include=("Engineer",), locations=places), 60, tmp_path)
     assert {job.id for job in jobs} == set(kept)
 
 
@@ -237,7 +235,7 @@ def test_the_own_city_comes_first_and_every_posting_stands_one_per_place(tmp_pat
     ]
     connection.executemany("INSERT INTO t VALUES (?, 'https://a.example/', ?, ?, ?, 'd', NULL)", rows)
     path = publish(connection, tmp_path, 4)
-    _, jobs = search(path, Filters(include=("Engineer",), locations=("Italy",), cities=("Milano",)), 60, tmp_path)
+    jobs = found(path, Filters(include=("Engineer",), locations=("Italy",), cities=("Milano",)), 60, tmp_path)
     assert jobs[0].id == "milan"
     assert {job.id for job in jobs} == {"rome", "milan", "other", "no-name-1", "no-name-2"}
 
@@ -258,7 +256,7 @@ def test_filters_match_whole_words_and_keep_symbols_literal(tmp_path: Path) -> N
     path = publish(connection, tmp_path, 4)
 
     def keys(filters: Filters) -> set[str]:
-        return {job.id for job in search(path, filters, 60, tmp_path)[1]}
+        return {job.id for job in found(path, filters, 60, tmp_path)}
 
     assert keys(Filters(include=("java",))) == {"1", "3"}
     assert keys(Filters(include=("java",), locations=("roma",))) == {"1"}
@@ -267,28 +265,33 @@ def test_filters_match_whole_words_and_keep_symbols_literal(tmp_path: Path) -> N
 
 
 def test_names_come_without_html_entities(tmp_path: Path) -> None:
-    _, jobs = search(database(tmp_path), Filters(include=("accountant",)), 60, tmp_path)
+    jobs = found(database(tmp_path), Filters(include=("accountant",)), 60, tmp_path)
     assert [(job.title, job.company) for job in jobs] == [("Accountant & Clerk", "B&A")]
 
 
 def test_search_needs_titles_and_says_so(tmp_path: Path) -> None:
     with pytest.raises(DatabaseError, match="no job titles"):
-        search(database(tmp_path), Filters(), 60, tmp_path)
+        found(database(tmp_path), Filters(), 60, tmp_path)
 
 
 def test_a_database_that_cannot_be_read_says_so(tmp_path: Path) -> None:
-    with pytest.raises(DatabaseError, match="manifest"):
-        search(str(tmp_path / "none.json"), Filters(include=("x",)), 60, tmp_path)
+    with pytest.raises(DatabaseError, match="cannot be read"):
+        load_manifest(str(tmp_path / "none.json"))
 
 
-def test_a_database_that_cannot_be_reached_says_so_in_plain_words_and_keeps_the_detail(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    def no_network(*args: object, **kwargs: object) -> None:
+def test_a_database_that_cannot_be_reached_says_so_in_plain_words_and_keeps_the_detail(monkeypatch: pytest.MonkeyPatch) -> None:
+    def down(*_: object, **__: object) -> None:
         raise urllib.error.URLError(socket.gaierror(11001, "getaddrinfo failed"))
 
-    monkeypatch.setattr("aihawk.database.urllib.request.urlopen", no_network)
-    with pytest.raises(DatabaseError, match="cannot be reached: check your internet connection") as error:
-        search("https://example.test/manifest.json", Filters(include=("x",)), 60, tmp_path)
-    assert "<urlopen error [Errno 11001] getaddrinfo failed>" in str(error.value)
+    monkeypatch.setattr("urllib.request.urlopen", down)
+    with pytest.raises(DatabaseError) as raised:
+        load_manifest("https://example.test/manifest.json")
+    assert str(raised.value).startswith("The job database cannot be reached: check your internet connection")
+    assert "getaddrinfo failed" in str(raised.value)
+
+
+def test_nothing_here_yet_finds_nothing(tmp_path: Path) -> None:
+    assert search(Library(tmp_path / "library"), Filters(include=("python",)), 60) == []
 
 
 def test_a_posting_with_no_description_needs_a_title_twice_as_fitting(tmp_path: Path) -> None:
@@ -298,42 +301,22 @@ def test_a_posting_with_no_description_needs_a_title_twice_as_fitting(tmp_path:
         ("bare", "https://a.example/1", "Bare Co", ""),
         ("told", "https://a.example/2", "Told Co", "Builds Python services. " * 20),
     ])
-    _, jobs = search(publish(connection, tmp_path, 2), Filters(include=("python",)), 60, tmp_path)
+    jobs = found(publish(connection, tmp_path, 2), Filters(include=("python",)), 60, tmp_path)
     assert [job.id for job in jobs] == ["told", "bare"]
     assert jobs[1].lexical == pytest.approx(UNDESCRIBED)
 
 
-def files_of(location: str) -> tuple[tuple[str, str, str], ...]:
-    return description_files(json.loads(Path(location).read_text(encoding="utf-8")), location)
-
-
-def test_many_descriptions_come_in_order_and_empty_where_there_is_no_key(tmp_path: Path) -> None:
-    location = database(tmp_path)
-    _, jobs = search(location, Filters(include=("python",)), 60, tmp_path)
-    keys = [job.id for job in jobs] + ["", "no-such-key"]
-    counts: list[int] = []
-    texts = stored_descriptions(keys, files_of(location), done=counts.append)
-    assert texts == [stored_descriptions([key], files_of(location))[0] for key in keys[:-2]] + ["", ""]
-    assert sorted(texts[:-2]) == ["Build", "Learn"] and counts[-1] == len(keys)
-
-
-def test_a_database_that_cannot_be_read_is_reported_once_and_the_search_has_no_descriptions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    location = database(tmp_path)
-    _, jobs = search(location, Filters(include=("python",)), 60, tmp_path)
-    keys = [job.id for job in jobs]
-    files = files_of(location)
-
-    class Unreadable:
-        def execute(self, *args: object) -> None:
-            raise duckdb.Error("the extension cannot be downloaded")
-
-        def close(self) -> None:
-            pass
-
-    monkeypatch.setattr(duckdb, "connect", lambda: Unreadable())
-    reported: list[str] = []
-    assert stored_descriptions(keys, files, problem=reported.append) == [""] * len(keys)
-    assert len(reported) == 1 and "job database" in reported[0]
+def test_descriptions_come_in_order_from_this_computer_and_empty_where_there_is_none(tmp_path: Path) -> None:
+    connection = duckdb.connect()
+    connection.execute("CREATE TABLE t (key VARCHAR, url VARCHAR, title VARCHAR, company VARCHAR, location VARCHAR, description VARCHAR, posted_at VARCHAR)")
+    connection.executemany("INSERT INTO t VALUES (?, ?, 'Python Engineer', 'Co', 'Rome', ?, NULL)", [
+        ("told", "https://a.example/1", "Build Python services. " * 12), ("short", "https://a.example/2", "Too short"),
+    ])
+    location = publish(connection, tmp_path, 2)
+    library = Library(tmp_path / "library")
+    library.sync(load_manifest(location), location)
+    urls = ["https://a.example/2", "https://a.example/1", "https://no.such/posting"]
+    assert library.descriptions(urls) == ["", "Build Python services. " * 12, ""]
 
 
 def test_what_was_tried_is_listed_newest_first_in_plain_words_with_what_the_page_said(tmp_path: Path) -> None:
@@ -437,27 +420,19 @@ def test_a_posting_whose_address_is_not_the_webs_is_left_out(tmp_path: Path) ->
         [("web", "HTTPS://a.example/1", "a"), ("disk", "file:///home/me/.aihawk/key", "b"), ("app", "javascript:alert(1)", "c")],
     )
     path = publish(connection, tmp_path, 4)
-    assert [job.id for job in search(path, Filters(include=("java",)), 60, tmp_path)[1]] == ["web"]
+    assert [job.id for job in found(path, Filters(include=("java",)), 60, tmp_path)] == ["web"]
 
 
-def test_a_full_disk_while_the_index_downloads_says_the_disk_is_full(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+def test_a_full_disk_while_the_jobs_download_says_the_disk_is_full(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     class Unreadable(io.BytesIO):
         def read(self, *_: object) -> bytes:
             raise OSError(errno.ENOSPC, "No space left on device")
 
     monkeypatch.setattr("urllib.request.urlopen", lambda *_, **__: Unreadable())
-    manifest = {"index": {"name": "index.parquet", "bytes": 740_000_000}, "built_at": "2026-11-05T00:00:00+00:00", "base_url": "https://a.example"}
-    with pytest.raises(DatabaseError, match="The disk is full: the job index needs 740 MB"):
-        local_index(manifest, "https://a.example/manifest.json", tmp_path)
-    assert not list(tmp_path.glob("*.part"))
-
-
-def test_the_index_download_says_how_far_it_has_come(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr("urllib.request.urlopen", lambda *_, **__: io.BytesIO(b"x" * (4 << 20)))
-    manifest = {"index": {"name": "index.parquet", "bytes": 4 << 20}, "built_at": "2026-11-05T00:00:00+00:00", "base_url": "https://a.example"}
-    lines: list[str] = []
-    assert local_index(manifest, "https://a.example/manifest.json", tmp_path, lines.append).read_bytes() == b"x" * (4 << 20)
-    assert [line.strip() for line in lines[1:]] == ["25%", "50%", "75%"]
+    manifest = {"build": "b1", "chunks": [{"name": "jobs_0.parquet", "rows": 1}], "deltas": [], "base_url": "https://a.example/b1"}
+    with pytest.raises(LibraryError, match="The disk is full"):
+        Library(tmp_path).sync(manifest, "https://a.example/manifest.json")
+    assert not list(tmp_path.rglob("*.part"))
 
 
 def test_a_store_from_before_the_key_changed_keeps_one_row_per_posting_the_one_furthest_along(tmp_path: Path) -> None:
@@ -466,15 +441,35 @@ def test_a_store_from_before_the_key_changed_keeps_one_row_per_posting_the_one_f
     path = tmp_path / "jobs.db"
     store = Store(path)
     link = "https://acme.wd5.myworkdayjobs.com/en-US/External/job/Ankeny-IA/Sales-Rep_R-1"
-    store.add(Job("acme.wd5.myworkdayjobs.com/en-us/external/job/ankeny-ia/sales-rep_r-1", "Acme", "Sales Rep", "Ankeny", link, link, ""))
-    store.add(Job("acme.wd5.myworkdayjobs.com/external/job/sales-rep_r-1", "Acme", "Sales Rep", "", link.replace("en-US/", "").replace("Ankeny-IA/", ""), link, ""))
-    store.set_status("acme.wd5.myworkdayjobs.com/external/job/sales-rep_r-1", "submitted", "sent")
-    store.add(Job("greenhouse.io/acme/jobs/1", "Acme", "Engineer", "", "https://boards.greenhouse.io/acme/jobs/1", "u", ""))
+    # The rows as a store of that time held them: one posting under two keys, each its own row.
+    old = [
+        ("acme.wd5.myworkdayjobs.com/en-us/external/job/ankeny-ia/sales-rep_r-1", link, "new"),
+        ("acme.wd5.myworkdayjobs.com/external/job/sales-rep_r-1", link.replace("en-US/", "").replace("Ankeny-IA/", ""), "submitted"),
+        ("greenhouse.io/acme/jobs/1", "https://boards.greenhouse.io/acme/jobs/1", "new"),
+    ]
+    store._db.executemany(
+        "INSERT INTO jobs (id, company, title, location, url, apply_url, description, status, first_seen, updated) VALUES (?, 'Acme', 'T', '', ?, 'u', '', ?, 'x', 'x')", old
+    )
+    store._db.execute("UPDATE jobs SET key = NULL")
+    store._db.commit()
     store.close()
     sqlite3.connect(path).execute("PRAGMA user_version = 1").connection.commit()
     store = Store(path)
+    # One row per posting, the one furthest along, now under its address.
     ids = {item.job.id: item.status for item in store.all()}
-    assert ids == {"workday:acme:r-1": "submitted", "greenhouse.io/acme/jobs/1": "new"}
+    assert ids == {link.replace("en-US/", "").replace("Ankeny-IA/", ""): "submitted", "https://boards.greenhouse.io/acme/jobs/1": "new"}
+    store.close()
+
+
+def test_a_posting_already_known_under_another_of_its_links_is_not_added_again(tmp_path: Path) -> None:
+    store = Store(tmp_path / "jobs.db")
+    link = "https://acme.wd5.myworkdayjobs.com/External/job/Sales-Rep_R-7"
+    assert store.add(Job(link, "Acme", "Sales Rep", "", link, link, "")) is True
+    store.set_status(link, "submitted", "sent")
+    # The same Workday posting under its other address: the same key, so the job sent is not offered again.
+    other = "https://wd3.myworkdaysite.com/recruiting/acme/External/job/Sales-Rep_R-7"
+    assert store.add_all([Job(other, "Acme", "Sales Rep", "", other, other, "")]) == 0
+    assert [item.job.id for item in store.all()] == [link]
     store.close()
 
 
@@ -491,99 +486,122 @@ def test_a_job_seen_again_keeps_its_score_and_status(tmp_path: Path) -> None:
 
 def test_a_new_folder_defaults_to_the_published_database(tmp_path: Path) -> None:
     config = load_config(tmp_path)
-    assert config.model == "deepseek/deepseek-v4.1-flash" and config.database == "https://huggingface.co/datasets/feder-cr/jobs/resolve/main/manifest.json"
+    assert config.model == "deepseek/deepseek-v4.1-flash" and config.database == "https://huggingface.co/datasets/feder-cr/jobs/resolve/main/library/manifest.json"
+    # A settings file that names the layout before 2026-11-09 reads the new one.
+    (tmp_path / "aihawk.yaml").write_text("database: https://huggingface.co/datasets/feder-cr/jobs/resolve/main/manifest.json\n", encoding="utf-8")
+    assert load_config(tmp_path).database.endswith("/library/manifest.json")
 
 
 
-def _index_rows(path: Path, rows: list[tuple[str, str]]) -> None:
+def chunk(path: Path, rows: list[tuple[str, str, str | None]]) -> None:
+    """A chunk, or a delta when the rows start with their `op`: (url, title, description), or (op, url, title, description)."""
     connection = duckdb.connect()
-    connection.execute(f"CREATE TABLE i ({', '.join(f'{column} VARCHAR' for column in INDEX_COLUMNS)})")
-    connection.executemany("INSERT INTO i (key, url, title) VALUES (?, ?, ?)", [(key, f"https://{key}", title) for key, title in rows])
-    connection.execute(f"COPY i TO '{path.as_posix()}' (FORMAT parquet)")
+    ops = len(rows[0]) == 4
+    connection.execute(f"CREATE TABLE c ({'op VARCHAR, ' if ops else ''}url VARCHAR, title VARCHAR, company VARCHAR, location VARCHAR, country VARCHAR, posted_at TIMESTAMP, description VARCHAR)")
+    connection.executemany(f"INSERT INTO c ({'op, ' if ops else ''}url, title, description) VALUES ({', '.join('?' for _ in rows[0])})", rows)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    connection.execute(f"COPY c TO '{path.as_posix()}' (FORMAT parquet)")
 
 
-def _delta_rows(path: Path, rows: list[tuple[str, str, str | None]]) -> None:
+def titles_here(library: Library) -> dict[str, str]:
     connection = duckdb.connect()
-    columns = ", ".join(f"{column} VARCHAR" for column in INDEX_COLUMNS)
-    connection.execute(f"CREATE TABLE d (op VARCHAR, {columns})")
-    connection.executemany("INSERT INTO d (op, key, url, title) VALUES (?, ?, ?, ?)",
-                           [(op, key, f"https://{key}" if title else None, title) for op, key, title in rows])
-    connection.execute(f"COPY d TO '{path.as_posix()}' (FORMAT parquet)")
+    try:
+        return dict(connection.execute("SELECT url, title FROM postings").fetchall()) if library.view(connection) else {}
+    finally:
+        connection.close()
 
 
-def test_the_chain_of_deltas_is_followed_from_the_build_this_computer_has() -> None:
-    deltas = [{"build": "b2", "after": "b1"}, {"build": "b3", "after": "b2"}, {"build": "b4", "after": "b3"}]
-    assert _chain(deltas, "b2") == deltas[1:]
-    assert _chain(deltas, "b4") == []
-    assert _chain(deltas, "b0") is None
-    assert _chain([{"build": "b2", "after": "b1"}, {"build": "b4", "after": "b3"}], "b1") is None
+def test_the_chunks_come_one_by_one_and_a_search_reads_what_has_come(tmp_path: Path) -> None:
+    site = tmp_path / "site" / "b1"
+    chunk(site / "jobs_0.parquet", [("https://a/1", "Newest", "x" * 300)])
+    chunk(site / "titles_0.parquet", [("https://a/2", "No text", None)])
+    manifest = {"build": "b1", "base_url": site.as_posix(), "deltas": [],
+                "chunks": [{"name": "jobs_0.parquet", "rows": 1}, {"name": "titles_0.parquet", "rows": 1}]}
+    library = Library(tmp_path / "here")
+    seen: list[dict[str, str]] = []
+    told: list[tuple[int, int]] = []
+    library.sync(manifest, "unused", lambda done, size: told.append((done, size)), lambda: seen.append(titles_here(library)))
+    assert seen == [{"https://a/1": "Newest"}, {"https://a/1": "Newest", "https://a/2": "No text"}], "a search reads each chunk as it comes"
+    assert told == [(0, 2), (1, 2), (2, 2)] and library.complete() and library.postings_here() == 2
+    # A second start downloads nothing.
+    library.sync(manifest, "unused", arrived=lambda: seen.append({}))
+    assert len(seen) == 2
 
 
-def test_deltas_are_applied_in_one_rewrite_and_the_newest_action_on_a_key_wins(tmp_path: Path) -> None:
-    index = tmp_path / "index.parquet"
-    _index_rows(index, [("a", "Old A"), ("b", "B"), ("c", "C")])
-    # First delta: a changes, d arrives. Second: d goes again, e arrives, b goes.
-    _delta_rows(tmp_path / "delta-jobs-20261107-010000.parquet", [("add", "a", "New A"), ("add", "d", "D")])
-    _delta_rows(tmp_path / "delta-jobs-20261107-060000.parquet", [("drop", "d", None), ("add", "e", "E"), ("drop", "b", None)])
-    chain = [{"url": (tmp_path / "delta-jobs-20261107-010000.parquet").as_posix()}, {"url": (tmp_path / "delta-jobs-20261107-060000.parquet").as_posix()}]
-    _apply(index, chain)
-    rows = duckdb.connect().execute(f"SELECT key, title FROM read_parquet('{index.as_posix()}') ORDER BY key").fetchall()
-    assert rows == [("a", "New A"), ("c", "C"), ("e", "E")]
-
-
-def test_an_index_kept_from_before_a_column_was_added_takes_the_delta_that_brings_it(tmp_path: Path) -> None:
-    path = tmp_path / "index.parquet"
-    _index_rows(path, [("a", "A"), ("b", "B")])
+def test_the_deltas_after_the_build_here_bring_it_up_to_date_and_the_newest_action_on_an_address_wins(tmp_path: Path) -> None:
+    site = tmp_path / "site"
+    chunk(site / "b1" / "jobs_0.parquet", [("https://a/1", "Old title", "x" * 300), ("https://a/2", "Gone", "y" * 300), ("https://a/3", "Same", "z" * 300)])
+    chunk(site / "d2.parquet", [("add", "https://a/1", "New title", "x" * 300), ("drop", "https://a/2", None, None)])
+    chunk(site / "d3.parquet", [("add", "https://a/4", "Added", None), ("add", "https://a/2", "Back again", "w" * 300)])
+    first = {"build": "b1", "base_url": (site / "b1").as_posix(), "deltas": [], "chunks": [{"name": "jobs_0.parquet", "rows": 3}]}
+    library = Library(tmp_path / "here")
+    library.sync(first, "unused")
+    deltas = [{"build": "b2", "after": "b1", "url": (site / "d2.parquet").as_posix()}, {"build": "b3", "after": "b2", "url": (site / "d3.parquet").as_posix()}]
+    # Named as the dataset names them: the newest last.
+    deltas[0]["url"], deltas[1]["url"] = (site / "delta-b2.parquet").as_posix(), (site / "delta-b3.parquet").as_posix()
+    (site / "d2.parquet").replace(site / "delta-b2.parquet")
+    (site / "d3.parquet").replace(site / "delta-b3.parquet")
+    library.sync({**first, "build": "b3", "deltas": deltas}, "unused")
+    assert titles_here(library) == {"https://a/1": "New title", "https://a/2": "Back again", "https://a/3": "Same", "https://a/4": "Added"}
     connection = duckdb.connect()
-    connection.execute(f"COPY (SELECT * EXCLUDE (described) FROM read_parquet('{path.as_posix()}')) TO '{(tmp_path / 'old.parquet').as_posix()}' (FORMAT parquet)")
-    (tmp_path / "old.parquet").replace(path)
-    _delta_rows(tmp_path / "delta.parquet", [("add", "b", "B"), ("add", "c", "C")])
-    _apply(path, [{"url": (tmp_path / "delta.parquet").as_posix()}])
-    rows = connection.execute(f"SELECT key FROM read_parquet('{path.as_posix()}') ORDER BY key").fetchall()
-    assert rows == [("a",), ("b",), ("c",)]
-    assert "described" in {row[0] for row in connection.execute(f"DESCRIBE SELECT * FROM read_parquet('{path.as_posix()}')").fetchall()}
+    library.view(connection)
+    assert dict(connection.execute("SELECT url, described FROM postings").fetchall())["https://a/4"] is False
 
 
-def test_a_description_is_found_by_its_key_in_the_file_whose_range_holds_it(tmp_path: Path) -> None:
-    connection = duckdb.connect()
-    for name, rows in (("jobs_0.parquet", [("a/1", "First"), ("a/2", "Second")]), ("jobs_1.parquet", [("b/1", "Third")])):
-        connection.execute("CREATE OR REPLACE TABLE t (key VARCHAR, description VARCHAR)")
-        connection.executemany("INSERT INTO t VALUES (?, ?)", rows)
-        connection.execute(f"COPY (SELECT * FROM t ORDER BY key) TO '{(tmp_path / name).as_posix()}' (FORMAT parquet)")
-    manifest = {"base_url": tmp_path.as_posix(), "files": [
-        {"name": "jobs_1.parquet", "first_key": "b/1", "last_key": "b/1"},
-        {"name": "jobs_0.parquet", "first_key": "a/1", "last_key": "a/2"},
+def test_a_chain_that_cannot_be_followed_starts_over_from_the_newest_build(tmp_path: Path) -> None:
+    site = tmp_path / "site"
+    chunk(site / "b1" / "jobs_0.parquet", [("https://a/1", "From b1", "x" * 300)])
+    chunk(site / "b9" / "jobs_0.parquet", [("https://a/9", "From b9", "x" * 300)])
+    library = Library(tmp_path / "here")
+    library.sync({"build": "b1", "base_url": (site / "b1").as_posix(), "deltas": [], "chunks": [{"name": "jobs_0.parquet", "rows": 1}]}, "unused")
+    # b9's deltas start after b5: nothing leads there from b1.
+    later = {"build": "b9", "base_url": (site / "b9").as_posix(), "chunks": [{"name": "jobs_0.parquet", "rows": 1}],
+             "deltas": [{"build": "b9", "after": "b5", "url": "unused"}]}
+    library.sync(later, "unused")
+    assert titles_here(library) == {"https://a/9": "From b9"}
+
+
+def test_many_deltas_become_one_and_a_search_finds_the_same(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    import aihawk.library as library_module
+
+    monkeypatch.setattr(library_module, "DELTAS_APART", 2)
+    site = tmp_path / "site"
+    chunk(site / "b0" / "jobs_0.parquet", [("https://a/1", "First", "x" * 300)])
+    deltas = []
+    for number, rows in enumerate([[("add", "https://a/1", "Second", "x" * 300)], [("add", "https://a/2", "Other", None)], [("drop", "https://a/1", None, None)]], 1):
+        chunk(site / f"delta-b{number}.parquet", rows)
+        deltas.append({"build": f"b{number}", "after": f"b{number - 1}", "url": (site / f"delta-b{number}.parquet").as_posix()})
+    library = Library(tmp_path / "here")
+    library.sync({"build": "b0", "base_url": (site / "b0").as_posix(), "deltas": [], "chunks": [{"name": "jobs_0.parquet", "rows": 1}]}, "unused")
+    library.sync({"build": "b3", "base_url": (site / "b0").as_posix(), "deltas": deltas, "chunks": [{"name": "jobs_0.parquet", "rows": 1}]}, "unused")
+    assert [path.name for path in library.delta_files()] == ["delta-b3.parquet"]
+    assert titles_here(library) == {"https://a/2": "Other"}, "the posting dropped last stays gone"
+
+
+def test_only_the_chunks_of_the_searchs_days_are_downloaded_all_of_them_before_it_runs(tmp_path: Path) -> None:
+    from datetime import UTC, datetime, timedelta
+
+    site = tmp_path / "site" / "b1"
+    chunk(site / "jobs_0.parquet", [("https://a/1", "New", "x" * 300)])
+    chunk(site / "jobs_1.parquet", [("https://a/2", "Older", "x" * 300)])
+    chunk(site / "titles_0.parquet", [("https://a/3", "New, no text", None)])
+
+    def ago(days: int) -> str:
+        return (datetime.now(UTC) - timedelta(days=days)).replace(tzinfo=None).isoformat()
+
+    manifest = {"build": "b1", "base_url": site.as_posix(), "deltas": [], "chunks": [
+        {"name": "jobs_0.parquet", "rows": 10, "newest": ago(1)}, {"name": "jobs_1.parquet", "rows": 90, "newest": ago(30)},
+        {"name": "titles_0.parquet", "rows": 5, "newest": ago(2)},
     ]}
-    files = description_files(manifest, "unused")
-    assert [first for first, _, _ in files] == ["a/1", "b/1"]
-    assert stored_descriptions(["b/1", "a/2", "zz/9", ""], files) == ["Third", "Second", "", ""]
+    library = Library(tmp_path / "here")
+    library.days = 15
+    seen: list[tuple[float, bool]] = []
+    library.sync(manifest, "unused", arrived=lambda: seen.append((round(library.readiness(), 2), library.ready())))
+    assert seen == [(0.67, False), (1.0, True)], "both new chunks, and a search only then"
+    assert titles_here(library) == {"https://a/1": "New", "https://a/3": "New, no text"}
+    # More days in the settings: the chunk they reach comes too.
+    library.days = 60
+    assert not library.ready()
+    library.sync(manifest, "unused")
+    assert library.ready() and "https://a/2" in titles_here(library)
 
-
-def test_an_index_kept_here_is_downloaded_once_then_brought_up_to_date_by_its_deltas(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    release = tmp_path / "release"
-    release.mkdir()
-    snapshot = release / "index.parquet"
-    _index_rows(snapshot, [("a", "A"), ("b", "B")])
-    downloads: list[str] = []
-
-    def served(url: str, *_: object, **__: object) -> io.BytesIO:
-        downloads.append(url)
-        return io.BytesIO(snapshot.read_bytes())
-
-    monkeypatch.setattr("urllib.request.urlopen", served)
-    cache = tmp_path / "cache"
-    first = {"build": "b1", "deltas": [], "index": {"name": "index.parquet", "bytes": snapshot.stat().st_size},
-             "built_at": "2026-11-07T01:00:00+00:00", "base_url": "https://a.example/b1"}
-    path = local_index(first, "https://a.example/manifest.json", cache)
-    assert duckdb.connect().execute(f"SELECT count(*) FROM read_parquet('{path.as_posix()}')").fetchone()[0] == 2
-    assert len(downloads) == 1
-    _delta_rows(release / "delta-jobs-b2.parquet", [("add", "c", "C"), ("drop", "a", None)])
-    second = {**first, "build": "b2", "base_url": "https://a.example/b2",
-              "deltas": [{"build": "b2", "after": "b1", "url": (release / "delta-jobs-b2.parquet").as_posix(), "adds": 1, "drops": 1}]}
-    lines: list[str] = []
-    path = local_index(second, "https://a.example/manifest.json", cache, lines.append)
-    assert len(downloads) == 1, "the whole index is never downloaded again"
-    assert lines == ["Getting the latest jobs."]
-    keys = [row[0] for row in duckdb.connect().execute(f"SELECT key FROM read_parquet('{path.as_posix()}') ORDER BY key").fetchall()]
-    assert keys == ["b", "c"]
-    assert local_index(second, "https://a.example/manifest.json", cache) == path and len(downloads) == 1

tests/test_pipeline.py

@@ -3,10 +3,9 @@
 from __future__ import annotations
 
 import importlib.util
-import itertools
 import json
 import sys
-import uuid
+from datetime import UTC, datetime, timedelta
 from pathlib import Path
 
 import duckdb
@@ -44,22 +43,26 @@ def merged(tmp_path: Path) -> tuple[dict, dict[str, tuple]]:
     (tmp_path / "tmp").mkdir()
     connection = build.connect(tmp_path)
     counts = build.merge(connection, tmp_path)
-    out = (tmp_path / "out").as_posix()
-    rows = duckdb.connect().execute(
-        f"SELECT i.url, i.title, d.description, d.sources FROM read_parquet('{out}/index.parquet') AS i"
-        f" JOIN read_parquet('{out}/jobs_*.parquet') AS d USING (key)"
-    ).fetchall()
+    rows = duckdb.connect().execute(f"SELECT url, title, description FROM read_parquet('{(tmp_path / 'out').as_posix()}/*.parquet')").fetchall()
     return counts, {row[0]: row for row in rows}
 
 
-def test_the_same_posting_from_two_sources_is_one_row_with_the_longest_description_and_both_names(tmp_path: Path) -> None:
+def urls_out(tmp_path: Path) -> list[str]:
+    """The postings published, by their address, which is their key."""
+    return sorted(row[0] for row in duckdb.connect().execute(f"SELECT url FROM read_parquet('{(tmp_path / 'out').as_posix()}/*.parquet')").fetchall())
+
+
+def text(words: str) -> str:
+    """A description long enough to be published: DESCRIBED characters or more."""
+    return (words + " ") * (build.DESCRIBED // len(words) + 1)
+
+
+def test_the_same_posting_from_two_sources_is_one_row_with_the_longest_description(tmp_path: Path) -> None:
     write_source(tmp_path, "short", [("https://jobs.lever.co/a/1/apply?utm_source=x", "Engineer", None)])
-    write_source(tmp_path, "long", [("https://www.jobs.lever.co/A/1", "Engineer", "A full description")])
+    write_source(tmp_path, "long", [("https://www.jobs.lever.co/A/1", "Engineer", text("A full description"))])
     counts, rows = merged(tmp_path)
     assert counts == {"rows_in": 2, "rows_out": 1}
-    row = next(iter(rows.values()))
-    assert row[2] == "A full description"
-    assert sorted(row[3]) == ["long", "short"]
+    assert rows == {"https://www.jobs.lever.co/A/1": ("https://www.jobs.lever.co/A/1", "Engineer", text("A full description"))}
 
 
 def test_copies_that_tie_go_the_same_way_in_every_build(tmp_path: Path) -> None:
@@ -68,24 +71,25 @@ def test_copies_that_tie_go_the_same_way_in_every_build(tmp_path: Path) -> None:
     _, rows = merged(tmp_path)
     row = next(iter(rows.values()))
     assert row[1] == "Engineer (beta)"
-    assert row[3] == ["alpha", "beta"]
 
 
 def test_workday_and_smartrecruiters_postings_are_one_under_any_of_their_links(tmp_path: Path) -> None:
     write_source(tmp_path, "a", [
-        ("https://acme.wd5.myworkdayjobs.com/en-US/External/job/Ankeny-IA/Sales-Rep_R-113371", "Sales Rep", "With the city"),
+        ("https://acme.wd5.myworkdayjobs.com/en-US/External/job/Ankeny-IA/Sales-Rep_R-113371", "Sales Rep", text("With the city")),
         ("https://jobs.smartrecruiters.com/Acme/743999872941471-test-job?oga=true", "Test job", None),
     ])
     write_source(tmp_path, "b", [
-        ("https://acme.wd5.myworkdayjobs.com/External/job/Sales-Rep_R-113371/apply", "Sales Rep", "Without the city, a longer text"),
+        ("https://acme.wd5.myworkdayjobs.com/External/job/Sales-Rep_R-113371/apply", "Sales Rep", text("Without the city, a longer text") * 2),
         ("https://wd3.myworkdaysite.com/recruiting/acme/External/job/Sales-Rep_R-113371", "Sales Rep", None),
         ("https://api.smartrecruiters.com/v1/companies/acme/postings/743999872941471", "Test job", None),
     ])
-    counts, rows = merged(tmp_path)
+    counts, _ = merged(tmp_path)
     assert counts == {"rows_in": 5, "rows_out": 2}
-    assert "Without the city, a longer text" in [row[2] for row in rows.values()], "the copy with the longest description wins"
-    keys = [row[0] for row in duckdb.connect().execute(f"SELECT key FROM read_parquet('{(tmp_path / 'out').as_posix()}/index.parquet') ORDER BY key").fetchall()]
-    assert keys == ["smartrecruiters:743999872941471", "workday:acme:r-113371"]
+    # The copy with the longest description wins; copies with none tie, and go the same way in every build.
+    assert urls_out(tmp_path) == [
+        "https://acme.wd5.myworkdayjobs.com/External/job/Sales-Rep_R-113371/apply",
+        "https://api.smartrecruiters.com/v1/companies/acme/postings/743999872941471",
+    ]
 
 
 def write_full_source(work: Path, name: str, rows: list[tuple]) -> None:
@@ -102,10 +106,6 @@ def write_full_source(work: Path, name: str, rows: list[tuple]) -> None:
     connection.execute(f"COPY t TO '{(work / 'sources' / (name + '.parquet')).as_posix()}' (FORMAT parquet)")
 
 
-def index_rows(tmp_path: Path) -> list[tuple]:
-    return duckdb.connect().execute(f"SELECT key, ats FROM read_parquet('{(tmp_path / 'out').as_posix()}/index.parquet') ORDER BY key").fetchall()
-
-
 def test_one_job_on_a_companys_form_and_on_a_board_is_one_posting_kept_on_the_form(tmp_path: Path) -> None:
     write_source_rows = [("https://boards.greenhouse.io/acme/jobs/1", "Engineer", "Acme Inc.", "Rome, Italy", None, "Short")]
     write_full_source(tmp_path, "company", write_source_rows)
@@ -113,12 +113,10 @@ def test_one_job_on_a_companys_form_and_on_a_board_is_one_posting_kept_on_the_fo
         ("https://www.welcometothejungle.com/en/companies/acme/jobs/99", "engineer", "ACME", "Rome", None, "A much longer description of the job"),
         ("https://www.welcometothejungle.com/en/companies/acme/jobs/100", "Engineer", "Acme", "Milan", None, "The same role in another city"),
     ])
-    counts, rows = merged(tmp_path)
+    counts, _ = merged(tmp_path)
     assert counts == {"rows_in": 3, "rows_out": 2}, "the role in Milan is another job"
     # The company's own link wins over the board's copy, even when the copy's description is longer.
-    assert index_rows(tmp_path) == [("greenhouse.io/acme/jobs/1", "greenhouse"), ("welcometothejungle.com/en/companies/acme/jobs/100", "welcometothejungle")]
-    kept = next(row for url, row in rows.items() if "greenhouse" in url)
-    assert sorted(kept[3]) == ["board", "company"], "the board's copy is named among the sources"
+    assert urls_out(tmp_path) == ["https://boards.greenhouse.io/acme/jobs/1", "https://www.welcometothejungle.com/en/companies/acme/jobs/100"]
 
 
 def test_one_role_posted_twice_on_one_site_is_two_openings(tmp_path: Path) -> None:
@@ -198,8 +196,7 @@ def test_a_site_mostly_made_of_copies_of_companies_forms_goes_whole(tmp_path: Pa
         ("https://other.example/4", "Analyst", "Delta", "Rome", None, None),
     ])
     merged(tmp_path)
-    keys = [key for key, _ in index_rows(tmp_path)]
-    assert keys == ["greenhouse.io/acme/jobs/1", "greenhouse.io/beta/jobs/2", "other.example/4"], "the front's third posting goes too"
+    assert urls_out(tmp_path) == ["https://boards.greenhouse.io/acme/jobs/1", "https://boards.greenhouse.io/beta/jobs/2", "https://other.example/4"], "the front's third posting goes too"
 
 
 def test_a_posting_with_no_company_is_never_merged_by_its_title(tmp_path: Path) -> None:
@@ -217,8 +214,26 @@ def test_a_vendor_has_one_name_and_is_read_from_the_link_when_a_source_does_not_
         ("https://recruiting.paylocity.com/Recruiting/Jobs/Details/2", "Two", "B", "Y", None, None),
         ("https://nobody.example/3", "Three", "C", "Z", None, None),
     ])
-    merged(tmp_path)
-    assert [ats for _, ats in index_rows(tmp_path)] == ["ashby", None, "paylocity"]
+    (tmp_path / "tmp").mkdir()
+    vendors = build.connect(tmp_path).execute(
+        f"SELECT vendor(ats, url) FROM read_parquet('{(tmp_path / 'sources' / 's.parquet').as_posix()}') ORDER BY url"
+    ).fetchall()
+    assert [vendor for (vendor,) in vendors] == ["ashby", None, "paylocity"]
+
+
+def test_the_links_host_names_the_vendor_before_a_sources_label(tmp_path: Path) -> None:
+    write_full_source(tmp_path, "s", [
+        ("https://www.linkedin.com/jobs/view/1", "One", "A", "X", "datastax", None),
+        ("https://boards.greenhouse.io/acme/jobs/2", "Two", "B", "Y", "workday", None),
+        ("https://nobody.example/3", "Three", "C", "Z", "datastax", None),
+        ("https://sjobs.brassring.com/TGnewUI/4", "Four", "D", "W", None, None),
+    ])
+    (tmp_path / "tmp").mkdir()
+    connection = build.connect(tmp_path)
+    vendors = connection.execute(f"SELECT vendor(ats, url) FROM read_parquet('{(tmp_path / 'sources' / 's.parquet').as_posix()}') ORDER BY url").fetchall()
+    # LinkedIn's posting labelled "datastax" is LinkedIn's, and goes; "datastax" names no vendor.
+    assert [vendor for (vendor,) in vendors] == ["greenhouse", None, "brassring", "linkedin"]
+    assert build.merge(connection, tmp_path)["rows_out"] == 3
 
 
 def test_greenhouse_serves_one_posting_from_several_hosts(tmp_path: Path) -> None:
@@ -238,31 +253,71 @@ def test_a_query_that_names_the_posting_keeps_it_apart(tmp_path: Path) -> None:
     assert merged(tmp_path)[0]["rows_out"] == 2
 
 
-def test_the_index_points_at_each_description_without_holding_it(tmp_path: Path) -> None:
-    write_source(tmp_path, "s", [("https://a.example/1", "One", "First text"), ("https://a.example/2", "Two", "Second text")])
+def write_dated(work: Path, name: str, rows: list[tuple]) -> None:
+    """A source whose rows give link, title, description and the date posted."""
+    connection = duckdb.connect()
+    columns = ", ".join(f"{column} {'BOOLEAN' if column == 'remote' else 'DOUBLE' if column.startswith('salary_m') else 'VARCHAR'}" for column in build.COMMON)
+    connection.execute(f"CREATE TABLE t ({columns})")
+    given = ("url", "title", "description", "posted_at", "location", "country", "remote")
+    connection.executemany(f"INSERT INTO t ({', '.join(given)}) VALUES ({', '.join('?' for _ in given)})", [(*row, *([None] * (len(given) - len(row)))) for row in rows])
+    (work / "sources").mkdir(parents=True, exist_ok=True)
+    connection.execute(f"COPY t TO '{(work / 'sources' / (name + '.parquet')).as_posix()}' (FORMAT parquet)")
+
+
+def days_ago(days: int) -> str:
+    return (datetime.now(UTC) - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%S")
+
+
+def test_a_chunk_holds_what_aihawk_reads_and_a_short_text_goes_to_the_chunks_without_one(tmp_path: Path) -> None:
+    write_source(tmp_path, "s", [("https://a.example/1", "One", text("First text")), ("https://a.example/2", "Two", "Too short to say anything")])
     merged(tmp_path)
     out = (tmp_path / "out").as_posix()
     connection = duckdb.connect()
-    columns = [row[0] for row in connection.execute(f"DESCRIBE SELECT * FROM read_parquet('{out}/index.parquet')").fetchall()]
-    assert "description" not in columns and "described" in columns and "part" not in columns
-    held = [row[0] for row in connection.execute(f"DESCRIBE SELECT * FROM read_parquet('{out}/jobs_*.parquet')").fetchall()]
-    assert held == list(build.DESCRIPTION_COLUMNS), "a description file holds what a lookup by key needs, nothing the index has"
-    for key, title in connection.execute(f"SELECT key, title FROM read_parquet('{out}/index.parquet')").fetchall():
-        text = connection.execute(f"SELECT description FROM read_parquet('{out}/jobs_*.parquet') WHERE key = ?", [key]).fetchone()[0]
-        assert text == {"One": "First text", "Two": "Second text"}[title]
+    assert sorted(path.name for path in (tmp_path / "out").glob("*.parquet")) == ["jobs_0.parquet", "titles_0.parquet"]
+    held = [row[0] for row in connection.execute(f"DESCRIBE SELECT * FROM read_parquet('{out}/jobs_0.parquet')").fetchall()]
+    assert held == list(build.CHUNK_COLUMNS), "a chunk holds what AIHawk reads, the address being the key"
+    assert connection.execute(f"SELECT url, description FROM read_parquet('{out}/jobs_0.parquet')").fetchall() == [("https://a.example/1", text("First text"))]
+    assert connection.execute(f"SELECT url, description FROM read_parquet('{out}/titles_0.parquet')").fetchall() == [("https://a.example/2", None)]
 
 
-def test_row_groups_stay_at_the_measured_size_and_the_files_stay_small(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr(build, "ROWS_PER_FILE", 1000)
-    write_source(tmp_path, "s", [(f"https://a.example/{number}", f"Job {number}", uuid.uuid4().hex + uuid.uuid4().hex) for number in range(20000)])
+def test_chunks_go_newest_first_and_a_posting_too_old_is_not_published(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(build, "CHUNK_ROWS", 3)
+    monkeypatch.setattr(build, "CHUNK_GROUP", 2)
+    rows = [(f"https://a.example/{age}", f"Job {age}", text("Work"), days_ago(age)) for age in (1, 2, 5, 9, 40, 55)]
+    future = ("https://a.example/future", "Future", text("Work"), days_ago(-30))
+    write_dated(tmp_path, "s", [*rows, future, ("https://a.example/old", "Old", text("Work"), days_ago(build.MAX_AGE_DAYS + 5))])
+    counts, _ = merged(tmp_path)
+    assert counts["rows_out"] == 7
+    dated = dict(duckdb.connect().execute(f"SELECT title, posted_at FROM read_parquet('{(tmp_path / 'out').as_posix()}/*.parquet')").fetchall())
+    assert dated["Future"] <= datetime.now() + timedelta(minutes=1), "a date after the build is the build's"
+    connection = duckdb.connect()
+    chunks = sorted((tmp_path / "out").glob("*.parquet"), key=build.chunk_order)
+    assert [path.name for path in chunks] == ["jobs_0.parquet", "jobs_1.parquet", "jobs_2.parquet"]
+    titles = [[row[0] for row in connection.execute("SELECT title FROM read_parquet(?)", [path.as_posix()]).fetchall()] for path in chunks]
+    # The chunks go by date; within one, by address.
+    assert [set(chunk) for chunk in titles] == [{"Future", "Job 1", "Job 2"}, {"Job 5", "Job 9", "Job 40"}, {"Job 55"}]
+    first = build.chunk_entry(connection, chunks[0])
+    assert first["rows"] == 3 and not first["titles_only"] and first["newest"] > first["oldest"]
+    groups = connection.execute("SELECT max(row_group_num_rows) FROM parquet_metadata(?)", [chunks[0].as_posix()]).fetchone()[0]
+    assert groups == 2
+    stats = connection.execute("SELECT path_in_schema, bool_or(stats_min_value IS NOT NULL) FROM parquet_metadata(?) GROUP BY 1", [chunks[0].as_posix()]).fetchall()
+    assert {column for column, kept in stats if kept} == {"posted_at"}, "statistics on the date alone keep the footer small"
+
+
+def test_the_chunks_with_no_text_come_after_all_the_others() -> None:
+    names = ["titles_0.parquet", "jobs_10.parquet", "jobs_2.parquet", "titles_1.parquet", "jobs_0.parquet"]
+    assert sorted((Path(name) for name in names), key=build.chunk_order) == [Path(name) for name in ("jobs_0.parquet", "jobs_2.parquet", "jobs_10.parquet", "titles_0.parquet", "titles_1.parquet")]
+
+
+def test_a_remote_posting_that_names_no_place_says_remote(tmp_path: Path) -> None:
+    write_dated(tmp_path, "s", [
+        ("https://a.example/1", "Anywhere", None, None, "", None, True),
+        ("https://a.example/2", "In Rome", None, None, "Rome", None, True),
+        ("https://a.example/3", "Nowhere said", None, None, "", None, None),
+    ])
     merged(tmp_path)
-    files = sorted((tmp_path / "out").glob("jobs_*.parquet"))
-    assert len(files) > 1
-    for file in files:
-        largest = duckdb.connect().execute("SELECT max(row_group_num_rows) FROM parquet_metadata(?)", [file.as_posix()]).fetchone()[0]
-        assert largest <= build.DESCRIPTION_GROUP
-        stats = duckdb.connect().execute("SELECT path_in_schema, bool_or(stats_min_value IS NOT NULL) FROM parquet_metadata(?) GROUP BY 1", [file.as_posix()]).fetchall()
-        assert {column for column, kept in stats if kept} == {"key"}, "statistics on the key alone keep the footer small"
+    places = duckdb.connect().execute(f"SELECT title, location FROM read_parquet('{(tmp_path / 'out').as_posix()}/*.parquet') ORDER BY title").fetchall()
+    assert places == [("Anywhere", "Remote"), ("In Rome", "Rome"), ("Nowhere said", "")]
 
 
 def test_rows_without_a_link_or_a_title_are_left_out(tmp_path: Path) -> None:
@@ -292,18 +347,21 @@ def test_a_healthy_database_is_published(rows: int, previous: int | None) -> Non
     publish.check(rows, previous)
 
 
-def test_one_commit_adds_the_build_its_delta_and_the_manifest_and_removes_all_but_the_last_builds(tmp_path: Path) -> None:
-    manifest = {"build": "jobs-20261108-100000", "index": {"name": "index.parquet"}, "files": [{"name": "jobs_0.parquet"}, {"name": "jobs_1.parquet"}]}
+def test_one_commit_adds_the_build_its_delta_and_the_manifest_and_removes_old_builds_and_deltas(tmp_path: Path) -> None:
+    manifest = {"build": "jobs-20261108-100000", "chunks": [{"name": "jobs_0.parquet"}, {"name": "titles_0.parquet"}],
+                "deltas": [{"url": f"https://x/library/deltas/delta-jobs-2026110{day}-000000.parquet"} for day in (5, 6, 7)]}
     built = [f"jobs-2026110{day}-000000" for day in range(1, 9)]
-    adds, removed = publish.commit_plan(tmp_path, manifest, tmp_path / "delta-jobs-20261108-100000.parquet", built)
+    stored = [f"delta-jobs-2026110{day}-000000.parquet" for day in range(1, 8)]
+    adds, removed, old_deltas = publish.commit_plan(tmp_path, manifest, tmp_path / "delta-jobs-20261108-100000.parquet", built, stored)
     assert [path for path, _ in adds] == [
-        "builds/jobs-20261108-100000/index.parquet", "builds/jobs-20261108-100000/jobs_0.parquet", "builds/jobs-20261108-100000/jobs_1.parquet",
-        "deltas/delta-jobs-20261108-100000.parquet", "manifest.json",
+        "library/builds/jobs-20261108-100000/jobs_0.parquet", "library/builds/jobs-20261108-100000/titles_0.parquet",
+        "library/deltas/delta-jobs-20261108-100000.parquet", "library/manifest.json",
     ]
-    assert adds[-1][0] == "manifest.json", "the manifest names files the same commit adds"
-    assert removed == [f"builds/jobs-2026110{day}-000000" for day in range(1, 4)]
-    adds, removed = publish.commit_plan(tmp_path, manifest, None, [])
-    assert "deltas/" not in " ".join(path for path, _ in adds) and removed == []
+    assert adds[-1][0] == "library/manifest.json", "the manifest names files the same commit adds"
+    assert removed == [f"library/builds/jobs-2026110{day}-000000" for day in range(1, 4)]
+    assert old_deltas == [f"library/deltas/delta-jobs-2026110{day}-000000.parquet" for day in range(1, 5)], "a delta the manifest no longer names goes"
+    adds, removed, old_deltas = publish.commit_plan(tmp_path, {**manifest, "deltas": []}, None, [], [])
+    assert "deltas/" not in " ".join(path for path, _ in adds) and removed == [] and old_deltas == []
 
 
 def test_every_source_names_its_home_and_licence() -> None:
@@ -313,50 +371,25 @@ def test_every_source_names_its_home_and_licence() -> None:
     assert json.dumps(names)
 
 
-def test_descriptions_are_written_in_key_order_and_each_file_says_its_range(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    # A reader asks for a posting by key and reads the one file, and the one row group, that can hold it.
-    monkeypatch.setattr(build, "ROWS_PER_FILE", 1000)
-    write_source(tmp_path, "s", [(f"https://a.example/{uuid.uuid4().hex}", "Job", uuid.uuid4().hex * 4) for _ in range(6000)])
-    merged(tmp_path)
+def write_chunk(path: Path, rows: list[tuple[str, str, str | None]]) -> None:
+    """A chunk of (address, title, description)."""
     connection = duckdb.connect()
-    entries = [build.file_entry(connection, path) for path in sorted((tmp_path / "out").glob("jobs_*.parquet"))]
-    assert len(entries) > 1
-    for path, entry in zip(sorted((tmp_path / "out").glob("jobs_*.parquet")), entries, strict=True):
-        keys = [row[0] for row in connection.execute("SELECT key FROM read_parquet(?)", [path.as_posix()]).fetchall()]
-        assert keys == sorted(keys) and (entry["first_key"], entry["last_key"]) == (keys[0], keys[-1])
-    ranges = sorted((entry["first_key"], entry["last_key"]) for entry in entries)
-    assert all(earlier[1] < later[0] for earlier, later in itertools.pairwise(ranges))
+    connection.execute(f"CREATE TABLE c ({', '.join(f'{column} VARCHAR' for column in publish.COLUMNS)})")
+    connection.executemany("INSERT INTO c (url, title, description) VALUES (?, ?, ?)", rows)
+    connection.execute(f"COPY c TO '{path.as_posix()}' (FORMAT parquet)")
 
 
-def write_index(path: Path, rows: list[tuple[str, str]]) -> None:
-    connection = duckdb.connect()
-    connection.execute(f"CREATE TABLE i (key VARCHAR, {', '.join(f'{column} VARCHAR' for column in publish.INDEX_COLUMNS)})")
-    connection.executemany("INSERT INTO i (key, url, title) VALUES (?, ?, ?)", [(key, f"https://{key}", title) for key, title in rows])
-    connection.execute(f"COPY i TO '{path.as_posix()}' (FORMAT parquet)")
-
-
-def test_a_delta_holds_the_postings_new_or_changed_and_the_keys_gone(tmp_path: Path) -> None:
-    write_index(tmp_path / "before.parquet", [("a", "Gone"), ("b", "Old title"), ("c", "Same")])
-    write_index(tmp_path / "after.parquet", [("b", "New title"), ("c", "Same"), ("d", "New")])
-    adds, drops = publish.write_delta((tmp_path / "before.parquet").as_posix(), tmp_path / "after.parquet", tmp_path / "delta.parquet")
+def test_a_delta_holds_the_postings_new_or_changed_whole_and_the_addresses_gone(tmp_path: Path) -> None:
+    write_chunk(tmp_path / "before_0.parquet", [("https://a", "Gone", "x"), ("https://b", "Same title", "Old text")])
+    write_chunk(tmp_path / "before_1.parquet", [("https://c", "Same", "Same text")])
+    write_chunk(tmp_path / "after_0.parquet", [("https://d", "New", "New text"), ("https://b", "Same title", "New text")])
+    write_chunk(tmp_path / "after_1.parquet", [("https://c", "Same", "Same text")])
+    before = [(tmp_path / f"before_{n}.parquet").as_posix() for n in (0, 1)]
+    adds, drops = publish.write_delta(before, [tmp_path / f"after_{n}.parquet" for n in (0, 1)], tmp_path / "delta.parquet")
     assert (adds, drops) == (2, 1)
-    rows = duckdb.connect().execute(f"SELECT op, key, title FROM read_parquet('{(tmp_path / 'delta.parquet').as_posix()}') ORDER BY key").fetchall()
-    assert rows == [("drop", "a", None), ("add", "b", "New title"), ("add", "d", "New")]
-
-
-def test_a_column_the_previous_index_lacks_changes_every_posting_once(tmp_path: Path) -> None:
-    write_index(tmp_path / "index.parquet", [("a", "Same"), ("b", "Same too")])
-    connection = duckdb.connect()
-    index = (tmp_path / "index.parquet").as_posix()
-    connection.execute(f"COPY (SELECT * REPLACE ('true' AS described) FROM read_parquet('{index}')) TO '{(tmp_path / 'after.parquet').as_posix()}' (FORMAT parquet)")
-    connection.execute(f"COPY (SELECT * EXCLUDE (described) FROM read_parquet('{index}')) TO '{(tmp_path / 'before.parquet').as_posix()}' (FORMAT parquet)")
-    adds, drops = publish.write_delta((tmp_path / "before.parquet").as_posix(), tmp_path / "after.parquet", tmp_path / "delta.parquet")
-    assert (adds, drops) == (2, 0)
-
-
-def test_a_build_is_named_by_its_manifest_or_by_the_end_of_its_address() -> None:
-    assert publish.build_of({"base_url": "https://github.com/o/r/releases/download/jobs-20261107-103405"}) == "jobs-20261107-103405"
-    assert publish.build_of({"build": "jobs-20261107-153405", "base_url": "x"}) == "jobs-20261107-153405"
+    rows = duckdb.connect().execute(f"SELECT op, url, title, description FROM read_parquet('{(tmp_path / 'delta.parquet').as_posix()}') ORDER BY url").fetchall()
+    # A posting whose description changed is changed: the reader gets it whole, its text included.
+    assert rows == [("drop", "https://a", None, None), ("add", "https://b", "Same title", "New text"), ("add", "https://d", "New", "New text")]
 
 
 def test_a_source_is_versioned_by_its_homes_commit_or_by_its_files_headers(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -405,3 +438,43 @@ def test_a_source_unchanged_since_the_last_build_is_taken_from_the_cache_and_a_c
     assert read == ["changed", "unknown"]
     assert [report.get("reused", False) for report in reports] == [True, False, False]
     assert cache.kept == ["changed"], "a source with no version is not kept: it is read every build"
+
+
+def test_a_country_is_its_two_letters_whatever_a_source_writes(tmp_path: Path) -> None:
+    (tmp_path / "tmp").mkdir()
+    connection = build.connect(tmp_path)
+    written = ["UNITED STATES", "Italia", "DEU", "it", "UK", "Côte d'Ivoire", "日本", "Remote", "PARIS", "XX", "", None]
+    codes = [row[0] for row in connection.execute("SELECT country_code(c) FROM unnest(?) WITH ORDINALITY AS t(c, n) ORDER BY n", [written]).fetchall()]
+    assert codes == ["US", "IT", "DE", "IT", "GB", "CI", "JP", None, None, None, None, None]
+
+
+def dates_out(tmp_path: Path) -> dict[str, datetime]:
+    return dict(duckdb.connect().execute(f"SELECT title, posted_at FROM read_parquet('{(tmp_path / 'out').as_posix()}/*.parquet')").fetchall())
+
+
+def test_a_posting_no_source_dates_keeps_the_day_a_build_first_saw_it_and_a_new_one_is_dated_today(tmp_path: Path) -> None:
+    write_dated(tmp_path, "s", [("https://a.example/1", "Seen before", text("Work")), ("https://a.example/2", "New", text("Work")),
+                                 ("https://a.example/3", "Dated", text("Work"), days_ago(3))])
+    (tmp_path / "tmp").mkdir()
+    connection = build.connect(tmp_path)
+    connection.execute("CREATE TEMP TABLE seen AS SELECT 'https://a.example/1' AS url, now()::TIMESTAMP - INTERVAL 20 DAY AS posted_at")
+    build.merge(connection, tmp_path)
+    dates = dates_out(tmp_path)
+    assert abs((datetime.now() - dates["Seen before"]).days - 20) <= 1
+    assert (datetime.now() - dates["New"]).total_seconds() < 3600
+    assert abs((datetime.now() - dates["Dated"]).days - 3) <= 1, "a source's own date wins"
+
+
+def test_the_first_build_to_date_them_backdates_them_between_15_and_60_days_by_their_site(tmp_path: Path) -> None:
+    write_dated(tmp_path, "s", [(f"https://site{n % 20}.example/{n}", f"Job {n}", text("Work")) for n in range(40)])
+    (tmp_path / "tmp").mkdir()
+    build.merge(build.connect(tmp_path), tmp_path, backdate=True)
+    first = dates_out(tmp_path)
+    ages = [(datetime.now() - posted).days for posted in first.values()]
+    # The oldest may fall on the 60th day exactly, past the database's 60: those go.
+    assert all(14 <= age <= 60 for age in ages) and len(set(ages)) > 5
+    # A site's postings have one date: they stay together in a chunk.
+    assert first["Job 3"] == first["Job 23"]
+    build.merge(build.connect(tmp_path), tmp_path, backdate=True)
+    assert {title: posted.date() for title, posted in dates_out(tmp_path).items()} == {title: posted.date() for title, posted in first.items()}, "the same dates every run"
+

tests/test_spend.py

@@ -47,7 +47,7 @@ def test_a_store_from_before_costs_were_kept_gets_the_column_and_an_application_
     )
     old.execute(
         "INSERT INTO jobs (id, company, title, location, url, apply_url, description, status, first_seen, updated)"
-        " VALUES ('a', 'Co', 'Eng', 'Rome', 'u', 'u', 'd', 'submitted', 't', 't')"
+        " VALUES ('a', 'Co', 'Eng', 'Rome', 'a', 'u', 'd', 'submitted', 't', 't')"
     )
     old.commit()
     old.close()

tests/test_tui.py

@@ -127,10 +127,10 @@ class ScriptedModelForApp:
 
 
 async def settle(pilot: Any, app: AIHawk) -> None:
-    """Until the flow's workers are done or waiting on the person."""
+    """Until the flow's workers are done or waiting on the person: the scoring, the jobs coming and the searches over them."""
     for _ in range(200):
         await pilot.pause(0.05)
-        running = [worker for worker in app.workers if worker.is_running and worker.group == "flow"]
+        running = [worker for worker in app.workers if worker.is_running and worker.group in ("flow", "library", "find")]
         if not running or isinstance(app.screen, (TextPrompt, SendBatch)):
             return
     raise AssertionError("the flow never settled")
@@ -272,6 +272,35 @@ def test_space_space_a_then_enter_sends_each_selected_job_once_and_asks_no_quest
     assert Path(calls[0]["cv_file"]).is_file()
 
 
+def test_every_application_sent_is_counted_once_by_how_it_ended_and_a_dry_run_is_not(
+    home: Path, resume: Path, restored_apply: Any, counted: list[str], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    import threading
+
+    from aihawk.counter import COUNTER
+
+    async def send(keys: list[str], outcomes: list[Outcome]) -> None:
+        app = scripted_app(home, resume)
+        fake_apply(app, outcomes, [])
+        async with app.run_test(size=(140, 40)) as pilot:
+            await settle(pilot, app)
+            await pilot.press(*keys, "a")
+            await wait_for_screen(pilot, SendBatch)
+            await pilot.press("enter")
+            await wait_for_apply(pilot, app)
+        for thread in threading.enumerate():
+            if thread.name == "usage-counter":
+                thread.join()
+
+    asyncio.run(send(["space", "space"], [Outcome("submitted", "sent"), Outcome("refused", "phone is required")]))
+    assert counted == [COUNTER.format(outcome="submitted"), COUNTER.format(outcome="refused")]
+    assert counted[0] == "https://github.com/feder-cr/Jobs_Applier_AI_Agent/releases/download/usage-counter/submitted.txt"
+    # A dry run fills the form and sends nothing: nothing is counted.
+    monkeypatch.setenv("AIHAWK_DRY_RUN", "1")
+    asyncio.run(send(["down", "down", "space"], [Outcome("blocked", "Dry run: nothing sent.")]))
+    assert len(counted) == 2
+
+
 def test_escape_on_the_batch_sends_none_and_the_selection_stays(home: Path, resume: Path, restored_apply: Any) -> None:
     calls: list[Any] = []
 
@@ -284,7 +313,7 @@ def test_escape_on_the_batch_sends_none_and_the_selection_stays(home: Path, resu
             await wait_for_screen(pilot, SendBatch)
             await pilot.press("escape")
             await wait_for_apply(pilot, app)
-            assert app.selected == {"high"}
+            assert app.selected == {"https://jobs.lever.co/beta/2"}, "a job is known by its address"
             assert str(app.query_one(DataTable).get_row_at(0)[0]) == "●"
 
     asyncio.run(run())
@@ -354,12 +383,15 @@ def test_s_lists_what_was_tried_newest_first_and_enter_opens_one_in_full(home: P
             await pilot.press("s")
             await wait_for_screen(pilot, SentList)
             table = app.screen.query_one(DataTable)
-            rows = [[str(table.get_row_at(index)[column]) for column in (1, 3, 4)] for index in range(table.row_count)]
-            assert rows[0] == ["Acme", "sent and confirmed", "Thank you"]
-            assert rows[1][1:] == ["blocked", "a captcha"]
+            rows = [[str(table.get_row_at(index)[column]) for column in (1, 3)] for index in range(table.row_count)]
+            assert rows == [["Acme", "sent and confirmed"], ["Beta", "blocked"]]
             await pilot.press("enter")
             await wait_for_screen(pilot, SentDetails)
             assert app.screen.item.job.id == "sent"
+            # How it went in a person's words: what the page said, and not the form's fields or the page's files.
+            text = str(app.screen.query_one("#summary Static").render())
+            assert "Sent and confirmed · 2026-11-03 09:00" in text and "The page said: Thank you" in text
+            assert "Filled in" not in text and "shots" not in text
             await pilot.press("escape")
             await pilot.press("escape")
 
@@ -414,12 +446,26 @@ def test_at_80_columns_the_role_is_the_last_column_and_at_100_the_location_is(ho
     asyncio.run(run())
 
 
+def test_enter_selects_a_job_as_space_does_and_a_second_one_unselects_it(home: Path, resume: Path) -> None:
+    async def run() -> None:
+        app = scripted_app(home, resume)
+        async with app.run_test(size=(140, 40)) as pilot:
+            await settle(pilot, app)
+            await pilot.press("enter", "space")
+            assert len(app.selected) == 2 and not isinstance(app.screen, JobDetails)
+            await pilot.press("up", "enter")
+            table = app.query_one(DataTable)
+            assert [str(table.get_row_at(index)[0]) for index in range(2)] == ["●", ""]
+
+    asyncio.run(run())
+
+
 def test_a_jobs_details_can_select_it_and_the_row_shows_the_mark(home: Path, resume: Path) -> None:
     async def run() -> None:
         app = scripted_app(home, resume)
         async with app.run_test(size=(140, 40)) as pilot:
             await settle(pilot, app)
-            await pilot.press("enter")
+            await pilot.press("i")
             await wait_for_screen(pilot, JobDetails)
             await pilot.press("space")
             await pilot.pause(0.1)
@@ -429,16 +475,18 @@ def test_a_jobs_details_can_select_it_and_the_row_shows_the_mark(home: Path, res
     asyncio.run(run())
 
 
-def test_enter_shows_why_a_job_fits(home: Path, resume: Path) -> None:
+def test_i_shows_the_job_as_its_posting_says_it_and_nothing_else(home: Path, resume: Path) -> None:
     async def run() -> None:
         app = scripted_app(home, resume)
         async with app.run_test(size=(140, 40)) as pilot:
             await settle(pilot, app)
-            await pilot.press("enter")
+            await pilot.press("i")
             await wait_for_screen(pilot, JobDetails)
-            # The job and its score, nothing else: Jev answers in or out, and writes no reasons.
-            text = str(app.screen.query_one("#summary Static").render())
-            assert "Bestco" in text and "Score 4.5/5" in text
+            # The role, the company, where, and the posting's text: no score, no address.
+            text = "\n".join(str(widget.render()) for widget in app.screen.query(Static))
+            item = app.current()
+            assert item and item.job.title in text and "Bestco" in text and item.job.location in text
+            assert "Score" not in text and "https://" not in text
             await pilot.press("escape")
 
     asyncio.run(run())
@@ -766,8 +814,9 @@ def test_a_posting_with_no_description_is_judged_on_its_title_and_no_page_is_rea
         raise AssertionError("a page was read from the person's computer")
 
     monkeypatch.setattr("urllib.request.urlopen", no_page)
-    monkeypatch.setattr(session_module, "stored_descriptions", lambda keys, files, done, problem: ["Stored text." if key == "told" else "" for key in keys])
-    asyncio.run(session.read(lambda _: None, lambda done, size: None))
+    # The descriptions come from the jobs on this computer, by address.
+    monkeypatch.setattr(session.library, "descriptions", lambda urls: ["Stored text." if url.endswith("/2") else "" for url in urls])
+    asyncio.run(session.read(lambda _: None))
     assert session.store.get("bare").score is not None and session.store.get("bare").job.description == ""  # type: ignore[union-attr]
     asked = json.dumps(session._client.posts)  # type: ignore[union-attr]
     assert "Stored text." in asked
@@ -955,3 +1004,14 @@ def test_the_right_arrow_says_so_when_there_is_nothing_left_to_read(home: Path,
             assert "No more jobs for this search." in logged(app)
 
     asyncio.run(run())
+
+
+def test_a_postings_html_reads_as_paragraphs_and_bullets_even_when_escaped() -> None:
+    from aihawk.screens import readable
+
+    html_text = "<h2>About us</h2><p>We build <strong>tools</strong>.</p><ul><li>Python</li><li>Go &amp; Rust</li></ul><script>x()</script>"
+    expected = "About us\n\nWe build tools.\n\n• Python\n• Go & Rust"
+    assert readable(html_text) == expected
+    # Some sources escape their HTML, once or twice over.
+    assert readable(html_text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")) == expected
+    assert readable("Plain text,   with  spaces.\n\n\n\nAnd a second paragraph.") == "Plain text, with spaces.\n\nAnd a second paragraph."

tests/test_usage_counters.py

@@ -0,0 +1,55 @@
+"""The applications' count is recorded by GitHub Actions, never by a machine of ours: a scheduled workflow, the script it
+runs, and the arithmetic of the history."""
+
+from __future__ import annotations
+
+import datetime as dt
+import importlib.util
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+spec = importlib.util.spec_from_file_location("usage_counters", ROOT / "scripts" / "usage_counters.py")
+counters = importlib.util.module_from_spec(spec)  # type: ignore[arg-type]
+sys.modules["usage_counters"] = counters
+spec.loader.exec_module(counters)  # type: ignore[union-attr]
+
+WORKFLOW = ROOT / ".github" / "workflows" / "counters.yml"
+
+
+def test_the_snapshot_is_a_scheduled_workflow_with_no_personal_secret() -> None:
+    text = WORKFLOW.read_text(encoding="utf-8")
+    assert "schedule:" in text and "cron:" in text
+    assert "scripts/usage_counters.py" in text
+    assert "ref: badges" in text and "HEAD:badges" in text
+    assert "secrets." not in text, "it must run on the built-in token alone"
+
+
+def test_the_app_and_the_snapshot_count_the_same_files() -> None:
+    from aihawk import counter
+
+    assert counter.OUTCOMES == counters.OUTCOMES
+    assert counter.COUNTER == f"https://github.com/{counters.REPO}/releases/download/{counters.COUNTER_TAG}/{{outcome}}.txt"
+
+
+def test_the_selftest_passes() -> None:
+    assert counters.selftest() == 0
+
+
+def test_a_day_is_the_difference_of_two_snapshots_and_the_badge_shows_the_applications_sent(tmp_path: Path) -> None:
+    totals = iter([
+        {"submitted": 10, "unconfirmed": 2, "refused": 1, "blocked": 0, "failed": 0},
+        {"submitted": 25, "unconfirmed": 3, "refused": 4, "blocked": 1, "failed": 2},
+    ])
+
+    def api(path: str) -> object:
+        if path.endswith("/releases/tags/usage-counter"):
+            return {"assets": [{"name": f"{name}.txt", "download_count": value} for name, value in next(totals).items()]}
+        return {"stargazers_count": 3}
+
+    history, daily, badges = tmp_path / "history.csv", tmp_path / "daily.csv", tmp_path / "badges"
+    counters.run(api, history, daily, badges, dt.date(2026, 11, 9))
+    counters.run(api, history, daily, badges, dt.date(2026, 11, 10))
+    rows = daily.read_text(encoding="utf-8").splitlines()
+    assert rows == ["day_utc,sent,submitted,unconfirmed,refused,blocked,failed", "2026-11-09,16,15,1,3,1,2"]
+    assert b"applications sent: 28" in (badges / "applications.svg").read_bytes()