From f8d7093c5ae5dca78919c10fee8de583572682d8 Mon Sep 17 00:00:00 2001 From: Federico Elia Date: Sat, 7 Nov 2026 03:34:57 +0100 Subject: [PATCH] Job database: rebuilt every five hours instead of once a day The biggest source, jobhive, publishes around 15:00 UTC and the new-grad lists commit every hour; the 04:23 build read them up to a day late. Cron 23 */5 * * *, off the top of the hour; timeout 200 minutes so a hung run does not take the next slot; six dated releases kept, about thirty hours. Texts say five hours, and the search log gives the build's hour. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/jobs-database.yml | 8 +++++--- README.md | 2 +- docs/data-sources.md | 4 ++-- pipeline/build.py | 4 ++-- pipeline/publish.py | 8 ++++---- src/aihawk/session.py | 2 +- tests/test_funnel.py | 2 +- 7 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/jobs-database.yml b/.github/workflows/jobs-database.yml index 9e282d26..4a26aa72 100644 --- a/.github/workflows/jobs-database.yml +++ b/.github/workflows/jobs-database.yml @@ -1,10 +1,12 @@ name: job database -# Every day: read the public job databases, merge them and publish the result as a release, which AIHawk reads when it searches. +# Every five hours: read the public job databases, merge them and publish the result as a release, which AIHawk reads when it searches. +# Five hours because the biggest source, jobhive, publishes around 15:00 UTC and the small new-grad lists every hour; a daily build read them up to a day late. +# Minute 23 because GitHub delays schedules at the top of the hour. # Users never scrape anything. on: schedule: - - cron: "23 4 * * *" + - cron: "23 */5 * * *" workflow_dispatch: permissions: @@ -17,7 +19,7 @@ concurrency: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 350 + timeout-minutes: 200 # shorter than the five hours to the next run, so a hung build does not take its slot steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/README.md b/README.md index d751a631..3e93b837 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Give it your resume. It finds the jobs that fit, you pick, and it applies. | Step | What happens | |---|---| | **1. Reads your resume** | Turns it into a search: the titles, the skills and the places that fit you. | -| **2. Searches** | 10 million postings from 24 public sources, rebuilt every day, searched on your computer. | +| **2. Searches** | 10 million postings from 24 public sources, rebuilt every five hours, searched on your computer. | | **3. Scores** | Reads the best 100 against your resume, then scores the top 25 from 1 to 5, with the gaps and the reasons. | | **4. You pick** | Select the jobs you want and say yes once. | | **5. Applies** | A real Firefox opens the company's form, fills every required field, attaches your CV and sends. | diff --git a/docs/data-sources.md b/docs/data-sources.md index 7e5fc4bf..16deb969 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -1,6 +1,6 @@ # Data sources -AIHawk's job database merges 24 public sources. `pipeline/build.py` reads each one every day, and the release keeps the postings that survive the merge. Each posting names the sources that carried it. +AIHawk's job database merges 24 public sources. `pipeline/build.py` reads each one every five hours, and the release keeps the postings that survive the merge. Each posting names the sources that carried it. The licence column says what the source's own page or dataset card states. "Not stated" means no licence was found on that page or in the repository's files. Nothing here is a legal opinion. Postings belong to their employers, and the link in each row goes to the page this table relies on. AIHawk's MIT licence covers its code, not this data. @@ -44,4 +44,4 @@ A source's owner can ask for it to be removed. There are two ways. - **In public:** open an issue with the [data source template](../.github/ISSUE_TEMPLATE/data_source.md). Say which source it is, who you are, and what to remove. - **In private:** if you would rather not write in public, use the private report in [SECURITY.md](../SECURITY.md). -What happens next: the source is removed from `SOURCES` in `pipeline/build.py` and from this table. The next daily build in [`.github/workflows/jobs-database.yml`](../.github/workflows/jobs-database.yml) does not read it. Releases already published keep their postings until the publish step drops them. It keeps the last three dated releases. +What happens next: the source is removed from `SOURCES` in `pipeline/build.py` and from this table. The next build, within five hours, in [`.github/workflows/jobs-database.yml`](../.github/workflows/jobs-database.yml) does not read it. Releases already published keep their postings until the publish step drops them. It keeps the last six dated releases, about thirty hours. diff --git a/pipeline/build.py b/pipeline/build.py index c0e09f4d..05880383 100644 --- a/pipeline/build.py +++ b/pipeline/build.py @@ -3,8 +3,8 @@ python pipeline/build.py --out work # everything python pipeline/build.py --out work --only riley # only these sources -Scheduled daily in .github/workflows/jobs-database.yml, which publishes work/out as a release. Each source is read into -work/sources/.parquet in the columns of COMMON. A source that fails is left out of the day's database and named in +Scheduled every five hours in .github/workflows/jobs-database.yml, which publishes work/out as a release. Each source is read into +work/sources/.parquet in the columns of COMMON. A source that fails is left out of that build's database and named in the manifest; publish.py refuses a database that lost too much of itself. Postings are the same posting when their links are the same after `KEY`, and of those the row with the longest description is kept, the other rows adding their source's name. The output is cut in files under GitHub's limit of 2 GiB per release file, with index.parquet beside them: every posting without its diff --git a/pipeline/publish.py b/pipeline/publish.py index 6bef6763..9ad81f1f 100644 --- a/pipeline/publish.py +++ b/pipeline/publish.py @@ -1,4 +1,4 @@ -"""Publishes work/out as today's release, and points the release `jobs-latest` at it. +"""Publishes work/out as this run's release, and points the release `jobs-latest` at it. A database much smaller than the one before it is refused: losing a big source must not silently empty everyone's search. The dated releases keep the files; `jobs-latest` holds only the manifest that names them, so a reader never sees a @@ -17,7 +17,7 @@ import urllib.request from pathlib import Path LATEST = "jobs-latest" -KEEP = 3 +KEEP = 6 # five-hourly builds: about 30 hours of releases, so a reader mid-search does not lose its files MIN_SHARE_OF_PREVIOUS = 0.7 @@ -67,14 +67,14 @@ def check(rows: int, previous: int | None) -> None: if rows == 0: sys.exit("the database is empty: nothing published") if previous and rows < previous * MIN_SHARE_OF_PREVIOUS: - sys.exit(f"{rows} postings against {previous} yesterday: nothing published. See the manifest's failed sources.") + sys.exit(f"{rows} postings against {previous} in the previous build: nothing published. See the manifest's failed sources.") def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--repo", required=True) - parser.add_argument("--allow-shrink", action="store_true", help="Publish a database much smaller than yesterday's on purpose.") + parser.add_argument("--allow-shrink", action="store_true", help="Publish a database much smaller than the previous build's on purpose.") arguments = parser.parse_args() manifest_path: Path = arguments.out / "manifest.json" diff --git a/src/aihawk/session.py b/src/aihawk/session.py index 710b7fc7..1019dc25 100644 --- a/src/aihawk/session.py +++ b/src/aihawk/session.py @@ -84,7 +84,7 @@ class Session: search, self.config.database, self.config.filters, self.config.max_age_days, index, lambda line: loop.call_soon_threadsafe(log, line) ) new = sum(self.store.add(job) for job in jobs) - log(f"{len(jobs)} postings fit your search, of {manifest['rows_out']:,} in the job database built {built_at(manifest):%Y-%m-%d}; {new} new") + log(f"{len(jobs)} postings fit your search, of {manifest['rows_out']:,} in the job database built {built_at(manifest):%Y-%m-%d %H:%M} UTC; {new} new") return new async def funnel(self, log: Log, status: Log) -> int: diff --git a/tests/test_funnel.py b/tests/test_funnel.py index d3ccedab..d908dc9c 100644 --- a/tests/test_funnel.py +++ b/tests/test_funnel.py @@ -150,7 +150,7 @@ def test_a_scan_that_logs_from_its_thread_says_it_on_the_loop(tmp_path: Path, mo loop_thread = threading.get_ident() asyncio.run(session.find(lambda line: seen.append((line, threading.get_ident())))) assert seen[0] == ("from the thread", loop_thread) - assert seen[1][0] == "0 postings fit your search, of 0 in the job database built 2026-11-04; 0 new" and seen[1][1] == loop_thread + assert seen[1][0] == "0 postings fit your search, of 0 in the job database built 2026-11-04 00:00 UTC; 0 new" and seen[1][1] == loop_thread def test_with_no_postings_left_the_funnel_does_nothing(tmp_path: Path) -> None: