feat(plugins): refresh the first-party catalog snapshot per release; replace only the embedded copy

scripts/refresh-first-party-marketplace.sh rewrites the embedded snapshot from
the public marketplace head (through sync-marketplace.py) after refusing a head
whose remote entries lack a sha256, a version or platforms, or follow a floating
release URL. With --check it fails when the files the catalog installs from
(marketplace.json, plugins/, skills/) differ between the snapshot revision and
the head; commits that touch none of them do not make it stale.
release-candidate.yml runs the check in its resolve job, before the long
parity and artifact builds.

MarketplaceStore now supersedes a saved "codewhale" catalog only when its source
is a 40-hex revision of the embedded snapshot's repository other than the
embedded one. A catalog the desktop added from its bundled file, one a person
added from their own document or URL, and a moving-branch URL are kept. The
replacement also applies before a write, so a stale embedded copy cannot block
re-adding the embedded source.

The computer-use pin assertion no longer hard-codes a release URL; it reads the
snapshot, so the pin can move without editing the test. New tests cover the
sha256 rule, the replacement and the kept catalogs.

The snapshot value itself is unchanged here: the Computer Use pin follows the
marketplace bump being merged.

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

.github/workflows/release-candidate.yml

@@ -68,6 +68,8 @@ jobs:
 
           echo "sha=${actual}" >> "${GITHUB_OUTPUT}"
           echo "version=${workspace_version}" >> "${GITHUB_OUTPUT}"
+      - name: Check the first-party catalog snapshot is current
+        run: ./scripts/refresh-first-party-marketplace.sh --check
       - name: Check version and OHOS release contracts
         # --require-dated-release, as auto-tag.yml and release.yml run it:
         # release.yml only accepts an RC receipt for the exact tag SHA, so an

crates/plugins/src/marketplace/store.rs

@@ -105,7 +105,9 @@ impl MarketplaceStore {
         };
         // Every surface consumes this same projection. Browsing is offline
         // and read-only; installation and updates use the reviewed installer.
-        // A user's local catalog or removal always takes precedence.
+        // A user's local catalog or removal always takes precedence; only a
+        // saved copy of an older embedded snapshot is superseded.
+        replace_superseded_embedded_catalog(&mut state)?;
         if !state.first_party_removed && !state.catalogs.contains_key("codewhale") {
             state
                 .catalogs
@@ -235,6 +237,7 @@ impl MarketplaceStore {
             .write()
             .map_err(|e| format!("failed to lock marketplace state for update: {e}"))?;
         let mut next = self.load_unlocked()?;
+        replace_superseded_embedded_catalog(&mut next)?;
         let result = mutate(&mut next)?;
         save_state_with_hardener(&self.path, &next, harden_plugin_state_file)?;
         Ok(result)
@@ -313,17 +316,57 @@ fn collapse_same_source_catalogs(mut state: MarketplaceState) -> MarketplaceStat
     state
 }
 
-fn first_party_catalog() -> Result<StoredMarketplaceCatalog, String> {
-    #[derive(Deserialize)]
-    struct Snapshot {
-        repository: String,
-        revision: String,
-        catalog: serde_json::Value,
-    }
-    let snapshot: Snapshot =
+#[derive(Deserialize)]
+struct EmbeddedSnapshot {
+    repository: String,
+    revision: String,
+    catalog: serde_json::Value,
+}
+
+impl EmbeddedSnapshot {
+    fn embedded() -> Result<Self, String> {
         serde_json::from_str(include_str!("../../assets/first-party-marketplace.json"))
-            .map_err(|error| format!("invalid bundled marketplace: {error}"))?;
-    let source = format!("{}/tree/{}", snapshot.repository, snapshot.revision);
+            .map_err(|error| format!("invalid bundled marketplace: {error}"))
+    }
+
+    fn source(&self) -> String {
+        format!("{}/tree/{}", self.repository, self.revision)
+    }
+
+    fn supersedes(&self, source_path: &str) -> bool {
+        source_path
+            .strip_prefix(&format!("{}/tree/", self.repository))
+            .is_some_and(|revision| {
+                revision != self.revision
+                    && revision.len() == 40
+                    && revision.bytes().all(|byte| byte.is_ascii_hexdigit())
+            })
+    }
+}
+
+/// A saved `codewhale` catalog whose source is an older revision of the
+/// embedded snapshot is replaced by the embedded one. A catalog added from any
+/// other source (the desktop's bundled file, a person's own document or URL)
+/// is never touched.
+fn replace_superseded_embedded_catalog(state: &mut MarketplaceState) -> Result<(), String> {
+    if state.first_party_removed {
+        return Ok(());
+    }
+    let Some(stored) = state.catalogs.get("codewhale") else {
+        return Ok(());
+    };
+    let snapshot = EmbeddedSnapshot::embedded()?;
+    if snapshot.supersedes(&stored.source_path) {
+        state
+            .catalogs
+            .insert("codewhale".into(), first_party_catalog()?);
+    }
+    Ok(())
+}
+
+fn first_party_catalog() -> Result<StoredMarketplaceCatalog, String> {
+    let snapshot = EmbeddedSnapshot::embedded()?;
+    let source = snapshot.source();
     let mut catalog = super::parsers::parse_catalog(super::parsers::MarketplaceDocument {
         catalog_id: MarketplaceCatalogId::new("codewhale"),
         format: super::types::MarketplaceFormat::Codewhale,
@@ -371,10 +414,8 @@ mod tests {
                 panic!("uninstallable candidate")
             };
             if candidate.name == "computer-use" {
-                assert_eq!(
-                    spec,
-                    "https://github.com/codewhale-hq/codewhale-cu-plugin/releases/download/v0.16.0/computer-use.tar.gz#sha256=11a4f5c97c32717fa5648d10c080aa0db48a4798ccdf0d17112d1ae5b9133ae4"
-                );
+                let (source, sha256) = snapshot_entry_pin("computer-use");
+                assert_eq!(spec, format!("{source}#sha256={sha256}"));
             } else {
                 assert!(spec.starts_with(
                     "https://codeload.github.com/codewhale-hq/codewhale-plugin-marketplace/"
@@ -401,6 +442,122 @@ mod tests {
         );
     }
 
+    fn snapshot_plugins() -> Vec<serde_json::Value> {
+        let snapshot: serde_json::Value =
+            serde_json::from_str(include_str!("../../assets/first-party-marketplace.json"))
+                .unwrap();
+        snapshot["catalog"]["plugins"].as_array().unwrap().clone()
+    }
+
+    fn snapshot_entry_pin(name: &str) -> (String, String) {
+        let entry = snapshot_plugins()
+            .into_iter()
+            .find(|entry| entry["name"] == name)
+            .unwrap();
+        (
+            entry["source"].as_str().unwrap().to_string(),
+            entry["sha256"].as_str().unwrap().to_string(),
+        )
+    }
+
+    #[test]
+    fn every_remote_snapshot_entry_carries_a_sha256() {
+        let snapshot = EmbeddedSnapshot::embedded().unwrap();
+        let pinned_bundle = format!(
+            "https://codeload.github.com/codewhale-hq/codewhale-plugin-marketplace/tar.gz/{}#",
+            snapshot.revision
+        );
+        let mut remote = 0;
+        for entry in snapshot_plugins() {
+            let source = entry["source"].as_str().unwrap();
+            assert!(
+                source.starts_with("https://"),
+                "{source} is not a pinned remote source"
+            );
+            if source.starts_with(&pinned_bundle) {
+                continue;
+            }
+            remote += 1;
+            let sha256 = entry["sha256"].as_str().unwrap_or_default();
+            assert!(
+                sha256.len() == 64
+                    && sha256
+                        .bytes()
+                        .all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')),
+                "{} has no sha256",
+                entry["name"]
+            );
+            assert!(
+                !source.contains("/latest/"),
+                "{} follows a floating release",
+                entry["name"]
+            );
+        }
+        assert!(remote >= 1, "the computer-use release asset is remote");
+    }
+
+    fn write_state(store: &MarketplaceStore, source_path: &str) {
+        let mut catalog = first_party_catalog().unwrap();
+        catalog.source_path = source_path.to_string();
+        catalog.catalog.candidates.clear();
+        let mut state = MarketplaceState::default();
+        state.catalogs.insert("codewhale".into(), catalog);
+        ensure_private_plugin_state_directory(store.path().parent().unwrap()).unwrap();
+        fs::write(store.path(), serde_json::to_string(&state).unwrap()).unwrap();
+    }
+
+    #[test]
+    fn a_saved_copy_of_an_older_embedded_snapshot_is_replaced() {
+        let root = tempfile::tempdir().unwrap();
+        let store = MarketplaceStore::open(Some(&root.path().join("plugins/state.json"))).unwrap();
+        let snapshot = EmbeddedSnapshot::embedded().unwrap();
+        let older = format!("{}/tree/{}", snapshot.repository, "0".repeat(40));
+        write_state(&store, &older);
+
+        let loaded = store.load().unwrap();
+        let kept = loaded.get("codewhale").unwrap();
+        assert_eq!(kept.source_path, snapshot.source());
+        assert!(kept.catalog.total_candidates() > 0);
+
+        store
+            .add(
+                &MarketplaceCatalogId::new("codewhale"),
+                first_party_catalog().unwrap(),
+            )
+            .unwrap();
+        let persisted: MarketplaceState =
+            serde_json::from_str(&fs::read_to_string(store.path()).unwrap()).unwrap();
+        assert_eq!(
+            persisted.get("codewhale").unwrap().source_path,
+            snapshot.source()
+        );
+    }
+
+    #[test]
+    fn desktop_bundled_and_person_added_catalogs_are_kept() {
+        let snapshot = EmbeddedSnapshot::embedded().unwrap();
+        let bundled_file = Path::new("resources")
+            .join("marketplace")
+            .join("marketplace.json")
+            .display()
+            .to_string();
+        let moving_branch = format!("{}/tree/main", snapshot.repository);
+        for source in [
+            bundled_file.as_str(),
+            "https://example.com/team/marketplace.json",
+            moving_branch.as_str(),
+        ] {
+            let root = tempfile::tempdir().unwrap();
+            let store =
+                MarketplaceStore::open(Some(&root.path().join("plugins/state.json"))).unwrap();
+            write_state(&store, source);
+            let loaded = store.load().unwrap();
+            let kept = loaded.get("codewhale").unwrap();
+            assert_eq!(kept.source_path, source);
+            assert_eq!(kept.catalog.total_candidates(), 0);
+        }
+    }
+
     #[test]
     fn malformed_store_is_not_hidden_by_bundled_catalog() {
         let root = tempfile::tempdir().unwrap();

scripts/refresh-first-party-marketplace.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Refresh or check the first-party catalog snapshot embedded in the Engine
+# (crates/plugins/assets/first-party-marketplace.json) against the head of the
+# public marketplace repository.
+#
+#   scripts/refresh-first-party-marketplace.sh            rewrite the snapshot from the marketplace head
+#   scripts/refresh-first-party-marketplace.sh --check    fail when the snapshot is older than the head
+#   --marketplace DIR                                     use this checkout instead of cloning the head
+#
+# Nothing is installed, trusted or executed. "Older" means the files the
+# catalog installs from (marketplace.json, plugins/, skills/) differ between the
+# snapshot's revision and the head; unrelated commits do not make it stale.
+set -euo pipefail
+
+root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+snapshot="${root}/crates/plugins/assets/first-party-marketplace.json"
+repo_url="https://github.com/codewhale-hq/codewhale-plugin-marketplace"
+mode=refresh
+checkout=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --check) mode=check; shift ;;
+    --marketplace)
+      [[ $# -ge 2 ]] || { echo "--marketplace needs a directory" >&2; exit 2; }
+      checkout="$2"; shift 2 ;;
+    -h|--help) sed -n '2,12p' "${BASH_SOURCE[0]}"; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "${checkout}" ]]; then
+  tmp="$(mktemp -d)"
+  trap 'rm -rf "${tmp}"' EXIT
+  git clone --quiet --filter=blob:none --no-checkout "${repo_url}" "${tmp}/marketplace"
+  checkout="${tmp}/marketplace"
+fi
+
+head_rev="$(git -C "${checkout}" rev-parse HEAD)"
+
+if [[ "${mode}" == refresh ]]; then
+  validate="$(cat <<'PY'
+import json, re, sys
+catalog = json.load(sys.stdin)
+problems = []
+for plugin in catalog["plugins"]:
+    name, source = plugin["name"], plugin["source"]
+    if source.startswith("path:") or re.fullmatch(r"github:[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", source):
+        continue
+    if "/latest/" in source:
+        problems.append(f"{name}: follows a floating release ({source})")
+    if not re.fullmatch(r"[0-9a-f]{64}", plugin.get("sha256", "")):
+        problems.append(f"{name}: remote source has no sha256")
+    for field in ("version", "platforms"):
+        if not plugin.get(field):
+            problems.append(f"{name}: remote source has no {field}")
+if problems:
+    sys.exit("marketplace head is not publishable as a snapshot:\n  " + "\n  ".join(problems))
+PY
+)"
+  git -C "${checkout}" show HEAD:marketplace.json | python3 -c "${validate}"
+  python3 "${root}/scripts/sync-marketplace.py" --marketplace "${checkout}"
+  exit 0
+fi
+
+snapshot_rev="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["revision"])' "${snapshot}")"
+if [[ "${snapshot_rev}" == "${head_rev}" ]]; then
+  echo "First-party catalog snapshot is at the marketplace head ${head_rev}."
+  exit 0
+fi
+if ! git -C "${checkout}" cat-file -e "${snapshot_rev}^{commit}" 2>/dev/null; then
+  echo "::error::Snapshot revision ${snapshot_rev} is not in the marketplace history; run scripts/refresh-first-party-marketplace.sh." >&2
+  exit 1
+fi
+tree_ids() {
+  local rev="$1" path
+  for path in marketplace.json plugins skills; do
+    git -C "${checkout}" rev-parse "${rev}:${path}" 2>/dev/null || echo "missing:${path}"
+  done
+}
+if [[ "$(tree_ids "${snapshot_rev}")" == "$(tree_ids "${head_rev}")" ]]; then
+  echo "First-party catalog snapshot (${snapshot_rev}) is current: nothing it installs from changed up to ${head_rev}."
+  exit 0
+fi
+echo "::error::The first-party catalog snapshot (${snapshot_rev}) is older than the marketplace head (${head_rev}). Run scripts/refresh-first-party-marketplace.sh, review the diff and commit it." >&2
+git -C "${checkout}" diff --stat "${snapshot_rev}" "${head_rev}" -- marketplace.json plugins skills >&2 || true
+exit 1