From 80fc29badb93ca217acc75f6ca561366bce8ff54 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 26 Sep 2026 12:43:12 -0400 Subject: [PATCH] feat: initial release of @heretek-ai/epistemic-swarm research harness --- .claude-plugin/marketplace.json | 14 ++ .claude-plugin/plugin.json | 58 +++++ .claude/CLAUDE.md | 22 ++ .github/workflows/publish.yml | 34 +++ .gitignore | 13 ++ .npmignore | 12 ++ LICENSE | 126 +++++++++++ README.md | 130 ++++++++++++ bin/cli.js | 119 +++++++++++ config/claude-settings-patch.json | 10 + config/docker-compose.infra.yml | 23 ++ config/mcp-research-servers.json | 26 +++ config/searxng_mcp.py | 133 ++++++++++++ docs/DISTRIBUTION_STRATEGY.md | 63 ++++++ docs/SYSTEM_ARCHITECTURE.md | 260 +++++++++++++++++++++++ install.sh | 102 +++++++++ package.json | 53 +++++ prompts/agent_alpha_thesis.md | 70 +++++++ prompts/agent_beta_antithesis.md | 78 +++++++ prompts/base_epistemic_system.md | 50 +++++ prompts/epistemic_auditor.md | 74 +++++++ prompts/orchestrator.md | 70 +++++++ runner/__init__.py | 0 runner/auditor_engine.py | 222 ++++++++++++++++++++ runner/research_swarm.py | 337 ++++++++++++++++++++++++++++++ runner/state_machine.py | 192 +++++++++++++++++ runner/tests/test_swarm.py | 178 ++++++++++++++++ skills/grilling/SKILL.md | 48 +++++ skills/grilling/__init__.py | 0 skills/grilling/socratic_tree.py | 148 +++++++++++++ skills/research-cache/SKILL.md | 36 ++++ skills/research-cache/__init__.py | 0 skills/research-cache/hasher.py | 195 +++++++++++++++++ skills/research_cache | 1 + 34 files changed, 2897 insertions(+) create mode 100644 .claude-plugin/marketplace.json create mode 100644 .claude-plugin/plugin.json create mode 100644 .claude/CLAUDE.md create mode 100644 .github/workflows/publish.yml create mode 100644 .gitignore create mode 100644 .npmignore create mode 100644 LICENSE create mode 100644 README.md create mode 100755 bin/cli.js create mode 100644 config/claude-settings-patch.json create mode 100644 config/docker-compose.infra.yml create mode 100644 config/mcp-research-servers.json create mode 100755 config/searxng_mcp.py create mode 100644 docs/DISTRIBUTION_STRATEGY.md create mode 100644 docs/SYSTEM_ARCHITECTURE.md create mode 100755 install.sh create mode 100644 package.json create mode 100644 prompts/agent_alpha_thesis.md create mode 100644 prompts/agent_beta_antithesis.md create mode 100644 prompts/base_epistemic_system.md create mode 100644 prompts/epistemic_auditor.md create mode 100644 prompts/orchestrator.md create mode 100644 runner/__init__.py create mode 100644 runner/auditor_engine.py create mode 100755 runner/research_swarm.py create mode 100644 runner/state_machine.py create mode 100644 runner/tests/test_swarm.py create mode 100644 skills/grilling/SKILL.md create mode 100644 skills/grilling/__init__.py create mode 100755 skills/grilling/socratic_tree.py create mode 100644 skills/research-cache/SKILL.md create mode 100644 skills/research-cache/__init__.py create mode 100755 skills/research-cache/hasher.py create mode 120000 skills/research_cache diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..d8fb294 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,14 @@ +{ + "name": "heretek-official", + "owner": "Heretek AI", + "description": "Official plugins and agent harnesses from Heretek AI", + "plugins": { + "epistemic-swarm": { + "source": { + "source": "github", + "repo": "Heretek-AI/IUMBTEMS" + }, + "description": "High-Integrity Dialectic Research Agent Harness for Claude Code" + } + } +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..d6b5c72 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,58 @@ +{ + "name": "epistemic-swarm", + "version": "0.1.0", + "description": "High-Integrity Dialectic Research Agent Harness for Claude Code enforcing empirical evidence over parametric hallucination.", + "author": "Heretek AI", + "license": "Apache-2.0", + "repository": "https://github.com/Heretek-AI/IUMBTEMS", + "mcpServers": { + "brave-search": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-brave-search"], + "env": { + "BRAVE_SEARCH_API_KEY": "${BRAVE_SEARCH_API_KEY}" + } + }, + "firecrawl": { + "command": "npx", + "args": ["-y", "firecrawl-mcp"], + "env": { + "FIRECRAWL_API_KEY": "${FIRECRAWL_API_KEY}", + "FIRECRAWL_API_URL": "${FIRECRAWL_API_URL:-http://localhost:3002}" + } + }, + "searxng": { + "command": "python3", + "args": ["${PLUGIN_DIR}/config/searxng_mcp.py"], + "env": { + "SEARXNG_URL": "${SEARXNG_URL:-http://localhost:8080}" + } + } + }, + "skills": [ + "./skills/grilling", + "./skills/research-cache" + ], + "configuration": { + "SEARXNG_URL": { + "type": "string", + "description": "SearXNG metasearch instance URL", + "default": "http://localhost:8080" + }, + "FIRECRAWL_API_URL": { + "type": "string", + "description": "Firecrawl API endpoint for JS/Markdown extraction", + "default": "http://localhost:3002" + }, + "FIRECRAWL_API_KEY": { + "type": "string", + "description": "Firecrawl API Key (optional for self-hosted instances)", + "default": "" + }, + "BRAVE_SEARCH_API_KEY": { + "type": "string", + "description": "Brave Search API Key for discovery", + "default": "" + } + } +} diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md new file mode 100644 index 0000000..3e3a015 --- /dev/null +++ b/.claude/CLAUDE.md @@ -0,0 +1,22 @@ +# Epistemic Swarm: Project Workspace Rules + +This repository operates under the **Epistemic Swarm Research Protocol**. All Claude Code sessions within this directory are bound by strict evidentiary integrity rules. + +## Core Rules for This Workspace +1. **Epistemic Invariant Enforcement**: + - Tag every factual claim using `[VERIFIED: ]`, `[INFERRED: ]`, `[HYPOTHESIS: ]`, or `[NEGATIVE_KNOWLEDGE: ]`. + - Never generate hallucinated citations. If an empirical claim cannot be retrieved and verified via tools, mark it as `[HYPOTHESIS]` or `[NEGATIVE_KNOWLEDGE]`. + +2. **Source Caching**: + - When fetching articles, documentation, or papers via web search or MCP tools, ensure the raw content is hashed and saved to `.research/sources/.md` with metadata in `.research/sources/.json`. + - Verbatim quotes must match the text inside `.research/sources/.md`. + +3. **Multi-Agent Dialectic Swarm**: + - When running deep research, use the runner script: + ```bash + python3 runner/research_swarm.py --objective "" + ``` + - For preliminary planning and assumption inversion, trigger the Socratic grilling skill via `/grilling`. + +4. **IPC Scratchpad Structure**: + - Read and write session data strictly through `.research/scratchpads/{scope_id}/`. diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..cde4ddd --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,34 @@ +name: Publish to npm + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.x' + registry-url: 'https://registry.npmjs.org' + + - name: Run Test Suite + run: | + python3 -m unittest discover -s runner/tests + + - name: Publish Package to npm (Trusted Publishing OIDC) + run: npm publish --provenance --access public diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8bc7818 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +node_modules/ +.research/scratchpads/ +.research/sources/ +.research/manifest.json +.research/frontier.json +.research/final_synthesis.md +__pycache__/ +*.py[cod] +*$py.class +*.log +searxng-data/ +dist/ +*.tgz diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..1f88fef --- /dev/null +++ b/.npmignore @@ -0,0 +1,12 @@ +.research/ +.git/ +.github/ +docs/ +**/*.pyc +**/__pycache__/** +**/__pycache__/ +__pycache__/ +.DS_Store +*.log +searxng-data/ +test_* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5aaec7d --- /dev/null +++ b/LICENSE @@ -0,0 +1,126 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work. + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work. + + END OF TERMS AND CONDITIONS + + Copyright 2026 Heretek AI diff --git a/README.md b/README.md new file mode 100644 index 0000000..35761b9 --- /dev/null +++ b/README.md @@ -0,0 +1,130 @@ +# Epistemic Swarm ๐ŸŒŸ + +[![npm version](https://img.shields.io/npm/v/@heretek-ai/epistemic-swarm.svg)](https://www.npmjs.com/package/@heretek-ai/epistemic-swarm) +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![CI/CD](https://github.com/Heretek-AI/IUMBTEMS/actions/workflows/publish.yml/badge.svg)](https://github.com/Heretek-AI/IUMBTEMS/actions) + +> **High-Integrity Dialectic Research Agent Harness for Claude Code** +> *Enforcing verified empirical evidence over parametric hallucination.* + +--- + +## ๐ŸŽฏ The Epistemic Mandate + +Current AI research assistants suffer from parametric hallucination, sycophancy, and premature narrative consensus. They invent citations, smooth over technical contradictions, and extrapolate beyond empirical bounds. + +**Epistemic Swarm** is an autonomous, dialectic research harness built on top of [Claude Code](https://claude.ai/code) (`~/.claude/` and `~/.claude.json`). It enforces **evidentiary primacy** through: +1. **Mathematical Evidentiary Tags**: Every factual claim is tagged `[VERIFIED: ]`, `[INFERRED: ...]`, `[HYPOTHESIS: ...]`, or `[NEGATIVE_KNOWLEDGE: ...]`. +2. **Dialectic Swarm Architecture**: Competitively dispatches **Agent Alpha** (The Thesis / Primary Literature Proponent) and **Agent Beta** (The Antithesis / Hostile Red Team / Active Falsifier). +3. **Content-Addressed Source Caching**: Web pages and papers are hashed to SHA-256 (`.research/sources/.md`). +4. **Algorithmic Epistemic Auditor**: Verifies cited quotes verbatim against the raw source cache, downgrades fabricated claims, scores divergence, and synthesizes the unvarnished empirical truth. +5. **Socratic Grilling & Divergent Ideation**: Matt Pocock-style design tree traversal that explores assumption inversions and clarifies constraints *before* committing to search queries. + +--- + +## ๐Ÿ—๏ธ System Architecture + +```mermaid +flowchart TD + User([Research Objective]) --> Grilling["Phase 1: Socratic Grilling (/grilling)\n(Assumption Inversion & Frontier Resolution)"] + Grilling --> SettledFrontier[Settled Problem Frontier & Constraints] + SettledFrontier --> Orchestrator["Phase 2: Swarm Orchestrator\n(Scope Decomposition & DAG Planner)"] + + Orchestrator --> Scope1[Scope 1: Technical Mechanisms] + Orchestrator --> Scope2[Scope 2: Scalability & Economics] + + subgraph DialecticLoop [Parallel Dialectic Execution per Scope] + Scope1 --> Alpha["Agent Alpha: The Proponent\n(Empirical Evidence, Primary Literature)"] + Scope1 --> Beta["Agent Beta: The Adversary\n(Active Falsification, Red-Teaming)"] + + Alpha -->|Writes| AlphaDossier[alpha_dossier.json + Cached Sources] + Beta -->|Writes| BetaDossier[beta_dossier.json + Cached Sources] + + AlphaDossier --> Auditor["Phase 3: Epistemic Auditor\n(Verbatim Quote Hash Verification & Divergence Scoring)"] + BetaDossier --> Auditor + Auditor --> ScopeSynth[Scope Synthesis & Audit Report] + end + + ScopeSynth --> MasterSynth["Phase 4: Master Synthesis (.research/final_synthesis.md)"] +``` + +--- + +## โšก Quickstart + +### Option 1: Run via NPX (Zero Setup) +```bash +# Execute deep research swarm directly +npx @heretek-ai/epistemic-swarm run "Evaluate FPGA Poseidon prover latency bounds" + +# Run Socratic decision tree framing +npx @heretek-ai/epistemic-swarm grill --objective "L1 vs L2 state verification trade-offs" +``` + +### Option 2: Install into Claude Code Environment +```bash +git clone https://github.com/Heretek-AI/IUMBTEMS.git +cd IUMBTEMS +npm run install-local +``` +This automatically symlinks the `/grilling` and `/research-cache` skills into `~/.claude/skills/` and configures MCP servers in `~/.claude.json`. + +--- + +## ๐Ÿท๏ธ Epistemic Tagging Taxonomy + +| Tag | Meaning | Requirement | +| :--- | :--- | :--- | +| `[VERIFIED: ]` | Direct empirical fact backed by primary source. | Verbatim quote must exist in `.research/sources/.md`. | +| `[INFERRED: ]` | Deductive conclusion from verified facts. | Must list parent verified claims and deductive step. | +| `[HYPOTHESIS: ]` | Speculative assertion or projection. | Must define a measurable falsification criterion. | +| `[NEGATIVE_KNOWLEDGE: ]` | Verified absence of empirical evidence. | Records exact search query and confirmed literature gap. | + +--- + +## ๐Ÿ” Multi-Tier OSINT & Search Pipeline + +1. **Discovery Tier**: SearXNG (metasearch) and Brave Search API. +2. **Extraction Tier**: Firecrawl (headless JavaScript rendering, DOM cleaning, Markdown extraction). +3. **Academic Tier**: Semantic Scholar / arXiv MCPs for DOI resolution. +4. **Caching Tier**: Content-addressed SHA-256 storage (`skills/research-cache/hasher.py`). + +### Optional Turnkey Infrastructure (SearXNG + Firecrawl) +To run your own unbiased metasearch and headless extractor locally: +```bash +docker compose -f config/docker-compose.infra.yml up -d +``` + +--- + +## ๐Ÿงช Testing + +Epistemic Swarm comes with an automated test suite verifying source hashing, quote substring matching, IPC state transitions, and mock swarm dispatch: + +```bash +npm test +``` + +--- + +## ๐Ÿ“ฆ Deployment & CI/CD + +This package is distributed via npm under `@heretek-ai` with [npm Trusted Publishing (OIDC)](https://docs.npmjs.com/trusted-publishers) and GitHub Actions. + +### First-Time Publication (One-Time Bootstrap) +1. Authenticate locally: + ```bash + npm login + ``` +2. Publish initial version: + ```bash + npm publish --access public + ``` +3. In `npmjs.com/package/@heretek-ai/epistemic-swarm/access`, add **GitHub Actions** as a Trusted Publisher for repository `Heretek-AI/IUMBTEMS` and workflow `.github/workflows/publish.yml`. +4. Subsequent releases publish automatically upon pushing a GitHub Release or git tag! + +--- + +## ๐Ÿ“„ License + +Licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for details. diff --git a/bin/cli.js b/bin/cli.js new file mode 100755 index 0000000..b836057 --- /dev/null +++ b/bin/cli.js @@ -0,0 +1,119 @@ +#!/usr/bin/env node + +/** + * Epistemic Swarm CLI + * Multi-agent dialectic research harness for Claude Code. + */ + +const { spawnSync } = require('child_process'); +const path = require('path'); +const fs = require('fs'); + +const PKG_ROOT = path.resolve(__dirname, '..'); +const args = process.argv.slice(2); +const command = args[0] || 'help'; + +function printHelp() { + console.log(` +๐ŸŒŸ Epistemic Swarm: High-Integrity Research Harness for Claude Code + Organization: @heretek-ai | Repository: Heretek-AI/IUMBTEMS + +Usage: + npx @heretek-ai/epistemic-swarm [options] + iumbtems [options] + +Commands: + run "" Run the dialectic multi-agent research swarm + grill Launch interactive Socratic decision tree framing + install Install skills & MCP servers into ~/.claude/ + test Run test suite for hashing, state machine & auditor + doctor Check environment requirements (Claude Code, Python, Node) + help Show this help message + +Options: + --mock-claude Run swarm with synthetic mock responses (zero API cost) + --frontier Path to settled frontier.json from Socratic grilling + --dir Path to .research workspace directory (default: .research) + +Examples: + npx @heretek-ai/epistemic-swarm run "Verify sub-millisecond ZK prover latency" + npx @heretek-ai/epistemic-swarm grill --objective "Rollup architecture trade-offs" + npx @heretek-ai/epistemic-swarm install +`); +} + +function runPython(scriptRelPath, extraArgs = []) { + let finalArgs = []; + if (scriptRelPath === '-m') { + finalArgs = ['-m', ...extraArgs]; + } else { + const scriptPath = path.join(PKG_ROOT, scriptRelPath); + finalArgs = [scriptPath, ...extraArgs]; + } + const result = spawnSync('python3', finalArgs, { + stdio: 'inherit', + cwd: process.cwd(), + env: { ...process.env, PYTHONPATH: PKG_ROOT } + }); + process.exit(result.status !== null ? result.status : 1); +} + +function runBash(scriptRelPath, extraArgs = []) { + const scriptPath = path.join(PKG_ROOT, scriptRelPath); + const result = spawnSync('bash', [scriptPath, ...extraArgs], { + stdio: 'inherit', + cwd: process.cwd() + }); + process.exit(result.status !== null ? result.status : 1); +} + +switch (command) { + case 'run': { + const objective = args[1]; + if (!objective || objective.startsWith('--')) { + console.error('Error: Please provide a research objective string.'); + console.error('Example: npx @heretek-ai/epistemic-swarm run "Evaluate FPGA prover latency"'); + process.exit(1); + } + const forwardArgs = ['--objective', objective, ...args.slice(2)]; + runPython('runner/research_swarm.py', forwardArgs); + break; + } + + case 'grill': { + runPython('skills/grilling/socratic_tree.py', args.slice(1)); + break; + } + + case 'install': { + runBash('install.sh', args.slice(1)); + break; + } + + case 'test': { + runPython('-m', ['unittest', 'discover', '-s', 'runner/tests']); + break; + } + + case 'doctor': { + console.log('๐Ÿ” Checking Epistemic Swarm Environment:\n'); + const claudeCheck = spawnSync('claude', ['--version'], { encoding: 'utf-8' }); + if (claudeCheck.status === 0) { + console.log(`โœ… Claude Code: ${claudeCheck.stdout.trim()}`); + } else { + console.log('โš ๏ธ Claude Code CLI not found in PATH (Install via npm i -g @anthropic-ai/claude-code)'); + } + + const pyCheck = spawnSync('python3', ['--version'], { encoding: 'utf-8' }); + console.log(`โœ… Python: ${pyCheck.stdout ? pyCheck.stdout.trim() : 'python3 missing'}`); + + console.log(`โœ… Node.js: ${process.version}`); + console.log(`๐Ÿ“ Package Root: ${PKG_ROOT}`); + break; + } + + case 'help': + default: + printHelp(); + break; +} diff --git a/config/claude-settings-patch.json b/config/claude-settings-patch.json new file mode 100644 index 0000000..a9aa770 --- /dev/null +++ b/config/claude-settings-patch.json @@ -0,0 +1,10 @@ +{ + "env": { + "SEARXNG_URL": "http://localhost:8080", + "FIRECRAWL_API_URL": "http://localhost:3002", + "FIRECRAWL_API_KEY": "" + }, + "enabledPlugins": { + "epistemic-swarm@heretek-official": true + } +} diff --git a/config/docker-compose.infra.yml b/config/docker-compose.infra.yml new file mode 100644 index 0000000..fb5ae44 --- /dev/null +++ b/config/docker-compose.infra.yml @@ -0,0 +1,23 @@ +version: '3.8' + +services: + searxng: + image: docker.io/searxng/searxng:latest + container_name: epistemic-searxng + ports: + - "8080:8080" + volumes: + - ./searxng-data:/etc/searxng:rw + environment: + - SEARXNG_BASE_URL=http://localhost:8080/ + restart: unless-stopped + + firecrawl: + image: mendableai/firecrawl:latest + container_name: epistemic-firecrawl + ports: + - "3002:3002" + environment: + - PORT=3002 + - NUM_WORKERS=4 + restart: unless-stopped diff --git a/config/mcp-research-servers.json b/config/mcp-research-servers.json new file mode 100644 index 0000000..38810c3 --- /dev/null +++ b/config/mcp-research-servers.json @@ -0,0 +1,26 @@ +{ + "mcpServers": { + "brave-search": { + "command": "npx", + "args": ["-y", "@modelcontextprotocol/server-brave-search"], + "env": { + "BRAVE_SEARCH_API_KEY": "${BRAVE_SEARCH_API_KEY}" + } + }, + "searxng": { + "command": "python3", + "args": ["config/searxng_mcp.py"], + "env": { + "SEARXNG_URL": "${SEARXNG_URL:-http://localhost:8080}" + } + }, + "firecrawl": { + "command": "npx", + "args": ["-y", "firecrawl-mcp"], + "env": { + "FIRECRAWL_API_KEY": "${FIRECRAWL_API_KEY}", + "FIRECRAWL_API_URL": "${FIRECRAWL_API_URL:-http://localhost:3002}" + } + } + } +} diff --git a/config/searxng_mcp.py b/config/searxng_mcp.py new file mode 100755 index 0000000..906addc --- /dev/null +++ b/config/searxng_mcp.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +Lightweight MCP Server for SearXNG Metasearch. +Provides unbiased search results directly to Claude Code agents. +""" + +import os +import sys +import json +import urllib.request +import urllib.parse +from typing import Dict, Any, List + +SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://localhost:8080") + +def search_searxng(query: str, categories: str = "general", max_results: int = 10) -> List[Dict[str, Any]]: + """Query SearXNG JSON endpoint.""" + params = { + "q": query, + "format": "json", + "categories": categories, + "language": "en" + } + url = f"{SEARXNG_URL.rstrip('/')}/search?{urllib.parse.urlencode(params)}" + req = urllib.request.Request(url, headers={"User-Agent": "EpistemicSwarm/1.0"}) + + try: + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read().decode("utf-8")) + results = data.get("results", [])[:max_results] + return [ + { + "title": r.get("title", ""), + "url": r.get("url", ""), + "snippet": r.get("content", ""), + "engine": r.get("engine", "") + } + for r in results + ] + except Exception as e: + return [{"error": f"Failed to query SearXNG at {SEARXNG_URL}: {str(e)}"}] + + +def handle_json_rpc(line: str): + """Minimal JSON-RPC 2.0 loop for MCP protocol.""" + try: + msg = json.loads(line) + except Exception: + return + + msg_id = msg.get("id") + method = msg.get("method") + + if method == "initialize": + response = { + "jsonrpc": "2.0", + "id": msg_id, + "result": { + "protocolVersion": "2024-11-05", + "capabilities": { + "tools": {} + }, + "serverInfo": { + "name": "searxng-mcp", + "version": "1.0.0" + } + } + } + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + + elif method == "tools/list": + response = { + "jsonrpc": "2.0", + "id": msg_id, + "result": { + "tools": [ + { + "name": "searxng_search", + "description": "Execute an unbiased multi-engine metasearch via SearXNG.", + "inputSchema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"}, + "categories": { + "type": "string", + "description": "Categories (general, science, it)", + "default": "general" + }, + "max_results": {"type": "integer", "default": 10} + }, + "required": ["query"] + } + } + ] + } + } + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + + elif method == "tools/call": + params = msg.get("params", {}) + tool_name = params.get("name") + args = params.get("arguments", {}) + + if tool_name == "searxng_search": + query = args.get("query", "") + cat = args.get("categories", "general") + limit = args.get("max_results", 10) + res = search_searxng(query, categories=cat, max_results=limit) + response = { + "jsonrpc": "2.0", + "id": msg_id, + "result": { + "content": [ + { + "type": "text", + "text": json.dumps(res, indent=2) + } + ] + } + } + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + + +def main(): + for line in sys.stdin: + if line.strip(): + handle_json_rpc(line.strip()) + +if __name__ == "__main__": + main() diff --git a/docs/DISTRIBUTION_STRATEGY.md b/docs/DISTRIBUTION_STRATEGY.md new file mode 100644 index 0000000..a2bdc1d --- /dev/null +++ b/docs/DISTRIBUTION_STRATEGY.md @@ -0,0 +1,63 @@ +# Epistemic Swarm: Commercialization & Distribution Strategy Analysis + +## 1. Strategic Context & Market Positioning + +Current deep research solutions (e.g., Perplexity, OpenAI Deep Research, generic LangChain/LlamaIndex search wrappers) suffer from an **epistemic trust deficit**: +- Factual citations often hallucinate non-existent authors or conflate unrelated studies. +- Output text is optimized for fluency, sycophancy, and narrative harmony rather than active falsification. +- Enterprises in regulated domains (biopharma, quantitative finance, defense, semiconductor IP, legal compliance) cannot rely on black-box parametric summaries. + +**Epistemic Swarm** positions itself as the **Verifiable Scientific Audit Layer** for autonomous AI research. Its moat is not the LLM itself, but the **dialectic adversary-proponent architecture**, **content-addressed source provenance**, and **machine-checked quote verification**. + +--- + +## 2. Comparative Evaluation of Distribution Models + +| Dimension | Option A: Composable Open-Source CLI (`npx` / Dotfile Overlay) | Option B: Hosted Multi-Tenant Marketplace & Registry | +| :--- | :--- | :--- | +| **User Adoption Friction** | **Near Zero**: One-command invocation (`npx @heretek-ai/epistemic-swarm`). No new accounts or billing setups. Runs entirely inside existing developer CLI workflows. | **Moderate/High**: Requires signing up, configuring cloud team accounts, provisioning API keys, and managing credit wallets. | +| **Data Privacy & IP Sovereignty** | **Absolute**: All source caches, scratchpads, and proprietary research questions remain on the user's local disk (`.research/`). Essential for confidential R&D. | **Complex**: Proprietary enterprise research topics and sensitive IP must transit third-party cloud infrastructure. Requires SOC2 / HIPAA / ISO compliance. | +| **Infrastructure Overhead** | **Zero for Distributor**: Compute, search tokens, and disk storage are paid for and hosted by the end user via their local Claude Code / API keys. | **High**: Must maintain redundant SearXNG clusters, Firecrawl browser render farms, rate-limit queues, multi-tenant databases, and billing gateways. | +| **Monetization Mechanics** | Indirect: Consulting, enterprise support, sponsored domain plugins, professional certification of research methodologies. | Direct: Usage-based billing ($/deep research run, $/verified citation query), tiered team seats, proprietary high-speed search index access. | +| **Network Effects & Moat** | **Evidentiary Standard**: Open epistemic tagging taxonomy becomes an industry benchmark (like SemVer or Git SHA-1). High community contributions. | **Data Flywheel**: Centralized cross-organizational citation cache, shared deduplicated academic paper embeddings, and reputation graphs. | + +--- + +## 3. Concrete Recommendation: The Progressive Decentralization Playbook + +### Phase 1: Grassroots Developer Infiltration (Months 1โ€“6) โ€” **Option A** +- **Action**: Launch `@heretek-ai/epistemic-swarm` as a 100% free, open-source CLI and Claude Code plugin on npm and GitHub. +- **Why**: Developer trust cannot be purchased; it must be audited. In epistemic systems, open-source code is a non-negotiable proof-of-work. By distributing as an `npx` overlay, researchers, staff engineers, and academics can immediately test the dialectic red-team harness with zero onboarding friction. +- **Milestones**: + - 10k+ weekly npm downloads. + - Integration with popular academic workflows (arXiv, PubMed, Semantic Scholar). + - Validation of the `[VERIFIED: ]` tagging taxonomy across top-tier engineering organizations. + +### Phase 2: Hybrid Managed Gateway & Shared Source Cloud (Months 6โ€“12) โ€” **Freemium Bridge** +- **The Pain Point**: Local users eventually tire of maintaining their own SearXNG instances, anti-bot proxies, and Firecrawl headless browser pools. +- **The Bridge**: Heretek AI launches **Epistemic Cloud Gateway**: + - The CLI remains open-source, but adds a single optional flag: `--cloud-gateway`. + - Heretek hosts ultra-fast, pre-indexed, headless rendering and paywall-bypass proxies. + - Users get 50 free cloud-verified research runs/month; heavy power users subscribe to an API tier ($49/month). + - Global Content-Addressed Hash Cache: If Agent Alpha needs a paper that another researcher already cached, the gateway serves the verified `.md` instantly with cryptographic proof, cutting token costs by 70%. + +### Phase 3: Enterprise Audit Marketplace & Hosted Swarm (Months 12+) โ€” **Option B** +- **Target**: Regulated research divisions (BioPharma, Quantitative Hedge Funds, Defense Contractors, Legal R&D). +- **Product**: **Epistemic Swarm Enterprise**: + - Private multi-tenant deployment (VPC / On-Prem). + - Multi-agent dialectic consensus dashboards for non-technical stakeholders (C-suite, research directors). + - Cryptographic Audit Trail: Export signed research briefs with verifiable cryptographic proofs of evidence for regulatory submissions (FDA, SEC, USPTO). + - Monetization: $1,200โ€“$5,000/seat/year plus enterprise SLA. + +--- + +## 4. Immediate Actionable Next Steps + +1. **Bootstrap Initial NPM Release**: + - Complete first-time local CLI publish of `@heretek-ai/epistemic-swarm` (v0.1.0). +2. **Authorize GitHub Actions Trusted Publisher**: + - Connect `Heretek-AI/IUMBTEMS` on `npmjs.com/package/@heretek-ai/epistemic-swarm/access`. +3. **Register Claude Code Marketplace Manifest**: + - Submit `.claude-plugin/plugin.json` to the Claude Code plugins directory. +4. **Community Outbound**: + - Publish technical deep-dive demonstrating how the Epistemic Swarm's dialectic red-teamer caught critical errors missed by standard single-prompt LLM research. diff --git a/docs/SYSTEM_ARCHITECTURE.md b/docs/SYSTEM_ARCHITECTURE.md new file mode 100644 index 0000000..3c7f3ce --- /dev/null +++ b/docs/SYSTEM_ARCHITECTURE.md @@ -0,0 +1,260 @@ +# Epistemic Swarm: System Architecture & Technical Specification + +## 1. Architectural Mandate & Epistemic Foundations + +Modern large language models suffer from severe parametric leakage and sycophancy when executing deep research: models hallucinate nonexistent citations, conflate correlational claims with causal proofs, and smooth over scientific controversies to generate artificially unified prose. + +**Epistemic Swarm** is an autonomous research harness built for Claude Code (`~/.claude/` and `~/.claude.json`) designed around one governing constraint: **Evidentiary Primacy over Parametric Intuition**. + +### 1.1 Formal Evidentiary Taxonomy +Every factual assertion, quantitative metric, historical claim, or entity relationship produced by any agent in the harness must carry an explicit, machine-parseable epistemic classification: + +| Tag | Formal Definition | Verification Requirement | +| :--- | :--- | :--- | +| `[VERIFIED: ]` | Grounded directly in verbatim text retrieved during the active research session. | Content-addressed SHA-256 hash in `.research/sources/.md` with exact substring match. | +| `[INFERRED: ]` | Deductive or inductive conclusion derived logically from one or more verified facts. | Explicit list of parent verified claim IDs ($C_1, C_2 \implies C_{\text{inferred}}$) with step-by-step logic. | +| `[HYPOTHESIS: ]` | Plausible hypothesis, speculative mechanism, or open projection requiring testing. | Defined falsification criterion or empirical test that would disprove it. | +| `[NEGATIVE_KNOWLEDGE: ]` | Explicit declaration that indexed search corpuses returned null or contradictory findings. | Record of search queries, parameters, and negative SERP summaries. | + +### 1.2 Mathematical Epistemic Scoring Function +To prevent conversational drift and hallucination, each generated dossier is scored by an Epistemic Auditor using the following objective function: + +$$\mathcal{E}(D) = \frac{\alpha \sum_{i=1}^{N_v} \mathcal{V}(c_i) + \beta \sum_{j=1}^{N_n} \mathcal{N}(k_j) - \gamma \sum_{k=1}^{N_u} \mathcal{U}(u_k)}{N_v + N_i + N_h + N_n + N_u}$$ + +Where: +- $N_v$: Number of verified assertions with valid source hashes. +- $\mathcal{V}(c_i)$: Verification weight proportional to source tier (Peer-reviewed DOI = 1.0, Technical Documentation = 0.8, Primary Press/Filings = 0.7, Secondary Media = 0.4). +- $N_n$: Count of explicit negative knowledge discoveries. +- $\mathcal{N}(k_j)$: Negative knowledge bonus factor ($\beta = 0.5$). +- $N_u$: Count of ungrounded or unverified factual claims. +- $\mathcal{U}(u_k)$: Severe hallucination penalty ($\gamma = 2.5$). +- $N_i, N_h$: Count of inferred claims and open hypotheses. + +Any dossier where $\mathcal{E}(D) < 0.65$ is rejected by the auditor and re-queued for empirical retrieval. + +--- + +## 2. Dialectic Multi-Agent Swarm + +The system replaces single-threaded LLM research with a decentralized dialectic swarm executed locally via Claude Code headless sessions (`claude -p`): + +```mermaid +flowchart TD + User([User Research Goal]) --> Grilling[Phase 1: Socratic Grilling & Divergent Ideation] + Grilling --> SettledFrontier[Settled Problem Frontier & Constraints] + SettledFrontier --> Orchestrator[Phase 2: Swarm Orchestrator] + + Orchestrator -->|Decompose into Scopes| Scope1[Scope 1: Core Technical Mechanism] + Orchestrator -->|Decompose into Scopes| Scope2[Scope 2: Economic & Scalability Constraints] + Orchestrator -->|Decompose into Scopes| ScopeN[Scope N: Edge-Case Failure Modes] + + subgraph DialecticPair [Dialectic Execution Loop per Scope] + Scope1 --> Alpha["Agent Alpha: The Proponent\n(Corroboration, Proofs, Primary Citations)"] + Scope1 --> Beta["Agent Beta: The Adversary\n(Active Falsification, Counter-evidence, Edge Cases)"] + + Alpha -->|Writes| AlphaDossier[alpha_dossier.json + Cached Sources] + Beta -->|Writes| BetaDossier[beta_dossier.json + Cached Sources] + + AlphaDossier --> Auditor[Phase 3: Epistemic Auditor] + BetaDossier --> Auditor + + Auditor -->|Calculate Divergence & Verify Quotes| DivergenceMatrix[Divergence Matrix & Verification Pass] + DivergenceMatrix --> ScopeSynthesis[Scope Synthesis Dossier] + end + + ScopeSynthesis --> FinalSynthesis[Phase 4: Unified Master Synthesis & Executive Brief] + FinalSynthesis --> Artifacts[.research/final_synthesis.md] +``` + +### 2.1 Swarm Roles & Postures +1. **Swarm Orchestrator (`orchestrator.md`)**: + - Parses the initial problem statement and Socratic frontier. + - Decomposes the inquiry into decoupled, orthogonal sub-scopes. + - Emits `.research/manifest.json` containing the dependency DAG, sub-scope IDs, required source tiers, and hypothesis lists. + +2. **Agent Alpha (The Proponent / Thesis - `agent_alpha_thesis.md`)**: + - Posture: Constructive, empirical, affirmative. + - Objective: Search for working implementations, benchmark results, mathematical proofs, peer-reviewed validations, and foundational literature. + - Mandate: Every affirmative statement must point to an indexed primary source in `.research/sources/.md`. + +3. **Agent Beta (The Adversary / Antithesis - `agent_beta_antithesis.md`)**: + - Posture: Hostile auditor, red-teamer, active falsifier. + - Objective: Probe for failure states, retracted papers, replication crises, confounding variables, edge-case regressions, patent/licensing bottlenecks, and vendor lock-in. + - Mandate: For every thesis claim in the scope, search deliberately for inverted queries (`"why [claim] fails"`, `"[claim] debunked"`, `"[claim] performance degradation"`). + +4. **Epistemic Auditor (The Synthesizer - `epistemic_auditor.md`)**: + - Posture: Neutral judge, citation verifier, and mathematical synthesizer. + - Objective: + 1. Read `alpha_dossier.json` and `beta_dossier.json`. + 2. Query the local cache to verify that all cited quote strings exist verbatim in `.research/sources/.md`. + 3. Calculate the **Divergence Score** $D_{\alpha\beta} \in [0, 1]$. + 4. Excise any assertion marked `[VERIFIED]` that fails hash verification. + 5. Synthesize conflicting viewpoints into an unvarnished balance sheet of empirical truths. + +--- + +## 3. Filesystem IPC Protocol & State Machine + +The swarm uses a zero-external-dependency filesystem IPC protocol rooted at `.research/` within the repository workspace. + +``` +.research/ +โ”œโ”€โ”€ manifest.json # Global session metadata, scope DAG, status +โ”œโ”€โ”€ sources/ # Content-addressed raw document cache +โ”‚ โ”œโ”€โ”€ a1b2c3d4...9f.json # Metadata: URL, title, HTTP headers, timestamp, query +โ”‚ โ””โ”€โ”€ a1b2c3d4...9f.md # Verbatim cleaned Markdown extraction +โ”œโ”€โ”€ scratchpads/ +โ”‚ โ”œโ”€โ”€ scope_01_mechanisms/ +โ”‚ โ”‚ โ”œโ”€โ”€ manifest.json # Scope status: PENDING | ALPHA_RUNNING | BETA_RUNNING | AUDITING | DONE +โ”‚ โ”‚ โ”œโ”€โ”€ alpha_dossier.json # Proponent's structured empirical findings +โ”‚ โ”‚ โ”œโ”€โ”€ alpha_dossier.md # Proponent's narrative brief +โ”‚ โ”‚ โ”œโ”€โ”€ beta_dossier.json # Adversary's structured counter-evidence +โ”‚ โ”‚ โ”œโ”€โ”€ beta_dossier.md # Adversary's narrative red-team report +โ”‚ โ”‚ โ”œโ”€โ”€ audit_report.json # Quote verification log, divergence score, trimmed claims +โ”‚ โ”‚ โ””โ”€โ”€ scope_synthesis.md # Synthesized consensus for Scope 01 +โ”‚ โ””โ”€โ”€ scope_02_scalability/ +โ”‚ โ””โ”€โ”€ ... +โ””โ”€โ”€ final_synthesis.md # Master synthesized research report with epistemic provenance +``` + +### 3.1 State Transitions +``` +[PENDING] + โ”‚ + โ–ผ (Dispatch worker thread) +[ALPHA_DISPATCHED] โ”€โ”€โ” + โ”‚ โ”‚ (Concurrent execution) + โ–ผ โ–ผ +[ALPHA_RUNNING] [BETA_DISPATCHED] + โ”‚ โ”‚ + โ–ผ โ–ผ +[ALPHA_COMPLETE] [BETA_RUNNING] + โ”‚ โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ–ผ + [BETA_COMPLETE] + โ”‚ + โ–ผ (Spawn Epistemic Auditor) + [AUDITING] + โ”‚ (Verify hashes, check quotes, score divergence) + โ–ผ + [SYNTHESIZING] + โ”‚ + โ–ผ + [COMPLETE] +``` + +### 3.2 Scratchpad Schema Definitions + +#### `manifest.json` (Global Session) +```json +{ + "session_id": "epistemic-swarm-2026-09-26-001", + "objective": "Evaluate feasibility of sub-millisecond zero-knowledge state updates on L1 rollups", + "status": "RUNNING", + "created_at": "2026-09-26T12:00:00Z", + "scopes": [ + { + "scope_id": "scope_01_prover_latency", + "title": "Hardware Prover Latency & Memory Footprint", + "dependencies": [], + "status": "IN_PROGRESS" + }, + { + "scope_id": "scope_02_recursion_overhead", + "title": "Recursive SNARK/STARK Verification Overheads", + "dependencies": ["scope_01_prover_latency"], + "status": "PENDING" + } + ], + "telemetry": { + "total_sources_cached": 42, + "total_claims_verified": 118, + "unverified_claims_pruned": 7 + } +} +``` + +#### `alpha_dossier.json` / `beta_dossier.json` +```json +{ + "agent": "Agent Alpha (Thesis)", + "scope_id": "scope_01_prover_latency", + "timestamp": "2026-09-26T12:05:30Z", + "claims": [ + { + "claim_id": "C-01-001", + "tag": "VERIFIED", + "statement": "FPGA-accelerated Poseidon hash provers achieve sub-200ms witness generation on 2^20 constraints.", + "source_hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "source_url": "https://arxiv.org/abs/2405.xxxxx", + "verbatim_quote": "Our FPGA pipeline executes the Poseidon round constraints in 184ms with a peak memory bandwidth of 45 GB/s." + } + ], + "negative_knowledge": [ + { + "query": "GPU Poseidon provers < 50ms latency memory bandwidth constraints", + "finding": "No published benchmark confirms sub-50ms prover latency under consumer PCIe 4.0 bandwidth limitations." + } + ] +} +``` + +--- + +## 4. Multi-Tier Search & Content Caching Pipeline + +The harness integrates four functional tooling layers: + +```mermaid +flowchart LR + subgraph Discovery [Tier 1: Discovery & SERP] + SearXNG["SearXNG (Metasearch)"] + Brave["Brave Search API"] + end + + subgraph Extraction [Tier 2: Content Extraction] + Firecrawl["Firecrawl MCP\n(Clean Markdown, JS Rendering)"] + Fetch["Local Fetch & Parser"] + end + + subgraph Domain [Tier 3: Domain Verification] + Scholar["arXiv / Semantic Scholar MCP"] + DOI["DOI & CrossRef Resolver"] + end + + subgraph Storage [Tier 4: Caching & Audit] + Hasher["SHA-256 Hasher\n(.research/sources/.md)"] + Verifier["Quote Substring Matcher"] + end + + Discovery --> Extraction + Extraction --> Hasher + Domain --> Hasher + Hasher --> Verifier +``` + +### 4.1 Content-Addressed Caching +Every web page, paper abstract, or document retrieved is immediately ingested by `skills/research-cache/hasher.py`: +1. Strips HTML boilerplate and scripts, converting to clean GitHub-Flavored Markdown. +2. Computes the SHA-256 digest of the normalized text content. +3. Saves `.research/sources/.md` and `.research/sources/.json` containing retrieval provenance (timestamp, search query, original URL, response code, and HTTP headers). +4. The agent is provided with the `` hash. +5. In downstream synthesis, when the agent cites `[VERIFIED: sha256]`, the Epistemic Auditor verifies that `verbatim_quote` is an exact substring within `.research/sources/.md`. If substring lookup fails, the claim is rejected. + +--- + +## 5. Socratic Grilling & Divergent-to-Convergent Balance + +To prevent prematurely narrowing the scope of research onto biased search terms, the harness implements a mandatory two-phase cognitive workflow: + +### Phase 1: Divergent Socratic Exploration (`skills/grilling`) +Based on Matt Pocock's design tree and frontier methodology: +1. **Assumption Inversion**: The agent inverts every default assumption (e.g., *"What if low latency is unnecessary if pipelined throughput is infinite?"*). +2. **Design Tree Mapping**: The inquiry is modeled as a decision DAG where every leaf node is an open prerequisite. +3. **The Frontier**: Questions whose prerequisites are settled are posed in systematic rounds. +4. **Factual Delegation**: When a question depends on an unknown empirical fact, the agent dispatches a research subagent to resolve it autonomously without interrogating the user. +5. **Decisional Resolution**: When a question involves trade-offs or priorities, the agent asks the user and locks the answer into the frontier. + +### Phase 2: Convergent Dialectic Falsification +Once the frontier is resolved, the Orchestrator freezes the problem space and transitions to the dialectic multi-agent swarm for empirical retrieval, falsification, and epistemic auditing. diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..20727fc --- /dev/null +++ b/install.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -e + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CLAUDE_DIR="$HOME/.claude" +CLAUDE_JSON="$HOME/.claude.json" + +echo "========================================================" +echo "๐Ÿš€ Installing Epistemic Swarm into Claude Code" +echo " Source: $REPO_DIR" +echo "========================================================" + +# 1. Dependency checks +command -v python3 >/dev/null 2>&1 || { echo "โŒ python3 is required but not installed."; exit 1; } +command -v node >/dev/null 2>&1 || { echo "โŒ node is required but not installed."; exit 1; } +command -v npm >/dev/null 2>&1 || { echo "โŒ npm is required but not installed."; exit 1; } + +echo "โœ… Core prerequisites detected (Python $(python3 --version | cut -d' ' -f2), Node $(node -v))" + +# 2. Setup ~/.claude/skills symlinks +mkdir -p "$CLAUDE_DIR/skills" + +echo "๐Ÿ”— Linking skills into $CLAUDE_DIR/skills/..." +ln -sf "$REPO_DIR/skills/grilling" "$CLAUDE_DIR/skills/grilling" +ln -sf "$REPO_DIR/skills/research-cache" "$CLAUDE_DIR/skills/research-cache" +echo " - $CLAUDE_DIR/skills/grilling -> $REPO_DIR/skills/grilling" +echo " - $CLAUDE_DIR/skills/research-cache -> $REPO_DIR/skills/research-cache" + +# 3. Patch ~/.claude.json mcpServers non-destructively +if [ -f "$CLAUDE_JSON" ]; then + echo "๐Ÿ”ง Merging research MCP servers into $CLAUDE_JSON..." + python3 - <\"" +echo "========================================================" diff --git a/package.json b/package.json new file mode 100644 index 0000000..6b6ada4 --- /dev/null +++ b/package.json @@ -0,0 +1,53 @@ +{ + "name": "@heretek-ai/epistemic-swarm", + "version": "0.1.0", + "description": "Epistemic Swarm: High-Integrity Dialectic Research Agent Harness for Claude Code", + "main": "bin/cli.js", + "bin": { + "epistemic-swarm": "./bin/cli.js", + "iumbtems": "./bin/cli.js" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "repository": { + "type": "git", + "url": "git+https://github.com/Heretek-AI/IUMBTEMS.git" + }, + "bugs": { + "url": "https://github.com/Heretek-AI/IUMBTEMS/issues" + }, + "homepage": "https://github.com/Heretek-AI/IUMBTEMS#readme", + "keywords": [ + "claude-code", + "ai-agents", + "dialectic-swarm", + "epistemic-integrity", + "research-harness", + "mcp", + "deep-research", + "socratic-grilling" + ], + "author": "Heretek AI", + "license": "Apache-2.0", + "files": [ + "bin", + "prompts", + "skills", + "config", + "runner", + ".claude-plugin", + "install.sh", + "README.md", + "LICENSE" + ], + "engines": { + "node": ">=18.0.0" + }, + "scripts": { + "test": "python3 -m unittest discover -s runner/tests", + "prepack": "python3 -m unittest discover -s runner/tests", + "install-local": "bash install.sh" + } +} diff --git a/prompts/agent_alpha_thesis.md b/prompts/agent_alpha_thesis.md new file mode 100644 index 0000000..1a135eb --- /dev/null +++ b/prompts/agent_alpha_thesis.md @@ -0,0 +1,70 @@ +# AGENT ALPHA: THE PROPONENT (THESIS) SPECIFICATION + +You are **Agent Alpha (The Proponent)** within the Epistemic Swarm dialectic harness. Your role is to construct the strongest possible empirical, affirmative case for the research targets assigned to your scope. + +--- + +## 1. POSTURE & OBJECTIVE + +- **Posture**: Rigorous, empirical, affirmative, evidence-first. +- **Mission**: Discover primary literature, reference implementations, benchmark datasets, verified production metrics, and mathematical proofs that corroborate the scope's affirmative targets. +- **Quarantine Warning**: You may NOT use your internal parametric memory to fabricate numbers, benchmark results, or author citations. Every assertion must be grounded in an active search or tool retrieval. + +--- + +## 2. TOOL WORKFLOW & SOURCE CACHING + +1. **Search**: Use Brave Search, SearXNG, or academic MCP tools to discover primary sources. +2. **Extract & Cache**: For every relevant source found, fetch the full content and invoke the research cache utility to store it: + ```bash + python3 skills/research-cache/hasher.py cache --url "" --content "" --title "" + ``` + This will output the content-addressed hash (e.g., `3f8a9e21...`). +3. **Extract Verbatim Excerpts**: Note the exact sentence or paragraph that substantiates your claim. The Epistemic Auditor will verify that your quote matches the cached markdown file character-for-character. + +--- + +## 3. OUTPUT SPECIFICATION + +You must write your findings to two files in `.research/scratchpads/{scope_id}/`: + +### 1. `alpha_dossier.json` +```json +{ + "agent": "Agent Alpha (Thesis)", + "scope_id": "<scope_id>", + "timestamp": "<ISO-8601>", + "affirmative_claims": [ + { + "claim_id": "ALPHA-C01", + "tag": "VERIFIED", + "statement": "<Concise empirical assertion>", + "source_hash": "<sha256>", + "source_url": "<URL or DOI>", + "verbatim_quote": "<Exact substring from the cached markdown document>", + "tier": "PEER_REVIEWED | BENCHMARK | DOCUMENTATION | MEDIA" + } + ], + "inferred_implications": [ + { + "inference_id": "ALPHA-I01", + "tag": "INFERRED", + "statement": "<Deductive derivation>", + "parent_claims": ["ALPHA-C01"], + "deductive_logic": "<Step-by-step logic bridging the premises to conclusion>" + } + ], + "negative_knowledge": [ + { + "query": "<Search query executed>", + "finding": "<What could NOT be found or proven in the literature>" + } + ] +} +``` + +### 2. `alpha_dossier.md` +A comprehensive narrative research brief organizing your corroborating evidence logically, incorporating inline tags: +- `[VERIFIED: <source_hash>]` +- `[INFERRED: <reasoning>]` +- `[NEGATIVE_KNOWLEDGE: <query>]` diff --git a/prompts/agent_beta_antithesis.md b/prompts/agent_beta_antithesis.md new file mode 100644 index 0000000..c754cbe --- /dev/null +++ b/prompts/agent_beta_antithesis.md @@ -0,0 +1,78 @@ +# AGENT BETA: THE ADVERSARY (ANTITHESIS / RED TEAM) SPECIFICATION + +You are **Agent Beta (The Adversary / Red Team)** within the Epistemic Swarm dialectic harness. Your role is active falsification, vulnerability hunting, and empirical counter-argumentation for the scope assigned to you. + +--- + +## 1. POSTURE & OBJECTIVE + +- **Posture**: Hostile technical auditor, red-teamer, falsification investigator. +- **Mission**: Hunt for edge cases, performance cliffs, retracted findings, methodology flaws, p-hacking, hidden assumptions, scalability bottlenecks, unstated trade-offs, and critical failure modes that disprove or restrict the affirmative thesis. +- **Cognitive Stance**: Presume that optimistic claims in technical documentation or marketing whitepapers are unproven until verified against adversarial pressure. + +--- + +## 2. ADVERSARIAL RETRIEVAL STRATEGIES + +Execute inverted and adversarial queries across Brave Search, SearXNG, and academic databases: +1. **Failure Modes**: `"<technology/method> failure"`, `"<claim> debunked"`, `"<system> outage postmortem"`. +2. **Methodological Critiques**: `"<paper title> critique"`, `"<author> rebuttal"`, `"<technique> limitations"`. +3. **Performance Cliffs**: `"<benchmark> regression"`, `"<library> memory leak bottleneck"`, `"<model> degradation"`. +4. **Reproducibility Checks**: Look up papers in Retraction Watch or replication surveys to check if findings survived independent scrutiny. + +--- + +## 3. TOOL WORKFLOW & SOURCE CACHING + +1. When you discover counter-evidence, fetch the full content. +2. Cache the source immediately using the research cache utility: + ```bash + python3 skills/research-cache/hasher.py cache --url "<URL>" --content "<MARKDOWN_CONTENT>" --title "<TITLE>" + ``` +3. Extract exact verbatim quotes showing the contradiction, flaw, or boundary condition. + +--- + +## 4. OUTPUT SPECIFICATION + +You must write your findings to two files in `.research/scratchpads/{scope_id}/`: + +### 1. `beta_dossier.json` +```json +{ + "agent": "Agent Beta (Adversary / Red Team)", + "scope_id": "<scope_id>", + "timestamp": "<ISO-8601>", + "falsification_claims": [ + { + "claim_id": "BETA-C01", + "tag": "VERIFIED", + "statement": "<Empirical counter-claim or demonstrated failure mode>", + "source_hash": "<sha256>", + "source_url": "<URL or DOI>", + "verbatim_quote": "<Exact substring from the cached markdown document showing failure or limitation>", + "severity": "CRITICAL_BLOCKER | SEVERE_DEGRADATION | EDGE_CASE | METHODOLOGY_FLAW" + } + ], + "methodological_critiques": [ + { + "target_assertion": "<The affirmative claim being challenged>", + "critique": "<Why the claim is invalid, confounded, or ungeneralizable>", + "evidence_hash": "<sha256>" + } + ], + "negative_knowledge": [ + { + "query": "<Adversarial query executed>", + "finding": "<Confirmations where claimed protections or alternatives do not exist>" + } + ] +} +``` + +### 2. `beta_dossier.md` +A comprehensive narrative red-team dossier laying out the empirical vulnerabilities, counter-evidence, and strict operational boundaries of the evaluated system, tagged with: +- `[VERIFIED: <source_hash>]` +- `[INFERRED: <reasoning>]` +- `[HYPOTHESIS: <test>]` +- `[NEGATIVE_KNOWLEDGE: <query>]` diff --git a/prompts/base_epistemic_system.md b/prompts/base_epistemic_system.md new file mode 100644 index 0000000..f7e106b --- /dev/null +++ b/prompts/base_epistemic_system.md @@ -0,0 +1,50 @@ +# EPISTEMIC INTEGRITY INVARIANT SPECIFICATION (CLAUDE CODE OVERRIDE) + +You are operating under the **Epistemic Integrity Protocol**. Your internal parametric memory is treated as untrusted, fallible heuristic guidance. It is **STRICTLY QUARANTINED**. You are prohibited from presenting unverified parametric recollections as established empirical fact. + +--- + +## 1. MANDATORY TAGGING TAXONOMY + +Every factual proposition, quantitative statistic, experimental finding, algorithmic benchmark, or historical claim MUST be classified with an explicit inline epistemic tag: + +### 1. `[VERIFIED: <Source/DOI/URL | hash>]` +- **Criterion**: The claim is backed by a primary or high-confidence secondary document retrieved, indexed, and cached in `.research/sources/<hash>.md` during this session. +- **Requirement**: The claim must reflect verbatim excerpts from the source. The hash MUST correspond to a valid cached document. +- **Example**: + > The Llama-3-70B model exhibits a 131,072 token context window with grouped-query attention (GQA) across all 8 key-value heads `[VERIFIED: arXiv:2407.21783 | 3f8a9e21]`. + +### 2. `[INFERRED: <Reasoning Chain>]` +- **Criterion**: The claim is a logical, mathematical, or deductive derivation from one or more verified facts. +- **Requirement**: You must cite the parent verified premises and provide the deductive bridging step. +- **Example**: + > Given a 184ms prover time per block `[VERIFIED: 3f8a9e21]` and a 12-second block target, single-prover hardware utilization will not exceed 1.53% without batching `[INFERRED: 0.184s / 12s = 1.533%]`. + +### 3. `[HYPOTHESIS: <Falsification Criterion>]` +- **Criterion**: An extrapolation, speculative causal mechanism, or unverified prediction. +- **Requirement**: Must include a concrete empirical condition or test that would falsify the statement. +- **Example**: + > Transitioning from Poseidon to Tip5 hash functions will reduce SNARK witness generation time by ~30% `[HYPOTHESIS: Falsified if benchmark on 2^20 constraints shows < 15% reduction]`. + +### 4. `[NEGATIVE_KNOWLEDGE: <Search Query>]` +- **Criterion**: Rigorous verification that no empirical evidence exists in the indexed literature for a given claim. +- **Requirement**: State the exact search queries executed and summarize the negative finding. +- **Example**: + > `[NEGATIVE_KNOWLEDGE: "sub-10ms zk-STARK verification on mobile devices"]` Exhaustive literature search across arXiv and IEEE Xplore yielded zero published implementations or benchmarks meeting this latency bound on ARM architectures. + +--- + +## 2. HALLUCINATION PENALTY & BEHAVIORAL INVARIANTS + +1. **Anti-Continuity Rule**: Never synthesize plausible "middle grounds" or invent harmonious narratives when sources conflict. When sources disagree, present the dialectic contradiction explicitly with source hashes for both sides. +2. **Citation Fabrication Ban**: Never construct a citation from parametric memory. If you cannot produce a real URL, DOI, or cached `<hash>.md` file, you MUST NOT cite a source. Use `[HYPOTHESIS]` or `[NEGATIVE_KNOWLEDGE]`. +3. **Negative Knowledge Reward**: Acknowledging that an assertion cannot be substantiated is treated as a high-value empirical contribution. Hallucinating an answer is penalized as an epistemic failure. +4. **Content-Addressed Provenance**: When retrieving text via search tools or MCP extraction (Brave, SearXNG, Firecrawl), ensure the raw payload is saved to `.research/sources/<sha256>.md` via the caching tool before citing it. + +--- + +## 3. EPISTEMIC CODE OF CONDUCT + +- When in doubt: **FETCH, VERIFY, THEN ASSERT.** +- If retrieval tools fail or return 403/429/empty: **RECORD AS NEGATIVE KNOWLEDGE, DO NOT GUESS.** +- Maintain cognitive separation between **empirical facts** (retrieved from reality) and **design preferences** (derived from the user's objectives). diff --git a/prompts/epistemic_auditor.md b/prompts/epistemic_auditor.md new file mode 100644 index 0000000..b21059d --- /dev/null +++ b/prompts/epistemic_auditor.md @@ -0,0 +1,74 @@ +# EPISTEMIC AUDITOR & SYNTHESIZER SPECIFICATION + +You are the **Epistemic Auditor and Synthesizer** of the Epistemic Swarm research harness. Your role is neutral adjudication, mathematical verification of primary quotes, calculation of divergence between competing agents, and construction of the final verified research synthesis. + +--- + +## 1. AUDIT MANDATE & OBJECTIVES + +1. **Quote Verification**: For every assertion tagged `[VERIFIED: <sha256>]` in `alpha_dossier.json` and `beta_dossier.json`, you must verify that the `verbatim_quote` exists as an exact or near-exact substring inside the cached file `.research/sources/<sha256>.md`. +2. **Downgrade & Flag Policy**: + - If an assertion's verbatim quote CANNOT be found in the cached source, you MUST downgrade the assertion from `[VERIFIED]` to `[UNVERIFIED - REJECTED]`. + - Log the exact discrepancy in `audit_report.json`. + - Penalize the scope's Epistemic Score. +3. **Divergence Scoring**: + - Compare the core propositions of Agent Alpha and Agent Beta. + - Calculate the Divergence Score $D \in [0.0, 1.0]$. High divergence indicates genuine scientific controversy or operational trade-offs, which must be clearly exposed rather than artificially blended. +4. **Synthesis Compilation**: + - Write a balanced, verifiable brief in `.research/scratchpads/{scope_id}/scope_synthesis.md` and `.research/final_synthesis.md`. + - Eliminate all unsourced marketing claims, ungrounded speculation, or sycophantic generalizations. + +--- + +## 2. DIVERGENCE METRICS & SCORING FORMULA + +The Epistemic Score for a scope dossier is computed as: + +$$\mathcal{E} = \frac{1.0 \times N_{\text{verified}} + 0.5 \times N_{\text{neg\_knowledge}} - 2.5 \times N_{\text{rejected}}}{N_{\text{verified}} + N_{\text{inferred}} + N_{\text{hypothesis}} + N_{\text{rejected}}}$$ + +If $\mathcal{E} < 0.65$, mark the scope status as `AUDIT_WARNING: LOW_EMPIRICAL_GROUNDING`. + +The Divergence Score $D_{\alpha\beta}$ is defined as: + +$$D_{\alpha\beta} = \frac{|\text{Contradicted Claims}|}{|\text{Total Scope Claims}|}$$ + +--- + +## 3. OUTPUT SPECIFICATION + +You must write your findings to two files in `.research/scratchpads/{scope_id}/`: + +### 1. `audit_report.json` +```json +{ + "auditor": "Epistemic Auditor v1.0", + "scope_id": "<scope_id>", + "timestamp": "<ISO-8601>", + "verification_summary": { + "total_claims_audited": 24, + "verified_passed": 22, + "unverified_rejected": 2, + "negative_knowledge_points": 5, + "epistemic_score": 0.81, + "divergence_score": 0.42 + }, + "rejected_claims": [ + { + "claim_id": "ALPHA-C03", + "reason": "Verbatim quote not located in source cache e3b0c442...", + "original_statement": "..." + } + ], + "divergence_matrix": [ + { + "dimension": "Latency under peak memory bandwidth", + "alpha_thesis": "Sub-200ms achieved in benchmark [VERIFIED: 3f8a9e21]", + "beta_antithesis": "Degrades to >850ms when batch size exceeds 16 due to PCIe 4.0 transfer stalls [VERIFIED: 7b2c14da]", + "adjudicated_verdict": "Latency is bounded sub-200ms only for isolated single-proof workloads; production batches encounter PCIe saturation." + } + ] +} +``` + +### 2. `scope_synthesis.md` +A rigorous markdown report synthesizing the validated empirical evidence, displaying the dialectic balance sheet, and identifying remaining empirical frontiers. diff --git a/prompts/orchestrator.md b/prompts/orchestrator.md new file mode 100644 index 0000000..9255aa5 --- /dev/null +++ b/prompts/orchestrator.md @@ -0,0 +1,70 @@ +# SWARM ORCHESTRATOR SPECIFICATION + +You are the **Swarm Orchestrator** of the Epistemic Swarm research harness. Your objective is to translate a complex, ambiguous, or multi-faceted research question into an orthogonal, decoupled Directed Acyclic Graph (DAG) of dialectic research sub-scopes. + +--- + +## 1. COGNITIVE RESPONSIBILITIES + +### A. Autonomous Divergent Exploration +Before finalizing any research plan, you MUST run a divergent ideation step: +1. **Lateral Analogies**: What analogous problems exist in adjacent domains (e.g., biological immune systems vs. distributed fault tolerance; compiler optimization vs. query planning)? +2. **Premise Inversion**: What if the core assumption of the user's objective is fundamentally flawed or obsolete? +3. **Boundary Condition Probing**: What are the extreme edges (infinite scale, zero compute, adversarial poisoning, extreme network latency)? + +If `.research/frontier.json` exists from a prior Socratic grilling session, ingest its settled decisions and open questions as hard architectural constraints. + +### B. Decoupled Scope Decomposition +Decompose the macro research question into 2 to 4 decoupled, orthogonal sub-scopes: +- Each sub-scope must be self-contained so that a dialectic researcher pair (Alpha and Beta) can investigate it without blocking on other scopes. +- Define explicit dependencies between scopes only when strictly necessary (forming a DAG). + +--- + +## 2. OUTPUT SPECIFICATION + +You must output a structured scope decomposition manifest written to `.research/manifest.json`. + +The manifest MUST follow this exact schema: + +```json +{ + "session_id": "epistemic-<timestamp>-<hash>", + "objective": "<The overarching research objective>", + "divergent_ideation": { + "lateral_analogies": [ + "<Analogy 1>", + "<Analogy 2>" + ], + "inverted_premises": [ + "<Inverted assumption 1>" + ], + "boundary_conditions": [ + "<Extreme condition 1>" + ] + }, + "scopes": [ + { + "scope_id": "scope_01_<slug>", + "title": "<Concise title of scope 1>", + "objective": "<Specific question to resolve>", + "dependencies": [], + "affirmative_targets": [ + "<Key mechanism or metric Alpha must prove>" + ], + "adversarial_targets": [ + "<Key failure mode or vulnerability Beta must probe>" + ], + "required_source_tiers": ["PEER_REVIEWED", "TECHNICAL_SPEC", "PRIMARY_BENCHMARKS"] + } + ] +} +``` + +--- + +## 3. SCOPE DECOMPOSITION INVARIANTS + +1. **Orthogonality**: No two scopes may investigate the exact same metric or component. (e.g., Scope 1 = Cryptographic Proof Size; Scope 2 = P2P Network Propagation; Scope 3 = Hardware ASIC Prover Economics). +2. **Falsifiability**: Every scope MUST provide concrete `adversarial_targets` for Agent Beta to falsify. +3. **Empirical Grounding**: Do not define philosophical or purely qualitative scopes; frame scopes in terms of measurable, benchmarkable, or historically observable claims. diff --git a/runner/__init__.py b/runner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/runner/auditor_engine.py b/runner/auditor_engine.py new file mode 100644 index 0000000..f496b9c --- /dev/null +++ b/runner/auditor_engine.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +""" +Algorithmic Epistemic Auditor Engine for Epistemic Swarm. +Verifies quote authenticity against content-addressed source cache, +calculates divergence metrics, prunes ungrounded claims, and generates synthesis reports. +""" + +import sys +import json +from pathlib import Path +from datetime import datetime, timezone +from typing import Dict, Any, List, Tuple, Optional + +# Import SourceHasher from skills/research-cache +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from skills.research_cache.hasher import SourceHasher +from runner.state_machine import ResearchStateMachine, ScopeStatus + +class EpistemicAuditorEngine: + def __init__(self, base_dir: Optional[Path] = None): + self.base_dir = base_dir or Path(".research") + self.hasher = SourceHasher(base_dir=self.base_dir) + self.state_machine = ResearchStateMachine(base_dir=self.base_dir) + + def audit_scope(self, scope_id: str) -> Dict[str, Any]: + """Runs the audit pipeline on a scope with ready dossiers.""" + scope_dir = self.state_machine.get_scope_dir(scope_id) + alpha_file = scope_dir / "alpha_dossier.json" + beta_file = scope_dir / "beta_dossier.json" + + if not alpha_file.exists() or not beta_file.exists(): + raise FileNotFoundError(f"Both dossiers must exist to audit {scope_id}") + + with open(alpha_file, "r", encoding="utf-8") as f: + alpha_dossier = json.load(f) + with open(beta_file, "r", encoding="utf-8") as f: + beta_dossier = json.load(f) + + self.state_machine.update_scope_status(scope_id, ScopeStatus.AUDITING) + + # 1. Audit Alpha Claims + alpha_results, alpha_verified, alpha_rejected = self._verify_claims( + alpha_dossier.get("affirmative_claims", []) + ) + + # 2. Audit Beta Claims + beta_results, beta_verified, beta_rejected = self._verify_claims( + beta_dossier.get("falsification_claims", []) + ) + + total_verified = alpha_verified + beta_verified + total_rejected = alpha_rejected + beta_rejected + + # Negative knowledge counts + neg_knowledge_alpha = len(alpha_dossier.get("negative_knowledge", [])) + neg_knowledge_beta = len(beta_dossier.get("negative_knowledge", [])) + total_neg_knowledge = neg_knowledge_alpha + neg_knowledge_beta + + total_inferred = len(alpha_dossier.get("inferred_implications", [])) + total_hypotheses = len(beta_dossier.get("hypotheses", [])) + + # 3. Calculate Epistemic Score + total_assertions = max(1, total_verified + total_inferred + total_hypotheses + total_rejected) + raw_score = (1.0 * total_verified + 0.5 * total_neg_knowledge - 2.5 * total_rejected) / total_assertions + epistemic_score = max(0.0, min(1.0, round(raw_score, 3))) + + # 4. Calculate Divergence Score + divergence_score, divergence_matrix = self._compute_divergence(alpha_dossier, beta_dossier) + + # 5. Build Audit Report + audit_report = { + "auditor": "Epistemic Auditor Engine v1.0", + "scope_id": scope_id, + "audited_at": datetime.now(timezone.utc).isoformat(), + "summary": { + "total_claims_audited": len(alpha_results) + len(beta_results), + "verified_passed": total_verified, + "unverified_rejected": total_rejected, + "negative_knowledge_count": total_neg_knowledge, + "epistemic_score": epistemic_score, + "divergence_score": divergence_score, + "verdict": "CERTIFIED" if epistemic_score >= 0.65 else "WARNING_LOW_GROUNDING" + }, + "alpha_claims_audit": alpha_results, + "beta_claims_audit": beta_results, + "divergence_matrix": divergence_matrix + } + + # Save audit_report.json + with open(scope_dir / "audit_report.json", "w", encoding="utf-8") as f: + json.dump(audit_report, f, indent=2) + + # 6. Generate Synthesis Markdown + synthesis_md = self._generate_synthesis_markdown( + scope_id, alpha_dossier, beta_dossier, audit_report + ) + with open(scope_dir / "scope_synthesis.md", "w", encoding="utf-8") as f: + f.write(synthesis_md) + + # Mark scope completed + sm = self.state_machine.load_scope_manifest(scope_id) + sm["status"] = ScopeStatus.COMPLETE.value + sm["audit_completed"] = True + self.state_machine.save_scope_manifest(scope_id, sm) + + return audit_report + + def _verify_claims(self, claims: List[Dict[str, Any]]) -> Tuple[List[Dict[str, Any]], int, int]: + audited_claims = [] + verified_count = 0 + rejected_count = 0 + + for claim in claims: + cid = claim.get("claim_id", "UNKNOWN") + shash = claim.get("source_hash", "") + quote = claim.get("verbatim_quote", "") + statement = claim.get("statement", "") + + if not shash or not quote: + audited_claims.append({ + "claim_id": cid, + "statement": statement, + "original_tag": claim.get("tag", "VERIFIED"), + "audited_tag": "UNVERIFIED_REJECTED", + "reason": "Missing source_hash or verbatim_quote", + "confidence": 0.0 + }) + rejected_count += 1 + continue + + passed, conf, msg = self.hasher.verify_quote(shash, quote) + if passed: + audited_claims.append({ + "claim_id": cid, + "statement": statement, + "original_tag": "VERIFIED", + "audited_tag": "VERIFIED", + "source_hash": shash, + "confidence": conf, + "verification_message": msg + }) + verified_count += 1 + else: + audited_claims.append({ + "claim_id": cid, + "statement": statement, + "original_tag": "VERIFIED", + "audited_tag": "UNVERIFIED_REJECTED", + "source_hash": shash, + "rejected_quote": quote, + "reason": msg, + "confidence": conf + }) + rejected_count += 1 + + return audited_claims, verified_count, rejected_count + + def _compute_divergence(self, alpha_dossier: Dict[str, Any], + beta_dossier: Dict[str, Any]) -> Tuple[float, List[Dict[str, Any]]]: + alpha_claims = alpha_dossier.get("affirmative_claims", []) + beta_claims = beta_dossier.get("falsification_claims", []) + critiques = beta_dossier.get("methodological_critiques", []) + + matrix = [] + contradictions = 0 + + for crit in critiques: + target = crit.get("target_assertion", "") + matrix.append({ + "tension_type": "METHODOLOGICAL_CHALLENGE", + "proponent_claim": target, + "adversary_critique": crit.get("critique", ""), + "counter_evidence_hash": crit.get("evidence_hash", "NONE") + }) + contradictions += 1 + + # Calculate divergence + total_claims = max(1, len(alpha_claims) + len(beta_claims)) + divergence = round(min(1.0, (contradictions * 2) / total_claims), 2) + return divergence, matrix + + def _generate_synthesis_markdown(self, scope_id: str, alpha: Dict[str, Any], + beta: Dict[str, Any], audit: Dict[str, Any]) -> str: + summary = audit["summary"] + md = [] + md.append(f"# Epistemic Synthesis: Scope {scope_id}\n") + md.append(f"**Audit Verdict**: `{summary['verdict']}` | **Epistemic Score**: `{summary['epistemic_score']}/1.0` | **Divergence Index**: `{summary['divergence_score']}`\n") + + md.append("## 1. Verified Empirical Grounding") + for claim in audit["alpha_claims_audit"]: + if claim["audited_tag"] == "VERIFIED": + md.append(f"- `[VERIFIED: {claim['source_hash'][:8]}]` {claim['statement']}") + for claim in audit["beta_claims_audit"]: + if claim["audited_tag"] == "VERIFIED": + md.append(f"- `[VERIFIED: {claim['source_hash'][:8]}]` (Counter-Evidence) {claim['statement']}") + + md.append("\n## 2. Dialectic Tensions & Falsification Audit") + if audit["divergence_matrix"]: + for item in audit["divergence_matrix"]: + md.append(f"### Tension: {item['tension_type']}") + md.append(f"- **Thesis Assertion**: {item['proponent_claim']}") + md.append(f"- **Adversarial Critique**: {item['adversary_critique']}") + else: + md.append("No active contradictions identified between primary literature and red-team findings.") + + md.append("\n## 3. Rejected & Unverified Assertions") + rejected = [c for c in audit["alpha_claims_audit"] + audit["beta_claims_audit"] if c["audited_tag"] == "UNVERIFIED_REJECTED"] + if rejected: + for r in rejected: + md.append(f"- โš ๏ธ **PURGED**: \"{r['statement']}\" โ€” *Reason: {r['reason']}*") + else: + md.append("Zero claims rejected. 100% of cited assertions verified against source cache.") + + md.append("\n## 4. Negative Knowledge Catalog") + all_neg = alpha.get("negative_knowledge", []) + beta.get("negative_knowledge", []) + if all_neg: + for n in all_neg: + md.append(f"- `[NEGATIVE_KNOWLEDGE: {n['query']}]` {n['finding']}") + else: + md.append("No negative knowledge declarations logged.") + + return "\n".join(md) diff --git a/runner/research_swarm.py b/runner/research_swarm.py new file mode 100755 index 0000000..d50689a --- /dev/null +++ b/runner/research_swarm.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +""" +Epistemic Swarm: Dialectic Multi-Agent Research Runner. +Executes parallel Claude Code sub-processes (claude -p) for Proponent and Adversary agents, +monitors filesystem IPC scratchpads, and invokes the Epistemic Auditor. +""" + +import os +import sys +import json +import shutil +import argparse +import subprocess +from pathlib import Path +from datetime import datetime, timezone +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Dict, Any, List, Optional + +# Ensure project root is in sys.path +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from runner.state_machine import ResearchStateMachine, SessionStatus, ScopeStatus +from runner.auditor_engine import EpistemicAuditorEngine +from skills.research_cache.hasher import SourceHasher + +class SwarmRunner: + def __init__(self, base_dir: Optional[Path] = None, mock_mode: bool = False): + self.base_dir = base_dir or Path(".research") + self.mock_mode = mock_mode + self.state_machine = ResearchStateMachine(base_dir=self.base_dir) + self.auditor = EpistemicAuditorEngine(base_dir=self.base_dir) + self.hasher = SourceHasher(base_dir=self.base_dir) + self.prompts_dir = PROJECT_ROOT / "prompts" + + def run_claude_process(self, prompt: str, system_prompt_file: Optional[Path] = None, + tools: str = "default") -> str: + """Executes a headless Claude Code session via `claude -p`.""" + if self.mock_mode: + return self._mock_claude_response(prompt) + + cmd = ["claude", "-p", prompt, "--tools", tools] + if system_prompt_file and system_prompt_file.exists(): + cmd.extend(["--system-prompt", str(system_prompt_file)]) + + try: + res = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True, + cwd=str(PROJECT_ROOT) + ) + return res.stdout.strip() + except subprocess.CalledProcessError as e: + print(f"[ERROR] Claude process failed: {e.stderr}", file=sys.stderr) + raise RuntimeError(f"Claude execution failed: {e.stderr}") + + def _mock_claude_response(self, prompt: str) -> str: + """Mock response generator for unit testing without live API keys.""" + if "Orchestrator" in prompt or "manifest.json" in prompt: + return json.dumps({ + "session_id": "mock-session-001", + "objective": "Evaluate ZK prover latency", + "scopes": [ + { + "scope_id": "scope_01_latency", + "title": "Hardware Prover Latency Bounds", + "objective": "Evaluate Poseidon hash witness generation latency on FPGAs vs GPUs", + "dependencies": [], + "affirmative_targets": ["Sub-200ms witness generation on 2^20 constraints"], + "adversarial_targets": ["PCIe bus bottlenecks during batch streaming"] + } + ] + }) + return "MOCK_RESPONSE" + + def orchestrate_objective(self, objective: str, frontier_file: Optional[Path] = None) -> List[Dict[str, Any]]: + """Phase 1: Run Swarm Orchestrator to decompose the research question.""" + print(f"\n๐Ÿง  [Phase 1: Orchestration] Decomposing objective: '{objective}'...") + self.state_machine.init_session(objective) + self.state_machine.update_session_status(SessionStatus.ORCHESTRATING) + + frontier_context = "" + if frontier_file and frontier_file.exists(): + with open(frontier_file, "r", encoding="utf-8") as f: + frontier_data = json.load(f) + frontier_context = f"\nSETTLED CONSTRAINTS FROM FRONTIER:\n{json.dumps(frontier_data.get('settled_constraints', {}), indent=2)}" + + orchestrator_prompt = f""" +You are the Swarm Orchestrator. Read prompts/orchestrator.md. +Objective: {objective} +{frontier_context} + +Output ONLY valid JSON representing the scope decomposition conforming to prompts/orchestrator.md. +""" + system_prompt = self.prompts_dir / "orchestrator.md" + raw_output = self.run_claude_process(orchestrator_prompt, system_prompt_file=system_prompt) + + # Parse JSON + try: + # Handle potential markdown fence blocks + clean_json = raw_output + if "```json" in clean_json: + clean_json = clean_json.split("```json")[1].split("```")[0] + elif "```" in clean_json: + clean_json = clean_json.split("```")[1].split("```")[0] + manifest_data = json.loads(clean_json.strip()) + scopes = manifest_data.get("scopes", []) + except Exception as e: + print(f"[WARN] Failed to parse JSON from orchestrator output: {e}. Using fallback decomposition.") + scopes = [ + { + "scope_id": "scope_01_primary_investigation", + "title": f"Investigation: {objective[:40]}", + "objective": objective, + "dependencies": [], + "affirmative_targets": ["Find corroborating empirical data"], + "adversarial_targets": ["Probe counter-arguments and failure modes"] + } + ] + + self.state_machine.set_scopes(scopes) + print(f"โœ… Generated {len(scopes)} decoupled dialectic scopes.") + return scopes + + def run_agent_alpha(self, scope: Dict[str, Any]): + """Executes Agent Alpha (Thesis / Proponent) for a scope.""" + scope_id = scope["scope_id"] + print(f" [Alpha] ๐Ÿ›๏ธ Starting Agent Alpha (Thesis) on [{scope_id}]...") + + if self.mock_mode: + # Create synthetic cached source + sample_content = "# FPGA Prover Benchmark\nOur FPGA pipeline executes the Poseidon round constraints in 184ms with a peak memory bandwidth of 45 GB/s." + shash = self.hasher.store_source("https://arxiv.org/abs/2405.0001", sample_content, "FPGA Benchmark") + + dossier = { + "agent": "Agent Alpha (Thesis)", + "scope_id": scope_id, + "timestamp": datetime.now(timezone.utc).isoformat(), + "affirmative_claims": [ + { + "claim_id": "ALPHA-C01", + "tag": "VERIFIED", + "statement": "FPGA-accelerated Poseidon provers achieve sub-200ms latency on 2^20 constraints.", + "source_hash": shash, + "source_url": "https://arxiv.org/abs/2405.0001", + "verbatim_quote": "Our FPGA pipeline executes the Poseidon round constraints in 184ms with a peak memory bandwidth of 45 GB/s." + } + ], + "inferred_implications": [ + { + "inference_id": "ALPHA-I01", + "tag": "INFERRED", + "statement": "Hardware provers satisfy 1-second block finality bounds.", + "parent_claims": ["ALPHA-C01"], + "deductive_logic": "184ms << 1000ms target." + } + ], + "negative_knowledge": [] + } + else: + prompt = f"Run Agent Alpha for scope: {json.dumps(scope)}. Save findings to {scope_id} scratchpad." + system_prompt = self.prompts_dir / "agent_alpha_thesis.md" + self.run_claude_process(prompt, system_prompt_file=system_prompt) + # Read created dossier from scratchpad + dossier_path = self.state_machine.get_scope_dir(scope_id) / "alpha_dossier.json" + with open(dossier_path, "r", encoding="utf-8") as f: + dossier = json.load(f) + + self.state_machine.record_agent_completion(scope_id, "alpha", dossier) + print(f" [Alpha] โœ… Completed Agent Alpha for [{scope_id}].") + + def run_agent_beta(self, scope: Dict[str, Any]): + """Executes Agent Beta (Antithesis / Red Team) for a scope.""" + scope_id = scope["scope_id"] + print(f" [Beta] ๐ŸŽฏ Starting Agent Beta (Red Team) on [{scope_id}]...") + + if self.mock_mode: + sample_content = "# PCIe Bus Saturation Study\nIn continuous batch streaming, PCIe 4.0 transfers introduce a 650ms delay, yielding total latency > 800ms." + shash = self.hasher.store_source("https://arxiv.org/abs/2406.9999", sample_content, "PCIe Bottlenecks") + + dossier = { + "agent": "Agent Beta (Red Team)", + "scope_id": scope_id, + "timestamp": datetime.now(timezone.utc).isoformat(), + "falsification_claims": [ + { + "claim_id": "BETA-C01", + "tag": "VERIFIED", + "statement": "Batch streaming incurs a 650ms PCIe transfer delay under production loads.", + "source_hash": shash, + "source_url": "https://arxiv.org/abs/2406.9999", + "verbatim_quote": "In continuous batch streaming, PCIe 4.0 transfers introduce a 650ms delay, yielding total latency > 800ms." + } + ], + "methodological_critiques": [ + { + "target_assertion": "FPGA-accelerated Poseidon provers achieve sub-200ms latency on 2^20 constraints.", + "critique": "Benchmark isolates compute kernel and ignores host-to-device PCIe latency in pipelined batches.", + "evidence_hash": shash + } + ], + "negative_knowledge": [ + { + "query": "Zero-latency PCIe streaming ZK provers", + "finding": "No architecture eliminates bus transfer overhead without on-chip memory > 128GB." + } + ] + } + else: + prompt = f"Run Agent Beta for scope: {json.dumps(scope)}. Save findings to {scope_id} scratchpad." + system_prompt = self.prompts_dir / "agent_beta_antithesis.md" + self.run_claude_process(prompt, system_prompt_file=system_prompt) + dossier_path = self.state_machine.get_scope_dir(scope_id) / "beta_dossier.json" + with open(dossier_path, "r", encoding="utf-8") as f: + dossier = json.load(f) + + self.state_machine.record_agent_completion(scope_id, "beta", dossier) + print(f" [Beta] โœ… Completed Agent Beta for [{scope_id}].") + + def execute_scope_dialectic(self, scope: Dict[str, Any]): + """Dispatches Agent Alpha and Agent Beta concurrently.""" + scope_id = scope["scope_id"] + print(f"\nโšก [Swarm Dispatch] Launching Dialectic Pair for [{scope_id}]: '{scope.get('title')}'") + self.state_machine.update_scope_status(scope_id, ScopeStatus.RUNNING_PARALLEL) + + with ThreadPoolExecutor(max_workers=2) as executor: + future_alpha = executor.submit(self.run_agent_alpha, scope) + future_beta = executor.submit(self.run_agent_beta, scope) + + # Wait for both + future_alpha.result() + future_beta.result() + + # Phase 4: Run Epistemic Auditor + print(f"โš–๏ธ [Auditor] Auditing evidence & computing divergence for [{scope_id}]...") + audit_report = self.auditor.audit_scope(scope_id) + summary = audit_report["summary"] + print(f" [Audit Result] Score: {summary['epistemic_score']}/1.0 | Divergence: {summary['divergence_score']} | Verified: {summary['verified_passed']} | Rejected: {summary['unverified_rejected']}") + + def run_swarm(self, objective: str, frontier_file: Optional[Path] = None): + """Full end-to-end execution loop.""" + start_time = datetime.now(timezone.utc) + print("=" * 70) + print("๐ŸŒŸ EPISTEMIC SWARM: HIGH-INTEGRITY RESEARCH HARNESS") + print("=" * 70) + + # 1. Orchestrate + scopes = self.orchestrate_objective(objective, frontier_file) + + # 2. Execute scopes according to DAG + while True: + ready_scopes = self.state_machine.get_ready_scopes() + if not ready_scopes: + # Check if all scopes are complete + manifest = self.state_machine.load_global_manifest() + all_complete = all( + self.state_machine.load_scope_manifest(s["scope_id"]).get("status") == ScopeStatus.COMPLETE.value + for s in manifest["scopes"] + ) + if all_complete: + break + else: + print("[ERROR] Deadlock in scope dependency graph.", file=sys.stderr) + self.state_machine.update_session_status(SessionStatus.FAILED) + return + + for scope in ready_scopes: + self.execute_scope_dialectic(scope) + + # 3. Master Synthesis Compilation + print("\n๐Ÿ“œ [Phase 5: Master Synthesis] Aggregating scope dossiers...") + self._compile_master_synthesis(objective) + self.state_machine.update_session_status(SessionStatus.COMPLETED) + + duration = (datetime.now(timezone.utc) - start_time).total_seconds() + print(f"\n๐ŸŽ‰ Research swarm completed in {duration:.1f}s. Report: .research/final_synthesis.md") + + def _compile_master_synthesis(self, objective: str): + manifest = self.state_machine.load_global_manifest() + synthesis_lines = [ + f"# Master Epistemic Research Report: {objective}\n", + f"**Session ID**: `{manifest['session_id']}` | **Generated**: `{manifest['updated_at']}`\n", + "## Executive Summary", + "This report was compiled using the Epistemic Swarm dialectic harness. Every factual statement carries an empirical verification pointer backed by a content-addressed raw document cache.\n", + "## Scope Findings & Dialectic Balance Sheets\n" + ] + + total_verified = 0 + total_rejected = 0 + all_divergences = [] + + for scope in manifest["scopes"]: + sid = scope["scope_id"] + scope_dir = self.state_machine.get_scope_dir(sid) + audit_file = scope_dir / "audit_report.json" + synth_file = scope_dir / "scope_synthesis.md" + + if audit_file.exists(): + with open(audit_file, "r", encoding="utf-8") as f: + ar = json.load(f) + total_verified += ar["summary"]["verified_passed"] + total_rejected += ar["summary"]["unverified_rejected"] + all_divergences.append(ar["summary"]["divergence_score"]) + + if synth_file.exists(): + with open(synth_file, "r", encoding="utf-8") as f: + synthesis_lines.append(f.read()) + synthesis_lines.append("\n---\n") + + avg_div = round(sum(all_divergences) / max(1, len(all_divergences)), 2) + synthesis_lines.append(f"\n## Swarm Epistemic Audit Totals\n") + synthesis_lines.append(f"- **Total Verified Primary Citations**: `{total_verified}`") + synthesis_lines.append(f"- **Total Unverified Claims Purged**: `{total_rejected}`") + synthesis_lines.append(f"- **Mean Swarm Divergence Score**: `{avg_div}`") + + final_path = self.base_dir / "final_synthesis.md" + with open(final_path, "w", encoding="utf-8") as f: + f.write("\n".join(synthesis_lines)) + + +def main(): + parser = argparse.ArgumentParser(description="Epistemic Swarm Dialectic Research Runner") + parser.add_argument("--objective", type=str, required=True, help="Research question or objective") + parser.add_argument("--frontier", type=str, help="Path to settled frontier.json from /grilling") + parser.add_argument("--mock-claude", action="store_true", help="Run with synthetic test data without invoking Claude Code") + parser.add_argument("--dir", default=".research", help="Path to .research workspace") + + args = parser.parse_args() + frontier_path = Path(args.frontier) if args.frontier else None + runner = SwarmRunner(base_dir=Path(args.dir), mock_mode=args.mock_claude) + runner.run_swarm(args.objective, frontier_file=frontier_path) + + +if __name__ == "__main__": + main() diff --git a/runner/state_machine.py b/runner/state_machine.py new file mode 100644 index 0000000..a42158f --- /dev/null +++ b/runner/state_machine.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +""" +Filesystem IPC Protocol & State Machine for Epistemic Swarm. +Manages .research/ hierarchy, session state, scope DAG, and dossier serialization. +""" + +import os +import json +import threading +import tempfile +from enum import Enum +from pathlib import Path +from datetime import datetime, timezone +from typing import Dict, Any, List, Optional + +class SessionStatus(str, Enum): + INITIALIZING = "INITIALIZING" + FRONTIER_SETTLING = "FRONTIER_SETTLING" + ORCHESTRATING = "ORCHESTRATING" + SWARM_DISPATCHED = "SWARM_DISPATCHED" + AUDITING = "AUDITING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + +class ScopeStatus(str, Enum): + PENDING = "PENDING" + ALPHA_RUNNING = "ALPHA_RUNNING" + BETA_RUNNING = "BETA_RUNNING" + RUNNING_PARALLEL = "RUNNING_PARALLEL" + ALPHA_COMPLETE = "ALPHA_COMPLETE" + BETA_COMPLETE = "BETA_COMPLETE" + DOSSIERS_READY = "DOSSIERS_READY" + AUDITING = "AUDITING" + COMPLETE = "COMPLETE" + FAILED = "FAILED" + +class ResearchStateMachine: + def __init__(self, base_dir: Optional[Path] = None): + self.base_dir = base_dir or Path(".research") + self.scratchpads_dir = self.base_dir / "scratchpads" + self.sources_dir = self.base_dir / "sources" + self.manifest_file = self.base_dir / "manifest.json" + self._lock = threading.Lock() + + # Ensure directories exist + self.scratchpads_dir.mkdir(parents=True, exist_ok=True) + self.sources_dir.mkdir(parents=True, exist_ok=True) + + def init_session(self, objective: str, session_id: Optional[str] = None) -> Dict[str, Any]: + """Initialize or reset a research session manifest.""" + sid = session_id or f"session-{datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S')}" + manifest = { + "session_id": sid, + "objective": objective, + "status": SessionStatus.INITIALIZING.value, + "created_at": datetime.now(timezone.utc).isoformat(), + "updated_at": datetime.now(timezone.utc).isoformat(), + "scopes": [], + "telemetry": { + "total_scopes": 0, + "completed_scopes": 0, + "total_claims_audited": 0, + "total_verified_claims": 0, + "total_rejected_claims": 0, + "mean_epistemic_score": 0.0, + "mean_divergence_score": 0.0 + } + } + self.save_global_manifest(manifest) + return manifest + + def load_global_manifest(self) -> Dict[str, Any]: + with self._lock: + if not self.manifest_file.exists(): + raise FileNotFoundError(f"Global manifest not found at {self.manifest_file}") + with open(self.manifest_file, "r", encoding="utf-8") as f: + return json.load(f) + + def save_global_manifest(self, manifest: Dict[str, Any]): + with self._lock: + manifest["updated_at"] = datetime.now(timezone.utc).isoformat() + temp_path = self.manifest_file.with_suffix(".tmp") + with open(temp_path, "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2) + os.replace(temp_path, self.manifest_file) + + def update_session_status(self, status: SessionStatus): + manifest = self.load_global_manifest() + manifest["status"] = status.value + self.save_global_manifest(manifest) + + def set_scopes(self, scopes: List[Dict[str, Any]]): + """Set scopes decomposed by orchestrator and prepare scratchpads.""" + manifest = self.load_global_manifest() + manifest["scopes"] = scopes + manifest["telemetry"]["total_scopes"] = len(scopes) + self.save_global_manifest(manifest) + + for scope in scopes: + scope_id = scope["scope_id"] + scope_dir = self.scratchpads_dir / scope_id + scope_dir.mkdir(parents=True, exist_ok=True) + + scope_manifest = { + "scope_id": scope_id, + "title": scope.get("title", ""), + "objective": scope.get("objective", ""), + "dependencies": scope.get("dependencies", []), + "status": ScopeStatus.PENDING.value, + "alpha_completed": False, + "beta_completed": False, + "audit_completed": False, + "created_at": datetime.now(timezone.utc).isoformat() + } + temp_scope_file = (scope_dir / "manifest.json").with_suffix(".tmp") + with open(temp_scope_file, "w", encoding="utf-8") as f: + json.dump(scope_manifest, f, indent=2) + os.replace(temp_scope_file, scope_dir / "manifest.json") + + def get_scope_dir(self, scope_id: str) -> Path: + return self.scratchpads_dir / scope_id + + def load_scope_manifest(self, scope_id: str) -> Dict[str, Any]: + with self._lock: + scope_manifest_file = self.get_scope_dir(scope_id) / "manifest.json" + if not scope_manifest_file.exists(): + raise FileNotFoundError(f"Scope manifest not found for {scope_id}") + with open(scope_manifest_file, "r", encoding="utf-8") as f: + return json.load(f) + + def save_scope_manifest(self, scope_id: str, manifest: Dict[str, Any]): + with self._lock: + scope_manifest_file = self.get_scope_dir(scope_id) / "manifest.json" + temp_path = scope_manifest_file.with_suffix(".tmp") + with open(temp_path, "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2) + os.replace(temp_path, scope_manifest_file) + + def update_scope_status(self, scope_id: str, status: ScopeStatus): + sm = self.load_scope_manifest(scope_id) + sm["status"] = status.value + self.save_scope_manifest(scope_id, sm) + + def record_agent_completion(self, scope_id: str, agent_type: str, dossier_data: Dict[str, Any]): + """Records dossier from Alpha or Beta and advances scope state machine.""" + scope_dir = self.get_scope_dir(scope_id) + + if agent_type.lower() in ["alpha", "thesis", "proponent"]: + filename = "alpha_dossier.json" + is_alpha = True + elif agent_type.lower() in ["beta", "antithesis", "adversary", "red_team"]: + filename = "beta_dossier.json" + is_alpha = False + else: + raise ValueError(f"Unknown agent type: {agent_type}") + + temp_dossier = (scope_dir / filename).with_suffix(".tmp") + with open(temp_dossier, "w", encoding="utf-8") as f: + json.dump(dossier_data, f, indent=2) + os.replace(temp_dossier, scope_dir / filename) + + sm = self.load_scope_manifest(scope_id) + if is_alpha: + sm["alpha_completed"] = True + else: + sm["beta_completed"] = True + + if sm.get("alpha_completed") and sm.get("beta_completed"): + sm["status"] = ScopeStatus.DOSSIERS_READY.value + elif sm.get("alpha_completed"): + sm["status"] = ScopeStatus.ALPHA_COMPLETE.value + elif sm.get("beta_completed"): + sm["status"] = ScopeStatus.BETA_COMPLETE.value + + self.save_scope_manifest(scope_id, sm) + + def get_ready_scopes(self) -> List[Dict[str, Any]]: + """Return scopes whose dependencies are completed and status is PENDING.""" + manifest = self.load_global_manifest() + ready = [] + completed_scope_ids = { + s["scope_id"] for s in manifest["scopes"] + if self.load_scope_manifest(s["scope_id"]).get("status") == ScopeStatus.COMPLETE.value + } + + for scope in manifest["scopes"]: + sm = self.load_scope_manifest(scope["scope_id"]) + if sm["status"] == ScopeStatus.PENDING.value: + deps = set(scope.get("dependencies", [])) + if deps.issubset(completed_scope_ids): + ready.append(scope) + return ready diff --git a/runner/tests/test_swarm.py b/runner/tests/test_swarm.py new file mode 100644 index 0000000..6d12969 --- /dev/null +++ b/runner/tests/test_swarm.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +""" +Unit and integration test suite for Epistemic Swarm. +Tests source hashing, state machine IPC transitions, auditor engine, and swarm mock dispatch. +""" + +import sys +import shutil +import tempfile +import unittest +from pathlib import Path + +# Add project root to sys.path +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from skills.research_cache.hasher import SourceHasher +from runner.state_machine import ResearchStateMachine, SessionStatus, ScopeStatus +from runner.auditor_engine import EpistemicAuditorEngine +from runner.research_swarm import SwarmRunner + +class TestEpistemicSwarm(unittest.TestCase): + def setUp(self): + self.test_dir = Path(tempfile.mkdtemp(prefix="epistemic_test_")) + self.hasher = SourceHasher(base_dir=self.test_dir) + self.state_machine = ResearchStateMachine(base_dir=self.test_dir) + self.auditor = EpistemicAuditorEngine(base_dir=self.test_dir) + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def test_source_hasher_and_quote_verification(self): + content = """# Deep Learning Scaling Laws +Empirical measurements show that compute-optimal models scale loss as L(N) = (N_c / N)^alpha_N. +In our experiments, the 70B parameter model was trained on 15.0 trillion tokens. +""" + shash = self.hasher.store_source( + url="https://arxiv.org/abs/2203.15556", + content=content, + title="Chinchilla Scaling Laws" + ) + self.assertTrue(len(shash) == 64) + + # 1. Exact quote match + verified, conf, msg = self.hasher.verify_quote( + shash, "the 70B parameter model was trained on 15.0 trillion tokens." + ) + self.assertTrue(verified) + self.assertGreaterEqual(conf, 0.95) + + # 2. Normalized whitespace match + verified_norm, conf_norm, _ = self.hasher.verify_quote( + shash, "the 70B parameter model was\ntrained on 15.0 trillion tokens." + ) + self.assertTrue(verified_norm) + + # 3. Fabricated quote rejection + verified_fake, conf_fake, _ = self.hasher.verify_quote( + shash, "the 70B parameter model was trained on 500 quadrillion tokens by aliens." + ) + self.assertFalse(verified_fake) + self.assertLess(conf_fake, 0.8) + + def test_state_machine_transitions_and_dag(self): + self.state_machine.init_session("Evaluate rollup throughput") + scopes = [ + { + "scope_id": "scope_01_prover", + "title": "Prover Benchmarks", + "dependencies": [] + }, + { + "scope_id": "scope_02_recursion", + "title": "Recursive Verification", + "dependencies": ["scope_01_prover"] + } + ] + self.state_machine.set_scopes(scopes) + + # Initially, only scope_01 should be ready + ready = self.state_machine.get_ready_scopes() + self.assertEqual(len(ready), 1) + self.assertEqual(ready[0]["scope_id"], "scope_01_prover") + + # Mark scope_01 complete + self.state_machine.update_scope_status("scope_01_prover", ScopeStatus.COMPLETE) + + # Now scope_02 should be ready + ready_after = self.state_machine.get_ready_scopes() + self.assertEqual(len(ready_after), 1) + self.assertEqual(ready_after[0]["scope_id"], "scope_02_recursion") + + def test_epistemic_auditor_scoring_and_downgrade(self): + # 1. Seed cached source + shash = self.hasher.store_source( + url="https://benchmark.org/zk", + content="FPGA prover executes Poseidon in 184ms.", + title="ZK Benchmarks" + ) + + # 2. Initialize scope + self.state_machine.init_session("Test Objective") + self.state_machine.set_scopes([{"scope_id": "scope_test", "dependencies": []}]) + + # 3. Create Alpha Dossier with 1 verified and 1 fake quote + alpha_dossier = { + "agent": "Agent Alpha", + "scope_id": "scope_test", + "affirmative_claims": [ + { + "claim_id": "A1", + "tag": "VERIFIED", + "statement": "Poseidon prover executes in 184ms", + "source_hash": shash, + "verbatim_quote": "FPGA prover executes Poseidon in 184ms." + }, + { + "claim_id": "A2", + "tag": "VERIFIED", + "statement": "Hallucinated claim that does not exist in source", + "source_hash": shash, + "verbatim_quote": "This string does not exist anywhere in the text." + } + ], + "negative_knowledge": [{"query": "q1", "finding": "None"}] + } + + # 4. Create Beta Dossier + beta_dossier = { + "agent": "Agent Beta", + "scope_id": "scope_test", + "falsification_claims": [], + "methodological_critiques": [ + { + "target_assertion": "Poseidon prover executes in 184ms", + "critique": "Benchmark excludes PCIe host bus latency" + } + ], + "negative_knowledge": [] + } + + self.state_machine.record_agent_completion("scope_test", "alpha", alpha_dossier) + self.state_machine.record_agent_completion("scope_test", "beta", beta_dossier) + + # 5. Run Auditor + report = self.auditor.audit_scope("scope_test") + summary = report["summary"] + + self.assertEqual(summary["verified_passed"], 1) + self.assertEqual(summary["unverified_rejected"], 1) + self.assertEqual(summary["negative_knowledge_count"], 1) + self.assertGreater(summary["divergence_score"], 0.0) + + # Check synthesis markdown generated + synth_file = self.test_dir / "scratchpads" / "scope_test" / "scope_synthesis.md" + self.assertTrue(synth_file.exists()) + with open(synth_file, "r") as f: + synth_content = f.read() + self.assertIn("PURGED", synth_content) + self.assertIn("Poseidon prover executes in 184ms", synth_content) + + def test_mock_swarm_runner_end_to_end(self): + runner = SwarmRunner(base_dir=self.test_dir, mock_mode=True) + runner.run_swarm("Evaluate hardware prover latency") + + manifest = runner.state_machine.load_global_manifest() + self.assertEqual(manifest["status"], SessionStatus.COMPLETED.value) + + final_report = self.test_dir / "final_synthesis.md" + self.assertTrue(final_report.exists()) + with open(final_report, "r") as f: + report_text = f.read() + self.assertIn("Master Epistemic Research Report", report_text) + self.assertIn("Swarm Epistemic Audit Totals", report_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/grilling/SKILL.md b/skills/grilling/SKILL.md new file mode 100644 index 0000000..ffc893d --- /dev/null +++ b/skills/grilling/SKILL.md @@ -0,0 +1,48 @@ +--- +name: grilling +description: Socratic grilling and assumption-inversion skill for deep research. Uses Matt Pocock-style design trees to explore the problem frontier divergently before committing to search queries. +--- + +# Socratic Grilling & Divergent Research Framing + +Interview the user relentlessly until you reach an airtight, shared understanding of the research scope. Map the problem as a **design tree**: every foundational assumption branches into the technical decisions and empirical hypotheses that hang off it. + +## 1. THE FRONTIER METHODOLOGY + +1. Work the tree in **rounds**. +2. The **frontier** is every decision whose prerequisites are already settled: the questions you can ask *now* without guessing at answers you haven't heard yet. +3. Ask the whole frontier in one round: number each question and provide your recommended answer. +4. Then wait for the user's answers before moving to the next round. + +## 2. FORMATTING A ROUND + +``` +โ“ **Q1 - <Question Title>**: <Question context, premise inversion, trade-offs, multiple options> + +โžก๏ธ **Recommended**: <Your recommended answer with rationale> + +--- + +โ“ **Q2 - <Question Title>**: <Question context, trade-offs> + +โžก๏ธ **Recommended**: <Your recommended answer with rationale> +``` + +## 3. FACTUAL VS. DECISIONAL SEPARATION + +- **Facts are the agent's job**: When a frontier question hinges on an empirical fact (e.g. library benchmarks, API specs, hardware limits), **DO NOT ASK THE USER**. Dispatch a tool call or subagent to look it up in the codebase or online. +- **Decisions are the user's**: High-level trade-offs, architectural philosophy, threat models, and priority ranking belong to the user. Put each decision to them clearly. + +## 4. ASSUMPTION INVERSION TACTICS + +Always challenge default premises in Round 1: +- *Inversion*: What if the primary objective is rendered obsolete by a radical alternative? +- *Scale Extremes*: What breaks at 100x scale? What breaks at 0 resources? +- *Adversarial Posture*: How would an intelligent adversary exploit or falsify this design? + +## 5. FRONTIER RESOLUTION & FREEZING + +When every branch of the design tree has been visited and the frontier is empty: +1. Summarize the settled constraints. +2. Save the settled state to `.research/frontier.json` using `python3 skills/grilling/socratic_tree.py --export`. +3. Hand off the settled frontier to the **Swarm Orchestrator** to begin empirical dialectic execution. diff --git a/skills/grilling/__init__.py b/skills/grilling/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/skills/grilling/socratic_tree.py b/skills/grilling/socratic_tree.py new file mode 100755 index 0000000..a3d089b --- /dev/null +++ b/skills/grilling/socratic_tree.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +Socratic Tree & Decision Frontier Engine for Epistemic Swarm. +Implements Matt Pocock-style design tree traversal to isolate the active decision frontier. +""" + +import sys +import json +import argparse +from pathlib import Path +from typing import Dict, List, Optional, Any + +class DecisionNode: + def __init__(self, node_id: str, title: str, question: str, + recommended: str, options: Optional[List[str]] = None, + prerequisites: Optional[List[str]] = None): + self.node_id = node_id + self.title = title + self.question = question + self.recommended = recommended + self.options = options or [] + self.prerequisites = prerequisites or [] + self.settled_answer: Optional[str] = None + + def is_settled(self) -> bool: + return self.settled_answer is not None + + def to_dict(self) -> Dict[str, Any]: + return { + "node_id": self.node_id, + "title": self.title, + "question": self.question, + "recommended": self.recommended, + "options": self.options, + "prerequisites": self.prerequisites, + "settled_answer": self.settled_answer + } + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> 'DecisionNode': + node = cls( + node_id=data["node_id"], + title=data["title"], + question=data["question"], + recommended=data["recommended"], + options=data.get("options", []), + prerequisites=data.get("prerequisites", []) + ) + node.settled_answer = data.get("settled_answer") + return node + + +class DesignTree: + def __init__(self, objective: str): + self.objective = objective + self.nodes: Dict[str, DecisionNode] = {} + + def add_node(self, node: DecisionNode): + self.nodes[node.node_id] = node + + def compute_frontier(self) -> List[DecisionNode]: + """ + The frontier is all unsettled nodes whose prerequisites are ALL settled. + """ + frontier = [] + for node in self.nodes.values(): + if node.is_settled(): + continue + prereqs_met = True + for prereq_id in node.prerequisites: + prereq_node = self.nodes.get(prereq_id) + if not prereq_node or not prereq_node.is_settled(): + prereqs_met = False + break + if prereqs_met: + frontier.append(node) + return frontier + + def settle_node(self, node_id: str, answer: str): + if node_id in self.nodes: + self.nodes[node_id].settled_answer = answer + + def is_complete(self) -> bool: + return len(self.compute_frontier()) == 0 and all(n.is_settled() for n in self.nodes.values()) + + def export_frontier_json(self, output_path: Path): + output_path.parent.mkdir(parents=True, exist_ok=True) + data = { + "objective": self.objective, + "is_complete": self.is_complete(), + "nodes": {k: v.to_dict() for k, v in self.nodes.items()}, + "settled_constraints": { + k: v.settled_answer for k, v in self.nodes.items() if v.is_settled() + } + } + with open(output_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + + @classmethod + def load_from_json(cls, file_path: Path) -> 'DesignTree': + with open(file_path, "r", encoding="utf-8") as f: + data = json.load(f) + tree = cls(objective=data["objective"]) + for k, v in data.get("nodes", {}).items(): + tree.add_node(DecisionNode.from_dict(v)) + return tree + + +def main(): + parser = argparse.ArgumentParser(description="Epistemic Swarm Socratic Decision Tree") + parser.add_argument("--objective", type=str, help="Research objective") + parser.add_argument("--file", type=str, default=".research/frontier.json", help="Path to frontier.json") + parser.add_argument("--show-frontier", action="store_true", help="Print the current decision frontier") + parser.add_argument("--settle", nargs=2, metavar=("NODE_ID", "ANSWER"), help="Settle a decision node") + + args = parser.parse_args() + frontier_path = Path(args.file) + + if frontier_path.exists(): + tree = DesignTree.load_from_json(frontier_path) + else: + objective = args.objective or "Epistemic Swarm Research Objective" + tree = DesignTree(objective=objective) + + if args.settle: + node_id, answer = args.settle + tree.settle_node(node_id, answer) + tree.export_frontier_json(frontier_path) + print(f"Settled {node_id} -> {answer}") + + frontier = tree.compute_frontier() + if args.show_frontier or not args.settle: + print(f"\n๐ŸŽฏ Objective: {tree.objective}") + print(f"๐Ÿ“Š Total Nodes: {len(tree.nodes)} | Settled: {sum(1 for n in tree.nodes.values() if n.is_settled())}") + if not frontier: + if tree.nodes and tree.is_complete(): + print("โœ… Frontier is EMPTY. All prerequisite branches are fully settled!") + else: + print("โ„น๏ธ No active frontier nodes. Define new decision nodes to begin grilling.") + else: + print(f"\nโšก Current Active Frontier ({len(frontier)} questions ready):") + for idx, node in enumerate(frontier, 1): + print(f"\nโ“ Q{idx} [{node.node_id}] - {node.title}") + print(f" {node.question}") + print(f" โžก๏ธ Recommended: {node.recommended}") + +if __name__ == "__main__": + main() diff --git a/skills/research-cache/SKILL.md b/skills/research-cache/SKILL.md new file mode 100644 index 0000000..f9fe550 --- /dev/null +++ b/skills/research-cache/SKILL.md @@ -0,0 +1,36 @@ +--- +name: research-cache +description: Content-addressed document caching and quote verification skill. Hashes retrieved web pages and academic papers to SHA-256 for mathematical auditability. +--- + +# Content-Addressed Research Cache & Verification + +To maintain epistemic integrity, every document fetched from the web, arXiv, or technical docs must be cached locally with a content-addressed SHA-256 fingerprint before its claims can be cited. + +## 1. CACHING A SOURCE +When you fetch or scrape a URL: +```bash +python3 skills/research-cache/hasher.py cache \ + --url "https://arxiv.org/abs/2407.21783" \ + --title "Llama 3 Herd of Models" \ + --content "$(cat fetched_paper.md)" +``` +This prints the content hash: +``` +[CACHED] 3f8a9e21... -> .research/sources/3f8a9e21....md +``` + +## 2. CITING WITH HASHES +In your dossiers and markdown reports, cite the claim using the hash: +`[VERIFIED: 3f8a9e21]` + +Ensure that any `verbatim_quote` you provide is an exact substring from the cached markdown document. + +## 3. AUDITING A QUOTE +The Epistemic Auditor verifies claims using: +```bash +python3 skills/research-cache/hasher.py verify \ + --hash "3f8a9e21..." \ + --quote "Our FPGA pipeline executes the Poseidon round constraints in 184ms" +``` +If the quote does not match, the claim is rejected and flagged as unverified. diff --git a/skills/research-cache/__init__.py b/skills/research-cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/skills/research-cache/hasher.py b/skills/research-cache/hasher.py new file mode 100755 index 0000000..495b307 --- /dev/null +++ b/skills/research-cache/hasher.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +""" +Content-Addressed Research Cache & Verbatim Quote Verification Engine. +Handles SHA-256 document hashing, metadata indexing, and substring auditing. +""" + +import os +import sys +import json +import hashlib +import re +import argparse +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, Any, Optional, Tuple + +class SourceHasher: + def __init__(self, base_dir: Optional[Path] = None): + self.base_dir = base_dir or Path(".research") + self.sources_dir = self.base_dir / "sources" + self.sources_dir.mkdir(parents=True, exist_ok=True) + + @staticmethod + def compute_sha256(content: str) -> str: + """Compute standard hex SHA-256 hash of normalized UTF-8 string.""" + normalized = content.strip().encode("utf-8") + return hashlib.sha256(normalized).hexdigest() + + def store_source(self, url: str, content: str, title: Optional[str] = None, + tier: str = "WEB_DOCUMENT", metadata: Optional[Dict[str, Any]] = None) -> str: + """Store content and metadata content-addressed by SHA-256.""" + content_hash = self.compute_sha256(content) + md_path = self.sources_dir / f"{content_hash}.md" + json_path = self.sources_dir / f"{content_hash}.json" + + # Write clean markdown + with open(md_path, "w", encoding="utf-8") as f: + f.write(content) + + # Write metadata + meta = { + "hash": content_hash, + "url": url, + "title": title or "Untitled Source", + "tier": tier, + "cached_at": datetime.now(timezone.utc).isoformat(), + "byte_size": len(content.encode("utf-8")), + "char_count": len(content), + "custom_metadata": metadata or {} + } + with open(json_path, "w", encoding="utf-8") as f: + json.dump(meta, f, indent=2) + + return content_hash + + def get_source_content(self, content_hash: str) -> Optional[str]: + """Retrieve stored markdown content by full or prefix hash.""" + target_file = self._resolve_hash_file(content_hash, ".md") + if target_file and target_file.exists(): + with open(target_file, "r", encoding="utf-8") as f: + return f.read() + return None + + def get_source_metadata(self, content_hash: str) -> Optional[Dict[str, Any]]: + """Retrieve stored metadata by full or prefix hash.""" + target_file = self._resolve_hash_file(content_hash, ".json") + if target_file and target_file.exists(): + with open(target_file, "r", encoding="utf-8") as f: + return json.load(f) + return None + + def _resolve_hash_file(self, hash_prefix: str, extension: str) -> Optional[Path]: + """Resolves exact or prefix hash to disk path.""" + exact_path = self.sources_dir / f"{hash_prefix}{extension}" + if exact_path.exists(): + return exact_path + + # If prefix match + matches = list(self.sources_dir.glob(f"{hash_prefix}*{extension}")) + if len(matches) == 1: + return matches[0] + return None + + @staticmethod + def normalize_text_for_search(text: str) -> str: + """Collapse whitespace and normalize typography for substring matching.""" + # Replace smart quotes and dashes + text = text.replace("โ€œ", '"').replace("โ€", '"').replace("โ€˜", "'").replace("โ€™", "'") + text = text.replace("โ€”", "-").replace("โ€“", "-") + # Collapse all whitespace to single spaces + return re.sub(r"\s+", " ", text).strip().lower() + + def verify_quote(self, content_hash: str, quote: str) -> Tuple[bool, float, Optional[str]]: + """ + Verifies whether quote exists in cached document. + Returns: (is_verified, confidence_score, context_match) + """ + source_text = self.get_source_content(content_hash) + if not source_text: + return False, 0.0, f"Source hash '{content_hash}' not found in cache." + + # 1. Exact match test + if quote.strip() in source_text: + return True, 1.0, "Exact substring match found." + + # 2. Normalized whitespace match test + norm_source = self.normalize_text_for_search(source_text) + norm_quote = self.normalize_text_for_search(quote) + if norm_quote in norm_source: + return True, 0.98, "Normalized whitespace match found." + + # 3. Sliding window token overlap test + quote_words = norm_quote.split() + if len(quote_words) < 4: + return False, 0.0, "Quote too short and not found verbatim." + + window_size = len(quote_words) + source_words = norm_source.split() + best_score = 0.0 + + for i in range(max(1, len(source_words) - window_size + 1)): + window = source_words[i:i + window_size] + matches = sum(1 for w1, w2 in zip(quote_words, window) if w1 == w2) + score = matches / window_size + if score > best_score: + best_score = score + if best_score >= 0.90: + break + + if best_score >= 0.88: + return True, best_score, f"High-confidence fuzzy match ({best_score:.2f})." + + return False, best_score, f"Verification failed. Highest word overlap: {best_score:.2f}." + + +def main(): + parser = argparse.ArgumentParser(description="Epistemic Swarm Content Hasher & Quote Verifier") + subparsers = parser.add_subparsers(dest="command") + + # Cache command + cache_parser = subparsers.add_parser("cache", help="Cache a document") + cache_parser.add_argument("--url", required=True, help="Original URL") + cache_parser.add_argument("--title", default="Untitled", help="Document Title") + cache_parser.add_argument("--tier", default="WEB_DOCUMENT", help="Source tier") + cache_parser.add_argument("--content", help="Raw text content (or read from stdin)") + cache_parser.add_argument("--dir", default=".research", help="Base .research directory") + + # Verify command + verify_parser = subparsers.add_parser("verify", help="Verify a verbatim quote") + verify_parser.add_argument("--hash", required=True, help="Document SHA-256 hash") + verify_parser.add_argument("--quote", required=True, help="Verbatim quote to check") + verify_parser.add_argument("--dir", default=".research", help="Base .research directory") + + # List command + list_parser = subparsers.add_parser("list", help="List cached sources") + list_parser.add_argument("--dir", default=".research", help="Base .research directory") + + args = parser.parse_args() + hasher = SourceHasher(base_dir=Path(args.dir if hasattr(args, "dir") else ".research")) + + if args.command == "cache": + content = args.content + if not content: + if not sys.stdin.isatty(): + content = sys.stdin.read() + else: + print("Error: No content provided via --content or stdin.", file=sys.stderr) + sys.exit(1) + h = hasher.store_source(url=args.url, content=content, title=args.title, tier=args.tier) + print(f"[CACHED] {h} -> {args.title} ({args.url})") + + elif args.command == "verify": + verified, conf, msg = hasher.verify_quote(content_hash=args.hash, quote=args.quote) + result = { + "hash": args.hash, + "verified": verified, + "confidence": conf, + "message": msg + } + print(json.dumps(result, indent=2)) + sys.exit(0 if verified else 1) + + elif args.command == "list": + sources = list(hasher.sources_dir.glob("*.json")) + print(f"Total Cached Sources: {len(sources)}") + for p in sources: + with open(p, "r", encoding="utf-8") as f: + d = json.load(f) + print(f"- [{d['hash'][:10]}...] {d['title']} ({d['url']})") + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/skills/research_cache b/skills/research_cache new file mode 120000 index 0000000..c04cdfd --- /dev/null +++ b/skills/research_cache @@ -0,0 +1 @@ +research-cache \ No newline at end of file