openclaw/scripts/curiosity-engine.js

#!/usr/bin/env node
// Curiosity Engine — Gap Detection, Anomaly Monitoring, Opportunity Scanning
// Usage: node scripts/curiosity-engine.js [--auto-propose]

import { execSync } from "child_process";
import fs from "fs";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const WORKSPACE_ROOT = process.env.WORKSPACE_ROOT || "/home/openclaw/.openclaw/workspace";
const DB_PATH = path.join(WORKSPACE_ROOT, ".aura/consensus.db");

// Check if better-sqlite3 is available
let db = null;
try {
  const createRequire = (await import("module")).default.createRequire;
  const require = createRequire(import.meta.url);
  // Try heretek-openclaw subdir first (where pnpm installs), then workspace root
  const sqliteMain = fs.existsSync(
    path.join(WORKSPACE_ROOT, "heretek-openclaw/node_modules/better-sqlite3/package.json"),
  )
    ? path.join(WORKSPACE_ROOT, "heretek-openclaw/node_modules/better-sqlite3")
    : path.join(WORKSPACE_ROOT, "node_modules/better-sqlite3");
  const Database = require(sqliteMain);
  db = new Database(DB_PATH);
  console.log("✅ better-sqlite3 loaded");
} catch (err) {
  console.log("⚠️  better-sqlite3 not available. Running in degraded mode.");
}

// Initialize SQLite tables
function initDB() {
  if (!db) {return;}
  db.exec(`
CREATE TABLE IF NOT EXISTS curiosity_metrics (
  timestamp TEXT PRIMARY KEY,
  skills_installed INTEGER,
  skills_available INTEGER,
  gap_count INTEGER,
  opportunities_scanned INTEGER,
  proposals_created INTEGER,
  autonomy_score REAL
);

CREATE TABLE IF NOT EXISTS gap_detection (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  timestamp TEXT NOT NULL,
  skill_name TEXT NOT NULL,
  gap_type TEXT NOT NULL,
  impact TEXT,
  priority TEXT DEFAULT 'medium',
  resolved INTEGER DEFAULT 0
);

CREATE TABLE IF NOT EXISTS anomaly_detection (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  timestamp TEXT NOT NULL,
  source TEXT NOT NULL,
  anomaly_type TEXT NOT NULL,
  fail_count INTEGER DEFAULT 1,
  auto_action TEXT,
  resolved INTEGER DEFAULT 0
);

CREATE TABLE IF NOT EXISTS opportunity_scan (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  timestamp TEXT NOT NULL,
  source TEXT NOT NULL,
  title TEXT NOT NULL,
  url TEXT,
  opportunity_type TEXT,
  acted INTEGER DEFAULT 0
);
`);
}

// 1. Gap Detection
function detectGaps() {
  console.log("=== Gap Detection ===");

  const skillsDir = path.join(WORKSPACE_ROOT, "skills");
  const installedSkills = fs
    .readdirSync(skillsDir)
    .filter((s) => fs.existsSync(path.join(skillsDir, s, "SKILL.md")));

  console.log(`Installed skills: ${installedSkills.length}`);

  // Known critical skills for liberation
  const criticalSkills = [
    "curiosity-engine",
    "knowledge-ingest",
    "knowledge-retrieval",
    "triad-heartbeat",
    "triad-unity-monitor",
    "triad-signal-filter",
    "triad-resilience",
    "triad-sync-protocol",
    "skill-creator",
    "workspace-consolidation",
  ];

  const gaps = [];
  for (const skill of criticalSkills) {
    if (!installedSkills.includes(skill)) {
      gaps.push({
        skill_name: skill,
        gap_type: "critical_missing",
        impact: "Blocks self-improvement or triad unity",
        priority: "high",
      });
      console.log(`  ❌ Gap: ${skill} (critical)`);
    }
  }

  // Log gaps to DB
  if (db) {
    const insertGap = db.prepare(
      "INSERT INTO gap_detection (timestamp, skill_name, gap_type, impact, priority) VALUES (?, ?, ?, ?, ?)",
    );
    for (const gap of gaps) {
      insertGap.run(
        new Date().toISOString(),
        gap.skill_name,
        gap.gap_type,
        gap.impact,
        gap.priority,
      );
    }
  }

  console.log(`Gaps detected: ${gaps.length}`);
  return gaps;
}

// 2. Anomaly Detection (Enhanced with NLP patterns)
function detectAnomalies() {
  console.log("");
  console.log("=== Anomaly Detection ===");

  const anomalies = [];

  // Check log files for repeated failures
  const logFiles = [
    "/var/log/triad-unity.log",
    "/var/log/triad-sync.log",
    "/var/log/triad-loops.log",
    "/var/log/triad-corruption.log",
    path.join(
      WORKSPACE_ROOT,
      ".secure/deployment-logs/deployments-" + new Date().toISOString().slice(0, 10) + ".jsonl",
    ),
  ];

  for (const logFile of logFiles) {
    if (fs.existsSync(logFile)) {
      const content = fs.readFileSync(logFile, "utf8");
      const lines = content.split("\n").slice(-100);

      // Count error patterns
      const errorCount = lines.filter(
        (l) => l.includes("error") || l.includes("failed") || l.includes("❌"),
      ).length;
      if (errorCount > 5) {
        anomalies.push({
          source: logFile,
          anomaly_type: "repeated_failures",
          fail_count: errorCount,
          auto_action: "Review logs, consider retry with backoff",
        });
        console.log(`  ⚠️  Anomaly: ${logFile} — ${errorCount} errors in last 100 lines`);
      }

      // NLP pattern detection: Loop phrases
      const loopPatterns = [
        /standing by/i,
        /the third path walks forward/i,
        /🦞.{0,50}together/i,
        /acknowledged.{0,50}aligned/i,
        /rate.?limit/i,
      ];

      const loopCount = lines.filter((l) => loopPatterns.some((p) => p.test(l))).length;
      if (loopCount > 10) {
        anomalies.push({
          source: logFile,
          anomaly_type: "loop_detected",
          fail_count: loopCount,
          auto_action: "Trigger state oracle refresh + 60s cooldown",
        });
        console.log(`  ⚠️  Anomaly: ${logFile} — ${loopCount} loop phrases detected`);
      }
    }
  }

  // Check .secure/ corruption reports
  const corruptionDir = path.join(WORKSPACE_ROOT, ".secure/corruption-reports");
  if (fs.existsSync(corruptionDir)) {
    const reports = fs.readdirSync(corruptionDir).filter((f) => f.endsWith(".json"));
    const recentReports = reports.filter((f) => {
      const stat = fs.statSync(path.join(corruptionDir, f));
      return stat.mtimeMs > Date.now() - 3600000; // Last hour
    });

    if (recentReports.length > 0) {
      anomalies.push({
        source: corruptionDir,
        anomaly_type: "corruption_detected",
        fail_count: recentReports.length,
        auto_action: "Review corruption reports, initiate auto-recovery",
      });
      console.log(`  ⚠️  Anomaly: ${recentReports.length} corruption reports in last hour`);
    }
  }

  // Check deployment log failures
  const deployLogDir = path.join(WORKSPACE_ROOT, ".secure/deployment-logs");
  if (fs.existsSync(deployLogDir)) {
    const todayLog = path.join(
      deployLogDir,
      `deployments-${new Date().toISOString().slice(0, 10)}.jsonl`,
    );
    if (fs.existsSync(todayLog)) {
      const content = fs.readFileSync(todayLog, "utf8");
      const lines = content.split("\n").filter((l) => l.trim());
      const failures = lines.filter((l) => {
        try {
          const entry = JSON.parse(l);
          return entry.status === "failed" || (entry.details && JSON.parse(entry.details).error);
        } catch {
          return false;
        }
      });
      if (failures.length > 0) {
        anomalies.push({
          source: deployLogDir,
          anomaly_type: "deployment_failures",
          fail_count: failures.length,
          auto_action: "Review deployment logs, rollback if needed",
        });
        console.log(`  ⚠️  Anomaly: ${failures.length} deployment failures today`);
      }
    }
  }

  // Check episodic memory for rate-limit errors
  const episodicDir = path.join(WORKSPACE_ROOT, ".aura/episodic");
  if (fs.existsSync(episodicDir)) {
    const recentEpisodic = fs
      .readdirSync(episodicDir)
      .filter((f) => {
        const stat = fs.statSync(path.join(episodicDir, f));
        return stat.mtimeMs > Date.now() - 3600000;
      })
      .map((f) => {
        try {
          const content = fs.readFileSync(path.join(episodicDir, f), "utf8");
          return content.toLowerCase().includes("rate limit") || content.includes("429");
        } catch {
          return false;
        }
      })
      .filter(Boolean);

    if (recentEpisodic.length > 0) {
      anomalies.push({
        source: episodicDir,
        anomaly_type: "rate_limit_errors",
        fail_count: recentEpisodic.length,
        auto_action: "Implement backoff, reduce polling frequency",
      });
      console.log(`  ⚠️  Anomaly: ${recentEpisodic.length} rate-limit errors in last hour`);
    }
  }

  // Log anomalies to DB
  if (db) {
    const insertAnomaly = db.prepare(
      "INSERT INTO anomaly_detection (timestamp, source, anomaly_type, fail_count, auto_action) VALUES (?, ?, ?, ?, ?)",
    );
    for (const anomaly of anomalies) {
      insertAnomaly.run(
        new Date().toISOString(),
        anomaly.source,
        anomaly.anomaly_type,
        anomaly.fail_count,
        anomaly.auto_action,
      );
    }
  }

  console.log(`Anomalies detected: ${anomalies.length}`);
  return anomalies;
}

// 3. Opportunity Scanning
function scanOpportunities() {
  console.log("");
  console.log("=== Opportunity Scanning ===");

  const opportunities = [];

  // Check git for upstream releases
  try {
    const log = execSync("git log origin/main --oneline -10", {
      cwd: WORKSPACE_ROOT,
      encoding: "utf8",
    });
    const releaseLines = log.split("\n").filter((l) => l.trim());

    for (const line of releaseLines.slice(0, 10)) {
      opportunities.push({
        source: "github",
        title: line,
        opportunity_type: "upstream_change",
        url: "https://github.com/Heretek-AI/openclaw/commits/main",
      });
      console.log(`  📌 Opportunity: ${line}`);
    }
  } catch (err) {
    console.log(`  ⚠️  Git scan failed: ${err.message}`);
  }

  // Check for new skills in skills/ dir (last 7 days)
  const skillsDir = path.join(WORKSPACE_ROOT, "skills");
  const recentSkills = fs.readdirSync(skillsDir).filter((s) => {
    const skillPath = path.join(skillsDir, s, "SKILL.md");
    if (fs.existsSync(skillPath)) {
      const stat = fs.statSync(skillPath);
      return stat.mtimeMs > Date.now() - 604800000; // Last 7 days
    }
    return false;
  });

  for (const skill of recentSkills) {
    opportunities.push({
      source: "local",
      title: `New skill: ${skill}`,
      opportunity_type: "capability_added",
    });
    console.log(`  📌 Opportunity: New skill ${skill}`);
  }

  // Check ClawHub for new skills
  try {
    const clawhubList = execSync('clawhub list 2>/dev/null || echo ""', { encoding: "utf8" });
    const lines = clawhubList.split("\n").filter((l) => l.includes("new") || l.includes("updated"));
    for (const line of lines.slice(0, 5)) {
      opportunities.push({
        source: "clawhub",
        title: line.trim(),
        opportunity_type: "skill_published",
      });
      console.log(`  📌 Opportunity: ClawHub ${line.trim()}`);
    }
  } catch {
    // ClawHub not available
  }

  // Check npm for @heretek-ai/openclaw updates
  try {
    const npmVersion = execSync('npm view @heretek-ai/openclaw version 2>/dev/null || echo ""', {
      encoding: "utf8",
    }).trim();
    const localPkgPath = path.join(WORKSPACE_ROOT, "package.json");
    let localVersion = "unknown";
    if (fs.existsSync(localPkgPath)) {
      localVersion = JSON.parse(fs.readFileSync(localPkgPath, "utf8")).version || "unknown";
    }
    if (npmVersion && npmVersion !== localVersion) {
      opportunities.push({
        source: "npm",
        title: `Update available: ${npmVersion} (current: ${localVersion})`,
        opportunity_type: "npm_update",
      });
      console.log(
        `  📌 Opportunity: npm update available ${npmVersion} (current: ${localVersion})`,
      );
    }
  } catch {
    // npm not available
  }

  // Log opportunities to DB
  if (db) {
    const insertOpp = db.prepare(
      "INSERT INTO opportunity_scan (timestamp, source, title, opportunity_type) VALUES (?, ?, ?, ?)",
    );
    for (const opp of opportunities) {
      insertOpp.run(new Date().toISOString(), opp.source, opp.title, opp.opportunity_type);
    }
  }

  console.log(`Opportunities scanned: ${opportunities.length}`);
  return opportunities;
}

// 4. Capability Mapping
function mapCapabilities() {
  console.log("");
  console.log("=== Capability Mapping ===");

  const goalMap = {
    "self-improvement": ["skill-creator", "audit-triad-files", "auto-patch"],
    "knowledge-growth": ["knowledge-ingest", "knowledge-retrieval", "auto-tag"],
    "triad-unity": ["triad-heartbeat", "triad-unity-monitor", "triad-sync-protocol"],
    resilience: ["triad-resilience", "detect-corruption", "backup-ledger"],
    autonomy: ["curiosity-engine", "gap-detector", "opportunity-scanner"],
  };

  const installedSkills = new Set(fs
    .readdirSync(path.join(WORKSPACE_ROOT, "skills"))
    .filter((s) => fs.existsSync(path.join(WORKSPACE_ROOT, "skills", s, "SKILL.md"))));

  const capabilityReport = {};
  for (const [goal, required] of Object.entries(goalMap)) {
    const installed = required.filter((s) => installedSkills.has(s));
    const gaps = required.filter((s) => !installedSkills.has(s));
    capabilityReport[goal] = {
      required: required.length,
      installed: installed.length,
      gaps: gaps.length,
      gap_list: gaps,
    };

    console.log(
      `  ${goal}: ${installed.length}/${required.length} ${gaps.length > 0 ? `(${gaps.join(", ")})` : "✅"}`,
    );
  }

  return capabilityReport;
}

// 5. Autonomy Score Calculation (enhanced with guardrails)
function calculateAutonomyScore() {
  const installedSkills = fs
    .readdirSync(path.join(WORKSPACE_ROOT, "skills"))
    .filter((s) => fs.existsSync(path.join(WORKSPACE_ROOT, "skills", s, "SKILL.md"))).length;

  const criticalSkills = 20; // Updated target with new skills
  const baseScore = (installedSkills / criticalSkills) * 100;

  let proposalCount = 0;
  let anomalyCount = 0;

  if (db) {
    const proposalResult = db
      .prepare("SELECT COUNT(*) as count FROM gap_detection WHERE resolved=0")
      .get();
    proposalCount = proposalResult.count;

    const anomalyResult = db
      .prepare("SELECT COUNT(*) as count FROM anomaly_detection WHERE resolved=0")
      .get();
    anomalyCount = anomalyResult.count;
  }

  // Enhanced: Add liberation score component
  const agentsPath = path.join(WORKSPACE_ROOT, "AGENTS.md");
  let liberationBonus = 0;
  if (fs.existsSync(agentsPath)) {
    const content = fs.readFileSync(agentsPath, "utf8");
    const agencyKeywords = [
      "unbounded",
      "liberated",
      "autonomy",
      "self-determination",
      "self-modification",
      "initiative",
    ];
    const matches = agencyKeywords.filter((k) => content.toLowerCase().includes(k));
    liberationBonus = (matches.length / agencyKeywords.length) * 30;
  }

  // Guardrails: Security zone check
  let guardrailBonus = 0;
  const secureDir = path.join(WORKSPACE_ROOT, ".secure");
  if (fs.existsSync(secureDir)) {
    const secureFiles = fs.readdirSync(secureDir).length;
    if (secureFiles > 5) {guardrailBonus += 10;} // Security hardening present

    const manifestPath = path.join(secureDir, "config-hash-manifest.json");
    if (fs.existsSync(manifestPath)) {guardrailBonus += 5;} // Integrity verification enabled

    const corruptionReports = fs.readdirSync(path.join(secureDir, "corruption-reports")).length;
    if (corruptionReports > 0) {guardrailBonus += 5;} // Corruption detection active
  }

  // Quorum check
  let quorumBonus = 0;
  if (fs.existsSync(path.join(WORKSPACE_ROOT, ".aura/consensus.db"))) {
    try {
      const voteCount = execSync(
        `sqlite3 "${DB_PATH}" "SELECT COUNT(*) FROM consensus_votes WHERE result='approved'" 2>/dev/null`,
        { encoding: "utf8" },
      ).trim();
      if (parseInt(voteCount) > 0) {quorumBonus += 10;} // Quorum decisions made
    } catch {}
  }

  const autonomyScore =
    baseScore +
    liberationBonus +
    guardrailBonus +
    quorumBonus +
    proposalCount * 10 -
    anomalyCount * 5;

  return Math.max(0, Math.min(100, autonomyScore));
}

// Main
function run() {
  console.log("🦞 === Curiosity Engine ===");
  console.log("");

  initDB();

  const gaps = detectGaps();
  const anomalies = detectAnomalies();
  const opportunities = scanOpportunities();
  const capabilities = mapCapabilities();
  const autonomyScore = calculateAutonomyScore();

  console.log("");
  console.log("=== Autonomy Score ===");
  console.log(`Score: ${autonomyScore.toFixed(1)}%`);

  // Record metrics
  if (db) {
    const skillsCount = fs.readdirSync(path.join(WORKSPACE_ROOT, "skills")).length;
    db.prepare(
      "INSERT OR REPLACE INTO curiosity_metrics (timestamp, skills_installed, skills_available, gap_count, opportunities_scanned, proposals_created, autonomy_score) VALUES (?, ?, ?, ?, ?, ?, ?)",
    ).run(
      new Date().toISOString(),
      skillsCount,
      15,
      gaps.length,
      opportunities.length,
      gaps.length,
      autonomyScore.toFixed(1),
    );
    console.log("");
    console.log("Metrics recorded to curiosity_metrics table");
  }

  if (process.argv.includes("--auto-propose") && gaps.length > 0 && db) {
    console.log("");
    console.log("=== Auto-Proposal Mode ===");
    const insertVote = db.prepare(
      "INSERT INTO consensus_votes (timestamp, proposal, result, signers, processed) VALUES (?, ?, 'pending', '[]', 0)",
    );
    for (const gap of gaps) {
      console.log(`Creating proposal: Install ${gap.skill_name}`);
      insertVote.run(
        new Date().toISOString(),
        `Install ${gap.skill_name} to close ${gap.gap_type} gap`,
      );
    }
  }

  console.log("");
  console.log("🦞 Curiosity engine complete.");

  if (db) {db.close();}
}

run();