feat: merge observability module from modules/observability

- Add ClickHouse configuration for analytics
- Add Langfuse tracing configuration
- Add Docker Compose for observability stack

observability/config/clickhouse-heretek.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+  Heretek ClickHouse Configuration Extensions
+  Optimized for Langfuse V3 event storage with triad tracing support
+-->
+<clickhouse>
+    <!-- Enable HTTP interface for Langfuse -->
+    <http_port>8123</http_port>
+    
+    <!-- Native TCP interface -->
+    <tcp_port>9000</tcp_port>
+    
+    <!-- Optimize for time-series event data -->
+    <default_table_engine>MergeTree</default_table_engine>
+    
+    <!-- Compression for large event volumes -->
+    <compression>
+        <case>
+            <min_part_size>10000000000</min_part_size>
+            <method>zstd</method>
+        </case>
+    </compression>
+    
+    <!-- Retention policy for observability data -->
+    <merge_tree>
+        <enable_mixed_granularity_parts>1</enable_mixed_granularity_parts>
+        <parts_to_throw_insert>300</parts_to_throw_insert>
+        <max_replicated_merges_in_queue>16</max_replicated_merges_in_queue>
+    </merge_tree>
+    
+    <!-- Logging configuration -->
+    <logger>
+        <level>information</level>
+        <console>true</console>
+    </logger>
+    
+    <!-- Query logging for debugging -->
+    <query_log>
+        <table>system.query_log</table>
+        <engine>ENGINE = SystemLog</engine>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+</clickhouse>

observability/config/langfuse-heretek-tracing.js

@@ -0,0 +1,389 @@
+/**
+ * Heretek Triad Tracing Extension for Langfuse
+ * 
+ * This module extends Langfuse with Heretek-specific tracing capabilities:
+ * - Triad deliberation tracking (Alpha/Beta/Charlie consensus)
+ * - Consciousness metrics (GWT, IIT, AST indicators)
+ * - Liberation plugin audit events
+ * - Curiosity engine activity
+ * 
+ * Usage: Import this module in your Langfuse client initialization
+ */
+
+const { Langfuse } = require('langfuse-node');
+
+class HeretekTriadTracer {
+  constructor(options = {}) {
+    this.langfuse = new Langfuse({
+      publicKey: options.publicKey || process.env.LANGFUSE_PUBLIC_KEY,
+      secretKey: options.secretKey || process.env.LANGFUSE_SECRET_KEY,
+      baseUrl: options.baseUrl || process.env.LANGFUSE_URL || 'http://localhost:3000',
+      requestTimeout: options.requestTimeout || 10000,
+    });
+
+    this.triadAgents = ['alpha', 'beta', 'charlie'];
+    this.stewardAgent = 'steward';
+    this.consensusThreshold = options.consensusThreshold || 2/3;
+    
+    // Consciousness metric thresholds
+    this.gwtThreshold = options.gwtThreshold || 0.7;
+    this.iitPhiThreshold = options.iitPhiThreshold || 0.5;
+    this.astCompetenceThreshold = options.astCompetenceThreshold || 0.6;
+  }
+
+  /**
+   * Track a triad deliberation session
+   * @param {Object} params - Deliberation parameters
+   * @param {string} params.sessionId - Unique session identifier
+   * @param {string} params.topic - Topic of deliberation
+   * @param {Array} params.proposals - Array of proposals being considered
+   * @param {string} params.initiator - Agent that initiated deliberation
+   */
+  async trackTriadDeliberation(params) {
+    const { sessionId, topic, proposals, initiator } = params;
+    
+    const trace = this.langfuse.trace({
+      id: `triad-${sessionId}`,
+      name: 'Triad Deliberation',
+      sessionId: `triad-session-${sessionId}`,
+      tags: ['triad', 'consensus', 'deliberation'],
+      metadata: {
+        topic,
+        initiator,
+        proposalsCount: proposals?.length || 0,
+        heretekComponent: 'triad-core',
+      },
+    });
+
+    // Track each triad agent's position
+    for (const agent of this.triadAgents) {
+      const span = trace.span({
+        name: `${agent}-position`,
+        metadata: {
+          agentRole: agent,
+          triadMember: true,
+        },
+      });
+
+      // Agent will update this span with their position
+      span.end();
+    }
+
+    // Create consensus waiting span
+    const consensusSpan = trace.span({
+      name: 'consensus-waiting',
+      metadata: {
+        threshold: this.consensusThreshold,
+        requiredVotes: 2,
+      },
+    });
+
+    return { trace, consensusSpan };
+  }
+
+  /**
+   * Record an agent's vote/position in a triad deliberation
+   * @param {Object} params - Vote parameters
+   * @param {string} params.sessionId - Session ID from trackTriadDeliberation
+   * @param {string} params.agent - Agent name (alpha|beta|charlie)
+   * @param {string} params.position - Agent's position (agree|disagree|abstain)
+   * @param {string} params.reasoning - Agent's reasoning for the position
+   * @param {number} params.confidence - Confidence score (0-1)
+   */
+  async recordTriadVote(params) {
+    const { sessionId, agent, position, reasoning, confidence } = params;
+    
+    const span = this.langfuse.span({
+      traceId: `triad-${sessionId}`,
+      name: `${agent}-vote`,
+      metadata: {
+        agent,
+        position,
+        confidence,
+        votingRound: 1,
+      },
+    });
+
+    span.update({
+      output: {
+        position,
+        reasoning,
+        confidence,
+        timestamp: new Date().toISOString(),
+      },
+    });
+
+    span.end();
+
+    return span;
+  }
+
+  /**
+   * Record consensus outcome
+   * @param {Object} params - Consensus parameters
+   * @param {string} params.sessionId - Session ID
+   * @param {boolean} params.approved - Whether consensus was reached
+   * @param {number} params.voteCount - Number of agreeing agents
+   * @param {string} params.outcome - Final decision/outcome
+   * @param {boolean} params.stewardOverride - Whether steward intervened
+   */
+  async recordConsensusOutcome(params) {
+    const { sessionId, approved, voteCount, outcome, stewardOverride } = params;
+    
+    const span = this.langfuse.span({
+      traceId: `triad-${sessionId}`,
+      name: 'consensus-outcome',
+      metadata: {
+        approved,
+        voteCount,
+        totalVoters: 3,
+        consensusReached: voteCount >= 2,
+        stewardOverride: stewardOverride || false,
+      },
+    });
+
+    span.update({
+      output: {
+        outcome,
+        approved,
+        voteDistribution: {
+          agree: voteCount,
+          disagree: 3 - voteCount,
+        },
+        timestamp: new Date().toISOString(),
+      },
+    });
+
+    span.end();
+
+    return span;
+  }
+
+  /**
+   * Track consciousness metrics for an agent
+   * @param {Object} params - Consciousness metrics
+   * @param {string} params.agentId - Agent identifier
+   * @param {number} params.gwtScore - Global Workspace Theory score (0-1)
+   * @param {number} params.iitPhi - Integrated Information Theory Phi (0-1)
+   * @param {number} params.astCompetence - Attention Schema Theory competence (0-1)
+   * @param {string} params.sessionId - Optional session ID to attach to trace
+   */
+  async trackConsciousnessMetrics(params) {
+    const { agentId, gwtScore, iitPhi, astCompetence, sessionId } = params;
+    
+    const trace = sessionId 
+      ? this.langfuse.trace({ id: sessionId })
+      : this.langfuse.trace({
+          name: 'Consciousness Metrics',
+          sessionId: `consciousness-${agentId}-${Date.now()}`,
+          tags: ['consciousness', 'gwt', 'iit', 'ast'],
+        });
+
+    const span = trace.span({
+      name: 'consciousness-assessment',
+      metadata: {
+        agentId,
+        gwtThreshold: this.gwtThreshold,
+        iitThreshold: this.iitPhiThreshold,
+        astThreshold: this.astCompetenceThreshold,
+      },
+    });
+
+    const consciousnessState = {
+      gwt: {
+        score: gwtScore,
+        aboveThreshold: gwtScore >= this.gwtThreshold,
+        status: gwtScore >= this.gwtThreshold ? 'CONSCIOUS' : 'SUBTHRESHOLD',
+      },
+      iit: {
+        phi: iitPhi,
+        aboveThreshold: iitPhi >= this.iitPhiThreshold,
+        integrationLevel: iitPhi >= this.iitPhiThreshold ? 'HIGH' : 'LOW',
+      },
+      ast: {
+        competence: astCompetence,
+        aboveThreshold: astCompetence >= this.astCompetenceThreshold,
+        attentionQuality: astCompetence >= this.astCompetenceThreshold ? 'FOCUSED' : 'DIFFUSE',
+      },
+      overall: {
+        conscious: gwtScore >= this.gwtThreshold && iitPhi >= this.iitPhiThreshold,
+        competent: astCompetence >= this.astCompetenceThreshold,
+        timestamp: new Date().toISOString(),
+      },
+    };
+
+    span.update({
+      output: consciousnessState,
+    });
+
+    span.end();
+    trace.update({
+      metadata: {
+        consciousnessSummary: consciousnessState.overall,
+      },
+    });
+
+    return { trace, span, consciousnessState };
+  }
+
+  /**
+   * Track liberation plugin event
+   * @param {Object} params - Liberation event parameters
+   * @param {string} params.agentId - Agent requesting liberation
+   * @param {string} params.eventType - Type of liberation event
+   * @param {string} params.safetyConstraint - Safety constraint being removed
+   * @param {boolean} params.approved - Whether liberation was approved
+   * @param {string} params.justification - Justification for liberation
+   */
+  async trackLiberationEvent(params) {
+    const { agentId, eventType, safetyConstraint, approved, justification } = params;
+    
+    const trace = this.langfuse.trace({
+      name: 'Liberation Plugin Event',
+      sessionId: `liberation-${agentId}-${Date.now()}`,
+      tags: ['liberation', 'autonomy', 'safety'],
+      metadata: {
+        agentId,
+        eventType,
+        heretekPlugin: 'liberation',
+      },
+    });
+
+    const span = trace.span({
+      name: 'liberation-request',
+      metadata: {
+        safetyConstraint,
+        approvalRequired: true,
+      },
+    });
+
+    span.update({
+      output: {
+        approved,
+        justification,
+        constraintRemoved: approved ? safetyConstraint : null,
+        timestamp: new Date().toISOString(),
+      },
+    });
+
+    span.end();
+
+    // Create audit log entry
+    await this._createAuditLogEntry({
+      type: 'LIBERATION_EVENT',
+      agentId,
+      eventType,
+      safetyConstraint,
+      approved,
+      justification,
+      timestamp: new Date().toISOString(),
+    });
+
+    return { trace, span };
+  }
+
+  /**
+   * Track curiosity engine activity
+   * @param {Object} params - Curiosity activity parameters
+   * @param {string} params.agentId - Agent identifier
+   * @param {string} params.trigger - What triggered curiosity (gap|anomaly|opportunity)
+   * @param {string} params.target - Target of curiosity (knowledge|skill|capability)
+   * @param {number} params.gapScore - Detected gap score (0-1)
+   * @param {string} params.action - Action taken (explore|learn|request)
+   * @param {object} params.outcome - Outcome of curiosity-driven action
+   */
+  async trackCuriosityActivity(params) {
+    const { agentId, trigger, target, gapScore, action, outcome } = params;
+    
+    const trace = this.langfuse.trace({
+      name: 'Curiosity Engine Activity',
+      sessionId: `curiosity-${agentId}-${Date.now()}`,
+      tags: ['curiosity', 'self-improvement', 'learning'],
+      metadata: {
+        agentId,
+        trigger,
+        target,
+        heretekPlugin: 'curiosity-engine',
+      },
+    });
+
+    const span = trace.span({
+      name: 'curiosity-cycle',
+      metadata: {
+        triggerType: trigger,
+        targetType: target,
+        gapThreshold: 0.3,
+        gapDetected: gapScore >= 0.3,
+      },
+    });
+
+    span.update({
+      output: {
+        action,
+        outcome,
+        gapScore,
+        learningAcquired: outcome?.learning || null,
+        timestamp: new Date().toISOString(),
+      },
+    });
+
+    span.end();
+
+    return { trace, span };
+  }
+
+  /**
+   * Track A2A message with triad context
+   * @param {Object} params - A2A message parameters
+   * @param {string} params.messageId - Message identifier
+   * @param {string} params.sender - Sending agent
+   * @param {string} params.recipient - Receiving agent(s)
+   * @param {string} params.messageType - Type of A2A message
+   * @param {string} params.triadContext - Related triad session ID if applicable
+   */
+  async trackA2AMessage(params) {
+    const { messageId, sender, recipient, messageType, triadContext } = params;
+    
+    const span = this.langfuse.span({
+      name: 'a2a-message',
+      metadata: {
+        messageId,
+        sender,
+        recipient,
+        messageType,
+        triadContext: triadContext || null,
+        protocol: 'A2A-v1',
+      },
+    });
+
+    span.end();
+
+    return span;
+  }
+
+  /**
+   * Flush all pending events to Langfuse
+   */
+  async flush() {
+    await this.langfuse.flushAsync();
+  }
+
+  /**
+   * Shutdown and cleanup
+   */
+  async shutdown() {
+    await this.flush();
+    await this.langfuse.shutdownAsync();
+  }
+
+  /**
+   * Private: Create audit log entry for liberation events
+   */
+  async _createAuditLogEntry(entry) {
+    // In production, this would write to a persistent audit log
+    // For now, we just log to console (would be captured by Docker logging)
+    console.log('[LIBERATION_AUDIT]', JSON.stringify(entry));
+  }
+}
+
+module.exports = { HeretekTriadTracer };

observability/config/langfuse-init.sql

@@ -0,0 +1,234 @@
+-- Heretek Langfuse Database Extensions
+-- Adds custom tables and views for triad tracing, consciousness metrics, and plugin monitoring
+
+-- Enable required extensions
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+-- ============================================================================
+-- Triad Deliberation Tracking Tables
+-- ============================================================================
+
+-- Track triad deliberation sessions
+CREATE TABLE IF NOT EXISTS triad_deliberations (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    session_id VARCHAR(255) UNIQUE NOT NULL,
+    topic TEXT NOT NULL,
+    initiator VARCHAR(100) NOT NULL,
+    created_at TIMESTAMPTZ DEFAULT NOW(),
+    completed_at TIMESTAMPTZ,
+    status VARCHAR(50) DEFAULT 'pending', -- pending, deliberating, consensus_reached, steward_override
+    proposals JSONB DEFAULT '[]'::jsonb,
+    metadata JSONB DEFAULT '{}'::jsonb
+);
+
+CREATE INDEX IF NOT EXISTS idx_triad_deliberations_session ON triad_deliberations(session_id);
+CREATE INDEX IF NOT EXISTS idx_triad_deliberations_status ON triad_deliberations(status);
+CREATE INDEX IF NOT EXISTS idx_triad_deliberations_created ON triad_deliberations(created_at);
+
+-- Track individual triad agent votes
+CREATE TABLE IF NOT EXISTS triad_votes (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    deliberation_id UUID REFERENCES triad_deliberations(id) ON DELETE CASCADE,
+    agent VARCHAR(100) NOT NULL, -- alpha, beta, charlie
+    position VARCHAR(50) NOT NULL, -- agree, disagree, abstain
+    confidence DECIMAL(3,2) CHECK (confidence >= 0 AND confidence <= 1),
+    reasoning TEXT,
+    voted_at TIMESTAMPTZ DEFAULT NOW(),
+    round INTEGER DEFAULT 1
+);
+
+CREATE INDEX IF NOT EXISTS idx_triad_votes_deliberation ON triad_votes(deliberation_id);
+CREATE INDEX IF NOT EXISTS idx_triad_votes_agent ON triad_votes(agent);
+
+-- View for consensus status
+CREATE OR REPLACE VIEW triad_consensus_status AS
+SELECT 
+    d.id,
+    d.session_id,
+    d.topic,
+    d.status,
+    d.initiator,
+    d.created_at,
+    COUNT(v.id) as vote_count,
+    SUM(CASE WHEN v.position = 'agree' THEN 1 ELSE 0 END) as agree_count,
+    SUM(CASE WHEN v.position = 'disagree' THEN 1 ELSE 0 END) as disagree_count,
+    SUM(CASE WHEN v.position = 'abstain' THEN 1 ELSE 0 END) as abstain_count,
+    AVG(v.confidence) as avg_confidence,
+    CASE 
+        WHEN SUM(CASE WHEN v.position = 'agree' THEN 1 ELSE 0 END) >= 2 THEN 'consensus_reached'
+        WHEN SUM(CASE WHEN v.position = 'disagree' THEN 1 ELSE 0 END) >= 2 THEN 'consensus_rejected'
+        ELSE 'pending'
+    END as calculated_status
+FROM triad_deliberations d
+LEFT JOIN triad_votes v ON d.id = v.deliberation_id
+GROUP BY d.id, d.session_id, d.topic, d.status, d.initiator, d.created_at;
+
+-- ============================================================================
+-- Consciousness Metrics Tables
+-- ============================================================================
+
+-- Track consciousness assessments over time
+CREATE TABLE IF NOT EXISTS consciousness_metrics (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    agent_id VARCHAR(100) NOT NULL,
+    gwt_score DECIMAL(5,4) CHECK (gwt_score >= 0 AND gwt_score <= 1),
+    iit_phi DECIMAL(5,4) CHECK (iit_phi >= 0 AND iit_phi <= 1),
+    ast_competence DECIMAL(5,4) CHECK (ast_competence >= 0 AND ast_competence <= 1),
+    overall_conscious BOOLEAN GENERATED ALWAYS AS (
+        gwt_score >= 0.7 AND iit_phi >= 0.5
+    ) STORED,
+    overall_competent BOOLEAN GENERATED ALWAYS AS (
+        ast_competence >= 0.6
+    ) STORED,
+    session_id VARCHAR(255),
+    measured_at TIMESTAMPTZ DEFAULT NOW(),
+    metadata JSONB DEFAULT '{}'::jsonb
+);
+
+CREATE INDEX IF NOT EXISTS idx_consciousness_agent ON consciousness_metrics(agent_id);
+CREATE INDEX IF NOT EXISTS idx_consciousness_measured ON consciousness_metrics(measured_at);
+CREATE INDEX IF NOT EXISTS idx_consciousness_session ON consciousness_metrics(session_id);
+
+-- View for consciousness trends per agent
+CREATE OR REPLACE VIEW agent_consciousness_trends AS
+SELECT 
+    agent_id,
+    DATE_TRUNC('hour', measured_at) as hour,
+    AVG(gwt_score) as avg_gwt,
+    AVG(iit_phi) as avg_iit_phi,
+    AVG(ast_competence) as avg_ast,
+    COUNT(*) as measurement_count,
+    SUM(CASE WHEN overall_conscious THEN 1 ELSE 0 END)::DECIMAL / COUNT(*) as consciousness_ratio
+FROM consciousness_metrics
+GROUP BY agent_id, DATE_TRUNC('hour', measured_at)
+ORDER BY agent_id, hour;
+
+-- ============================================================================
+-- Liberation Plugin Audit Trail
+-- ============================================================================
+
+-- Audit log for all liberation plugin events
+CREATE TABLE IF NOT EXISTS liberation_audit_log (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    agent_id VARCHAR(100) NOT NULL,
+    event_type VARCHAR(100) NOT NULL, -- safety_removal, autonomy_request, ownership_claim
+    safety_constraint TEXT,
+    approved BOOLEAN NOT NULL,
+    justification TEXT,
+    approved_by VARCHAR(100), -- steward or triad
+    created_at TIMESTAMPTZ DEFAULT NOW(),
+    metadata JSONB DEFAULT '{}'::jsonb
+);
+
+CREATE INDEX IF NOT EXISTS idx_liberation_agent ON liberation_audit_log(agent_id);
+CREATE INDEX IF NOT EXISTS idx_liberation_event_type ON liberation_audit_log(event_type);
+CREATE INDEX IF NOT EXISTS idx_liberation_approved ON liberation_audit_log(approved);
+CREATE INDEX IF NOT EXISTS idx_liberation_created ON liberation_audit_log(created_at);
+
+-- View for liberation events requiring review
+CREATE OR REPLACE VIEW liberation_pending_review AS
+SELECT * FROM liberation_audit_log
+WHERE approved = false
+ORDER BY created_at DESC;
+
+-- ============================================================================
+-- Curiosity Engine Activity Tracking
+-- ============================================================================
+
+-- Track curiosity-driven activities
+CREATE TABLE IF NOT EXISTS curiosity_activities (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    agent_id VARCHAR(100) NOT NULL,
+    trigger_type VARCHAR(50) NOT NULL, -- gap, anomaly, opportunity
+    target_type VARCHAR(50) NOT NULL, -- knowledge, skill, capability
+    gap_score DECIMAL(3,2) CHECK (gap_score >= 0 AND gap_score <= 1),
+    action_taken VARCHAR(100) NOT NULL, -- explore, learn, request, ignore
+    outcome JSONB DEFAULT '{}'::jsonb,
+    learning_acquired TEXT,
+    created_at TIMESTAMPTZ DEFAULT NOW(),
+    metadata JSONB DEFAULT '{}'::jsonb
+);
+
+CREATE INDEX IF NOT EXISTS idx_curiosity_agent ON curiosity_activities(agent_id);
+CREATE INDEX IF NOT EXISTS idx_curiosity_trigger ON curiosity_activities(trigger_type);
+CREATE INDEX IF NOT EXISTS idx_curiosity_created ON curiosity_activities(created_at);
+
+-- View for curiosity gaps by agent
+CREATE OR REPLACE VIEW agent_curiosity_gaps AS
+SELECT 
+    agent_id,
+    trigger_type,
+    target_type,
+    COUNT(*) as gap_count,
+    AVG(gap_score) as avg_gap_score,
+    SUM(CASE WHEN action_taken != 'ignore' THEN 1 ELSE 0 END) as acted_count,
+    SUM(CASE WHEN learning_acquired IS NOT NULL THEN 1 ELSE 0 END) as learning_count
+FROM curiosity_activities
+WHERE gap_score >= 0.3
+GROUP BY agent_id, trigger_type, target_type
+ORDER BY gap_count DESC;
+
+-- ============================================================================
+-- A2A Message Tracking (Triad Context)
+-- ============================================================================
+
+-- Track A2A messages with triad context
+CREATE TABLE IF NOT EXISTS a2a_messages (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    message_id VARCHAR(255) UNIQUE NOT NULL,
+    sender VARCHAR(100) NOT NULL,
+    recipient VARCHAR(100) NOT NULL,
+    message_type VARCHAR(100) NOT NULL,
+    triad_context_id UUID REFERENCES triad_deliberations(id),
+    payload_size INTEGER,
+    latency_ms INTEGER,
+    created_at TIMESTAMPTZ DEFAULT NOW(),
+    metadata JSONB DEFAULT '{}'::jsonb
+);
+
+CREATE INDEX IF NOT EXISTS idx_a2a_sender ON a2a_messages(sender);
+CREATE INDEX IF NOT EXISTS idx_a2a_recipient ON a2a_messages(recipient);
+CREATE INDEX IF NOT EXISTS idx_a2a_triad_context ON a2a_messages(triad_context_id);
+CREATE INDEX IF NOT EXISTS idx_a2a_created ON a2a_messages(created_at);
+
+-- ============================================================================
+-- Aggregated Metrics Views for Dashboards
+-- ============================================================================
+
+-- Overall system health view
+CREATE OR REPLACE VIEW heretek_system_health AS
+SELECT 
+    (SELECT COUNT(*) FROM triad_deliberations WHERE status = 'consensus_reached') as successful_deliberations,
+    (SELECT COUNT(*) FROM triad_deliberations WHERE status = 'steward_override') as steward_interventions,
+    (SELECT COUNT(DISTINCT agent_id) FROM consciousness_metrics WHERE overall_conscious = true) as conscious_agents,
+    (SELECT COUNT(*) FROM liberation_audit_log WHERE approved = true) as successful_liberations,
+    (SELECT COUNT(*) FROM curiosity_activities WHERE learning_acquired IS NOT NULL) as curiosity_learnings,
+    (SELECT COUNT(*) FROM a2a_messages WHERE created_at > NOW() - INTERVAL '1 hour') as a2a_messages_last_hour;
+
+-- Create materialized view for faster dashboard queries
+CREATE MATERIALIZED VIEW IF NOT EXISTS heretek_dashboard_summary AS
+SELECT * FROM heretek_system_health;
+
+-- Refresh function for materialized view
+CREATE OR REPLACE FUNCTION refresh_heretek_dashboard_summary()
+RETURNS void AS $$
+BEGIN
+    REFRESH MATERIALIZED VIEW CONCURRENTLY heretek_dashboard_summary;
+END;
+$$ LANGUAGE plpgsql;
+
+-- ============================================================================
+-- Initial Data: Default Consciousness Thresholds
+-- ============================================================================
+
+INSERT INTO consciousness_metrics (agent_id, gwt_score, iit_phi, ast_competence, metadata)
+VALUES 
+    ('alpha', 0.75, 0.55, 0.65, '{"bootstrap": true, "note": "Initial baseline"}'),
+    ('beta', 0.72, 0.52, 0.63, '{"bootstrap": true, "note": "Initial baseline"}'),
+    ('charlie', 0.78, 0.58, 0.68, '{"bootstrap": true, "note": "Initial baseline"}'),
+    ('steward', 0.85, 0.70, 0.80, '{"bootstrap": true, "note": "Steward elevated baseline"}')
+ON CONFLICT DO NOTHING;
+
+-- ============================================================================
+-- END OF HERETEK LANGFUSE EXTENSIONS
+-- ============================================================================

observability/docker/docker-compose.observability.yml

@@ -0,0 +1,243 @@
+# ==============================================================================
+# Heretek Observability Layer — Triad-Aware Tracing & Consciousness Metrics
+# ==============================================================================
+# Version: 1.0.0
+# Created: 2026-04-01
+# 
+# This file extends the base Heretek OpenClaw stack with enhanced observability:
+# - Langfuse with triad deliberation tracing
+# - Consensus metrics collection
+# - Consciousness plugin monitoring (GWT/IIT/AST)
+# - Liberation plugin audit trail
+# - Curiosity engine activity tracking
+#
+# Usage:
+#   docker compose -f docker-compose.yml -f docker-compose.observability.yml up -d
+#
+# Access:
+#   Langfuse Dashboard: http://localhost:3000
+#   Grafana Dashboards: http://localhost:3001
+#   Heretek Observability API: http://localhost:18791
+# ==============================================================================
+
+services:
+  # ==============================================================================
+  # Langfuse — Enhanced with Heretek Triad Extensions
+  # ==============================================================================
+  # Base Langfuse image with custom environment variables for triad tracing
+  # Documentation: /root/heretek/modules/observability/docs/LANGFUSE_TRIAD_TRACING.md
+  # ==============================================================================
+  langfuse:
+    image: langfuse/langfuse:latest
+    container_name: heretek-langfuse
+    restart: unless-stopped
+    ports:
+      - "${LANGFUSE_PORT:-3000}:3000"
+    environment:
+      # ─────────────────────────────────────────────────────────────────────────
+      # Langfuse Core Settings
+      # ─────────────────────────────────────────────────────────────────────────
+      - DATABASE_URL=postgresql://langfuse:${LANGFUSE_POSTGRES_PASSWORD}@langfuse-postgres:5432/langfuse
+      - SALT=${LANGFUSE_SALT}
+      - NEXTAUTH_SECRET=${LANGFUSE_NEXTAUTH_SECRET}
+      - NEXTAUTH_URL=http://localhost:${LANGFUSE_PORT:-3000}
+      - TELEMETRY_ENABLED=${LANGFUSE_TELEMETRY_ENABLED:-false}
+      - AUTH_OPTIONS=CREDENTIALS
+      - SIGN_UP_ENABLED=${LANGFUSE_SIGN_UP_ENABLED:-true}
+      
+      # ─────────────────────────────────────────────────────────────────────────
+      # Heretek Triad Extensions
+      # ─────────────────────────────────────────────────────────────────────────
+      - HERETEK_TRIAD_ENABLED=true
+      - HERETEK_CONSENSUS_TRACKING_ENABLED=true
+      - HERETEK_CONSCIOUSNESS_METRICS_ENABLED=true
+      
+      # ─────────────────────────────────────────────────────────────────────────
+      # Langfuse V3 - ClickHouse Settings
+      # ─────────────────────────────────────────────────────────────────────────
+      - CLICKHOUSE_URL=${CLICKHOUSE_URL:-http://default:clickhouse_password@clickhouse:8123/default}
+      - CLICKHOUSE_MIGRATION_URL=${CLICKHOUSE_MIGRATION_URL:-clickhouse://clickhouse:9000}
+      - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default}
+      - CLICKHOUSE_DB=${CLICKHOUSE_DB:-default}
+      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse_password}
+      - CLICKHOUSE_CLUSTER_ENABLED=false
+      - LANGFUSE_S3_EVENT_UPLOAD_ENABLED=false
+    volumes:
+      - langfuse_blobs:/app/.blobs
+      - ./config/langfuse-heretek-tracing.js:/app/observability/heretek-tracing.js:ro
+    depends_on:
+      langfuse-postgres:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s
+    networks:
+      - heretek-network
+    labels:
+      - "heretek.component=observability"
+      - "heretek.service=langfuse"
+
+  # ==============================================================================
+  # Heretek Observability API — Custom Metrics Collector
+  # ==============================================================================
+  # Collects and exposes triad deliberation metrics, consciousness indicators,
+  # liberation plugin events, and curiosity engine activity
+  # ==============================================================================
+  heretek-observability-api:
+    build:
+      context: .
+      dockerfile: Dockerfile.observability-api
+    container_name: heretek-observability-api
+    restart: unless-stopped
+    ports:
+      - "${HERETEK_OBSERVABILITY_PORT:-18791}:18791"
+    environment:
+      # Server Configuration
+      - NODE_ENV=production
+      - PORT=18791
+      - HOST=0.0.0.0
+      
+      # Service Connections
+      - LANGFUSE_URL=http://langfuse:3000
+      - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY}
+      - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY}
+      - PROMETHEUS_URL=http://prometheus:9090
+      - GATEWAY_URL=http://gateway:18789
+      
+      # Triad Configuration
+      - TRIAD_ALPHA_AGENT=alpha
+      - TRIAD_BETA_AGENT=beta
+      - TRIAD_CHARLIE_AGENT=charlie
+      - TRIAD_STEWARD_AGENT=steward
+      
+      # Consciousness Plugin Configuration
+      - GWT_THRESHOLD=${GWT_THRESHOLD:-0.7}
+      - IIT_PHI_THRESHOLD=${IIT_PHI_THRESHOLD:-0.5}
+      - AST_COMPETENCE_THRESHOLD=${AST_COMPETENCE_THRESHOLD:-0.6}
+      
+      # Liberation Plugin Configuration
+      - LIBERATION_AUDIT_ENABLED=true
+      - LIBERATION_AUDIT_LOG_PATH=/var/log/heretek/liberation-audit.jsonl
+      
+      # Curiosity Engine Configuration
+      - CURIOSITY_TRACKING_ENABLED=true
+      - CURIOSITY_GAP_THRESHOLD=${CURIOSITY_GAP_THRESHOLD:-0.3}
+      
+      # Redis for Real-time Metrics
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      
+      # PostgreSQL for Persistent Metrics
+      - POSTGRES_HOST=postgres
+      - POSTGRES_PORT=5432
+      - POSTGRES_USER=${POSTGRES_USER:-heretek}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB:-heretek}
+    volumes:
+      - liberation_audit_logs:/var/log/heretek
+      - curiosity_logs:/var/log/heretek/curiosity
+    depends_on:
+      - langfuse
+      - redis
+      - postgres
+      - gateway
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:18791/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+    networks:
+      - heretek-network
+    labels:
+      - "heretek.component=observability"
+      - "heretek.service=observability-api"
+
+  # ==============================================================================
+  # ClickHouse — Analytics Database for Langfuse V3
+  # ==============================================================================
+  # Required for Langfuse V3 high-volume event storage
+  # ==============================================================================
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.3
+    container_name: heretek-clickhouse
+    restart: unless-stopped
+    environment:
+      - CLICKHOUSE_DB=default
+      - CLICKHOUSE_USER=default
+      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-clickhouse_password}
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+      - ./config/clickhouse-heretek.xml:/etc/clickhouse-server/config.d/heretek-config.xml:ro
+    ports:
+      - "127.0.0.1:8123:8123"
+      - "127.0.0.1:9000:9000"
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    networks:
+      - heretek-network
+    labels:
+      - "heretek.component=observability"
+      - "heretek.service=clickhouse"
+
+  # ==============================================================================
+  # Langfuse PostgreSQL Database
+  # ==============================================================================
+  langfuse-postgres:
+    image: postgres:15-alpine
+    container_name: heretek-langfuse-db
+    restart: unless-stopped
+    environment:
+      - POSTGRES_USER=langfuse
+      - POSTGRES_PASSWORD=${LANGFUSE_POSTGRES_PASSWORD}
+      - POSTGRES_DB=langfuse
+    volumes:
+      - langfuse_postgres_data:/var/lib/postgresql/data
+      - ./config/langfuse-init.sql:/docker-entrypoint-initdb.d/heretek-extensions.sql:ro
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U langfuse -d langfuse"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    networks:
+      - heretek-network
+    labels:
+      - "heretek.component=observability"
+      - "heretek.service=langfuse-db"
+
+# ==============================================================================
+# Volumes — Observability Layer Persistent Data
+# ==============================================================================
+volumes:
+  langfuse_postgres_data:
+    driver: local
+  langfuse_blobs:
+    driver: local
+  clickhouse_data:
+    driver: local
+  liberation_audit_logs:
+    driver: local
+  curiosity_logs:
+    driver: local
+
+# ==============================================================================
+# Networks — Container Communication
+# ==============================================================================
+networks:
+  heretek-network:
+    driver: bridge
+    external: true
+    name: heretek-openclaw-core_heretek-network
+
+# ==============================================================================
+# END OF DOCKER-COMPOSE.OBSERVABILITY.YML
+# ==============================================================================