Merge branch 'main' of https://github.com/community-scripts/telemetry-service

2026-07-01 20:54:03 -04:00 · 2026-02-17 17:15:02 +01:00
parent ddf1fb5434 dcf8d17e9e
commit bbf391895b
6 changed files with 244 additions and 17 deletions
@@ -98,6 +98,11 @@ This service is designed with privacy in mind and is **GDPR/DSGVO compliant**:
 - ✅ **No tracking** - Session IDs are randomly generated and cannot be linked to users
 - ✅ **No third parties** - Data is only stored in our self-hosted PocketBase instance

+For full details, see:
+- **[Privacy & Telemetry Documentation](docs/PRIVACY.md)** — What we collect, how, and why
+- **[Records of Processing Activities (ROPA)](docs/ROPA.md)** — GDPR Art. 30
+- **[Technical & Organizational Measures (TOMS)](docs/TOMS.md)** — GDPR Art. 32
+
 ## License

 MIT License - see [LICENSE](LICENSE) file.
@@ -1731,8 +1731,8 @@ func (p *PBClient) FetchDashboardData(ctx context.Context, days int, repoSource

 		// === Extended metrics tracking ===

-		// Track tool executions (type="tool", tool name is in nsapp)
-		if r.Type == "tool" && r.NSAPP != "" {
+		// Track PVE tool executions (type="pve", tool name is in nsapp)
+		if r.Type == "pve" && r.NSAPP != "" {
 			toolCounts[r.NSAPP]++
 			data.TotalTools++
 		}
@@ -0,0 +1,155 @@
+# Privacy & Telemetry Documentation
+
+**Community-Scripts Telemetry Service**
+
+---
+
+## Overview
+
+The [Community-Scripts](https://github.com/community-scripts/ProxmoxVE) project includes an **opt-in** telemetry system that collects anonymous technical data when users install applications via Proxmox VE Helper-Scripts. This document explains what is collected, how it is processed, and how users can control it.
+
+**Dashboard:** [https://telemetry.community-scripts.org](https://telemetry.community-scripts.org)
+
+---
+
+## Consent & Opt-In
+
+Telemetry is **strictly opt-in**. On first use, a dialog asks users whether they want to share anonymous data:
+
+- **No option is pre-selected** — users must actively choose
+- Pressing Exit/Cancel defaults to **opt-out**
+- The choice is persisted in `/usr/local/community-scripts/diagnostics`
+- Users can change their preference at any time via the **Settings menu** during installation or by editing the config file directly
+
+### Changing Your Preference
+
+**Via Settings menu:** During any script installation, navigate to the Settings menu and select "Telemetry".
+
+**Via command line (on PVE host):**
+```bash
+# Opt out
+sed -i 's/^DIAGNOSTICS=.*/DIAGNOSTICS=no/' /usr/local/community-scripts/diagnostics
+
+# Opt in
+sed -i 's/^DIAGNOSTICS=.*/DIAGNOSTICS=yes/' /usr/local/community-scripts/diagnostics
+```
+
+**Inside a container (for addon scripts):**
+```bash
+# The same file exists inside containers created after opting in
+sed -i 's/^DIAGNOSTICS=.*/DIAGNOSTICS=no/' /usr/local/community-scripts/diagnostics
+```
+
+> **Note:** Changing the setting on the host affects all **new** containers. Existing containers retain the preference set at creation time.
+
+---
+
+## What We Collect
+
+All data is **purely technical** and **anonymous**. No field can identify a natural person.
+
+### Container & VM Installations (type: `lxc`, `vm`)
+
+| Field | Example | Purpose |
+|-------|---------|---------|
+| `nsapp` | `docker`, `homeassistant` | Which application was installed |
+| `status` | `success`, `failed` | Whether the installation succeeded |
+| `exit_code` | `0`, `1`, `130` | Exit code for error categorization |
+| `error` | `dpkg: error ...` | Truncated error message (failed installs only) |
+| `ct_type` | `1` (unprivileged) | Container privilege type |
+| `disk_size` | `8` | Disk size in GB |
+| `core_count` | `2` | Number of CPU cores |
+| `ram_size` | `2048` | RAM in MiB |
+| `os_type` | `debian` | Operating system |
+| `os_version` | `12` | OS version |
+| `pve_version` | `8.3` | Proxmox VE version |
+| `method` | `default`, `advanced` | Installation method used |
+| `install_duration` | `45` | Duration in seconds |
+| `random_id` | `a1b2c3d4-...` | Random UUID (not linked to user/system) |
+| `execution_id` | `a1b2c3d4-...` | Unique execution identifier |
+
+### PVE Tools (type: `pve`)
+
+| Field | Example | Purpose |
+|-------|---------|---------|
+| `nsapp` | `post-pve-install`, `microcode` | Which tool was executed |
+| `status` | `success`, `failed` | Execution result |
+| `exit_code` | `0` | Exit code |
+| `pve_version` | `8.3` | Proxmox VE version |
+| `install_duration` | `12` | Duration in seconds |
+
+### Addon Scripts (type: `addon`)
+
+| Field | Example | Purpose |
+|-------|---------|---------|
+| `nsapp` | `filebrowser`, `netdata` | Which addon was installed |
+| `status` | `success`, `failed` | Installation result |
+| `exit_code` | `0` | Exit code |
+| `os_type` | `debian` | Container OS |
+| `os_version` | `12` | Container OS version |
+| `install_duration` | `30` | Duration in seconds |
+
+---
+
+## What We Do NOT Collect
+
+- **No IP addresses** — not logged, not stored, not forwarded
+- **No hostnames** or domain names
+- **No MAC addresses** or hardware serial numbers
+- **No user credentials**, passwords, or API tokens
+- **No network configuration** or internal IP addresses
+- **No file paths**, directory listings, or file contents
+- **No personal data** of any kind (GDPR Art. 4 Nr. 1)
+
+---
+
+## How Data Is Used
+
+The collected data serves the following purposes **exclusively**:
+
+1. **Script Quality** — Identify scripts with high failure rates and fix them
+2. **Popularity Ranking** — Understand which scripts are most used to prioritize maintenance
+3. **Resource Trends** — Analyze typical CPU, RAM, and disk allocations
+4. **OS Compatibility** — Track which OS versions are in use
+5. **Error Analysis** — Categorize common failure patterns for faster debugging
+
+All aggregated statistics are publicly visible on the [dashboard](https://telemetry.community-scripts.org).
+
+---
+
+## Data Processing & Storage
+
+| Aspect | Details |
+|--------|---------|
+| **Processor** | Self-hosted on IONOS VPS in Germany (Frankfurt/Berlin) |
+| **Database** | PocketBase (SQLite-based, self-hosted) |
+| **Retention** | Data is stored indefinitely for trend analysis |
+| **Encryption** | TLS 1.3 in transit, encrypted storage at rest |
+| **Access** | Write: telemetry service only. Read: aggregated dashboard (public) |
+| **Third parties** | None — no data is shared with or sold to third parties |
+| **Backups** | Automated, encrypted, same data center |
+
+---
+
+## Legal Basis (GDPR)
+
+**Art. 6(1)(f) GDPR — Legitimate Interest**
+
+The legitimate interest lies in improving open-source software for the community. Since:
+- No personal data is collected (Art. 4 Nr. 1 GDPR)
+- Data collection is opt-in with active consent
+- Users can withdraw at any time
+- All data is anonymous and cannot identify individuals
+
+the processing is GDPR-compliant. For detailed documentation, see:
+- [Records of Processing Activities (ROPA)](ROPA.md) — Art. 30 GDPR
+- [Technical & Organizational Measures (TOMS)](TOMS.md) — Art. 32 GDPR
+
+---
+
+## Contact & Issues
+
+- **Questions:** Open an issue at [telemetry-service](https://github.com/community-scripts/telemetry-service/issues)
+- **Privacy concerns:** Open an issue with the `privacy` label
+- **Source code:** Fully open source — review the [service implementation](../service.go)
+- **Discussion:** [ProxmoxVE Discussions](https://github.com/community-scripts/ProxmoxVE/discussions)
@@ -7,4 +7,5 @@ require github.com/redis/go-redis/v9 v9.17.3
 require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/mattn/go-sqlite3 v1.14.34 // indirect
 )
@@ -6,5 +6,7 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
+github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/redis/go-redis/v9 v9.17.3 h1:fN29NdNrE17KttK5Ndf20buqfDZwGNgoUr9qjl1DQx4=
 github.com/redis/go-redis/v9 v9.17.3/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
@@ -73,10 +73,11 @@ type Config struct {
 // TelemetryIn matches payload from api.func (bash client)
 type TelemetryIn struct {
 	// Required
-	RandomID string `json:"random_id"`         // Session UUID
-	Type     string `json:"type"`              // "lxc", "vm", "tool", "addon"
-	NSAPP    string `json:"nsapp"`             // Application name (e.g., "jellyfin")
-	Status   string `json:"status"`            // "installing", "success", "failed", "aborted", "unknown"
+	RandomID    string `json:"random_id"`         // Session UUID
+	ExecutionID string `json:"execution_id,omitempty"` // Unique execution ID (unique-indexed in PocketBase)
+	Type        string `json:"type"`              // "lxc", "vm", "pve", "addon"
+	NSAPP       string `json:"nsapp"`             // Application name (e.g., "jellyfin")
+	Status      string `json:"status"`            // "installing", "success", "failed", "aborted", "unknown"

 	// Container/VM specs
 	CTType    int `json:"ct_type,omitempty"`    // 1=unprivileged, 2=privileged/VM
@@ -120,11 +121,12 @@ type TelemetryIn struct {

 // TelemetryOut is sent to PocketBase (matches telemetry collection)
 type TelemetryOut struct {
-	RandomID  string `json:"random_id"`
-	Type      string `json:"type"`
-	NSAPP     string `json:"nsapp"`
-	Status    string `json:"status"`
-	CTType    int    `json:"ct_type,omitempty"`
+	RandomID    string `json:"random_id"`
+	ExecutionID string `json:"execution_id,omitempty"`
+	Type        string `json:"type"`
+	NSAPP       string `json:"nsapp"`
+	Status      string `json:"status"`
+	CTType      int    `json:"ct_type,omitempty"`
 	DiskSize  int    `json:"disk_size,omitempty"`
 	CoreCount int    `json:"core_count,omitempty"`
 	RAMSize   int    `json:"ram_size,omitempty"`
@@ -152,6 +154,7 @@ type TelemetryOut struct {
 // TelemetryStatusUpdate contains only fields needed for status updates
 type TelemetryStatusUpdate struct {
 	Status          string `json:"status"`
+	ExecutionID     string `json:"execution_id,omitempty"`
 	Error           string `json:"error,omitempty"`
 	ExitCode        int    `json:"exit_code"`
 	InstallDuration int    `json:"install_duration,omitempty"`
@@ -254,11 +257,11 @@ func (p *PBClient) FindRecordByRandomID(ctx context.Context, randomID string) (s
 		return "", err
 	}

-	// URL encode the filter
+	// URL encode the filter to ensure special characters are handled correctly
 	filter := fmt.Sprintf("random_id='%s'", randomID)
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet,
 		fmt.Sprintf("%s/api/collections/%s/records?filter=%s&fields=id&perPage=1",
-			p.baseURL, p.targetColl, filter),
+			p.baseURL, p.targetColl, url.QueryEscape(filter)),
 		nil,
 	)
 	if err != nil {
@@ -291,6 +294,48 @@ func (p *PBClient) FindRecordByRandomID(ctx context.Context, randomID string) (s
 	return result.Items[0].ID, nil
 }

+// FindRecordByExecutionID searches for an existing record by execution_id (unique-indexed, O(1) lookup)
+func (p *PBClient) FindRecordByExecutionID(ctx context.Context, executionID string) (string, error) {
+	if err := p.ensureAuth(ctx); err != nil {
+		return "", err
+	}
+
+	filter := fmt.Sprintf("execution_id='%s'", executionID)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet,
+		fmt.Sprintf("%s/api/collections/%s/records?filter=%s&fields=id&perPage=1",
+			p.baseURL, p.targetColl, url.QueryEscape(filter)),
+		nil,
+	)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Authorization", "Bearer "+p.token)
+
+	resp, err := p.http.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", fmt.Errorf("pocketbase search by execution_id failed: %s", resp.Status)
+	}
+
+	var result struct {
+		Items []struct {
+			ID string `json:"id"`
+		} `json:"items"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		return "", err
+	}
+
+	if len(result.Items) == 0 {
+		return "", nil // Not found
+	}
+	return result.Items[0].ID, nil
+}
+
 // UpdateTelemetryStatus updates only status, error, and exit_code of an existing record
 func (p *PBClient) UpdateTelemetryStatus(ctx context.Context, recordID string, update TelemetryStatusUpdate) error {
 	if err := p.ensureAuth(ctx); err != nil {
@@ -422,7 +467,9 @@ func (p *PBClient) FetchRecordsPaginated(ctx context.Context, page, limit int, s
 // All records go to the same collection; repo_source is stored as a field.
 //
 // For status="installing": always creates a new record.
-// For status!="installing": updates existing record (found by random_id).
+// For status!="installing": updates existing record.
+//   - Prefers execution_id lookup (unique-indexed, O(1)) when available.
+//   - Falls back to random_id lookup (filter query) for old clients.
 func (p *PBClient) UpsertTelemetry(ctx context.Context, payload TelemetryOut) error {
 	// For "installing" status, always create new record
 	if payload.Status == "installing" {
@@ -430,7 +477,21 @@ func (p *PBClient) UpsertTelemetry(ctx context.Context, payload TelemetryOut) er
 	}

 	// For status updates (success/failed/unknown), find and update existing record
-	recordID, err := p.FindRecordByRandomID(ctx, payload.RandomID)
+	// Prefer execution_id (unique-indexed) over random_id (filter query) for faster lookups
+	var recordID string
+	var err error
+
+	if payload.ExecutionID != "" {
+		recordID, err = p.FindRecordByExecutionID(ctx, payload.ExecutionID)
+		if err != nil {
+			// Execution ID lookup failed, fall back to random_id
+			recordID, err = p.FindRecordByRandomID(ctx, payload.RandomID)
+		}
+	} else {
+		// Old client without execution_id — use random_id lookup
+		recordID, err = p.FindRecordByRandomID(ctx, payload.RandomID)
+	}
+
 	if err != nil {
 		// Search failed, log and return error
 		return fmt.Errorf("cannot find record to update: %w", err)
@@ -445,6 +506,7 @@ func (p *PBClient) UpsertTelemetry(ctx context.Context, payload TelemetryOut) er
 	// Update only status, error, exit_code, and new metrics fields
 	update := TelemetryStatusUpdate{
 		Status:          payload.Status,
+		ExecutionID:     payload.ExecutionID,
 		Error:           payload.Error,
 		ExitCode:        payload.ExitCode,
 		InstallDuration: payload.InstallDuration,
@@ -609,7 +671,7 @@ func getClientIP(r *http.Request, pt *ProxyTrust) net.IP {

 var (
 	// Allowed values for 'type' field
-	allowedType = map[string]bool{"lxc": true, "vm": true, "tool": true, "addon": true}
+	allowedType = map[string]bool{"lxc": true, "vm": true, "pve": true, "addon": true}

 	// Allowed values for 'status' field
 	allowedStatus = map[string]bool{"installing": true, "configuring": true, "success": true, "failed": true, "aborted": true, "unknown": true}
@@ -744,6 +806,7 @@ func sanitizeMultiLine(s string, max int) string {
 func validate(in *TelemetryIn) error {
 	// Sanitize all string fields
 	in.RandomID = sanitizeShort(in.RandomID, 64)
+	in.ExecutionID = sanitizeShort(in.ExecutionID, 64)
 	in.Type = sanitizeShort(in.Type, 8)
 	in.NSAPP = sanitizeShort(in.NSAPP, 64)
 	in.Status = sanitizeShort(in.Status, 16)
@@ -1058,7 +1121,7 @@ func main() {
 	cleaner := NewCleaner(CleanupConfig{
 		Enabled:          envBool("CLEANUP_ENABLED", true),
 		CheckInterval:    time.Duration(envInt("CLEANUP_INTERVAL_MIN", 15)) * time.Minute,
-		StuckAfterHours:  envInt("CLEANUP_STUCK_HOURS", 2),
+		StuckAfterHours:  envInt("CLEANUP_STUCK_HOURS", 1),
 		RetentionEnabled: envBool("RETENTION_ENABLED", false),
 		RetentionDays:    envInt("RETENTION_DAYS", 365),
 	}, pb)
@@ -1784,6 +1847,7 @@ func main() {
 		// Map input to PocketBase schema
 		out := TelemetryOut{
 			RandomID:        in.RandomID,
+			ExecutionID:     in.ExecutionID,
 			Type:            in.Type,
 			NSAPP:           in.NSAPP,
 			Status:          in.Status,