diff --git a/dashboard.go b/dashboard.go index 3b6364f..fbfff5f 100644 --- a/dashboard.go +++ b/dashboard.go @@ -13,6 +13,8 @@ import ( // DashboardData holds aggregated statistics for the dashboard type DashboardData struct { TotalInstalls int `json:"total_installs"` + TotalAllTime int `json:"total_all_time"` // Total records in DB (not limited) + SampleSize int `json:"sample_size"` // How many records were sampled SuccessCount int `json:"success_count"` FailedCount int `json:"failed_count"` InstallingCount int `json:"installing_count"` @@ -132,10 +134,15 @@ func (p *PBClient) FetchDashboardData(ctx context.Context, days int, repoSource } // Fetch all records for the period - records, err := p.fetchRecords(ctx, filter) + result, err := p.fetchRecords(ctx, filter) if err != nil { return nil, err } + records := result.Records + + // Set total counts + data.TotalAllTime = result.TotalItems // Actual total in database + data.SampleSize = len(records) // How many we actually processed // Aggregate statistics appCounts := make(map[string]int) @@ -257,18 +264,18 @@ func (p *PBClient) FetchDashboardData(ctx context.Context, days int, repoSource data.SuccessRate = float64(data.SuccessCount) / float64(completed) * 100 } - // Convert maps to sorted slices (top 10) - data.TopApps = topN(appCounts, 10) - data.OsDistribution = topNOs(osCounts, 10) + // Convert maps to sorted slices (increased limits for better analytics) + data.TopApps = topN(appCounts, 20) + data.OsDistribution = topNOs(osCounts, 15) data.MethodStats = topNMethod(methodCounts, 10) - data.PveVersions = topNPve(pveCounts, 10) + data.PveVersions = topNPve(pveCounts, 15) data.TypeStats = topNType(typeCounts, 10) // Error analysis - data.ErrorAnalysis = buildErrorAnalysis(errorPatterns, 10) + data.ErrorAnalysis = buildErrorAnalysis(errorPatterns, 15) - // Failed apps with failure rates - data.FailedApps = buildFailedApps(appCounts, appFailures, 10) + // Failed apps with failure rates (min 10 installs threshold) + data.FailedApps = buildFailedApps(appCounts, appFailures, 15) // Daily stats for chart data.DailyStats = buildDailyStats(dailySuccess, dailyFailed, days) @@ -282,10 +289,10 @@ func (p *PBClient) FetchDashboardData(ctx context.Context, days int, repoSource data.ErrorCategories = buildErrorCategories(errorCatCounts) // Top tools - data.TopTools = buildToolStats(toolCounts, 10) + data.TopTools = buildToolStats(toolCounts, 15) // Top addons - data.TopAddons = buildAddonStats(addonCounts, 10) + data.TopAddons = buildAddonStats(addonCounts, 15) // Average install duration if durationCount > 0 { @@ -308,23 +315,30 @@ type TelemetryRecord struct { Created string `json:"created"` } -func (p *PBClient) fetchRecords(ctx context.Context, filter string) ([]TelemetryRecord, error) { +// fetchRecordsResult contains records and total count +type fetchRecordsResult struct { + Records []TelemetryRecord + TotalItems int // Actual total in database (not limited) +} + +func (p *PBClient) fetchRecords(ctx context.Context, filter string) (*fetchRecordsResult, error) { var allRecords []TelemetryRecord page := 1 perPage := 500 maxRecords := 100000 // Limit to prevent timeout with large datasets + totalItems := 0 for { - var url string + var reqURL string if filter != "" { - url = fmt.Sprintf("%s/api/collections/%s/records?filter=%s&sort=-created&page=%d&perPage=%d", + reqURL = fmt.Sprintf("%s/api/collections/%s/records?filter=%s&sort=-created&page=%d&perPage=%d", p.baseURL, p.targetColl, filter, page, perPage) } else { - url = fmt.Sprintf("%s/api/collections/%s/records?sort=-created&page=%d&perPage=%d", + reqURL = fmt.Sprintf("%s/api/collections/%s/records?sort=-created&page=%d&perPage=%d", p.baseURL, p.targetColl, page, perPage) } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) if err != nil { return nil, err } @@ -345,6 +359,11 @@ func (p *PBClient) fetchRecords(ctx context.Context, filter string) ([]Telemetry } resp.Body.Close() + // Store total on first page + if page == 1 { + totalItems = result.TotalItems + } + allRecords = append(allRecords, result.Items...) // Stop if we have enough records or reached the end @@ -354,7 +373,10 @@ func (p *PBClient) fetchRecords(ctx context.Context, filter string) ([]Telemetry page++ } - return allRecords, nil + return &fetchRecordsResult{ + Records: allRecords, + TotalItems: totalItems, + }, nil } func topN(m map[string]int, n int) []AppCount { @@ -533,11 +555,12 @@ func buildErrorAnalysis(patterns map[string]map[string]bool, n int) []ErrorGroup func buildFailedApps(total, failed map[string]int, n int) []AppFailure { result := make([]AppFailure, 0) + minInstalls := 10 // Minimum installations to be considered (avoid noise from rare apps) for app, failCount := range failed { totalCount := total[app] - if totalCount == 0 { - continue + if totalCount < minInstalls { + continue // Skip apps with too few installations } rate := float64(failCount) / float64(totalCount) * 100 @@ -1808,13 +1831,15 @@ func DashboardHTML() string {
-
- - - - - - Ctrl+K +
+ +
@@ -1864,8 +1889,8 @@ func DashboardHTML() string {
- - + + @@ -2231,21 +2256,45 @@ func DashboardHTML() string { async function fetchData() { const activeBtn = document.querySelector('.filter-btn.active'); - const days = activeBtn ? activeBtn.dataset.days : '30'; + const days = activeBtn ? activeBtn.dataset.days : '7'; const repo = document.getElementById('repoFilter').value; + + // Show loading indicator + document.getElementById('loadingIndicator').style.display = 'flex'; + document.getElementById('cacheStatus').textContent = ''; + try { const response = await fetch('/api/dashboard?days=' + days + '&repo=' + repo); if (!response.ok) throw new Error('Failed to fetch data'); + + // Check cache status from header + const cacheHit = response.headers.get('X-Cache') === 'HIT'; + document.getElementById('cacheStatus').textContent = cacheHit ? '(cached)' : '(fresh)'; + return await response.json(); } catch (error) { document.getElementById('error').style.display = 'flex'; document.getElementById('errorText').textContent = error.message; throw error; + } finally { + document.getElementById('loadingIndicator').style.display = 'none'; } } function updateStats(data) { - document.getElementById('totalInstalls').textContent = data.total_installs.toLocaleString(); + // Use total_all_time for display if available, otherwise total_installs + const displayTotal = data.total_all_time || data.total_installs; + document.getElementById('totalInstalls').textContent = displayTotal.toLocaleString(); + + // Show sample info if data was sampled + const sampleInfo = document.getElementById('sampleInfo'); + if (sampleInfo && data.sample_size && data.sample_size < data.total_all_time) { + sampleInfo.textContent = '(based on ' + data.sample_size.toLocaleString() + ' recent records)'; + sampleInfo.style.display = 'block'; + } else if (sampleInfo) { + sampleInfo.style.display = 'none'; + } + document.getElementById('failedCount').textContent = data.failed_count.toLocaleString(); document.getElementById('successRate').textContent = data.success_rate.toFixed(1) + '%'; document.getElementById('successSubtitle').textContent = data.success_count.toLocaleString() + ' successful installations'; diff --git a/service.go b/service.go index 9f10d26..3e3e836 100644 --- a/service.go +++ b/service.go @@ -886,12 +886,12 @@ func main() { // Dashboard API endpoint (with caching) mux.HandleFunc("/api/dashboard", func(w http.ResponseWriter, r *http.Request) { - days := 30 + days := 7 // Default: 7 days if d := r.URL.Query().Get("days"); d != "" { fmt.Sscanf(d, "%d", &days) // days=0 means "all entries", negative values are invalid if days < 0 { - days = 30 + days = 7 } } @@ -1117,6 +1117,22 @@ func main() { ReadHeaderTimeout: 3 * time.Second, } + // Background cache warmup job - pre-populates cache for common dashboard queries + if cfg.CacheEnabled { + go func() { + // Initial warmup after startup + time.Sleep(10 * time.Second) + warmupDashboardCache(pb, cache, cfg) + + // Periodic refresh (every 4 minutes, before 5-minute TTL expires) + ticker := time.NewTicker(4 * time.Minute) + for range ticker.C { + warmupDashboardCache(pb, cache, cfg) + } + }() + log.Println("background cache warmup enabled") + } + log.Printf("telemetry-ingest listening on %s", cfg.ListenAddr) log.Fatal(srv.ListenAndServe()) } @@ -1202,4 +1218,44 @@ func splitCSV(s string) []string { } } return out +} + +// warmupDashboardCache pre-populates the cache with common dashboard queries +func warmupDashboardCache(pb *PBClient, cache *Cache, cfg Config) { + log.Println("[CACHE] Starting dashboard cache warmup...") + + // Common day ranges and repos to pre-cache + dayRanges := []int{7, 30, 90} + repos := []string{"ProxmoxVE", ""} // ProxmoxVE and "all" + + warmed := 0 + for _, days := range dayRanges { + for _, repo := range repos { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + + cacheKey := fmt.Sprintf("dashboard:%d:%s", days, repo) + + // Check if already cached + var existing *DashboardData + if cache.Get(ctx, cacheKey, &existing) { + cancel() + continue // Already cached, skip + } + + // Fetch and cache + data, err := pb.FetchDashboardData(ctx, days, repo) + cancel() + + if err != nil { + log.Printf("[CACHE] Warmup failed for days=%d repo=%s: %v", days, repo, err) + continue + } + + _ = cache.Set(context.Background(), cacheKey, data, cfg.CacheTTL) + warmed++ + log.Printf("[CACHE] Warmed cache for days=%d repo=%s (%d installs)", days, repo, data.TotalAllTime) + } + } + + log.Printf("[CACHE] Dashboard cache warmup complete (%d entries)", warmed) } \ No newline at end of file