fix(hub): keep the updater running when the system record lookup fails (#2588)

getRecord removed the system from the manager on any error, so a query that
timed out while the hub was stalled stopped the updater for good. The system
stayed 'up' in the database and was never polled again until a restart.
Only a record that is really gone removes the system now. (#2544)

internal/hub/systems/system.go

@@ -618,8 +618,13 @@ func createContainerRecords(app core.App, data []*container.Stats, systemId stri
 
 // getRecord retrieves the system record from the database.
 // If the record is not found, it removes the system from the manager.
+// Other errors, like a query timing out while the hub is stalled, leave the
+// system in place so the updater retries on the next tick (issue #2544).
 func (sys *System) getRecord(app core.App) (*core.Record, error) {
 	record, err := app.FindRecordById("systems", sys.Id)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return nil, err
+	}
 	if err != nil || record == nil {
 		_ = sys.manager.RemoveSystem(sys.Id)
 		if err == nil {

internal/hub/systems/system_get_record_test.go

@@ -0,0 +1,57 @@
+//go:build testing
+
+package systems
+
+import (
+	"context"
+	"testing"
+
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// failingRecordHub fails record lookups with err, like a query that times out
+// while the hub's VM is frozen.
+type failingRecordHub struct {
+	stubHub
+	err error
+}
+
+func (h failingRecordHub) FindRecordById(collectionModelOrIdentifier any, recordId string, optFilters ...func(q *dbx.SelectQuery) error) (*core.Record, error) {
+	return nil, h.err
+}
+
+func TestGetRecordKeepsSystemOnTransientError(t *testing.T) {
+	sys, app := newTestSystemWithHub(t)
+	sm := NewSystemManager(failingRecordHub{stubHub: stubHub{app}, err: context.DeadlineExceeded})
+	t.Cleanup(sm.cancel)
+	sys.manager = sm
+	sys.ctx, sys.cancel = sys.getContext(sm.ctx)
+	sm.systems.Set(sys.Id, sys)
+
+	record, err := sys.getRecord(sm.hub)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	assert.Nil(t, record)
+	assert.True(t, sm.HasSystem(sys.Id), "system should stay in the manager")
+	assert.NoError(t, sys.ctx.Err(), "updater context should not be cancelled")
+}
+
+func TestGetRecordRemovesDeletedSystem(t *testing.T) {
+	sys, app := newTestSystemWithHub(t)
+	sm := NewSystemManager(stubHub{app})
+	t.Cleanup(sm.cancel)
+	sys.manager = sm
+	sys.ctx, sys.cancel = sys.getContext(sm.ctx)
+	sm.systems.Set(sys.Id, sys)
+
+	record, err := app.FindRecordById("systems", sys.Id)
+	require.NoError(t, err)
+	require.NoError(t, app.Delete(record))
+
+	_, err = sys.getRecord(sm.hub)
+	require.Error(t, err)
+	assert.False(t, sm.HasSystem(sys.Id), "deleted system should be removed")
+	assert.Error(t, sys.ctx.Err(), "updater context should be cancelled")
+}