perf(devex): Improve Kafka/Redpanda reliability and performance for local development (#38641)

bin/mprocs.yaml

@@ -13,6 +13,7 @@ procs:
 
     frontend:
         shell: './bin/start-frontend'
+        autorestart: true
 
     temporal-worker-general-purpose:
         shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue general-purpose-task-queue'

docker-compose.base.yml

@@ -136,13 +136,15 @@ services:
             - --rpc-addr kafka:33145
             - --advertise-rpc-addr kafka:33145
             - --mode dev-container
-            - --smp 1
-            - --memory 1G
-            - --reserve-memory 200M
+            - --smp 2
+            - --memory 3G
+            - --reserve-memory 500M
             - --overprovisioned
             - --set redpanda.empty_seed_starts_cluster=false
             - --seeds kafka:33145
             - --set redpanda.auto_create_topics_enabled=true
+        volumes:
+            - redpanda-data:/var/lib/redpanda/data
         environment:
             ALLOW_PLAINTEXT_LISTENER: 'true'
         healthcheck:
@@ -443,3 +445,6 @@ services:
 networks:
     otel_network:
         driver: bridge
+
+volumes:
+    redpanda-data:

docker-compose.dev-coordinator.yml

@@ -230,3 +230,6 @@ services:
         depends_on:
             db:
                 condition: service_healthy
+
+volumes:
+    redpanda-data:

docker-compose.dev-full.yml

@@ -261,3 +261,6 @@ services:
 networks:
     otel_network:
         driver: bridge
+
+volumes:
+    redpanda-data:

docker-compose.dev-minimal.yml

@@ -164,3 +164,6 @@ services:
         depends_on:
             - redis
             - db
+
+volumes:
+    redpanda-data:

docker-compose.dev.yml

@@ -251,3 +251,6 @@ services:
 networks:
     otel_network:
         driver: bridge
+
+volumes:
+    redpanda-data:

docker-compose.hobby.yml

@@ -303,3 +303,4 @@ volumes:
     redis-data:
     redis7-data:
     kafka-data:
+    redpanda-data:

posthog/kafka_client/client.py

@@ -195,6 +195,9 @@ def can_connect():
     insignificant, even if it is occuring from, say, 30 separate pods, say,
     every 10 seconds.
     """
+    if settings.DEBUG and not settings.TEST:
+        return True  # Skip check in development - assume Kafka is "good enough"
+
     try:
         _KafkaProducer(test=settings.TEST)
     except Exception:
@@ -222,7 +225,7 @@ def build_kafka_consumer(
     auto_offset_reset="latest",
     test=False,
     group_id=None,
-    consumer_timeout_ms=float("inf"),
+    consumer_timeout_ms=5000 if (settings.DEBUG and not settings.TEST) else 305000,
 ):
     if settings.TEST:
         test = True  # Set at runtime so that overriden settings.TEST is supported

posthog/kafka_client/helper.py

@@ -109,6 +109,7 @@ def get_kafka_consumer(topic=None, value_deserializer=lambda v: json.loads(v.dec
         security_protocol="SSL",
         ssl_context=get_kafka_ssl_context(),
         value_deserializer=value_deserializer,
+        consumer_timeout_ms=5000 if (settings.DEBUG and not settings.TEST) else 305000,
         **kwargs,
     )