diff --git a/bin/mprocs.yaml b/bin/mprocs.yaml index 1979873c86..6187e1fce4 100755 --- a/bin/mprocs.yaml +++ b/bin/mprocs.yaml @@ -13,6 +13,7 @@ procs: frontend: shell: './bin/start-frontend' + autorestart: true temporal-worker-general-purpose: shell: 'bin/check_kafka_clickhouse_up && bin/check_temporal_up && python manage.py start_temporal_worker --task-queue general-purpose-task-queue' diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 00b3658b94..11b24cacaa 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -136,13 +136,15 @@ services: - --rpc-addr kafka:33145 - --advertise-rpc-addr kafka:33145 - --mode dev-container - - --smp 1 - - --memory 1G - - --reserve-memory 200M + - --smp 2 + - --memory 3G + - --reserve-memory 500M - --overprovisioned - --set redpanda.empty_seed_starts_cluster=false - --seeds kafka:33145 - --set redpanda.auto_create_topics_enabled=true + volumes: + - redpanda-data:/var/lib/redpanda/data environment: ALLOW_PLAINTEXT_LISTENER: 'true' healthcheck: @@ -443,3 +445,6 @@ services: networks: otel_network: driver: bridge + +volumes: + redpanda-data: diff --git a/docker-compose.dev-coordinator.yml b/docker-compose.dev-coordinator.yml index 9856b1aaf6..3f89f35275 100644 --- a/docker-compose.dev-coordinator.yml +++ b/docker-compose.dev-coordinator.yml @@ -230,3 +230,6 @@ services: depends_on: db: condition: service_healthy + +volumes: + redpanda-data: diff --git a/docker-compose.dev-full.yml b/docker-compose.dev-full.yml index e568eca24e..8a6088d2ae 100644 --- a/docker-compose.dev-full.yml +++ b/docker-compose.dev-full.yml @@ -261,3 +261,6 @@ services: networks: otel_network: driver: bridge + +volumes: + redpanda-data: diff --git a/docker-compose.dev-minimal.yml b/docker-compose.dev-minimal.yml index 34b9aed551..693269941f 100644 --- a/docker-compose.dev-minimal.yml +++ b/docker-compose.dev-minimal.yml @@ -164,3 +164,6 @@ services: depends_on: - redis - db + +volumes: + redpanda-data: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 649f9b8c89..a7d24a2ce1 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -251,3 +251,6 @@ services: networks: otel_network: driver: bridge + +volumes: + redpanda-data: diff --git a/docker-compose.hobby.yml b/docker-compose.hobby.yml index 61d6bb6a3e..2ebd81c88f 100644 --- a/docker-compose.hobby.yml +++ b/docker-compose.hobby.yml @@ -303,3 +303,4 @@ volumes: redis-data: redis7-data: kafka-data: + redpanda-data: diff --git a/posthog/kafka_client/client.py b/posthog/kafka_client/client.py index 5b91ea41f9..c53274b5c3 100644 --- a/posthog/kafka_client/client.py +++ b/posthog/kafka_client/client.py @@ -195,6 +195,9 @@ def can_connect(): insignificant, even if it is occuring from, say, 30 separate pods, say, every 10 seconds. """ + if settings.DEBUG and not settings.TEST: + return True # Skip check in development - assume Kafka is "good enough" + try: _KafkaProducer(test=settings.TEST) except Exception: @@ -222,7 +225,7 @@ def build_kafka_consumer( auto_offset_reset="latest", test=False, group_id=None, - consumer_timeout_ms=float("inf"), + consumer_timeout_ms=5000 if (settings.DEBUG and not settings.TEST) else 305000, ): if settings.TEST: test = True # Set at runtime so that overriden settings.TEST is supported diff --git a/posthog/kafka_client/helper.py b/posthog/kafka_client/helper.py index f9eb6cab1d..a2bffa5570 100644 --- a/posthog/kafka_client/helper.py +++ b/posthog/kafka_client/helper.py @@ -109,6 +109,7 @@ def get_kafka_consumer(topic=None, value_deserializer=lambda v: json.loads(v.dec security_protocol="SSL", ssl_context=get_kafka_ssl_context(), value_deserializer=value_deserializer, + consumer_timeout_ms=5000 if (settings.DEBUG and not settings.TEST) else 305000, **kwargs, )