Add rabbit as broker for celery

PostHog · Aug 15, 2023 · 28bd2c5 · 28bd2c5
1 parent fdf3ed9
commit 28bd2c5
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 1 deletion.
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -6,6 +6,7 @@ services:
     environment: &django_env
       DEBUG: 1
       REDIS_URL: redis://redis:6379
+      RABBITMQ_URL: amqp://posthog:posthog@rabbitmq:5672
       DATABASE_URL: postgres://housewatch:housewatch@db:5432/housewatch
       CLICKHOUSE_HOST: clickhouse
       CLICKHOUSE_DATABASE: default
@@ -29,6 +30,7 @@ services:
       - clickhouse
       - db
       - redis
+      - rabbitmq
 
   web:
     build:
@@ -72,6 +74,7 @@ services:
       - clickhouse
       - db
       - redis
+      - rabbitmq
 
   clickhouse:
     image: ${CLICKHOUSE_SERVER_IMAGE:-clickhouse/clickhouse-server:23.4.2.11}
@@ -94,3 +97,12 @@ services:
     depends_on:
       - web
       - app
+
+  rabbitmq:
+    image: rabbitmq:3.12.2-management-alpine
+    ports:
+      - "15672:15672" # Web management UI
+      - "5672:5672" # Default RabbitMQ broker port
+    environment:
+      RABBITMQ_DEFAULT_USER: posthog
+      RABBITMQ_DEFAULT_PASS: posthog
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -5,6 +5,7 @@ services:
     build: .
     environment: &django_env
       DATABASE_URL: postgres://housewatch:housewatch@db:5432/housewatch
+      RABBITMQ_URL: amqp://posthog:posthog@rabbitmq:5672
       REDIS_URL: redis://redis:6379
       CLICKHOUSE_HOST: $CLICKHOUSE_HOST
       CLICKHOUSE_DATABASE: $CLICKHOUSE_DATABASE
@@ -24,6 +25,12 @@ services:
       - .:/code
     ports:
       - "8000:8000"
+    depends_on:
+      - db
+      - redis
+      - rabbitmq
+      - clickhouse
+
   web:
     build: ./frontend
     ports:
@@ -37,6 +44,12 @@ services:
       - ./bin/celery
     volumes:
       - .:/code
+    depends_on:
+      - db
+      - redis
+      - rabbitmq
+      - clickhouse
+
   redis:
     image: redis:6.2.7-alpine
     restart: on-failure
@@ -65,3 +78,12 @@ services:
     depends_on:
       - web
       - app
+
+  rabbitmq:
+    image: rabbitmq:3.12.2-management-alpine
+    ports:
+      - "15672:15672" # Web management UI
+      - "5672:5672" # Default RabbitMQ broker port
+    environment:
+      RABBITMQ_DEFAULT_USER: posthog
+      RABBITMQ_DEFAULT_PASS: posthog
diff --git a/housewatch/settings/__init__.py b/housewatch/settings/__init__.py
@@ -207,8 +207,10 @@ def get_from_env(key: str, default: Any = None, *, optional: bool = False, type_
 
 if TEST or DEBUG:
     REDIS_URL = get_from_env("REDIS_URL", "redis://localhost:6379")
+    RABBITMQ_URL = get_from_env("RABBITMQ_URL", "amqp://localhost:5672")
 else:
     REDIS_URL = get_from_env("REDIS_URL")
+    RABBITMQ_URL = get_from_env("RABBITMQ_URL")
 
 
 CACHES = {
@@ -225,7 +227,7 @@ def get_from_env(key: str, default: Any = None, *, optional: bool = False, type_
 CELERY_QUEUES = (Queue("celery", Exchange("celery"), "celery"),)
 CELERY_DEFAULT_QUEUE = "celery"
 CELERY_IMPORTS = []  # type: ignore
-CELERY_BROKER_URL = REDIS_URL  # celery connects to redis
+CELERY_BROKER_URL = RABBITMQ_URL  # celery connects to rabbitmq
 CELERY_BEAT_MAX_LOOP_INTERVAL = 30  # sleep max 30sec before checking for new periodic events
 CELERY_RESULT_BACKEND = REDIS_URL  # stores results for lookup when processing
 CELERY_IGNORE_RESULT = True  # only applies to delay(), must do @shared_task(ignore_result=True) for apply_async