python: add celery distributed tracing example

python/celery4.1/consumer.py → python/celery/celery4.1/consumer.py

@@ -1,6 +1,3 @@
-import time
-import random
-
 from celery import Celery, shared_task
 from ddtrace import tracer
 
@@ -10,5 +7,5 @@ def add(x, y):
     return x + y
 
 
-app = Celery('consumer', broker='redis://localhost')
+app = Celery("consumer", broker="redis://localhost")
 app.autodiscover_tasks()

python/celery4.1/producer.py → python/celery/celery4.1/producer.py

@@ -1,8 +1,4 @@
 import time
-import random
-
-from celery import shared_task
-from ddtrace import tracer
 
 from consumer import add
 

python/celery/celery_distributed_tracing/.gitignore

@@ -0,0 +1,2 @@
+.env
+.venv

python/celery/celery_distributed_tracing/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.8-slim
+
+RUN apt-get update && apt-get install -y git
+
+ENV PYTHONUNBUFFERED 1
+
+WORKDIR /code
+COPY ./requirements.txt /code
+
+RUN pip install -r requirements.txt --src /usr/local/src

python/celery/celery_distributed_tracing/consumer.py

@@ -0,0 +1,20 @@
+import random
+import time
+
+from celery import Celery, shared_task
+from ddtrace import tracer
+
+
+@tracer.wrap()
+def consumer_work():
+    time.sleep(random.uniform(0, 0.02))
+
+
+@shared_task
+def add(x, y):
+    for _ in range(0, 5):
+        consumer_work()
+    return x + y
+
+
+app = Celery("consumer", broker="redis://redis", backend="redis://redis")

python/celery/celery_distributed_tracing/docker-compose.yml

@@ -0,0 +1,52 @@
+version: '3'
+
+services:
+  redis:
+    image: redis:5.0.6-alpine
+
+  producer:
+    build:
+        context: ./
+        dockerfile: ./Dockerfile
+    depends_on:
+        - redis
+    volumes:
+        - .:/code
+    environment:
+        - DATADOG_TRACE_AGENT_HOSTNAME=datadog
+        - DD_CELERY_DISTRIBUTED_TRACING_ENABLED=true
+        - DATADOG_TRACE_DEBUG=true
+        - DD_SERVICE=producer
+        - DD_ENV=celery-example
+    command: ddtrace-run python producer.py
+
+  consumer:
+    build:
+        context: ./
+        dockerfile: ./Dockerfile
+    depends_on:
+        - redis
+    volumes:
+        - .:/code
+    environment:
+        - DATADOG_TRACE_AGENT_HOSTNAME=datadog
+        - DD_CELERY_DISTRIBUTED_TRACING_ENABLED=true
+        - DATADOG_TRACE_DEBUG=true
+        - DD_SERVICE=consumer
+        - DD_ENV=celery-example
+    command: ddtrace-run celery -A consumer worker --loglevel=info
+
+  datadog:
+    image: datadog/agent:latest
+    environment:
+      - DD_HOSTNAME=trace-example
+      - DD_API_KEY=${DD_API_KEY}
+      - DD_APM_ENABLED=true
+      - DD_BIND_HOST=0.0.0.0
+    ports:
+      - 8125
+      - 8126
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    #   - /proc/:/host/proc/:ro
+    #   - /sys/fs/cgroup:/host/sys/fs/cgroup:r

python/celery/celery_distributed_tracing/producer.py

@@ -0,0 +1,30 @@
+import random
+import time
+
+from ddtrace import tracer
+
+from consumer import add
+
+
+@tracer.wrap()
+def prod_work():
+    time.sleep(random.uniform(0, 0.05))
+
+
+while True:
+    with tracer.trace("myapp.request"):
+        # Send off the task
+        result = add.apply_async((4, 4))
+
+        # Do some work in the meantime
+        for _ in range(3):
+            prod_work()
+
+        # Get the result
+        result.get()
+
+        # Do some more work
+        for _ in range(2):
+            prod_work()
+
+    time.sleep(0.5)

python/celery/celery_distributed_tracing/requirements.txt

@@ -0,0 +1,11 @@
+amqp==2.5.2
+billiard==3.6.3.0
+celery==4.4.2
+-e git://github.com/thieman/dd-trace-py.git@69d6cfe419468e8a22710068cb74d55694f1d311#egg=ddtrace
+importlib-metadata==1.6.0
+kombu==4.6.8
+msgpack==1.0.0
+pytz==2019.3
+redis==3.4.1
+vine==1.3.0
+zipp==3.1.0