From b80d8b1e755b7964d29b9da47f957469e43999b0 Mon Sep 17 00:00:00 2001 From: Kyle Verhoog Date: Tue, 31 Mar 2020 00:52:51 -0400 Subject: [PATCH] python: add celery distributed tracing example --- python/{ => celery}/celery4.1/.gitignore | 0 python/{ => celery}/celery4.1/Pipfile | 0 python/{ => celery}/celery4.1/Pipfile.lock | 0 python/{ => celery}/celery4.1/README.md | 0 python/{ => celery}/celery4.1/consumer.py | 5 +- .../{ => celery}/celery4.1/docker-compose.yml | 0 python/{ => celery}/celery4.1/producer.py | 4 -- .../celery_distributed_tracing/.gitignore | 2 + .../celery_distributed_tracing/Dockerfile | 10 ++++ .../celery_distributed_tracing/consumer.py | 20 +++++++ .../docker-compose.yml | 52 +++++++++++++++++++ .../celery_distributed_tracing/producer.py | 30 +++++++++++ .../requirements.txt | 11 ++++ 13 files changed, 126 insertions(+), 8 deletions(-) rename python/{ => celery}/celery4.1/.gitignore (100%) rename python/{ => celery}/celery4.1/Pipfile (100%) rename python/{ => celery}/celery4.1/Pipfile.lock (100%) rename python/{ => celery}/celery4.1/README.md (100%) rename python/{ => celery}/celery4.1/consumer.py (63%) rename python/{ => celery}/celery4.1/docker-compose.yml (100%) rename python/{ => celery}/celery4.1/producer.py (55%) create mode 100644 python/celery/celery_distributed_tracing/.gitignore create mode 100644 python/celery/celery_distributed_tracing/Dockerfile create mode 100644 python/celery/celery_distributed_tracing/consumer.py create mode 100644 python/celery/celery_distributed_tracing/docker-compose.yml create mode 100644 python/celery/celery_distributed_tracing/producer.py create mode 100644 python/celery/celery_distributed_tracing/requirements.txt diff --git a/python/celery4.1/.gitignore b/python/celery/celery4.1/.gitignore similarity index 100% rename from python/celery4.1/.gitignore rename to python/celery/celery4.1/.gitignore diff --git a/python/celery4.1/Pipfile b/python/celery/celery4.1/Pipfile similarity index 100% rename from python/celery4.1/Pipfile rename to python/celery/celery4.1/Pipfile diff --git a/python/celery4.1/Pipfile.lock b/python/celery/celery4.1/Pipfile.lock similarity index 100% rename from python/celery4.1/Pipfile.lock rename to python/celery/celery4.1/Pipfile.lock diff --git a/python/celery4.1/README.md b/python/celery/celery4.1/README.md similarity index 100% rename from python/celery4.1/README.md rename to python/celery/celery4.1/README.md diff --git a/python/celery4.1/consumer.py b/python/celery/celery4.1/consumer.py similarity index 63% rename from python/celery4.1/consumer.py rename to python/celery/celery4.1/consumer.py index b2f3500d..fd42405b 100644 --- a/python/celery4.1/consumer.py +++ b/python/celery/celery4.1/consumer.py @@ -1,6 +1,3 @@ -import time -import random - from celery import Celery, shared_task from ddtrace import tracer @@ -10,5 +7,5 @@ def add(x, y): return x + y -app = Celery('consumer', broker='redis://localhost') +app = Celery("consumer", broker="redis://localhost") app.autodiscover_tasks() diff --git a/python/celery4.1/docker-compose.yml b/python/celery/celery4.1/docker-compose.yml similarity index 100% rename from python/celery4.1/docker-compose.yml rename to python/celery/celery4.1/docker-compose.yml diff --git a/python/celery4.1/producer.py b/python/celery/celery4.1/producer.py similarity index 55% rename from python/celery4.1/producer.py rename to python/celery/celery4.1/producer.py index df9a91ab..4995950e 100644 --- a/python/celery4.1/producer.py +++ b/python/celery/celery4.1/producer.py @@ -1,8 +1,4 @@ import time -import random - -from celery import shared_task -from ddtrace import tracer from consumer import add diff --git a/python/celery/celery_distributed_tracing/.gitignore b/python/celery/celery_distributed_tracing/.gitignore new file mode 100644 index 00000000..85c55ebc --- /dev/null +++ b/python/celery/celery_distributed_tracing/.gitignore @@ -0,0 +1,2 @@ +.env +.venv diff --git a/python/celery/celery_distributed_tracing/Dockerfile b/python/celery/celery_distributed_tracing/Dockerfile new file mode 100644 index 00000000..c7c12c74 --- /dev/null +++ b/python/celery/celery_distributed_tracing/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.8-slim + +RUN apt-get update && apt-get install -y git + +ENV PYTHONUNBUFFERED 1 + +WORKDIR /code +COPY ./requirements.txt /code + +RUN pip install -r requirements.txt --src /usr/local/src diff --git a/python/celery/celery_distributed_tracing/consumer.py b/python/celery/celery_distributed_tracing/consumer.py new file mode 100644 index 00000000..bc34335c --- /dev/null +++ b/python/celery/celery_distributed_tracing/consumer.py @@ -0,0 +1,20 @@ +import random +import time + +from celery import Celery, shared_task +from ddtrace import tracer + + +@tracer.wrap() +def consumer_work(): + time.sleep(random.uniform(0, 0.02)) + + +@shared_task +def add(x, y): + for _ in range(0, 5): + consumer_work() + return x + y + + +app = Celery("consumer", broker="redis://redis", backend="redis://redis") diff --git a/python/celery/celery_distributed_tracing/docker-compose.yml b/python/celery/celery_distributed_tracing/docker-compose.yml new file mode 100644 index 00000000..d8f100a3 --- /dev/null +++ b/python/celery/celery_distributed_tracing/docker-compose.yml @@ -0,0 +1,52 @@ +version: '3' + +services: + redis: + image: redis:5.0.6-alpine + + producer: + build: + context: ./ + dockerfile: ./Dockerfile + depends_on: + - redis + volumes: + - .:/code + environment: + - DATADOG_TRACE_AGENT_HOSTNAME=datadog + - DD_CELERY_DISTRIBUTED_TRACING_ENABLED=true + - DATADOG_TRACE_DEBUG=true + - DD_SERVICE=producer + - DD_ENV=celery-example + command: ddtrace-run python producer.py + + consumer: + build: + context: ./ + dockerfile: ./Dockerfile + depends_on: + - redis + volumes: + - .:/code + environment: + - DATADOG_TRACE_AGENT_HOSTNAME=datadog + - DD_CELERY_DISTRIBUTED_TRACING_ENABLED=true + - DATADOG_TRACE_DEBUG=true + - DD_SERVICE=consumer + - DD_ENV=celery-example + command: ddtrace-run celery -A consumer worker --loglevel=info + + datadog: + image: datadog/agent:latest + environment: + - DD_HOSTNAME=trace-example + - DD_API_KEY=${DD_API_KEY} + - DD_APM_ENABLED=true + - DD_BIND_HOST=0.0.0.0 + ports: + - 8125 + - 8126 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + # - /proc/:/host/proc/:ro + # - /sys/fs/cgroup:/host/sys/fs/cgroup:r diff --git a/python/celery/celery_distributed_tracing/producer.py b/python/celery/celery_distributed_tracing/producer.py new file mode 100644 index 00000000..f696f582 --- /dev/null +++ b/python/celery/celery_distributed_tracing/producer.py @@ -0,0 +1,30 @@ +import random +import time + +from ddtrace import tracer + +from consumer import add + + +@tracer.wrap() +def prod_work(): + time.sleep(random.uniform(0, 0.05)) + + +while True: + with tracer.trace("myapp.request"): + # Send off the task + result = add.apply_async((4, 4)) + + # Do some work in the meantime + for _ in range(3): + prod_work() + + # Get the result + result.get() + + # Do some more work + for _ in range(2): + prod_work() + + time.sleep(0.5) diff --git a/python/celery/celery_distributed_tracing/requirements.txt b/python/celery/celery_distributed_tracing/requirements.txt new file mode 100644 index 00000000..54a4e10f --- /dev/null +++ b/python/celery/celery_distributed_tracing/requirements.txt @@ -0,0 +1,11 @@ +amqp==2.5.2 +billiard==3.6.3.0 +celery==4.4.2 +-e git://github.com/thieman/dd-trace-py.git@69d6cfe419468e8a22710068cb74d55694f1d311#egg=ddtrace +importlib-metadata==1.6.0 +kombu==4.6.8 +msgpack==1.0.0 +pytz==2019.3 +redis==3.4.1 +vine==1.3.0 +zipp==3.1.0