From d35851cfb53b397776ba5174f0f8f7d790e837ba Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Tue, 24 Mar 2020 21:52:50 -0600 Subject: [PATCH 1/9] Celery configuration --- codeandomexico/__init__.py | 5 +++++ codeandomexico/celery_app.py | 23 +++++++++++++++++++++++ codeandomexico/settings.py | 10 ++++++++++ 3 files changed, 38 insertions(+) create mode 100644 codeandomexico/celery_app.py diff --git a/codeandomexico/__init__.py b/codeandomexico/__init__.py index e69de29..eb5b19e 100644 --- a/codeandomexico/__init__.py +++ b/codeandomexico/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery_app import app as celery_app \ No newline at end of file diff --git a/codeandomexico/celery_app.py b/codeandomexico/celery_app.py new file mode 100644 index 0000000..6e4b5fa --- /dev/null +++ b/codeandomexico/celery_app.py @@ -0,0 +1,23 @@ + +""" +======================= +Celery config +======================= +First steps with Django +https://docs.celeryproject.org/en/latest/django/first-steps-with-django.html +""" + +from __future__ import absolute_import +from django.conf import settings +import os +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'codeandomexico.settings') + +app = Celery(broker=settings.CELERY_BROKER_URL, backend='amqp://', namespace='CELERY') +app.config_from_object('django.conf:settings') +# app.autodiscover_tasks(settings.INSTALLED_APPS) + +if __name__ == '__main__': + app.start() diff --git a/codeandomexico/settings.py b/codeandomexico/settings.py index 148ce77..03557fe 100644 --- a/codeandomexico/settings.py +++ b/codeandomexico/settings.py @@ -82,6 +82,16 @@ } } +# Django Celery +# https://pypi.org/project/django-celery/ + +CELERY_BROKER_URL = os.environ.get('CLOUDAMQP_URL') +CELERY_IMPORTS = ('api_covid19.tasks',) +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = "America/Mexico_City" + # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators From 0452db0b7346b40ab3f4a3185abf10c7ec2bc1e7 Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Tue, 24 Mar 2020 21:53:21 -0600 Subject: [PATCH 2/9] Update dependencies --- requirements.txt | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c161d28..1afbe05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,39 @@ +amqp==2.5.2 +asgiref==3.2.7 beautifulsoup4==4.8.2 +billiard==3.6.3.0 bs4==0.0.1 camelot-py==0.7.3 +celery==4.4.2 +certifi==2019.11.28 +chardet==3.0.4 +click==7.1.1 Django==3.0.4 django-appconf==1.0.3 django-compressor==2.3 django-extensions==2.2.3 +et-xmlfile==1.0.1 +gunicorn==20.0.4 +idna==2.8 +importlib-metadata==1.5.0 +jdcal==1.4.1 +kombu==4.6.8 +numpy==1.18.2 +opencv-python==4.2.0.32 +openpyxl==3.0.3 pandas==1.0.1 pdfminer.six==20200124 +pycryptodome==3.9.7 PyPDF2==1.26.0 +python-dateutil==2.8.1 pytz==2019.3 +rcssmin==1.0.6 requests==2.22.0 -six==1.14.0 \ No newline at end of file +rjsmin==1.1.0 +six==1.14.0 +sortedcontainers==2.1.0 +soupsieve==2.0 +sqlparse==0.3.1 +urllib3==1.25.8 +vine==1.3.0 +zipp==3.1.0 From e4d1337dfe230d435cbbc42bcae9f7c8e3b3beb4 Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Tue, 24 Mar 2020 21:53:52 -0600 Subject: [PATCH 3/9] Update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b6e4761..673feba 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,7 @@ venv.bak/ .mypy_cache/ .dmypy.json dmypy.json - +.vscode +celerybeat* # Pyre type checker .pyre/ From da5ec62d65b0b7d6710f37421ea36a8cdd697e7e Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Tue, 24 Mar 2020 21:54:19 -0600 Subject: [PATCH 4/9] Create example contrab tasks --- api_covid19/tasks.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 api_covid19/tasks.py diff --git a/api_covid19/tasks.py b/api_covid19/tasks.py new file mode 100644 index 0000000..45a44ac --- /dev/null +++ b/api_covid19/tasks.py @@ -0,0 +1,27 @@ + +""" +======================== +Celery Tasks +======================== +""" + +# celery imports +from celery.decorators import task +from celery.task.schedules import crontab +from celery.decorators import periodic_task + +from .models import * +from .scripts.fetch_data import run + +@periodic_task(run_every=(crontab(hour=2)), name="update_info: Scrap https://www.gob.mx/ website every 2 hrs") +def update_info(): + print("Updating data...") + run() + +@periodic_task(run_every=(crontab()), name="example_task: This is example task running every minute") +def example_task(): + # do some stuff + # use models + # make requests + print("Running every minute...") + run() \ No newline at end of file From 5a9465822a3299f7132cfcdc35a693c47e107b0f Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Tue, 24 Mar 2020 21:54:49 -0600 Subject: [PATCH 5/9] .env file example for broker url --- .env.example | 1 + 1 file changed, 1 insertion(+) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f3ffd7d --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +CLOUDAMQP_URL=amqp://guest:guest@127.0.0.1:5672/ \ No newline at end of file From f3c0b0ee174c027078f754bc7df9f9c892954a45 Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Tue, 24 Mar 2020 21:55:25 -0600 Subject: [PATCH 6/9] Docker setup added --- Dockerfile | 11 +++++++++++ docker-compose.yml | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b0f637c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.6.6 + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +WORKDIR /app +COPY requirements.txt ./ +RUN apt-get update && apt-get -y install ghostscript && apt-get clean +RUN pip install --upgrade pip +RUN pip install --no-cache-dir -r requirements.txt +COPY . /app diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..dc15ca4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3' +services: + ### + # services + ### + web: + build: . + network_mode: host + command: gunicorn codeandomexico.wsgi --log-file - + volumes: + - .:/app + ports: + - "8000:8000" + tty: true + env_file: + - ./.env + stdin_open: true + + worker: + build: . + depends_on: + - rabbitmq + network_mode: host + command: celery worker -A codeandomexico -B --loglevel=INFO + env_file: + - ./.env + volumes: + - .:/app + + rabbitmq: + image: rabbitmq:3-management + restart: always + ports: + - "15672:15672" + - "5672:5672" + network_mode: host From 1e975a75851984aa45ab56d482b0d745bb5dc9a0 Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Tue, 24 Mar 2020 22:31:04 -0600 Subject: [PATCH 7/9] fix: change amqp backend to rpc --- codeandomexico/celery_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeandomexico/celery_app.py b/codeandomexico/celery_app.py index 6e4b5fa..c49afc9 100644 --- a/codeandomexico/celery_app.py +++ b/codeandomexico/celery_app.py @@ -15,7 +15,7 @@ # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'codeandomexico.settings') -app = Celery(broker=settings.CELERY_BROKER_URL, backend='amqp://', namespace='CELERY') +app = Celery(broker=settings.CELERY_BROKER_URL, backend='rpc://', namespace='CELERY') app.config_from_object('django.conf:settings') # app.autodiscover_tasks(settings.INSTALLED_APPS) From efa090483fcfdb3d78e4e3e43183b0695d6152b6 Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Tue, 24 Mar 2020 22:39:48 -0600 Subject: [PATCH 8/9] Docs for tasks --- api_covid19/tasks.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/api_covid19/tasks.py b/api_covid19/tasks.py index 45a44ac..2e0332c 100644 --- a/api_covid19/tasks.py +++ b/api_covid19/tasks.py @@ -3,6 +3,8 @@ ======================== Celery Tasks ======================== +For more celery crontab info visit: + https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#crontab-schedules """ # celery imports @@ -15,13 +17,19 @@ @periodic_task(run_every=(crontab(hour=2)), name="update_info: Scrap https://www.gob.mx/ website every 2 hrs") def update_info(): + """ + This is a real world task. + """ print("Updating data...") run() @periodic_task(run_every=(crontab()), name="example_task: This is example task running every minute") def example_task(): + """ + This is a example task + """ # do some stuff # use models # make requests print("Running every minute...") - run() \ No newline at end of file + run() # just for test. May be deleted later \ No newline at end of file From 8887b70e3fb6691587bef53f050c9cd72e5a4b58 Mon Sep 17 00:00:00 2001 From: Raul Novelo Cruz Date: Wed, 25 Mar 2020 02:30:25 -0600 Subject: [PATCH 9/9] At minute 0 past every 2nd hour --- api_covid19/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_covid19/tasks.py b/api_covid19/tasks.py index 2e0332c..7216769 100644 --- a/api_covid19/tasks.py +++ b/api_covid19/tasks.py @@ -15,7 +15,7 @@ from .models import * from .scripts.fetch_data import run -@periodic_task(run_every=(crontab(hour=2)), name="update_info: Scrap https://www.gob.mx/ website every 2 hrs") +@periodic_task(run_every=(crontab(minute=0, hour='*/2')), name="update_info: Scrap https://www.gob.mx/ website every 2 hrs") def update_info(): """ This is a real world task.