diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f3ffd7d --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +CLOUDAMQP_URL=amqp://guest:guest@127.0.0.1:5672/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index b6e4761..673feba 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,7 @@ venv.bak/ .mypy_cache/ .dmypy.json dmypy.json - +.vscode +celerybeat* # Pyre type checker .pyre/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b0f637c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.6.6 + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +WORKDIR /app +COPY requirements.txt ./ +RUN apt-get update && apt-get -y install ghostscript && apt-get clean +RUN pip install --upgrade pip +RUN pip install --no-cache-dir -r requirements.txt +COPY . /app diff --git a/api_covid19/tasks.py b/api_covid19/tasks.py new file mode 100644 index 0000000..7216769 --- /dev/null +++ b/api_covid19/tasks.py @@ -0,0 +1,35 @@ + +""" +======================== +Celery Tasks +======================== +For more celery crontab info visit: + https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#crontab-schedules +""" + +# celery imports +from celery.decorators import task +from celery.task.schedules import crontab +from celery.decorators import periodic_task + +from .models import * +from .scripts.fetch_data import run + +@periodic_task(run_every=(crontab(minute=0, hour='*/2')), name="update_info: Scrap https://www.gob.mx/ website every 2 hrs") +def update_info(): + """ + This is a real world task. + """ + print("Updating data...") + run() + +@periodic_task(run_every=(crontab()), name="example_task: This is example task running every minute") +def example_task(): + """ + This is a example task + """ + # do some stuff + # use models + # make requests + print("Running every minute...") + run() # just for test. May be deleted later \ No newline at end of file diff --git a/codeandomexico/__init__.py b/codeandomexico/__init__.py index e69de29..eb5b19e 100644 --- a/codeandomexico/__init__.py +++ b/codeandomexico/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery_app import app as celery_app \ No newline at end of file diff --git a/codeandomexico/celery_app.py b/codeandomexico/celery_app.py new file mode 100644 index 0000000..c49afc9 --- /dev/null +++ b/codeandomexico/celery_app.py @@ -0,0 +1,23 @@ + +""" +======================= +Celery config +======================= +First steps with Django +https://docs.celeryproject.org/en/latest/django/first-steps-with-django.html +""" + +from __future__ import absolute_import +from django.conf import settings +import os +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'codeandomexico.settings') + +app = Celery(broker=settings.CELERY_BROKER_URL, backend='rpc://', namespace='CELERY') +app.config_from_object('django.conf:settings') +# app.autodiscover_tasks(settings.INSTALLED_APPS) + +if __name__ == '__main__': + app.start() diff --git a/codeandomexico/settings.py b/codeandomexico/settings.py index 148ce77..03557fe 100644 --- a/codeandomexico/settings.py +++ b/codeandomexico/settings.py @@ -82,6 +82,16 @@ } } +# Django Celery +# https://pypi.org/project/django-celery/ + +CELERY_BROKER_URL = os.environ.get('CLOUDAMQP_URL') +CELERY_IMPORTS = ('api_covid19.tasks',) +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = "America/Mexico_City" + # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..dc15ca4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3' +services: + ### + # services + ### + web: + build: . + network_mode: host + command: gunicorn codeandomexico.wsgi --log-file - + volumes: + - .:/app + ports: + - "8000:8000" + tty: true + env_file: + - ./.env + stdin_open: true + + worker: + build: . + depends_on: + - rabbitmq + network_mode: host + command: celery worker -A codeandomexico -B --loglevel=INFO + env_file: + - ./.env + volumes: + - .:/app + + rabbitmq: + image: rabbitmq:3-management + restart: always + ports: + - "15672:15672" + - "5672:5672" + network_mode: host diff --git a/requirements.txt b/requirements.txt index c161d28..1afbe05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,39 @@ +amqp==2.5.2 +asgiref==3.2.7 beautifulsoup4==4.8.2 +billiard==3.6.3.0 bs4==0.0.1 camelot-py==0.7.3 +celery==4.4.2 +certifi==2019.11.28 +chardet==3.0.4 +click==7.1.1 Django==3.0.4 django-appconf==1.0.3 django-compressor==2.3 django-extensions==2.2.3 +et-xmlfile==1.0.1 +gunicorn==20.0.4 +idna==2.8 +importlib-metadata==1.5.0 +jdcal==1.4.1 +kombu==4.6.8 +numpy==1.18.2 +opencv-python==4.2.0.32 +openpyxl==3.0.3 pandas==1.0.1 pdfminer.six==20200124 +pycryptodome==3.9.7 PyPDF2==1.26.0 +python-dateutil==2.8.1 pytz==2019.3 +rcssmin==1.0.6 requests==2.22.0 -six==1.14.0 \ No newline at end of file +rjsmin==1.1.0 +six==1.14.0 +sortedcontainers==2.1.0 +soupsieve==2.0 +sqlparse==0.3.1 +urllib3==1.25.8 +vine==1.3.0 +zipp==3.1.0