From fb9112871a54688e63b52b0defae8d28eb2cfc1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Krist=C3=B3f=20Torma?= <tormakristof@tormakristof.eu>
Date: Mon, 13 Mar 2023 13:33:40 +0100
Subject: [PATCH] use celery
---
Dockerfile | 3 +-
k8s/deployment.yml | 16 ++
k8s/redis.values.yaml | 11 ++
requirements/base.in | 9 +-
requirements/production.in | 20 +--
requirements/production.txt | 232 +++++++++++++++++++--------
src/account/auth_pipeline.py | 4 +-
src/account/serializers.py | 10 +-
src/common/email.py | 18 ++-
src/homework/serializers.py | 17 +-
src/kszkepzes/__init__.py | 3 +
src/kszkepzes/celery.py | 9 ++
src/kszkepzes/settings/local.py | 2 +
src/kszkepzes/settings/production.py | 4 +
14 files changed, 253 insertions(+), 105 deletions(-)
create mode 100644 k8s/redis.values.yaml
create mode 100644 src/kszkepzes/celery.py
diff --git a/Dockerfile b/Dockerfile
index 030717c..a16be46 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.8.1
+FROM python:3.11
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONUNBUFFERED 1
@@ -13,7 +13,6 @@ WORKDIR $APP_HOME
COPY ./requirements/production.txt requirements.txt
-RUN apt-get -y update && apt-get install -y python python-pip python-dev python-django-extensions postgresql-client netcat
RUN pip install -r requirements.txt
COPY ./src $APP_HOME
diff --git a/k8s/deployment.yml b/k8s/deployment.yml
index b979cc6..5139785 100644
--- a/k8s/deployment.yml
+++ b/k8s/deployment.yml
@@ -107,6 +107,22 @@ spec:
limits:
memory: 200Mi
cpu: "2"
+ - name: worker
+ image: harbor.sch.bme.hu/kszk/kszkepzes-backend:##IMAGETAG##
+ imagePullPolicy: "IfNotPresent"
+ envFrom:
+ - configMapRef:
+ name: kszkepzes-config
+ - secretRef:
+ name: kszkepzes-secret-config
+ command: ["python3"]
+ args: ["-m", "celery", "-A", "kszkepzes", "worker", "-l", "info"]
+ resources:
+ requests:
+ cpu: "100m"
+ limits:
+ memory: 600Mi
+ cpu: "2"
volumes:
- name: kszkepzes-media-volume
persistentVolumeClaim:
diff --git a/k8s/redis.values.yaml b/k8s/redis.values.yaml
new file mode 100644
index 0000000..8791d69
--- /dev/null
+++ b/k8s/redis.values.yaml
@@ -0,0 +1,11 @@
+# https://artifacthub.io/packages/helm/bitnami/redis
+master:
+ resources:
+ limits:
+ memory: 2Gi
+ cpu: 2
+replica:
+ resources:
+ limits:
+ memory: 2Gi
+ cpu: 2
diff --git a/requirements/base.in b/requirements/base.in
index 962a298..b983771 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -1,4 +1,5 @@
-Django==2.2.4
-djangorestframework==3.10.2
-django-solo==1.1.3
-django-import-export==1.2.0
+Django~=4.1
+djangorestframework~=3.14
+django-solo~=2.0
+django-import-export~=3.1
+celery~=5.2
diff --git a/requirements/production.in b/requirements/production.in
index 2ef446c..5fa215f 100644
--- a/requirements/production.in
+++ b/requirements/production.in
@@ -1,11 +1,13 @@
-r base.in
psycopg2-binary
-gunicorn==19.7.1
-flake8==3.7.8
-pip-tools==4.1.0
-django-extensions==2.2.1
-python-language-server==0.28.2
-drf-yasg==1.16.1
-packaging==19.1
-Pillow==7.0.0
-djangorestframework-api-key==1.4.1
\ No newline at end of file
+redis
+gunicorn~=20.1
+flake8~=6.0
+pip-tools~=6.12
+django-extensions~=3.2
+python-language-server~=0.36
+drf-yasg~=1.21
+packaging~=23.0
+Pillow~=9.4
+djangorestframework-api-key~=2.3
+
diff --git a/requirements/production.txt b/requirements/production.txt
index 7cdc80b..75af8c7 100644
--- a/requirements/production.txt
+++ b/requirements/production.txt
@@ -1,70 +1,170 @@
#
-# This file is autogenerated by pip-compile
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
#
# pip-compile --output-file=requirements/production.txt requirements/production.in
#
-attrs==19.3.0 # via packaging
-certifi==2018.1.18 # via requests
-chardet==3.0.4 # via requests
-click==7.0 # via pip-tools
-coreapi==2.3.3 # via drf-yasg
-coreschema==0.0.4 # via coreapi, drf-yasg
-defusedxml==0.5.0 # via python3-openid, social-auth-core
-diff-match-patch==20121119 # via django-import-export
-django-extensions==2.2.1
-django-import-export==1.2.0
-django-social-authsch==0.1
-django-solo==1.1.3
-django==2.2.4
-djangorestframework-api-key==1.4.1
-djangorestframework==3.10.2
-drf-yasg==1.16.1
-entrypoints==0.3 # via flake8
-et-xmlfile==1.0.1 # via openpyxl
-flake8==3.7.8
-future==0.18.2 # via python-language-server
-gunicorn==19.7.1
-idna==2.6 # via requests
-importlib-metadata==1.5.0 # via pluggy
-inflection==0.3.1 # via drf-yasg
-itypes==1.1.0 # via coreapi
-jdcal==1.3 # via openpyxl
-jedi==0.14.1 # via python-language-server
-jinja2==2.11.1 # via coreschema
-markupsafe==1.1.1 # via jinja2
-mccabe==0.6.1 # via flake8
-oauthlib==2.0.6 # via requests-oauthlib, social-auth-core
-odfpy==1.3.6 # via tablib
-openpyxl==2.5.0 # via tablib
-packaging==19.1
-parso==0.6.1 # via jedi
-pillow==7.0.0
-pip-tools==4.1.0
-pluggy==0.13.1 # via python-language-server
-psycopg2-binary==2.8.4
-pycodestyle==2.5.0 # via flake8
-pyflakes==2.1.1 # via flake8
-pyjwt==1.5.3 # via social-auth-core
-pyparsing==2.4.6 # via packaging
-python-jsonrpc-server==0.3.4 # via python-language-server
-python-language-server==0.28.2
-python3-openid==3.1.0 # via social-auth-core
-pytz==2017.3 # via django
-pyyaml==5.3 # via tablib
-requests-oauthlib==0.8.0 # via social-auth-core
-requests==2.22.0 # via coreapi, requests-oauthlib, social-auth-core
-ruamel.yaml.clib==0.2.0 # via ruamel.yaml
-ruamel.yaml==0.16.7 # via drf-yasg
-six==1.11.0 # via django-extensions, drf-yasg, packaging, pip-tools, social-auth-app-django, social-auth-core
-social-auth-app-django==2.1.0 # via django-social-authsch
-social-auth-core==1.6.0 # via django-social-authsch, social-auth-app-django
-sqlparse==0.3.0 # via django
-tablib==0.12.1 # via django-import-export
-ujson==1.35 # via python-jsonrpc-server
-unicodecsv==0.14.1 # via tablib
-uritemplate==3.0.1 # via coreapi, drf-yasg
-urllib3==1.25.8 # via requests
-xlrd==1.1.0 # via tablib
-xlwt==1.3.0 # via tablib
-zipp==2.1.0 # via importlib-metadata
+amqp==5.1.1
+ # via kombu
+asgiref==3.6.0
+ # via django
+billiard==3.6.4.0
+ # via celery
+build==0.10.0
+ # via pip-tools
+celery==5.2.7
+ # via -r requirements/production.in
+certifi==2018.1.18
+ # via requests
+chardet==3.0.4
+ # via requests
+click==8.1.3
+ # via
+ # celery
+ # click-didyoumean
+ # click-plugins
+ # click-repl
+ # pip-tools
+click-didyoumean==0.3.0
+ # via celery
+click-plugins==1.1.1
+ # via celery
+click-repl==0.2.0
+ # via celery
+coreapi==2.3.3
+ # via drf-yasg
+coreschema==0.0.4
+ # via
+ # coreapi
+ # drf-yasg
+diff-match-patch==20121119
+ # via django-import-export
+django==4.1.7
+ # via
+ # -r requirements/base.in
+ # django-extensions
+ # django-import-export
+ # django-solo
+ # djangorestframework
+ # drf-yasg
+django-extensions==3.2.1
+ # via -r requirements/production.in
+django-import-export==3.1.0
+ # via -r requirements/base.in
+django-solo==2.0.0
+ # via -r requirements/base.in
+djangorestframework==3.14.0
+ # via
+ # -r requirements/base.in
+ # drf-yasg
+djangorestframework-api-key==2.3.0
+ # via -r requirements/production.in
+drf-yasg==1.21.5
+ # via -r requirements/production.in
+et-xmlfile==1.0.1
+ # via openpyxl
+flake8==6.0.0
+ # via -r requirements/production.in
+gunicorn==20.1.0
+ # via -r requirements/production.in
+idna==2.6
+ # via requests
+inflection==0.3.1
+ # via drf-yasg
+itypes==1.1.0
+ # via coreapi
+jedi==0.17.2
+ # via python-language-server
+jinja2==2.11.1
+ # via coreschema
+kombu==5.2.4
+ # via celery
+markuppy==1.14
+ # via tablib
+markupsafe==1.1.1
+ # via jinja2
+mccabe==0.7.0
+ # via flake8
+odfpy==1.3.6
+ # via tablib
+openpyxl==3.1.2
+ # via tablib
+packaging==23.0
+ # via
+ # -r requirements/production.in
+ # build
+ # drf-yasg
+parso==0.7.1
+ # via jedi
+pillow==9.4.0
+ # via -r requirements/production.in
+pip-tools==6.12.3
+ # via -r requirements/production.in
+pluggy==0.13.1
+ # via python-language-server
+prompt-toolkit==3.0.38
+ # via click-repl
+pycodestyle==2.10.0
+ # via flake8
+pyflakes==3.0.1
+ # via flake8
+pyproject-hooks==1.0.0
+ # via build
+python-jsonrpc-server==0.4.0
+ # via python-language-server
+python-language-server==0.36.2
+ # via -r requirements/production.in
+pytz==2022.7.1
+ # via
+ # celery
+ # djangorestframework
+ # drf-yasg
+pyyaml==5.3
+ # via tablib
+requests==2.22.0
+ # via coreapi
+ruamel-yaml==0.17.21
+ # via drf-yasg
+ruamel-yaml-clib==0.2.7
+ # via ruamel-yaml
+six==1.11.0
+ # via click-repl
+sqlparse==0.3.0
+ # via django
+tablib[html,ods,xls,xlsx,yaml]==3.3.0
+ # via django-import-export
+tomli==2.0.1
+ # via
+ # build
+ # pyproject-hooks
+ujson==5.7.0
+ # via
+ # python-jsonrpc-server
+ # python-language-server
+uritemplate==3.0.1
+ # via
+ # coreapi
+ # drf-yasg
+urllib3==1.25.8
+ # via requests
+vine==5.0.0
+ # via
+ # amqp
+ # celery
+ # kombu
+wcwidth==0.2.6
+ # via prompt-toolkit
+wheel==0.38.4
+ # via pip-tools
+xlrd==1.1.0
+ # via tablib
+xlwt==1.3.0
+ # via tablib
+psycopg2-binary==2.9.5
+ # via -r requirements/production.in
+redis==4.5.1
+ # via -r requirements/production.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools
diff --git a/src/account/auth_pipeline.py b/src/account/auth_pipeline.py
index 34ca018..7516991 100644
--- a/src/account/auth_pipeline.py
+++ b/src/account/auth_pipeline.py
@@ -1,5 +1,5 @@
from django.core import exceptions
-from common import email
+from common.email import registration
from . import models
@@ -11,4 +11,4 @@ def create_profile(backend, user, response, *args, **kwargs):
except exceptions.ObjectDoesNotExist:
models.Profile.objects.create(user=user)
if user.email is not None:
- email.registration(user)
+ registration.delay(user)
diff --git a/src/account/serializers.py b/src/account/serializers.py
index 728074e..b0b04f6 100644
--- a/src/account/serializers.py
+++ b/src/account/serializers.py
@@ -1,6 +1,6 @@
from rest_framework import serializers
from account import models
-from common import email
+from common.email import admitted, denied
class ChoiceSerializer(serializers.ModelSerializer):
@@ -59,9 +59,9 @@ class ProfileSerializer_User(serializers.ModelSerializer):
new_role = validated_data.get('role', instance.role)
if instance.role != new_role:
if new_role == 'Student':
- email.admitted(instance.user)
+ admitted.delay(instance.user)
if new_role == 'Denied':
- email.denied(instance.user)
+ denied.delay(instance.user)
return super().update(instance, validated_data)
def get_full_name(self, obj):
@@ -118,9 +118,9 @@ class ProfileSerializer_Staff(serializers.ModelSerializer):
new_role = validated_data.get('role', instance.role)
if instance.role != new_role:
if new_role == 'Student':
- email.admitted(instance.user)
+ admitted.delay(instance.user)
if new_role == 'Denied':
- email.denied(instance.user)
+ denied.delay(instance.user)
return super().update(instance, validated_data)
def get_full_name(self, obj):
diff --git a/src/common/email.py b/src/common/email.py
index 53b3ba1..4c89adb 100644
--- a/src/common/email.py
+++ b/src/common/email.py
@@ -1,7 +1,9 @@
import os
-from django.core.mail import EmailMessage
-import codecs
import sys
+import codecs
+from django.core.mail import EmailMessage
+from celery import shared_task
+
SENDER_EMAIL = os.getenv('SENDER_MAIL', 'kepzes@kszk.bme.hu')
HOMEWORK_LINK = os.getenv(
@@ -13,7 +15,7 @@ def get_full_name(user):
def read_email(name):
- with codecs.open('common/emails/' + name, 'r', 'utf-8') as myfile:
+ with codecs.open('worker/emails/' + name, 'r', 'utf-8') as myfile:
data = myfile.read()
return data
@@ -34,30 +36,31 @@ def send_out_mail(subject, message, SENDER_EMAIL, receiver_email):
email.send()
+@shared_task()
def registration(user):
subject = "KSZKépzés regisztráció"
message = read_email('registration.txt')
message = str.format(message % {'name': get_full_name(user)})
send_out_mail(subject, message, SENDER_EMAIL, [user.email, ])
- pass
+@shared_task()
def admitted(user):
subject = "Jelentkezés eredménye"
message = read_email('admitted.txt')
message = str.format(message % {'name': get_full_name(user)})
send_out_mail(subject, message, SENDER_EMAIL, [user.email, ])
- pass
+@shared_task()
def denied(user):
subject = "Jelentkezés eredménye"
message = read_email('denied.txt')
message = str.format(message % {'name': get_full_name(user)})
send_out_mail(subject, message, SENDER_EMAIL, [user.email, ])
- pass
+@shared_task()
def new_homework(user, deadline):
deadline = deadline.strftime('%Y-%m-%d %H:%M')
subject = "Új házifeladat"
@@ -65,9 +68,9 @@ def new_homework(user, deadline):
message = str.format(
message % {'name': get_full_name(user), 'link': HOMEWORK_LINK, 'deadline': deadline})
send_out_mail(subject, message, SENDER_EMAIL, [user.email, ])
- pass
+@shared_task()
def homework_corrected(user, title, accepted):
subject = "Házifeladat eredménye"
if accepted:
@@ -80,4 +83,3 @@ def homework_corrected(user, title, accepted):
'status': status,
'title': title})
send_out_mail(subject, message, SENDER_EMAIL, [user.email, ])
- pass
diff --git a/src/homework/serializers.py b/src/homework/serializers.py
index d2ccfb1..498e4e0 100755
--- a/src/homework/serializers.py
+++ b/src/homework/serializers.py
@@ -1,7 +1,7 @@
from rest_framework import serializers
from django.utils import timezone
from . import models
-from common import email
+from common.email import homework_corrected, new_homework
class TaskSerializer(serializers.ModelSerializer):
@@ -17,12 +17,11 @@ class TaskSerializer(serializers.ModelSerializer):
return data
def create(self, validated_data):
- # TODO: Worker timeout. Do something with outlook request limitations
- # profiles = Profile.objects.filter(
- # role="Student").exclude(
- # user__email='')
- # for profile in profiles:
- # email.new_homework(profile.user, validated_data.get('deadline'))
+ profiles = Profile.objects.filter(
+ role="Student").exclude(
+ user__email='')
+ for profile in profiles:
+ new_homework.delay(profile.user, validated_data.get('deadline'))
return self.Meta.model.objects.create(**validated_data)
@@ -61,7 +60,7 @@ class SolutionSerializer_Student(serializers.ModelSerializer):
def update(self, instance, validated_data):
if instance.corrected is not True and validated_data.get(
'corrected', instance.corrected) is True:
- email.homework_corrected(
+ homework_corrected.delay(
instance.created_by.user,
instance.task.title,
validated_data.get('accepted', instance.accepted)
@@ -105,7 +104,7 @@ class SolutionSerializer_Staff(serializers.ModelSerializer):
def update(self, instance, validated_data):
if instance.corrected is not True and validated_data.get(
'corrected', instance.corrected) is True:
- email.homework_corrected(
+ homework_corrected.delay(
instance.created_by.user,
instance.task.title,
validated_data.get('accepted', instance.accepted)
diff --git a/src/kszkepzes/__init__.py b/src/kszkepzes/__init__.py
index e69de29..53f4ccb 100644
--- a/src/kszkepzes/__init__.py
+++ b/src/kszkepzes/__init__.py
@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)
diff --git a/src/kszkepzes/celery.py b/src/kszkepzes/celery.py
new file mode 100644
index 0000000..99d24f1
--- /dev/null
+++ b/src/kszkepzes/celery.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+
+import os
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kszkepzes.settings.local")
+app = Celery("kszkepzes_worker")
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks()
diff --git a/src/kszkepzes/settings/local.py b/src/kszkepzes/settings/local.py
index 1c4e0b5..b33d4ee 100644
--- a/src/kszkepzes/settings/local.py
+++ b/src/kszkepzes/settings/local.py
@@ -9,3 +9,5 @@ EMAIL_USE_TLS = True
EMAIL_PORT = 587
EMAIL_HOST_USER = os.getenv('EMAIL')
EMAIL_HOST_PASSWORD = os.getenv('EMAIL_PASSWORD')
+CELERY_BROKER_URL = "redis://localhost:6379"
+CELERY_RESULT_BACKEND = "redis://localhost:6379"
diff --git a/src/kszkepzes/settings/production.py b/src/kszkepzes/settings/production.py
index 708db64..8ece79a 100644
--- a/src/kszkepzes/settings/production.py
+++ b/src/kszkepzes/settings/production.py
@@ -30,3 +30,7 @@ EMAIL_PORT = 587
EMAIL_USE_TLS = True
EMAIL_HOST_USER = os.getenv('SMTP_USER')
EMAIL_HOST_PASSWORD = os.getenv('SMTP_PASSWORD')
+
+CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL')
+CELERY_RESULT_BACKEND = os.getenv('CELERY_BROKER_URL')
+
--
GitLab