diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ed84308aaeac6362b450ff5ecc15d8cf844448b..13c9dcf0885b43522a028ffffe3e9abd8232bdc7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,11 +4,7 @@ image: python:3.11 # - postgres:13 variables: - POSTGRES_DB: kszkepzes - CONTAINER_IMAGE: 'harbor.sch.bme.hu/kszk/$CI_PROJECT_NAME:$CI_COMMIT_REF_NAME' - CI_REGISTRY_IMAGE: '$CONTAINER_IMAGE' - CS_REGISTRY_USER: '$REGISTRY_USER' - CS_REGISTRY_PASSWORD: '$REGISTRY_PASSWORD' + CONTAINER_IMAGE: 'harbor.sch.bme.hu/kszk-kepzes/backend:$CI_COMMIT_REF_NAME' cache: key: '$CI_JOB_NAME-$CI_COMMIT_REF_SLUG' @@ -19,32 +15,13 @@ cache: stages: - Test and Flake8 - Docker Build - - Scan Container - deploy -container_scanning: - stage: Scan Container - tags: [kszk] - rules: - - if: '$CI_COMMIT_TAG' - variables: - CS_DEFAULT_BRANCH_IMAGE: $CONTAINER_IMAGE - -sast: - tags: [kszk] - stage: Test and Flake8 - -secret_detection: - tags: [kszk] - rules: - - if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_TAG || $CI_COMMIT_REF_NAME == "dev"' - stage: Test and Flake8 Flake8: stage: Test and Flake8 rules: - if: '$CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_TAG || $CI_COMMIT_REF_NAME == "dev"' - tags: [kszk] script: - python -V - python3 -m venv env @@ -73,7 +50,6 @@ Flake8: Docker build to KSZK registry: stage: Docker Build - tags: [kszk] rules: - if: '$CI_COMMIT_TAG' image: @@ -83,27 +59,22 @@ Docker build to KSZK registry: - echo "{\"auths\":{\"harbor.sch.bme.hu\":{\"username\":\"$REGISTRY_USER\",\"password\":\"$REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CONTAINER_IMAGE -Deploy to Kubernetes: - stage: deploy - image: alpine - tags: [kszk] - environment: - name: master - before_script: - - chmod 600 $KUBECONFIG - - apk add --no-cache curl - - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - - chmod +x ./kubectl - - mv ./kubectl /usr/local/bin/kubectl - - kubectl version - script: - - cd k8s - - sed -i "s|##IMAGETAG##|${CI_COMMIT_REF_NAME}|" deployment.yml - - kubectl apply -f deployment.yml - rules: - - if: '$CI_COMMIT_TAG' - -include: -- template: Security/SAST.gitlab-ci.yml -- template: Jobs/Secret-Detection.gitlab-ci.yml -- template: Jobs/Container-Scanning.gitlab-ci.yml +# Deploy to Kubernetes: +# stage: deploy +# image: alpine +# tags: [k9r] +# environment: +# name: master +# before_script: +# - chmod 600 $KUBECONFIG +# - apk add --no-cache curl +# - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +# - chmod +x ./kubectl +# - mv ./kubectl /usr/local/bin/kubectl +# - kubectl version +# script: +# - cd k8s +# - sed -i "s|##IMAGETAG##|${CI_COMMIT_REF_NAME}|" deployment.yml +# - kubectl apply -f deployment.yml +# rules: +# - if: '$CI_COMMIT_TAG' diff --git a/Dockerfile b/Dockerfile index a16be4644e4aa73edef391ca20b95acf9cf654bd..5b37aaf3807625d3fc3352ee23ae749f8bbac8a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,9 @@ mkdir -p /home/app && mkdir $APP_HOME && \ mkdir $APP_HOME/staticfiles && mkdir $APP_HOME/mediafiles WORKDIR $APP_HOME -COPY ./requirements/production.txt requirements.txt +COPY ./requirements . +RUN python -m pip install pip-tools +RUN pip-compile --output-file=requirements.txt production.in RUN pip install -r requirements.txt diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..023d131d3621079eaf02c701177a0f279fdf45d1 --- /dev/null +++ b/k8s/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: kszkepzes +description: + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "prod" diff --git a/k8s/deployment.yml b/k8s/deployment.yml deleted file mode 100644 index be0e71650ff87f1635d36238bf54d6f09a23d4a5..0000000000000000000000000000000000000000 --- a/k8s/deployment.yml +++ /dev/null @@ -1,155 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kszkepzes-backend - namespace: kszk-kepzes-site -spec: - strategy: - type: Recreate - replicas: 1 - selector: - matchLabels: - app: kszkepzes-backend - template: - metadata: - labels: - app: kszkepzes-backend - spec: - initContainers: - - name: volume-permission-fix - image: busybox - command: - - "sh" - - "-c" - - | - chown -R 1000:1000 /mediafiles - chown -R 1000:1000 /staticfiles - volumeMounts: - - name: kszkepzes-media-volume - mountPath: /mediafiles - - name: kszkepzes-static-volume - mountPath: /staticfiles - resources: - limits: - memory: 30Mi - - name: kszkepzes-backend-collectstatic - image: harbor.sch.bme.hu/kszk/kszkepzes-backend:##IMAGETAG## - imagePullPolicy: "IfNotPresent" - command: ["python", "manage.py", "collectstatic", "--noinput"] - volumeMounts: - - mountPath: /home/app/kszkepzes-backend/staticfiles - name: kszkepzes-static-volume - envFrom: - - configMapRef: - name: kszkepzes-config - - secretRef: - name: kszkepzes-secret-config - resources: - limits: - memory: 200Mi - - name: kszkepzes-backend-migrate - image: harbor.sch.bme.hu/kszk/kszkepzes-backend:##IMAGETAG## - imagePullPolicy: "IfNotPresent" - command: ["python", "manage.py", "migrate", "--noinput"] - envFrom: - - configMapRef: - name: kszkepzes-config - - secretRef: - name: kszkepzes-secret-config - resources: - limits: - memory: 200Mi - containers: - - name: kszkepzes-backend - image: harbor.sch.bme.hu/kszk/kszkepzes-backend:##IMAGETAG## - imagePullPolicy: "IfNotPresent" - volumeMounts: - - mountPath: /home/app/kszkepzes-backend/mediafiles - name: kszkepzes-media-volume - - mountPath: /home/app/kszkepzes-backend/staticfiles - name: kszkepzes-static-volume - ports: - - containerPort: 8000 - envFrom: - - configMapRef: - name: kszkepzes-config - - secretRef: - name: kszkepzes-secret-config - resources: - requests: - cpu: "80m" - limits: - memory: 600Mi - cpu: "2" - - name: kszkepzes-files-serve - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - image: halverneus/static-file-server:latest - imagePullPolicy: "Always" - volumeMounts: - - mountPath: /var/www/html/mediafiles - name: kszkepzes-media-volume - - mountPath: /var/www/html/staticfiles - name: kszkepzes-static-volume - ports: - - containerPort: 8080 - env: - - name: PORT - value: "8080" - - name: SHOW_LISTING - value: "false" - - name: FOLDER - value: "/var/www/html" - resources: - requests: - cpu: "20m" - limits: - memory: 200Mi - cpu: "2" - volumes: - - name: kszkepzes-media-volume - persistentVolumeClaim: - claimName: kszkepzes-media-pv-claim - - name: kszkepzes-static-volume - persistentVolumeClaim: - claimName: kszkepzes-static-pv-claim - imagePullSecrets: - - name: harbor ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kszkepzes-backend-worker - namespace: kszk-kepzes-site -spec: - strategy: - type: Recreate - replicas: 1 - selector: - matchLabels: - app: kszkepzes-backend-worker - template: - metadata: - labels: - app: kszkepzes-backend-worker - spec: - containers: - - name: worker - image: harbor.sch.bme.hu/kszk/kszkepzes-backend:##IMAGETAG## - imagePullPolicy: "IfNotPresent" - envFrom: - - configMapRef: - name: kszkepzes-config - - secretRef: - name: kszkepzes-secret-config - command: ["python3"] - args: ["-m", "celery", "-A", "kszkepzes", "worker", "-l", "debug"] - resources: - requests: - cpu: "100m" - limits: - memory: 600Mi - cpu: "2" - imagePullSecrets: - - name: harbor diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml deleted file mode 100644 index bbde98c0f9620b369055de31a1bfeeefa324887c..0000000000000000000000000000000000000000 --- a/k8s/ingress.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod - kubernetes.io/ingress.class: nginx - kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/proxy-body-size: 20m - name: kszkepzes - namespace: kszk-kepzes-site -spec: - rules: - - host: ujonc.kszk.bme.hu - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: kszkepzes-frontend - port: - number: 3000 - - path: /api - pathType: Prefix - backend: - service: - name: kszkepzes-backend - port: - number: 8000 - - path: /admin - pathType: Prefix - backend: - service: - name: kszkepzes-backend - port: - number: 8000 - - path: /staticfiles - pathType: Prefix - backend: - service: - name: kszkepzes-backend - port: - number: 8080 - - path: /mediafiles - pathType: Prefix - backend: - service: - name: kszkepzes-backend - port: - number: 8080 - tls: - - hosts: - - ujonc.kszk.bme.hu - secretName: kszkepzes-cert diff --git a/k8s/pvc.yaml b/k8s/pvc.yaml deleted file mode 100644 index f314143c98b4ea9790aff6ac8c46a54ad4dc1cff..0000000000000000000000000000000000000000 --- a/k8s/pvc.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - app: kszkepzes - name: kszkepzes-media-pv-claim - namespace: kszk-kepzes-site -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: vsphere - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - app: kszkepzes - name: kszkepzes-static-pv-claim - namespace: kszk-kepzes-site -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - storageClassName: vsphere diff --git a/k8s/redis.values.yaml b/k8s/redis.values.yaml deleted file mode 100644 index bfcb6b819adb30ea50de9ebddfeca6b879b1992a..0000000000000000000000000000000000000000 --- a/k8s/redis.values.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# https://artifacthub.io/packages/helm/bitnami/redis -master: - resources: - requests: - memory: 100Mi - cpu: 80m - limits: - memory: 0.75Gi - cpu: 1 - persistence: - size: 1Gi -replica: - resources: - requests: - memory: 100Mi - cpu: 80m - limits: - memory: 0.75Gi - cpu: 1 - persistence: - size: 1Gi diff --git a/k8s/service.yml b/k8s/service.yml deleted file mode 100644 index f0f2242bb949041f9a50d974cbe6d28b220e6051..0000000000000000000000000000000000000000 --- a/k8s/service.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: kszkepzes-backend - namespace: kszk-kepzes-site -spec: - type: ClusterIP - ports: - - port: 8000 - targetPort: 8000 - name: backend - protocol: TCP - - port: 8080 - targetPort: 8080 - name: files - protocol: TCP - selector: - app: kszkepzes-backend diff --git a/k8s/templates/backend.yaml b/k8s/templates/backend.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72b654b96de13591589b1bb8cc81b9f500ed2960 --- /dev/null +++ b/k8s/templates/backend.yaml @@ -0,0 +1,213 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-backend +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-backend + template: + metadata: + labels: + app: {{ .Release.Name }}-backend + spec: + initContainers: + - name: kszkepzes-backend-collectstatic + image: {{ .Values.backend.image }}:{{ .Values.backend.tag }} + imagePullPolicy: "Always" + command: ["python", "manage.py", "collectstatic", "--noinput"] + envFrom: + - configMapRef: + name: {{ .Release.Name }}-config + - secretRef: + name: {{ .Release.Name }}-secret-config + resources: + limits: + memory: 200Mi + - name: kszkepzes-backend-migrate + image: {{ .Values.backend.image }}:{{ .Values.backend.tag }} + imagePullPolicy: "Always" + command: ["python", "manage.py", "migrate", "--noinput"] + envFrom: + - configMapRef: + name: {{ .Release.Name }}-config + - secretRef: + name: {{ .Release.Name }}-secret-config + resources: + limits: + memory: 200Mi + containers: + - name: kszkepzes-backend + image: {{ .Values.backend.image }}:{{ .Values.backend.tag }} + imagePullPolicy: "Always" + ports: + - containerPort: 8000 + envFrom: + - configMapRef: + name: {{ .Release.Name }}-config + - secretRef: + name: {{ .Release.Name }}-secret-config + resources: + requests: + cpu: "80m" + limits: + memory: 600Mi + cpu: "2" + startupProbe: + httpGet: + path: /healthz/ + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 5 + timeoutSeconds: 4 + livenessProbe: + httpGet: + port: 8000 + path: /healthz/ + periodSeconds: 10 + failureThreshold: 2 + timeoutSeconds: 4 + - name: kszkepzes-files-serve + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + image: halverneus/static-file-server:latest + imagePullPolicy: "Always" + ports: + - containerPort: 8080 + env: + - name: PORT + value: "8080" + - name: SHOW_LISTING + value: "false" + - name: FOLDER + value: "/var/www/html" + resources: + requests: + cpu: "20m" + limits: + memory: 200Mi + cpu: "2" + imagePullSecrets: + - name: {{ .Values.backend.imagePullSecretName }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-backend-worker +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-backend-worker + template: + metadata: + labels: + app: {{ .Release.Name }}-backend-worker + spec: + containers: + - name: worker + image: {{ .Values.backend.image }}:{{ .Values.backend.tag }} + imagePullPolicy: "Always" + envFrom: + - configMapRef: + name: {{ .Release.Name }}-config + - secretRef: + name: {{ .Release.Name }}-secret-config + command: ["python3"] + args: ["-m", "celery", "-A", "kszkepzes", "worker", "-l", "debug"] + livenessProbe: + exec: + command: + - celery + - inspect + - ping + periodSeconds: 10 + failureThreshold: 2 + timeoutSeconds: 4 + startupProbe: + exec: + command: + - celery + - inspect + - ping + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 5 + timeoutSeconds: 4 + resources: + requests: + cpu: "100m" + limits: + memory: 2Gi + cpu: "2" + imagePullSecrets: + - name: {{ .Values.backend.imagePullSecretName }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-config +data: + # DEBUG: "1" + DB_NAME: {{ .Values.db.name }} + DB_HOST: {{ .Values.db.host }} + CELERY_BROKER_URL: {{ printf "redis://%s-redis:6379" .Release.Name }} + DJANGO_SETTINGS_MODULE: "kszkepzes.settings.production" + + {{ if .Values.minio.enabled }} + MINIO_EXTERNAL_ENDPOINT: {{ .Values.url }} + MINIO_ACCESS_ENDPOINT: {{ .Release.Name }}-minio:9000 + {{ else }} + MINIO_EXTERNAL_ENDPOINT: {{ .Values.minio.MINIO_EXTERNAL_ENDPOINT }} + MINIO_ACCESS_ENDPOINT: {{ .Values.minio.MINIO_ACCESS_ENDPOINT }} + {{ end }} + + MINIO_STATIC_BUCKET: "kepzes-static" + MINIO_MEDIA_BUCKET: "kepzes-media" + CSRF_TRUSTED_ORIGINS: {{ printf "https://%s" .Values.url }} + SECRET_KEY: {{ .Values.backend.secretKey }} + OIDC_CLIENT_ID: {{ .Values.backend.oidc.OIDC_CLIENT_ID }} + OIDC_AUTHORIZATION_ENDPOINT: {{ .Values.backend.oidc.OIDC_AUTHORIZATION_ENDPOINT }} + OIDC_TOKEN_ENDPOINT: {{ .Values.backend.oidc.OIDC_TOKEN_ENDPOINT }} + OIDC_USERINFO_ENDPOINT: {{ .Values.backend.oidc.OIDC_USERINFO_ENDPOINT }} + OIDC_JWKS_ENDPOINT: {{ .Values.backend.oidc.OIDC_JWKS_ENDPOINT }} + OIDC_SIGN_ALGO: {{ .Values.backend.oidc.OIDC_SIGN_ALGO }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-secret-config +data: + DB_USER: {{ .Values.db.user | b64enc }} + DB_PASSWORD: {{ .Values.db.password | b64enc }} + MINIO_ACCESS_KEY: {{ .Values.minio.user | b64enc }} + MINIO_SECRET_KEY: {{ .Values.minio.password | b64enc }} + OIDC_CLIENT_SECRET: {{ .Values.backend.oidc.OIDC_CLIENT_SECRET | b64enc}} + + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-backend +spec: + type: ClusterIP + ports: + - port: 8000 + targetPort: 8000 + name: backend + protocol: TCP + - port: 8080 + targetPort: 8080 + name: files + protocol: TCP + selector: + app: {{ .Release.Name }}-backend diff --git a/k8s/templates/frontend.yaml b/k8s/templates/frontend.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ed8fb37eaddb1dcc24af931026bdd5449adf046 --- /dev/null +++ b/k8s/templates/frontend.yaml @@ -0,0 +1,42 @@ +{{ if .Values.frontend.enabled}} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-frontend +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-frontend + template: + metadata: + labels: + app: {{ .Release.Name }}-frontend + spec: + containers: + - name: kszkepzes-frontend + image: {{ .Values.frontend.image }}:{{ .Values.frontend.tag }} + imagePullPolicy: 'Always' + ports: + - containerPort: 3000 + resources: + limits: + memory: 200Mi + imagePullSecrets: + - name: {{ .Values.frontend.imagePullSecretName }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-frontend +spec: + type: ClusterIP + ports: + - port: 3000 + targetPort: 3000 + name: front + protocol: TCP + selector: + app: {{ .Release.Name }}-frontend +{{ end }} \ No newline at end of file diff --git a/k8s/templates/ingress.yaml b/k8s/templates/ingress.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f88da14c836308c07a29f8f90ef059fa0fd19a9d --- /dev/null +++ b/k8s/templates/ingress.yaml @@ -0,0 +1,95 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + cert-manager.io/cluster-issuer: letsencrypt + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/proxy-body-size: 1000m + name: {{ .Release.Name }} +spec: + rules: + - host: {{ .Values.url }} + http: + paths: + - path: /api + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-backend + port: + number: 8000 + - path: /admin + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-backend + port: + number: 8000 + - path: /staticfiles + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-backend + port: + number: 8080 + - path: /mediafiles + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-backend + port: + number: 8080 + - path: /oidc + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-backend + port: + number: 8000 + {{ if .Values.minio.enabled }} + - path: /kepzes-static + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-minio + port: + number: 9000 + - path: /kepzes-media + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-minio + port: + number: 9000 + {{end}} + tls: + - hosts: + - {{ .Values.url }} + secretName: {{ .Release.Name }}-cert + +{{ if .Values.frontend.enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + cert-manager.io/cluster-issuer: letsencrypt + kubernetes.io/tls-acme: "true" + name: {{ .Release.Name }}-frontend +spec: + rules: + - host: {{ .Values.url }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ .Release.Name }}-frontend + port: + number: 3000 + tls: + - hosts: + - {{ .Values.url }} + secretName: {{ .Release.Name }}-cert +{{ end }} \ No newline at end of file diff --git a/k8s/templates/minio.yaml b/k8s/templates/minio.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e5d9dc10e2b53c796abf2fb154831db136375fe --- /dev/null +++ b/k8s/templates/minio.yaml @@ -0,0 +1,89 @@ +{{ if .Values.minio.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-minio +spec: + selector: + matchLabels: + app: {{ .Release.Name }}-minio + replicas: 1 + template: + metadata: + labels: + app: {{ .Release.Name }}-minio + spec: + volumes: + - name: kszkepzes-minio-volume + persistentVolumeClaim: + claimName: {{ .Release.Name }}-minio-pvc + containers: + - name: {{ .Release.Name}}-minio + image: {{ .Values.minio.image }}:{{ .Values.minio.tag }} + args: ["server", "/data" ,"--console-address", ":9001"] + ports: + - containerPort: 9000 + protocol: TCP + - containerPort: 9001 + protocol: TCP + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-minio-credentials + key: MINIO_ROOT_USER + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-minio-credentials + key: MINIO_ROOT_PASSWORD + volumeMounts: + - mountPath: /data + name: kszkepzes-minio-volume + resources: + requests: + cpu: "200m" + memory: 500Mi + limits: + cpu: "2" + memory: 1Gi +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-minio-credentials +type: Opaque +data: + MINIO_ROOT_USER: {{ .Values.minio.user | b64enc }} + MINIO_ROOT_PASSWORD: {{ .Values.minio.password | b64enc }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Release.Name }}-minio-pvc +spec: + storageClassName: {{ .Values.minio.storageClassName }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-minio +spec: + type: ClusterIP + ports: + - port: 9000 + targetPort: 9000 + name: api + protocol: TCP + - port: 9001 + targetPort: 9001 + name: ui + protocol: TCP + selector: + app: {{ .Release.Name }}-minio +{{ end }} \ No newline at end of file diff --git a/k8s/templates/redis.yaml b/k8s/templates/redis.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ec18fae1bdde405ecb866441bc4207cf8a703f6 --- /dev/null +++ b/k8s/templates/redis.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-redis +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-redis + template: + metadata: + labels: + app: {{ .Release.Name }}-redis + spec: + containers: + - name: redis + image: {{ .Values.redis.image }}:{{ .Values.redis.tag }} + ports: + - containerPort: 6379 + args: + - "--save" + - "" + - "--appendonly" + - "no" + resources: + requests: + memory: 100Mi + cpu: "100m" + limits: + memory: 750Mi + cpu: "1" + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-redis +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + selector: + app: {{ .Release.Name }}-redis \ No newline at end of file diff --git a/k8s/values.yaml b/k8s/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..960d49561abc5b003f76dff68e94781284b67782 --- /dev/null +++ b/k8s/values.yaml @@ -0,0 +1,49 @@ +url: "ujonc.kszk.bme.hu" + +backend: + image: "harbor.sch.bme.hu/kszk-kepzes/backend" + tag: "test" + storageClassName: "local-path" + imagePullSecretName: "harbor" + secretKey: "" + oidc: + OIDC_CLIENT_ID: "a56088b8-c381-4da6-8e17-d7d129716b5a" + OIDC_CLIENT_SECRET: "" + OIDC_AUTHORIZATION_ENDPOINT: "https://login.microsoftonline.com/79f0ae63-ef51-49f5-9f51-78a3346e1507/oauth2/v2.0/authorize" + OIDC_TOKEN_ENDPOINT: "https://login.microsoftonline.com/79f0ae63-ef51-49f5-9f51-78a3346e1507/oauth2/v2.0/token" + OIDC_USERINFO_ENDPOINT: "https://graph.microsoft.com/oidc/userinfo" + OIDC_JWKS_ENDPOINT: "https://login.microsoftonline.com/79f0ae63-ef51-49f5-9f51-78a3346e1507/discovery/v2.0/keys" + OIDC_SIGN_ALGO: "RS256" + +frontend: + enabled: false + image: "harbor.sch.bme.hu/kszk-kepzes/frontend" + tag: "latest" + imagePullSecretName: "harbor" + +db: + host: "dbsch.internal" + name: "" + user: "" + password: "" + +minio: + # Deployoljon e saját miniot + enabled: true + # Ha enabled = false, milyen endpoint on érje el a backend a miniot + MINIO_ACCESS_ENDPOINT: "1.2.3.4:9000" + # Ha enabled = false, milyen (https) endpoint on lehet publikusan elérni a miniot + MINIO_EXTERNAL_ENDPOINT: "1.2.3.4:9000" + # Helyi vagy külső minio user + user: "user" + # Helyi vagy külső minio pass + password: "" + + # Csak ha enabled = true + image: "quay.io/minio/minio" + tag: "RELEASE.2024-02-14T21-36-02Z" + storageClassName: "local-path" + +redis: + image: "redis" + tag: "latest" diff --git a/requirements/base.in b/requirements/base.in index adb572227b7d9c760872e3ece624430d1ef2f2f9..cd4092b59be37f9e690b940b8172f59dbad8ef47 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -3,4 +3,8 @@ djangorestframework~=3.14 django-solo~=2.0 django-import-export~=3.1 celery~=5.2 -social-auth-app-django~=5.0 \ No newline at end of file +django-health-check +sentry-sdk[django] +django-extensions +drf-oidc-auth +django-minio-backend \ No newline at end of file diff --git a/requirements/production.in b/requirements/production.in index 5fa215f62e6cfdc67d77ce8aae0f3f086e83c8c3..a92c0abdfccbc837ba7657abef4c108807f7acdd 100644 --- a/requirements/production.in +++ b/requirements/production.in @@ -10,4 +10,9 @@ drf-yasg~=1.21 packaging~=23.0 Pillow~=9.4 djangorestframework-api-key~=2.3 - +mozilla_django_oidc +drf_spectacular +django-health-check +sentry-sdk[django] +drf-oidc-auth +django-minio-backend \ No newline at end of file diff --git a/requirements/production.txt b/requirements/production.txt index b58f177cc3494f44304d9717bd0989f07210d14c..8a15d11f17845cee85fa30170433f49ea223a0a5 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -2,24 +2,35 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --output-file=requirements/production.txt requirements/production.in +# pip-compile --output-file=production.txt production.in # amqp==5.1.1 # via kombu +argon2-cffi==23.1.0 + # via minio +argon2-cffi-bindings==21.2.0 + # via argon2-cffi asgiref==3.6.0 # via django async-timeout==4.0.2 # via redis +authlib==1.3.0 + # via drf-oidc-auth billiard==3.6.4.0 # via celery build==0.10.0 # via pip-tools celery==5.2.7 - # via -r requirements/base.in + # via -r base.in certifi==2018.1.18 - # via requests + # via + # minio + # requests + # sentry-sdk cffi==1.15.1 - # via cryptography + # via + # argon2-cffi-bindings + # cryptography chardet==3.0.4 # via requests click==8.1.3 @@ -42,41 +53,56 @@ coreschema==0.0.4 # coreapi # drf-yasg cryptography==39.0.2 - # via social-auth-core -defusedxml==0.7.1 # via - # python3-openid - # social-auth-core + # authlib + # drf-oidc-auth + # josepy + # mozilla-django-oidc + # pyopenssl diff-match-patch==20121119 # via django-import-export django==4.1.7 # via - # -r requirements/base.in + # -r base.in # django-extensions + # django-health-check # django-import-export + # django-minio-backend # django-solo # djangorestframework + # drf-oidc-auth # drf-yasg + # mozilla-django-oidc + # sentry-sdk django-extensions==3.2.1 - # via -r requirements/production.in + # via + # -r base.in + # -r production.in +django-health-check==3.18.1 + # via -r base.in django-import-export==3.1.0 - # via -r requirements/base.in + # via -r base.in +django-minio-backend==3.5.0 + # via -r base.in django-solo==2.0.0 - # via -r requirements/base.in + # via -r base.in djangorestframework==3.14.0 # via - # -r requirements/base.in + # -r base.in + # drf-oidc-auth # drf-yasg djangorestframework-api-key==2.3.0 - # via -r requirements/production.in + # via -r production.in +drf-oidc-auth==3.0.0 + # via -r base.in drf-yasg==1.21.5 - # via -r requirements/production.in + # via -r production.in et-xmlfile==1.0.1 # via openpyxl flake8==6.0.0 - # via -r requirements/production.in + # via -r production.in gunicorn==20.1.0 - # via -r requirements/production.in + # via -r production.in idna==2.6 # via requests inflection==0.3.1 @@ -87,6 +113,8 @@ jedi==0.17.2 # via python-language-server jinja2==2.11.1 # via coreschema +josepy==1.14.0 + # via mozilla-django-oidc kombu==5.2.4 # via celery markuppy==1.14 @@ -95,47 +123,47 @@ markupsafe==1.1.1 # via jinja2 mccabe==0.7.0 # via flake8 -oauthlib==3.2.2 - # via - # requests-oauthlib - # social-auth-core +minio==7.2.4 + # via django-minio-backend +mozilla-django-oidc==4.0.0 + # via -r production.in odfpy==1.3.6 # via tablib openpyxl==3.1.2 # via tablib packaging==23.0 # via - # -r requirements/production.in + # -r production.in # build # drf-yasg parso==0.7.1 # via jedi pillow==9.4.0 - # via -r requirements/production.in + # via -r production.in pip-tools==6.12.3 - # via -r requirements/production.in + # via -r production.in pluggy==0.13.1 # via python-language-server prompt-toolkit==3.0.38 # via click-repl psycopg2-binary==2.9.5 - # via -r requirements/production.in + # via -r production.in pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi +pycryptodome==3.20.0 + # via minio pyflakes==3.0.1 # via flake8 -pyjwt==2.6.0 - # via social-auth-core +pyopenssl==23.2.0 + # via josepy pyproject-hooks==1.0.0 # via build python-jsonrpc-server==0.4.0 # via python-language-server python-language-server==0.36.2 - # via -r requirements/production.in -python3-openid==3.2.0 - # via social-auth-core + # via -r production.in pytz==2022.7.1 # via # celery @@ -144,32 +172,32 @@ pytz==2022.7.1 pyyaml==5.3 # via tablib redis==4.5.1 - # via -r requirements/production.in + # via -r production.in requests==2.22.0 # via # coreapi - # requests-oauthlib - # social-auth-core -requests-oauthlib==1.3.1 - # via social-auth-core + # drf-oidc-auth + # mozilla-django-oidc ruamel-yaml==0.17.21 # via drf-yasg ruamel-yaml-clib==0.2.7 # via ruamel-yaml +sentry-sdk[django]==1.9.0 + # via -r base.in six==1.11.0 # via click-repl -social-auth-app-django==5.0.0 - # via -r requirements/base.in -social-auth-core==4.3.0 - # via social-auth-app-django sqlparse==0.3.0 # via django tablib[html,ods,xls,xlsx,yaml]==3.3.0 - # via django-import-export + # via + # django-import-export + # tablib tomli==2.0.1 # via # build # pyproject-hooks +typing-extensions==4.9.0 + # via minio ujson==5.7.0 # via # python-jsonrpc-server @@ -179,7 +207,10 @@ uritemplate==3.0.1 # coreapi # drf-yasg urllib3==1.25.8 - # via requests + # via + # minio + # requests + # sentry-sdk vine==5.0.0 # via # amqp diff --git a/src/authsch/__init__.py b/src/authsch/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/src/authsch/apps.py b/src/authsch/apps.py deleted file mode 100644 index 13882bac31bad2b60b80e7ab11c2e860621bfcd0..0000000000000000000000000000000000000000 --- a/src/authsch/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -# Original: https://git.sch.bme.hu/kszk/devteam/django-authsch -from django.apps import AppConfig - - -class AuthschConfig(AppConfig): - name = 'authsch' diff --git a/src/authsch/authentication.py b/src/authsch/authentication.py deleted file mode 100644 index 86d633478edb207219b12f991a41c0398e5d1580..0000000000000000000000000000000000000000 --- a/src/authsch/authentication.py +++ /dev/null @@ -1,35 +0,0 @@ -# Original: https://git.sch.bme.hu/kszk/devteam/django-authsch -# store schacc in username field -from social_core.backends.oauth import BaseOAuth2 - - -class AuthSCHOAuth2(BaseOAuth2): - """AuthSCH OAuth2 authentication backend""" - name = 'authsch' - ID_KEY = 'internal_id' - AUTHORIZATION_URL = 'https://auth.sch.bme.hu/site/login' - ACCESS_TOKEN_URL = 'https://auth.sch.bme.hu/oauth2/token' - ACCESS_TOKEN_METHOD = 'POST' - REFRESH_TOKEN_URL = 'https://auth.sch.bme.hu/oauth2/token' - DEFAULT_SCOPE = ['basic', 'mail', 'givenName', 'sn', 'linkedAccounts'] - EXTRA_DATA = [ - ('internal_id', 'id'), - ('expires_in', 'expires'), - ('refresh_token', 'refresh_token'), - ] - - def get_user_details(self, response): - """Return user details from AuthSCH account""" - return { - 'username': response.get('linkedAccounts').get('schacc'), - 'email': response.get('mail'), - 'first_name': response.get('givenName'), - 'last_name': response.get('sn') - } - - def user_data(self, access_token, *args, **kwargs): - """Loads user data from service""" - return self.get_json( - 'https://auth.sch.bme.hu/api/profile/', - params={'access_token': access_token} - ) diff --git a/src/kszkepzes/oidc_auth.py b/src/kszkepzes/oidc_auth.py new file mode 100644 index 0000000000000000000000000000000000000000..1bf22f24cd3dcdb26baa69cc15c4a54274db25df --- /dev/null +++ b/src/kszkepzes/oidc_auth.py @@ -0,0 +1,26 @@ +from mozilla_django_oidc.auth import OIDCAuthenticationBackend +from account import models +from common.email import registration + + +class JSOIDCAuthBackend(OIDCAuthenticationBackend): + def create_user(self, claims): + user = super(JSOIDCAuthBackend, self).create_user(claims) + user.first_name = claims.get('given_name', '') + user.last_name = claims.get('family_name', '') + user.save() + models.Profile.objects.create(user=user) + if user.email is not None: + registration.delay( + {'last_name': user.last_name, 'first_name': user.first_name, 'email': user.email}) + return user + + def update_user(self, user, claims): + user.first_name = claims.get('given_name', '') + user.last_name = claims.get('family_name', '') + user.save() + return user + + +def generate_username(email: str): + return email.split('@')[0] diff --git a/src/kszkepzes/settings/base.py b/src/kszkepzes/settings/base.py index b84b0721e5370b8d149606acacc17d56f86e0055..74c77ff90d3bbe25990a0b6f7a5b98944909a1f8 100644 --- a/src/kszkepzes/settings/base.py +++ b/src/kszkepzes/settings/base.py @@ -43,9 +43,11 @@ INSTALLED_APPS = [ 'import_export', 'rest_framework', "rest_framework_api_key", - 'social_django', + "rest_framework.authtoken", 'solo', - 'authsch', + 'oidc_auth', + 'mozilla_django_oidc', + 'django_minio_backend.apps.DjangoMinioBackendConfig', 'homework', 'account', 'stats', @@ -55,6 +57,12 @@ INSTALLED_APPS = [ 'images', 'groups', 'drf_yasg', + 'health_check', # required + 'health_check.contrib.migrations', + 'health_check.contrib.redis', # requires Redis broker + 'health_check.db', # stock Django health checkers + 'health_check.contrib.celery', # requires celery + 'health_check.contrib.celery_ping', # requires celery ] MIDDLEWARE = [ @@ -65,6 +73,7 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'mozilla_django_oidc.middleware.SessionRefresh', ] ROOT_URLCONF = 'kszkepzes.urls' @@ -117,31 +126,39 @@ AUTH_PASSWORD_VALIDATORS = [ }, ] -SOCIAL_AUTH_URL_NAMESPACE = 'social' AUTHENTICATION_BACKENDS = [ - 'authsch.authentication.AuthSCHOAuth2', - 'django.contrib.auth.backends.ModelBackend', + 'kszkepzes.oidc_auth.JSOIDCAuthBackend', ] -SOCIAL_AUTH_PIPELINE = ( - 'social_core.pipeline.social_auth.social_details', - 'social_core.pipeline.social_auth.social_uid', - 'social_core.pipeline.social_auth.auth_allowed', - 'social_core.pipeline.social_auth.social_user', - 'social_core.pipeline.user.get_username', - 'social_core.pipeline.user.create_user', - 'account.auth_pipeline.create_profile', - 'social_core.pipeline.social_auth.associate_user', - 'social_core.pipeline.social_auth.load_extra_data', - 'social_core.pipeline.user.user_details', -) - -SOCIAL_AUTH_AUTHSCH_KEY = os.getenv('AUTHSCH_KEY') -SOCIAL_AUTH_AUTHSCH_SECRET = os.getenv('AUTHSCH_SECRET') -SOCIAL_AUTH_LOGIN_REDIRECT_URL = '/' -LOGIN_URL = "login/authsch/" -LOGOUT_REDIRECT_URL = '/' + +REST_FRAMEWORK = { + 'DEFAULT_RENDERER_CLASSES': ( + 'rest_framework.renderers.JSONRenderer', + ), + 'DEFAULT_PARSER_CLASSES': ( + 'rest_framework.parsers.JSONParser', + ), + 'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema', + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'mozilla_django_oidc.contrib.drf.OIDCAuthentication', + 'rest_framework.authentication.SessionAuthentication', + ], +} + + +OIDC_RP_CLIENT_ID = os.environ.get('OIDC_CLIENT_ID') +OIDC_RP_CLIENT_SECRET = os.environ.get('OIDC_CLIENT_SECRET') +OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get('OIDC_AUTHORIZATION_ENDPOINT') +OIDC_OP_TOKEN_ENDPOINT = os.environ.get('OIDC_TOKEN_ENDPOINT') +OIDC_OP_USER_ENDPOINT = os.environ.get('OIDC_USERINFO_ENDPOINT') +OIDC_OP_JWKS_ENDPOINT = os.environ.get('OIDC_JWKS_ENDPOINT') +OIDC_RP_SIGN_ALGO = os.environ.get('OIDC_SIGN_ALGO', 'RS256') +OIDC_EXEMPT_URLS = ["/healthz/", "/admin/", "/oidc/"] +LOGIN_REDIRECT_URL = "/" +LOGOUT_REDIRECT_URL = "/" +OIDC_USERNAME_ALGO = 'kszkepzes.oidc_auth.generate_username' + # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ @@ -165,3 +182,4 @@ STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") MEDIA_URL = "/mediafiles/" MEDIA_ROOT = os.path.join(BASE_DIR, "mediafiles") + diff --git a/src/kszkepzes/settings/production.py b/src/kszkepzes/settings/production.py index 37fa5eb19ff3ca8e3f663283087127e4ac24dcf5..4d70d696ab206677e8017e425b3905304ff7f480 100644 --- a/src/kszkepzes/settings/production.py +++ b/src/kszkepzes/settings/production.py @@ -1,3 +1,5 @@ +from datetime import timedelta +from typing import List, Tuple from .base import * DEBUG = int(os.environ.get("DEBUG", default=0)) @@ -15,15 +17,6 @@ DATABASES = { } } -REST_FRAMEWORK = { - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - ), - 'DEFAULT_PARSER_CLASSES': ( - 'rest_framework.parsers.JSONParser', - ) -} - EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' EMAIL_HOST = os.getenv('SMTP_HOST', 'smtp.office365.com') EMAIL_PORT = 587 @@ -33,5 +26,31 @@ EMAIL_HOST_PASSWORD = os.getenv('SMTP_PASSWORD') CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL') CELERY_RESULT_BACKEND = os.getenv('CELERY_BROKER_URL') +REDIS_URL = os.getenv('CELERY_BROKER_URL') CSRF_TRUSTED_ORIGINS = [os.getenv('CSRF_TRUSTED_ORIGINS')] + +FILE_UPLOAD_DIR = os.environ.get("FILE_UPLOAD_DIR") + +#Minio Stuff +MINIO_CONSISTENCY_CHECK_ON_START = True +MINIO_EXTERNAL_ENDPOINT = os.environ.get("MINIO_EXTERNAL_ENDPOINT") +MINIO_EXTERNAL_ENDPOINT_USE_HTTPS = True +MINIO_ENDPOINT = os.environ.get("MINIO_ACCESS_ENDPOINT") +MINIO_USE_HTTPS = False +MINIO_REGION = 'us-east-1' +MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY") +MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY") +MINIO_URL_EXPIRY_HOURS = timedelta(hours=2) +MINIO_PUBLIC_BUCKETS = [ + os.environ.get('MINIO_STATIC_BUCKET'), + os.environ.get('MINIO_MEDIA_BUCKET') +] +MINIO_POLICY_HOOKS: List[Tuple[str, dict]] = [] +MINIO_MEDIA_FILES_BUCKET = os.environ.get('MINIO_MEDIA_BUCKET') +MINIO_STATIC_FILES_BUCKET = os.environ.get('MINIO_STATIC_BUCKET') +MINIO_BUCKET_CHECK_ON_SAVE = True +DEFAULT_FILE_STORAGE = 'django_minio_backend.models.MinioBackend' +STATICFILES_STORAGE = 'django_minio_backend.models.MinioBackendStatic' + +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") \ No newline at end of file diff --git a/src/kszkepzes/urls.py b/src/kszkepzes/urls.py index ff01eab0b222a59e74c79672a1e875d010a54e7f..89b5af48b3991556131b92132955469b48554185 100644 --- a/src/kszkepzes/urls.py +++ b/src/kszkepzes/urls.py @@ -29,7 +29,6 @@ urlpatterns = [ re_path(r'^api/v1/redoc/$', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), re_path(r'^admin/', admin.site.urls), - re_path(r'^api/v1/', include('social_django.urls', namespace='social')), re_path(r'^api/v1/homework/', include('homework.urls')), re_path(r'^api/v1/', include('stats.urls')), re_path(r'^api/v1/', include('account.urls')), @@ -38,7 +37,9 @@ urlpatterns = [ re_path(r'^api/v1/', include('groups.urls')), re_path(r'^api/v1/', include('mentors.urls')), re_path(r'^api/v1/', include('images.urls')), - re_path(r'^api/v1/logout/$', auth_views.LogoutView.as_view(), name='logout'), + re_path(r'^healthz/', include('health_check.urls')), + re_path(r'^oidc/', include('mozilla_django_oidc.urls')), + ] if settings.DEBUG: