diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f8f3dc671..dc118a1298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to - ✨(frontend) comment side panel #2279 - ✨(buildpack) add PaaS deployment support, tested with Scalingo #2293 - 🔧(backend) allow configuring settings OIDC_OP_USER_ENDPOINT_FORMAT +- ⚡️(helm) create a dedicated svc and deployment for yprovider converter #2368 ### Changed diff --git a/Makefile b/Makefile index e529312244..d1defd6f26 100644 --- a/Makefile +++ b/Makefile @@ -232,6 +232,7 @@ run-backend: ## Start only the backend application and all needed services @$(COMPOSE) up --force-recreate -d docspec @$(COMPOSE) up --force-recreate -d celery-dev @$(COMPOSE) up --force-recreate -d y-provider-development + @$(COMPOSE) up --force-recreate -d y-provider-development-converter @$(COMPOSE) up --force-recreate -d nginx .PHONY: run-backend @@ -247,6 +248,7 @@ run-e2e: @$(COMPOSE_E2E) stop y-provider-development @$(COMPOSE_E2E) up --force-recreate -d frontend @$(COMPOSE_E2E) up --force-recreate -d y-provider + @$(COMPOSE_E2E) up --force-recreate -d y-provider-converter .PHONY: run-e2e status: ## an alias for "docker compose ps" diff --git a/compose-e2e.yml b/compose-e2e.yml index e940294a58..f918b5cc1e 100644 --- a/compose-e2e.yml +++ b/compose-e2e.yml @@ -1,8 +1,7 @@ services: - frontend: user: "${DOCKER_USER:-1000}" - build: + build: context: . dockerfile: ./src/frontend/Dockerfile target: frontend-production @@ -16,7 +15,7 @@ services: y-provider: user: ${DOCKER_USER:-1000} - build: + build: context: . dockerfile: ./src/frontend/servers/y-provider/Dockerfile target: y-provider @@ -26,4 +25,15 @@ services: - env.d/development/common - env.d/development/common.local ports: - - "4444:4444" \ No newline at end of file + - "4444:4444" + + y-provider-converter: + user: ${DOCKER_USER:-1000} + image: impress:y-provider-production + restart: unless-stopped + env_file: + - env.d/development/common + - env.d/development/common.local + depends_on: + y-provider: + condition: service_started diff --git a/compose.yml b/compose.yml index 7821beb117..1480304ffb 100644 --- a/compose.yml +++ b/compose.yml @@ -199,6 +199,21 @@ services: - /home/frontend/node_modules - /home/frontend/servers/y-provider/node_modules + y-provider-development-converter: + user: ${DOCKER_USER:-1000} + image: impress:y-provider-development + restart: unless-stopped + env_file: + - env.d/development/common + - env.d/development/common.local + volumes: + - ./src/frontend/:/home/frontend + - /home/frontend/node_modules + - /home/frontend/servers/y-provider/node_modules + depends_on: + y-provider-development: + condition: service_started + kc_postgresql: image: postgres:14.3 healthcheck: diff --git a/env.d/development/common b/env.d/development/common index 804a3976e2..e187e2c6cc 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -81,7 +81,7 @@ COLLABORATION_WS_URL=ws://localhost:4444/collaboration/ws/ COLLABORATION_WS_INACTIVITY_TIMEOUT=15 # Seconds DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token -Y_PROVIDER_API_BASE_URL=http://y-provider-development:4444/api/ +Y_PROVIDER_API_BASE_URL=http://y-provider-development-converter:4444/api/ Y_PROVIDER_API_KEY=yprovider-api-key DOCSPEC_API_URL=http://docspec:4000/conversion diff --git a/env.d/development/common.e2e b/env.d/development/common.e2e index b69b5b97c8..73fb517dde 100644 --- a/env.d/development/common.e2e +++ b/env.d/development/common.e2e @@ -2,7 +2,7 @@ BURST_THROTTLE_RATES="1000/minute" COLLABORATION_API_URL=http://y-provider:4444/collaboration/api/ SUSTAINED_THROTTLE_RATES="1000/minute" -Y_PROVIDER_API_BASE_URL=http://y-provider:4444/api/ +Y_PROVIDER_API_BASE_URL=http://y-provider-converter:4444/api/ # Throttle API_DOCUMENT_THROTTLE_RATE=1000/min diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index 19aba4c28b..6dfead8e44 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -16,6 +16,9 @@ backend: replicas: 1 envVars: COLLABORATION_SERVER_SECRET: my-secret + COLLABORATION_API_URL: https://docs.127.0.0.1.nip.io/collaboration/api/ + COLLABORATION_WS_NOT_CONNECTED_READY_ONLY: False + CONVERSION_UPLOAD_ENABLED: True DJANGO_CSRF_TRUSTED_ORIGINS: https://docs.127.0.0.1.nip.io DJANGO_CONFIGURATION: Feature DJANGO_ALLOWED_HOSTS: docs.127.0.0.1.nip.io @@ -71,7 +74,7 @@ backend: STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage DOCSPEC_API_URL: http://impress-docs-docspec:4000/conversion USER_RECONCILIATION_FORM_URL: https://docs.127.0.0.1.nip.io - Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider:443/api/ + Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider-converter:443/api/ Y_PROVIDER_API_KEY: my-secret CACHES_KEY_PREFIX: "{{ now | unixEpoch }}" django: @@ -153,6 +156,11 @@ frontend: runAsNonRoot: false yProvider: + + converter: + enabled: true + replicas: 2 + replicas: 1 image: diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index 4ba687c9f2..fb25acd53d 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -17,6 +17,9 @@ backend: replicas: 1 envVars: COLLABORATION_SERVER_SECRET: my-secret + CONVERSION_UPLOAD_ENABLED: True + COLLABORATION_API_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }}/collaboration/api/ + COLLABORATION_WS_NOT_CONNECTED_READY_ONLY: True DJANGO_CSRF_TRUSTED_ORIGINS: https://{{ .Values.feature }}-docs.{{ .Values.domain }} DJANGO_CONFIGURATION: Feature DJANGO_ALLOWED_HOSTS: {{ .Values.feature }}-docs.{{ .Values.domain }} @@ -71,7 +74,7 @@ backend: STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage DOCSPEC_API_URL: http://impress-docs-docspec:4000/conversion USER_RECONCILIATION_FORM_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }} - Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider:443/api/ + Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider-converter:443/api/ Y_PROVIDER_API_KEY: my-secret CACHES_KEY_PREFIX: "{{ now | unixEpoch }}" migrate: @@ -131,6 +134,11 @@ frontend: tag: *tag yProvider: + + converter: + enabled: true + replicas: 1 + replicas: 1 image: diff --git a/src/helm/impress/templates/_helpers.tpl b/src/helm/impress/templates/_helpers.tpl index 29530f9052..c4fe19048b 100644 --- a/src/helm/impress/templates/_helpers.tpl +++ b/src/helm/impress/templates/_helpers.tpl @@ -186,6 +186,15 @@ Requires top level scope {{ include "impress.fullname" . }}-y-provider {{- end }} +{{/* +Full name for the yProvider converter + +Requires top level scope +*/}} +{{- define "impress.yProvider.converter.fullname" -}} +{{ include "impress.yProvider.fullname" . }}-converter +{{- end }} + {{/* Full name for the docSpec diff --git a/src/helm/impress/templates/yprovider_deployment_converter.yaml b/src/helm/impress/templates/yprovider_deployment_converter.yaml new file mode 100644 index 0000000000..0eb3dbe3ca --- /dev/null +++ b/src/helm/impress/templates/yprovider_deployment_converter.yaml @@ -0,0 +1,189 @@ +{{ if .Values.yProvider.converter.enabled -}} +{{- $yProvider := .Values.yProvider -}} +{{- $converter := .Values.yProvider.converter -}} +{{- $service := mergeOverwrite (dict) (default dict $yProvider.service) (default dict $converter.service) -}} +{{- $image := mergeOverwrite (dict) (default dict $yProvider.image) (default dict $converter.image) -}} +{{- $probes := mergeOverwrite (dict) (default dict $yProvider.probes) (default dict $converter.probes) -}} +{{- $pdb := mergeOverwrite (dict) (default dict $yProvider.pdb) (default dict $converter.pdb) -}} +{{- $dpAnnotations := mergeOverwrite (dict) (default dict $yProvider.dpAnnotations) (default dict $converter.dpAnnotations) -}} +{{- $podAnnotations := mergeOverwrite (dict) (default dict $yProvider.podAnnotations) (default dict $converter.podAnnotations) -}} +{{- $replicas := default $yProvider.replicas $converter.replicas -}} +{{- $serviceAccountName := default $yProvider.serviceAccountName $converter.serviceAccountName -}} +{{- $shareProcessNamespace := default $yProvider.shareProcessNamespace $converter.shareProcessNamespace -}} +{{- $sidecars := default $yProvider.sidecars $converter.sidecars -}} +{{- $command := default $yProvider.command $converter.command -}} +{{- $args := default $yProvider.args $converter.args -}} +{{- $envFrom := default $yProvider.envFrom $converter.envFrom -}} +{{- $securityContext := mergeOverwrite (dict) (default dict $yProvider.securityContext) (default dict $converter.securityContext) -}} +{{- $resources := mergeOverwrite (dict) (default dict $yProvider.resources) (default dict $converter.resources) -}} +{{- $nodeSelector := mergeOverwrite (dict) (default dict $yProvider.nodeSelector) (default dict $converter.nodeSelector) -}} +{{- $affinity := mergeOverwrite (dict) (default dict $yProvider.affinity) (default dict $converter.affinity) -}} +{{- $tolerations := default $yProvider.tolerations $converter.tolerations -}} +{{- $persistence := mergeOverwrite (dict) (default dict $yProvider.persistence) (default dict $converter.persistence) -}} +{{- $extraVolumeMounts := default $yProvider.extraVolumeMounts $converter.extraVolumeMounts -}} +{{- $extraVolumes := default $yProvider.extraVolumes $converter.extraVolumes -}} +{{- $envVarsScope := dict "envVars" (mergeOverwrite (dict) (default dict $yProvider.envVars) (default dict $converter.envVars)) -}} +{{- $envVars := include "impress.common.env" (list . $envVarsScope) -}} +{{- $fullName := include "impress.yProvider.converter.fullname" . -}} +{{- $component := "yProvider-converter" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + annotations: + {{- with $dpAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +spec: + replicas: {{ $replicas }} + selector: + matchLabels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} + template: + metadata: + annotations: + {{- with $podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + {{- if $serviceAccountName }} + serviceAccountName: {{ $serviceAccountName }} + {{- end }} + shareProcessNamespace: {{ $shareProcessNamespace }} + containers: + {{- with $sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ $image.repository | default $.Values.image.repository }}:{{ $image.tag | default $.Values.image.tag }}" + imagePullPolicy: {{ $image.pullPolicy | default $.Values.image.pullPolicy }} + {{- with $command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with $args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if $envVars}} + env: + {{- $envVars | indent 12 }} + {{- end }} + {{- if $envFrom }} + envFrom: + {{- toYaml $envFrom | nindent 12 }} + {{- end }} + {{- with $securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ $service.targetPort }} + protocol: TCP + {{- if $probes.liveness }} + livenessProbe: + {{- include "impress.probes.abstract" (merge $probes.liveness (dict "targetPort" $service.targetPort )) | nindent 12 }} + {{- end }} + {{- if $probes.readiness }} + readinessProbe: + {{- include "impress.probes.abstract" (merge $probes.readiness (dict "targetPort" $service.targetPort )) | nindent 12 }} + {{- end }} + {{- if $probes.startup }} + startupProbe: + {{- include "impress.probes.abstract" (merge $probes.startup (dict "targetPort" $service.targetPort )) | nindent 12 }} + {{- end }} + {{- with $resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range $name, $volume := $persistence }} + - name: "{{ $name }}" + mountPath: "{{ $volume.mountPath }}" + {{- end }} + {{- range $extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with $nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "impress.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range $name, $volume := $persistence }} + - name: "{{ $name }}" + {{- if eq $volume.type "emptyDir" }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: "{{ $fullName }}-{{ $name }}" + {{- end }} + {{- end }} + {{- range $extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .secret }} + secret: + {{ toYaml .secret | nindent 12 }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +--- +{{ if $pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} +{{ end }} +{{ end }} diff --git a/src/helm/impress/templates/yprovider_svc_converter.yaml b/src/helm/impress/templates/yprovider_svc_converter.yaml new file mode 100644 index 0000000000..45cd1e7bf2 --- /dev/null +++ b/src/helm/impress/templates/yprovider_svc_converter.yaml @@ -0,0 +1,25 @@ +{{ if .Values.yProvider.converter.enabled -}} +{{- $yProvider := .Values.yProvider -}} +{{- $converter := .Values.yProvider.converter -}} +{{- $service := mergeOverwrite (dict) (default dict $yProvider.service) (default dict $converter.service) -}} +{{- $fullName := include "impress.yProvider.converter.fullname" . -}} +{{- $component := "yProvider-converter" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} + annotations: + {{- toYaml $service.annotations | nindent 4 }} +spec: + type: {{ $service.type }} + ports: + - port: {{ $service.port }} + targetPort: {{ $service.targetPort }} + protocol: TCP + name: http + selector: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 4 }} +{{ end -}} diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index d218ad71d7..a64084647f 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -647,6 +647,77 @@ yProvider: pullPolicy: IfNotPresent tag: "latest" + converter: + ## @param yProvider.converter.enabled Enable the yProvider converter deployment and service + enabled: false + + ## @param yProvider.converter.replicas Amount of yProvider replicas + replicas: 3 + + ## @param yProvider.converter.resources Resource requirements for the yProvider container + resources: {} + + ## @param yProvider.converter.service.type yProvider converter Service type + ## @param yProvider.converter.service.port yProvider converter Service listening port + ## @param yProvider.converter.service.targetPort yProvider converter container listening port + ## @param yProvider.converter.service.annotations Annotations to add to the yProvider converter Service + service: {} + + ## @param yProvider.converter.command Override the yProvider converter container command + command: [] + + ## @param yProvider.converter.args Override the yProvider converter container args + args: [] + + ## @param yProvider.converter.shareProcessNamespace Enable share process namespace between containers + shareProcessNamespace: false + + ## @param yProvider.converter.sidecars Add sidecars containers to yProvider converter deployment + sidecars: [] + + ## @skip yProvider.converter.securityContext + securityContext: {} + + ## @skip yProvider.converter.envVars + envVars: {} + + ## @skip yProvider.converter.envFrom + envFrom: [] + + ## @param yProvider.converter.podAnnotations Annotations to add to the yProvider converter Pod + podAnnotations: {} + + ## @param yProvider.converter.dpAnnotations Annotations to add to the yProvider converter Deployment + dpAnnotations: {} + + ## @skip yProvider.converter.probes + probes: {} + + ## @param yProvider.converter.nodeSelector Node selector for the yProvider converter Pod + nodeSelector: {} + + ## @param yProvider.converter.tolerations Tolerations for the yProvider converter Pod + tolerations: [] + + ## @param yProvider.converter.affinity Affinity for the yProvider converter Pod + affinity: {} + + ## @param yProvider.converter.persistence Additional volumes to create and mount on the yProvider converter + persistence: {} + + ## @param yProvider.converter.extraVolumeMounts Additional volumes to mount on the yProvider converter + extraVolumeMounts: [] + + ## @param yProvider.converter.extraVolumes Additional volumes to mount on the yProvider converter + extraVolumes: [] + + ## @param yProvider.converter.pdb.enabled Enable pdb on yProvider converter + pdb: + enabled: true + + ## @param yProvider.converter.serviceAccountName Optional service account name to use for yProvider converter pods + serviceAccountName: null + ## @param yProvider.command Override the yProvider container command command: []