From 6916d396608f24374bd9686f51679e65bef1153a Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Thu, 28 Aug 2025 15:43:21 +0200 Subject: [PATCH 01/48] Make stack ready to deploy --- helm/arranger/values.yaml | 10 +++++----- helm/databases/values.yaml | 4 ++-- helm/elasticsearch/values.yaml | 10 +++++----- helm/keycloak-db/values.yaml | 4 ++-- helm/keycloak/templates/deployment.yaml | 2 +- helm/keycloak/values.yaml | 10 ++++++---- helm/maestro/values.yaml | 4 ++-- helm/minio/values.yaml | 6 +++--- helm/score/values.yaml | 8 ++++---- helm/song-db/values.yaml | 4 ++-- helm/song/templates/deployment.yaml | 2 -- helm/song/values.yaml | 10 +++++----- 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/helm/arranger/values.yaml b/helm/arranger/values.yaml index e352b54..a3281cd 100644 --- a/helm/arranger/values.yaml +++ b/helm/arranger/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 fullnameOverride: "arranger" url: - hostname: "arranger.local" + hostname: "arranger-ilifu.openup.org.za" image: repository: ghcr.io/overture-stack/arranger-server @@ -26,7 +26,7 @@ arranger: compatibilityMode: true stage: - baseUrl: "http://arranger.local:3000" + baseUrl: "http://arranger-ilifu.openup.org.za:3000" adminRoot: "http://arranger-server:5050/graphql" resources: @@ -43,7 +43,7 @@ ingress: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "50m" hosts: - - host: arranger.local + - host: arranger-ilifu.openup.org.za paths: - path: / pathType: Prefix @@ -64,7 +64,7 @@ probes: password: "myelasticpassword" stage: - baseUrl: "http://arranger.local:3000" + baseUrl: "http://arranger-ilifu.openup.org.za:3000" adminRoot: "http://arranger-server:5050/graphql" resources: @@ -81,7 +81,7 @@ ingress: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "50m" hosts: - - host: arranger.local + - host: arranger-ilifu.openup.org.za paths: - path: / pathType: Prefix diff --git a/helm/databases/values.yaml b/helm/databases/values.yaml index 6332ad3..3950285 100644 --- a/helm/databases/values.yaml +++ b/helm/databases/values.yaml @@ -10,8 +10,8 @@ fullnameOverride: "" podAnnotations: {} -podSecurityContext: {} - # fsGroup: 2000 +podSecurityContext: + fsGroup: 999 securityContext: # capabilities: diff --git a/helm/elasticsearch/values.yaml b/helm/elasticsearch/values.yaml index 6b6cc54..e799d77 100644 --- a/helm/elasticsearch/values.yaml +++ b/helm/elasticsearch/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 fullnameOverride: "elasticsearch" url: - hostname: "elasticsearch.local" + hostname: "es-ilifu.openup.org.za" image: repository: docker.elastic.co/elasticsearch/elasticsearch @@ -55,11 +55,11 @@ probes: ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: elasticsearch.local + - host: es-ilifu.openup.org.za paths: - path: / pathType: Prefix @@ -92,11 +92,11 @@ resources: ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: elasticsearch.local + - host: es-ilifu.openup.org.za paths: - path: / pathType: Prefix diff --git a/helm/keycloak-db/values.yaml b/helm/keycloak-db/values.yaml index 3c4c8f0..b75aa64 100644 --- a/helm/keycloak-db/values.yaml +++ b/helm/keycloak-db/values.yaml @@ -10,8 +10,8 @@ fullnameOverride: "keycloak-db" podAnnotations: {} -podSecurityContext: {} - # fsGroup: 2000 +podSecurityContext: + fsGroup: 999 securityContext: # capabilities: diff --git a/helm/keycloak/templates/deployment.yaml b/helm/keycloak/templates/deployment.yaml index 4ee0d04..f5da4c0 100644 --- a/helm/keycloak/templates/deployment.yaml +++ b/helm/keycloak/templates/deployment.yaml @@ -68,7 +68,7 @@ spec: - name: KC_HOSTNAME_STRICT value: "false" - name: KC_HOSTNAME_STRICT_HTTPS - value: "false" + value: "true" {{- if .Values.extraEnv }} {{- range .Values.extraEnv }} - name: {{ .name }} diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index 3de1d73..32578a1 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -1,7 +1,7 @@ replicaCount: 1 url: - hostname: "keycloak.local" + hostname: "keycloak-ilifu.openup.org.za" image: repository: quay.io/keycloak/keycloak @@ -17,13 +17,13 @@ service: ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-buffer-size: "128k" nginx.ingress.kubernetes.io/proxy-buffering: "on" nginx.ingress.kubernetes.io/proxy-buffers-number: "20" hosts: - - host: keycloak.local + - host: keycloak-ilifu.openup.org.za paths: - path: / pathType: Prefix @@ -38,6 +38,8 @@ extraEnv: value: "false" - name: JAVA_OPTS_APPEND value: "-Djava.awt.headless=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.net.preferIPv4Stack=true -server -Xms512m -Xmx768m" + - name: KC_CORS_ORIGINS + value: "*" probes: liveness: @@ -66,7 +68,7 @@ keycloak: password: admin123 # Keycloak settings - hostname: keycloak.local + hostname: keycloak-ilifu.openup.org.za healthEnabled: true startupMode: start-dev diff --git a/helm/maestro/values.yaml b/helm/maestro/values.yaml index a3eaf21..42f9b44 100644 --- a/helm/maestro/values.yaml +++ b/helm/maestro/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 fullnameOverride: "maestro" url: - hostname: "maestro.local" + hostname: "maestro-ilifu.openup.org.za" image: repository: ghcr.io/overture-stack/maestro @@ -117,7 +117,7 @@ ingress: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: maestro.local + - host: maestro-ilifu.openup.org.za paths: - path: / pathType: Prefix diff --git a/helm/minio/values.yaml b/helm/minio/values.yaml index b14dd75..9e51b50 100644 --- a/helm/minio/values.yaml +++ b/helm/minio/values.yaml @@ -3,11 +3,11 @@ replicaCount: 1 fullnameOverride: "minio" url: - hostname: "minio.local" + hostname: "minio-ilifu.openup.org.za" console: url: - hostname: "minio-console.local" + hostname: "minio-console-ilifu.openup.org.za" ingress: enabled: true className: "nginx" @@ -34,7 +34,7 @@ ingress: nginx.ingress.kubernetes.io/proxy-body-size: "1000m" nginx.ingress.kubernetes.io/proxy-request-buffering: "off" hosts: - - host: minio.local + - host: minio-ilifu.openup.org.za paths: - path: / pathType: Prefix diff --git a/helm/score/values.yaml b/helm/score/values.yaml index 000a6ca..207c91b 100644 --- a/helm/score/values.yaml +++ b/helm/score/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 fullnameOverride: "score" url: - hostname: "score.local" + hostname: "score-ilifu.openup.org.za" image: repository: ghcr.io/overture-stack/score-server @@ -86,7 +86,7 @@ score: resourceserver: jwt: jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" - issuerUri: "http://keycloak.local/realms/agari" + issuerUri: "http://keycloak-ilifu.openup.org.za/realms/agari" ingress: enabled: true @@ -94,7 +94,7 @@ ingress: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: score.local + - host: score-ilifu.openup.org.za paths: - path: / pathType: Prefix @@ -112,7 +112,7 @@ ingress: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: score.local + - host: score-ilifu.openup.org.za paths: - path: / pathType: Prefix diff --git a/helm/song-db/values.yaml b/helm/song-db/values.yaml index 1b396d9..cdd3bce 100644 --- a/helm/song-db/values.yaml +++ b/helm/song-db/values.yaml @@ -10,8 +10,8 @@ fullnameOverride: "song-db" podAnnotations: {} -podSecurityContext: {} - # fsGroup: 2000 +podSecurityContext: + fsGroup: 999 securityContext: # capabilities: diff --git a/helm/song/templates/deployment.yaml b/helm/song/templates/deployment.yaml index 1303c55..82db2c4 100644 --- a/helm/song/templates/deployment.yaml +++ b/helm/song/templates/deployment.yaml @@ -98,8 +98,6 @@ spec: value: {{ .Values.song.auth.oauth2.resourceserver.jwt.jwkSetUri | quote }} - name: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI value: {{ .Values.song.auth.oauth2.resourceserver.jwt.issuerUri | quote }} - - name: SPRING_APPLICATION_JSON - value: '{"auth":{"server":{"provider":"{{ .Values.song.auth.server.provider }}","clientID":"{{ .Values.song.auth.server.clientId }}","clientSecret":"{{ .Values.song.auth.server.clientSecret }}","tokenName":"{{ .Values.song.auth.server.tokenName }}","keycloak":{"host":"{{ .Values.song.auth.server.keycloak.host }}","realm":"{{ .Values.song.auth.server.keycloak.realm }}"},"scope":{"study":{"prefix":"{{ .Values.song.auth.server.scope.study.prefix }}","suffix":"{{ .Values.song.auth.server.scope.study.suffix }}"},"system":"{{ .Values.song.auth.server.scope.system }}"},"introspectionUri":"{{ .Values.song.auth.oauth2.introspectionUri }}"}},"score":{"clientCredentials":{"id":"{{ .Values.song.score.clientCredentials.id }}","secret":"{{ .Values.song.score.clientCredentials.secret }}","tokenUrl":"{{ .Values.song.score.clientCredentials.tokenUrl }}","systemScope":"{{ .Values.song.score.clientCredentials.systemScope }}"}},"spring":{"security":{"oauth2":{"resourceserver":{"jwt":{"jwk-set-uri":"{{ .Values.song.auth.oauth2.resourceserver.jwt.jwkSetUri }}","issuer-uri":"{{ .Values.song.auth.oauth2.resourceserver.jwt.issuerUri }}"}}}},"basic":{"enabled":false}},"logging":{"level":{"bio.overture.song.server.security":"DEBUG","org.springframework.security":"DEBUG"}}}' - name: AUTH_SERVER_INTROSPECTIONURI value: {{ .Values.song.auth.oauth2.introspectionUri | quote }} # Postgres Variables diff --git a/helm/song/values.yaml b/helm/song/values.yaml index 122f6fc..dd1b64b 100644 --- a/helm/song/values.yaml +++ b/helm/song/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 fullnameOverride: "song" url: - hostname: "song.local" + hostname: "song-ilifu.openup.org.za" image: repository: ghcr.io/overture-stack/song-server @@ -27,7 +27,7 @@ song: resourceserver: jwt: jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" - issuerUri: "http://keycloak.local/realms/agari" + issuerUri: "http://keycloak-ilifu.openup.org.za/realms/agari" client: study: maxUploadFileSize: 10737418240 @@ -73,7 +73,7 @@ song: resourceserver: jwt: jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" - issuerUri: "http://keycloak.local/realms/agari" + issuerUri: "http://keycloak-ilifu.openup.org.za/realms/agari" introspectionUri: "http://keycloak:8080/realms/agari/apikey/check_api_key/" # Kafka Variables (enabled) @@ -109,11 +109,11 @@ probes: ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: song.local + - host: song-ilifu.openup.org.za paths: - path: / pathType: Prefix From d7a9e9e91d638ea906cdd4a2a2fafe5055247a62 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Fri, 29 Aug 2025 12:32:09 +0200 Subject: [PATCH 02/48] Make stack ready to deploy --- helm/arranger/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/arranger/values.yaml b/helm/arranger/values.yaml index a3281cd..7470d3b 100644 --- a/helm/arranger/values.yaml +++ b/helm/arranger/values.yaml @@ -39,7 +39,7 @@ resources: ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "50m" hosts: @@ -70,14 +70,14 @@ probes: resources: limits: cpu: 200m - memory: 512Mi + memory: 1Gi requests: cpu: 100m memory: 256Mi ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "50m" hosts: From f13452e9c050511b4219927259b445fd60f81d79 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 2 Sep 2025 15:56:12 +0200 Subject: [PATCH 03/48] Make stack ready to deploy --- helm/keycloak/templates/deployment.yaml | 2 +- helm/keycloak/values.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/helm/keycloak/templates/deployment.yaml b/helm/keycloak/templates/deployment.yaml index f5da4c0..4ee0d04 100644 --- a/helm/keycloak/templates/deployment.yaml +++ b/helm/keycloak/templates/deployment.yaml @@ -68,7 +68,7 @@ spec: - name: KC_HOSTNAME_STRICT value: "false" - name: KC_HOSTNAME_STRICT_HTTPS - value: "true" + value: "false" {{- if .Values.extraEnv }} {{- range .Values.extraEnv }} - name: {{ .name }} diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index 32578a1..f98940b 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -38,6 +38,10 @@ extraEnv: value: "false" - name: JAVA_OPTS_APPEND value: "-Djava.awt.headless=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.net.preferIPv4Stack=true -server -Xms512m -Xmx768m" + - name: KC_HOSTNAME + value: keycloak-ilifu.openup.org.za + - name: KC_PROXY + value: "edge" - name: KC_CORS_ORIGINS value: "*" From 28f185221c2f38bb97b6171cca801b4eb075ae94 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 2 Sep 2025 16:31:24 +0200 Subject: [PATCH 04/48] Make ready for deploy --- helm/folio-db/values.yaml | 4 ++-- helm/folio/values.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/folio-db/values.yaml b/helm/folio-db/values.yaml index cf69752..c86c1a2 100644 --- a/helm/folio-db/values.yaml +++ b/helm/folio-db/values.yaml @@ -10,8 +10,8 @@ fullnameOverride: "folio-db" podAnnotations: {} -podSecurityContext: {} - # fsGroup: 2000 +podSecurityContext: + fsGroup: 999 securityContext: # capabilities: diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index d294ec5..54a272e 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -16,7 +16,7 @@ folio: keycloak: host: "http://keycloak:8080" realm: "agari" - issuer: "http://keycloak.local/realms/agari" + issuer: "http://keycloak-ilifu.openup.org.za/realms/agari" clientId: "dms" clientSecret: "dms-secret" database: @@ -28,10 +28,10 @@ folio: ingress: enabled: true - className: nginx + className: private annotations: {} hosts: - - host: folio.local + - host: folio-ilifu.openup.org.za paths: - path: / pathType: Prefix From c32bb3046fecee8be05b058044bf90fafd11d0a8 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 9 Sep 2025 17:10:21 +0200 Subject: [PATCH 05/48] Increase ES ephemeral storage --- helm/elasticsearch/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/elasticsearch/values.yaml b/helm/elasticsearch/values.yaml index e799d77..11b1a72 100644 --- a/helm/elasticsearch/values.yaml +++ b/helm/elasticsearch/values.yaml @@ -86,9 +86,11 @@ resources: limits: cpu: 500m memory: 768Mi + ephemeral-storage: "4Gi" requests: cpu: 200m memory: 512Mi + ephemeral-storage: "2Gi" ingress: enabled: true From 92c4a3e5113fede2ef1bdaa57b14d060395e5a39 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Wed, 10 Sep 2025 11:47:23 +0200 Subject: [PATCH 06/48] Increase ephemeral storage --- helm/arranger/values.yaml | 2 ++ helm/keycloak/values.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/helm/arranger/values.yaml b/helm/arranger/values.yaml index 7470d3b..7305b1f 100644 --- a/helm/arranger/values.yaml +++ b/helm/arranger/values.yaml @@ -33,9 +33,11 @@ resources: limits: cpu: 500m memory: 1Gi + ephemeral-storage: "4Gi" requests: cpu: 200m memory: 512Mi + ephemeral-storage: "2Gi" ingress: enabled: true diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index f98940b..61e57c7 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -81,9 +81,11 @@ resources: limits: cpu: 500m memory: 1Gi + ephemeral-storage: "4Gi" requests: cpu: 200m memory: 512Mi + ephemeral-storage: "2Gi" # Node selection nodeSelector: {} From 36195fc522e1623ce59c09c0366f4816d939d005 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Fri, 12 Sep 2025 07:38:12 +0200 Subject: [PATCH 07/48] Make ready for deploy --- helm/maestro/values.yaml | 2 +- helm/minio/values.yaml | 4 ++-- helm/score/values.yaml | 14 +++----------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/helm/maestro/values.yaml b/helm/maestro/values.yaml index 42f9b44..81328fc 100644 --- a/helm/maestro/values.yaml +++ b/helm/maestro/values.yaml @@ -113,7 +113,7 @@ probes: ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: diff --git a/helm/minio/values.yaml b/helm/minio/values.yaml index 9e51b50..465ed81 100644 --- a/helm/minio/values.yaml +++ b/helm/minio/values.yaml @@ -10,7 +10,7 @@ console: hostname: "minio-console-ilifu.openup.org.za" ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "1000m" tls: [] @@ -29,7 +29,7 @@ service: ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "1000m" nginx.ingress.kubernetes.io/proxy-request-buffering: "off" diff --git a/helm/score/values.yaml b/helm/score/values.yaml index 207c91b..4a10db2 100644 --- a/helm/score/values.yaml +++ b/helm/score/values.yaml @@ -90,7 +90,7 @@ score: ingress: enabled: true - className: "nginx" + className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: @@ -98,6 +98,8 @@ ingress: paths: - path: / pathType: Prefix + tls: [] + resources: limits: cpu: 500m @@ -106,16 +108,6 @@ resources: cpu: 200m memory: 512Mi -ingress: - enabled: true - className: "nginx" - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "100m" - hosts: - - host: score-ilifu.openup.org.za - paths: - - path: / - pathType: Prefix nodeSelector: {} tolerations: [] From 86b49113183337c1b2f6a477e29bd1dd4ca39a1b Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 22 Sep 2025 16:28:46 +0200 Subject: [PATCH 08/48] Make ready for prod deploy --- helm/arranger/values.yaml | 2 ++ helm/folio-db/sql/init.sql | 37 ++++++++++------------------ helm/folio-db/values.yaml | 4 +-- helm/folio/templates/deployment.yaml | 18 ++++++++------ helm/folio/values.yaml | 6 ++++- helm/maestro/values.yaml | 6 +++-- helm/score/values.yaml | 7 +++--- helm/song/values.yaml | 8 ++++++ 8 files changed, 49 insertions(+), 39 deletions(-) diff --git a/helm/arranger/values.yaml b/helm/arranger/values.yaml index 7305b1f..e804b14 100644 --- a/helm/arranger/values.yaml +++ b/helm/arranger/values.yaml @@ -73,9 +73,11 @@ resources: limits: cpu: 200m memory: 1Gi + ephemeral-storage: 2Gi requests: cpu: 100m memory: 256Mi + ephemeral-storage: 1Gi ingress: enabled: true diff --git a/helm/folio-db/sql/init.sql b/helm/folio-db/sql/init.sql index 0e745a1..14155a5 100644 --- a/helm/folio-db/sql/init.sql +++ b/helm/folio-db/sql/init.sql @@ -10,8 +10,7 @@ CREATE TABLE IF NOT EXISTS pathogens ( name VARCHAR(255) NOT NULL UNIQUE, scientific_name VARCHAR(255), description TEXT, - created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_T - IMESTAMP, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, deleted_at TIMESTAMP WITH TIME ZONE NULL ); @@ -19,13 +18,12 @@ CREATE TABLE IF NOT EXISTS pathogens ( -- Create projects table CREATE TABLE IF NOT EXISTS projects ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - slug VARCHAR(255) NOT NULL UNIQUE, name VARCHAR(255) NOT NULL, description TEXT, - organization_id VARCHAR(255) NOT NULL, -- Keycloak organization ID + organisation_id VARCHAR(255) NOT NULL DEFAULT 'default-org', -- Keycloak organisation ID user_id VARCHAR(255) NOT NULL, -- Keycloak user ID of creator pathogen_id UUID REFERENCES pathogens(id), - status VARCHAR(50) DEFAULT 'active' CHECK (status IN ('active', 'inactive', 'completed', 'archived')), + privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private')), created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, deleted_at TIMESTAMP WITH TIME ZONE NULL @@ -37,24 +35,21 @@ CREATE TABLE IF NOT EXISTS studies ( study_id VARCHAR(255) NOT NULL UNIQUE, name VARCHAR(255) NOT NULL, description TEXT, + status VARCHAR(50) DEFAULT 'draft' CHECK (status IN ('draft', 'published')), + privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private')), project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, - start_date DATE, - end_date DATE, - status VARCHAR(50) DEFAULT 'active' CHECK (status IN ('active', 'inactive', 'completed', 'archived')), created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, deleted_at TIMESTAMP WITH TIME ZONE NULL ); -- Create indexes for better performance -CREATE INDEX IF NOT EXISTS idx_projects_slug ON projects(slug); CREATE INDEX IF NOT EXISTS idx_projects_pathogen ON projects(pathogen_id); -CREATE INDEX IF NOT EXISTS idx_projects_status ON projects(status); -CREATE INDEX IF NOT EXISTS idx_projects_organization ON projects(organization_id); +CREATE INDEX IF NOT EXISTS idx_projects_organisation ON projects(organisation_id); CREATE INDEX IF NOT EXISTS idx_projects_user ON projects(user_id); +CREATE INDEX IF NOT EXISTS idx_projects_privacy ON projects(privacy); CREATE INDEX IF NOT EXISTS idx_studies_project ON studies(project_id); CREATE INDEX IF NOT EXISTS idx_studies_study_id ON studies(study_id); -CREATE INDEX IF NOT EXISTS idx_studies_status ON studies(status); CREATE INDEX IF NOT EXISTS idx_pathogens_name ON pathogens(name); -- Create updated_at trigger function @@ -85,12 +80,11 @@ CREATE TRIGGER update_studies_updated_at CREATE OR REPLACE VIEW project_details AS SELECT p.id, - p.slug, p.name, p.description, - p.organization_id, + p.organisation_id, p.user_id, - p.status, + p.privacy, p.created_at, p.updated_at, p.deleted_at, @@ -101,8 +95,8 @@ FROM projects p LEFT JOIN pathogens pat ON p.pathogen_id = pat.id AND pat.deleted_at IS NULL LEFT JOIN studies s ON p.id = s.project_id AND s.deleted_at IS NULL WHERE p.deleted_at IS NULL -GROUP BY p.id, p.slug, p.name, p.description, p.organization_id, - p.user_id, p.status, p.created_at, p.updated_at, p.deleted_at, +GROUP BY p.id, p.name, p.description, p.organisation_id, + p.user_id, p.privacy, p.created_at, p.updated_at, p.deleted_at, pat.name, pat.scientific_name; CREATE OR REPLACE VIEW study_details AS @@ -111,13 +105,12 @@ SELECT s.study_id, s.name, s.description, - s.start_date, - s.end_date, s.status, + s.privacy, + s.project_id, s.created_at, s.updated_at, s.deleted_at, - p.slug as project_slug, p.name as project_name, pat.name as pathogen_name FROM studies s @@ -125,10 +118,6 @@ JOIN projects p ON s.project_id = p.id AND p.deleted_at IS NULL LEFT JOIN pathogens pat ON p.pathogen_id = pat.id AND pat.deleted_at IS NULL WHERE s.deleted_at IS NULL; --- Grant permissions to the folio application user (if needed) --- Note: This assumes the folio app connects with the same user as the database owner --- In production, you might want to create a separate application user with limited permissions - COMMENT ON TABLE pathogens IS 'Reference table for pathogen information'; COMMENT ON TABLE projects IS 'Main projects table containing project metadata'; COMMENT ON TABLE studies IS 'Studies table containing study information linked to projects'; diff --git a/helm/folio-db/values.yaml b/helm/folio-db/values.yaml index c86c1a2..cf69752 100644 --- a/helm/folio-db/values.yaml +++ b/helm/folio-db/values.yaml @@ -10,8 +10,8 @@ fullnameOverride: "folio-db" podAnnotations: {} -podSecurityContext: - fsGroup: 999 +podSecurityContext: {} + # fsGroup: 2000 securityContext: # capabilities: diff --git a/helm/folio/templates/deployment.yaml b/helm/folio/templates/deployment.yaml index ee1e875..0a96e91 100644 --- a/helm/folio/templates/deployment.yaml +++ b/helm/folio/templates/deployment.yaml @@ -34,25 +34,29 @@ spec: value: {{ .Values.folio.auth.keycloak.clientId | quote }} - name: KEYCLOAK_CLIENT_SECRET value: {{ .Values.folio.auth.keycloak.clientSecret | quote }} - - name: FOLIO_DB_HOST + - name: DB_HOST value: {{ .Values.folio.database.host | quote }} - - name: FOLIO_DB_PORT + - name: DB_PORT value: {{ .Values.folio.database.port | quote }} - - name: FOLIO_DB_NAME + - name: DB_NAME value: {{ .Values.folio.database.name | quote }} - - name: FOLIO_DB_USER + - name: DB_USER value: {{ .Values.folio.database.user | quote }} - - name: FOLIO_DB_PASSWORD + - name: DB_PASSWORD value: {{ .Values.folio.database.password | quote }} + - name: OVERTURE_SCORE + value: {{ .Values.folio.overture.score | quote }} + - name: OVERTURE_SONG + value: {{ .Values.folio.overture.song | quote }} livenessProbe: httpGet: - path: /health + path: /info/health port: http initialDelaySeconds: 30 periodSeconds: 10 readinessProbe: httpGet: - path: /health + path: /info/health port: http initialDelaySeconds: 5 periodSeconds: 5 diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index 54a272e..0d9e0d5 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -16,7 +16,7 @@ folio: keycloak: host: "http://keycloak:8080" realm: "agari" - issuer: "http://keycloak-ilifu.openup.org.za/realms/agari" + issuer: "http://keycloak.local/realms/agari" clientId: "dms" clientSecret: "dms-secret" database: @@ -26,6 +26,10 @@ folio: user: "admin" password: "folio-db-pass-123" + overture: + score: "http://score:8087" + song: "http://song:8080" + ingress: enabled: true className: private diff --git a/helm/maestro/values.yaml b/helm/maestro/values.yaml index 81328fc..4ef798d 100644 --- a/helm/maestro/values.yaml +++ b/helm/maestro/values.yaml @@ -86,10 +86,12 @@ maestro: resources: limits: cpu: 500m - memory: 1.5Gi + memory: 1.5Gi + ephemeral-storage: 2Gi requests: cpu: 100m memory: 512Mi + ephemeral-storage: 1Gi probes: liveness: @@ -127,4 +129,4 @@ persistence: nodeSelector: {} tolerations: [] -affinity: {} \ No newline at end of file +affinity: {} diff --git a/helm/score/values.yaml b/helm/score/values.yaml index 4a10db2..d212cd5 100644 --- a/helm/score/values.yaml +++ b/helm/score/values.yaml @@ -85,8 +85,8 @@ score: oauth2: resourceserver: jwt: - jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" - issuerUri: "http://keycloak-ilifu.openup.org.za/realms/agari" + jwkSetUri: "https://keycloak-ilifu.openup.org.za/realms/agari/protocol/openid-connect/certs" + issuerUri: "https://keycloak-ilifu.openup.org.za/realms/agari" ingress: enabled: true @@ -104,10 +104,11 @@ resources: limits: cpu: 500m memory: 1Gi + ephemeral-storage: 2Gi requests: cpu: 200m memory: 512Mi - + ephemeral-storage: 1Gi nodeSelector: {} tolerations: [] diff --git a/helm/song/values.yaml b/helm/song/values.yaml index dd1b64b..6cd3e17 100644 --- a/helm/song/values.yaml +++ b/helm/song/values.yaml @@ -28,6 +28,12 @@ song: jwt: jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" issuerUri: "http://keycloak-ilifu.openup.org.za/realms/agari" + cors: + allowedOrigins: "*" + allowedMethods: "*" + allowedHeaders: "*" + allowCredentials: false + maxAge: 3600 client: study: maxUploadFileSize: 10737418240 @@ -124,9 +130,11 @@ resources: limits: cpu: 500m memory: 1.5Gi + ephemeral-storage: 2Gi requests: cpu: 150m memory: 768Mi + ephemeral-storage: 1Gi # Node selection nodeSelector: {} From 4f7456b12fd7101bc1225538563c64b2111b0493 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 30 Sep 2025 21:20:35 +0200 Subject: [PATCH 09/48] CORS fixes and KC update --- helm/folio/values.yaml | 22 ++++++++++++++++++++-- helm/keycloak/templates/deployment.yaml | 16 ++++++++-------- helm/keycloak/values.yaml | 14 +++++++++++++- helm/score/values.yaml | 8 +++++++- helm/song/values.yaml | 11 +++++------ 5 files changed, 53 insertions(+), 18 deletions(-) diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index 0d9e0d5..3f7a8d7 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -20,11 +20,19 @@ folio: clientId: "dms" clientSecret: "dms-secret" database: + vendor: postgres host: "folio-db" port: "5432" name: "folio" user: "admin" password: "folio-db-pass-123" + pool: + size: 10 + maxOverflow: 20 + recycle: 3600 + preping: true + timeout: 30 + options: "?connect_timeout=10&application_name=folio" overture: score: "http://score:8087" @@ -33,7 +41,11 @@ folio: ingress: enabled: true className: private - annotations: {} + annotations: + nginx.ingress.kubernetes.io/enable-cors: "true" + nginx.ingress.kubernetes.io/cors-allow-origin: "*" + nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" + nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" hosts: - host: folio-ilifu.openup.org.za paths: @@ -41,7 +53,13 @@ ingress: pathType: Prefix tls: [] -resources: {} +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 200m + memory: 256Mi nodeSelector: {} diff --git a/helm/keycloak/templates/deployment.yaml b/helm/keycloak/templates/deployment.yaml index 4ee0d04..88b146b 100644 --- a/helm/keycloak/templates/deployment.yaml +++ b/helm/keycloak/templates/deployment.yaml @@ -58,17 +58,17 @@ spec: - name: KEYCLOAK_ADMIN_PASSWORD value: {{ .Values.keycloak.admin.password }} # Keycloak settings - - name: KC_HOSTNAME - value: {{ .Values.url.hostname }} + #- name: KC_HOSTNAME + # value: {{ .Values.url.hostname }} - name: KC_HEALTH_ENABLED value: "{{ .Values.keycloak.healthEnabled }}" # Proxy settings for Kubernetes - - name: KC_PROXY - value: "edge" - - name: KC_HOSTNAME_STRICT - value: "false" - - name: KC_HOSTNAME_STRICT_HTTPS - value: "false" + #- name: KC_PROXY + # value: "edge" + #- name: KC_HOSTNAME_STRICT + # value: "false" + #- name: KC_HOSTNAME_STRICT_HTTPS + # value: "false" {{- if .Values.extraEnv }} {{- range .Values.extraEnv }} - name: {{ .name }} diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index 61e57c7..a02a8da 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -6,7 +6,7 @@ url: image: repository: quay.io/keycloak/keycloak pullPolicy: IfNotPresent - tag: "22.0.5" + tag: "26.2.5" nameOverride: "" fullnameOverride: "keycloak" @@ -44,6 +44,16 @@ extraEnv: value: "edge" - name: KC_CORS_ORIGINS value: "*" + - name: KC_HOSTNAME_URL + value: "https://keycloak-ilifu.openup.org.za" + - name: KC_HOSTNAME_ADMIN_URL + value: "https://keycloak-ilifu.openup.org.za" + - name: KC_HTTP_ENABLED + value: "true" + - name: KC_HTTPS_CERTIFICATE_FILE + value: "" + - name: KC_HTTPS_CERTIFICATE_KEY_FILE + value: "" probes: liveness: @@ -75,6 +85,8 @@ keycloak: hostname: keycloak-ilifu.openup.org.za healthEnabled: true startupMode: start-dev + frontendUrl: "https://keycloak-ilifu.openup.org.za" + adminUrl: "https://keycloak-ilifu.openup.org.za" # Resource limits - better configuration from values-keycloak.yaml resources: diff --git a/helm/score/values.yaml b/helm/score/values.yaml index d212cd5..1c024c2 100644 --- a/helm/score/values.yaml +++ b/helm/score/values.yaml @@ -42,7 +42,7 @@ score: systemScope: "song.READ" s3: - endpoint: "http://minio:9000" + endpoint: "https://minio-ilifu.openup.org.za" accesskey: "admin" secretkey: "admin123" sigv4enabled: true @@ -93,6 +93,10 @@ ingress: className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" + nginx.ingress.kubernetes.io/enable-cors: "true" + nginx.ingress.kubernetes.io/cors-allow-origin: "*" + nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" + nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" hosts: - host: score-ilifu.openup.org.za paths: @@ -113,3 +117,5 @@ resources: nodeSelector: {} tolerations: [] affinity: {} + + diff --git a/helm/song/values.yaml b/helm/song/values.yaml index 6cd3e17..724ea1a 100644 --- a/helm/song/values.yaml +++ b/helm/song/values.yaml @@ -28,12 +28,7 @@ song: jwt: jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" issuerUri: "http://keycloak-ilifu.openup.org.za/realms/agari" - cors: - allowedOrigins: "*" - allowedMethods: "*" - allowedHeaders: "*" - allowCredentials: false - maxAge: 3600 + client: study: maxUploadFileSize: 10737418240 @@ -118,6 +113,10 @@ ingress: className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" + nginx.ingress.kubernetes.io/enable-cors: "true" + nginx.ingress.kubernetes.io/cors-allow-origin: "*" + nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" + nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" hosts: - host: song-ilifu.openup.org.za paths: From ceeb3c129473dbf24a3369fe2ef332059f1dfa9e Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 30 Sep 2025 21:21:05 +0200 Subject: [PATCH 10/48] Create helm chart for frontend --- helm/frontend/Chart.yaml | 6 +++ helm/frontend/templates/_helpers.tpl | 7 ++++ helm/frontend/templates/deployment.yaml | 32 +++++++++++++++ helm/frontend/templates/ingress.yaml | 26 +++++++++++++ helm/frontend/templates/service.yaml | 17 ++++++++ helm/frontend/values.yaml | 52 +++++++++++++++++++++++++ 6 files changed, 140 insertions(+) create mode 100644 helm/frontend/Chart.yaml create mode 100644 helm/frontend/templates/_helpers.tpl create mode 100644 helm/frontend/templates/deployment.yaml create mode 100644 helm/frontend/templates/ingress.yaml create mode 100644 helm/frontend/templates/service.yaml create mode 100644 helm/frontend/values.yaml diff --git a/helm/frontend/Chart.yaml b/helm/frontend/Chart.yaml new file mode 100644 index 0000000..5a6da13 --- /dev/null +++ b/helm/frontend/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: agari-frontend +description: A Helm chart for the Agari frontend +type: application +version: 0.0.1 +appVersion: "0.0.1" \ No newline at end of file diff --git a/helm/frontend/templates/_helpers.tpl b/helm/frontend/templates/_helpers.tpl new file mode 100644 index 0000000..6eaa7a5 --- /dev/null +++ b/helm/frontend/templates/_helpers.tpl @@ -0,0 +1,7 @@ +{{- define "agari-frontend.name" -}} +{{- default .Chart.Name .Values.nameOverride -}} +{{- end -}} + +{{- define "agari-frontend.fullname" -}} +{{- printf "%s-%s" (include "agari-frontend.name" .) .Release.Name -}} +{{- end -}} \ No newline at end of file diff --git a/helm/frontend/templates/deployment.yaml b/helm/frontend/templates/deployment.yaml new file mode 100644 index 0000000..a0b5db0 --- /dev/null +++ b/helm/frontend/templates/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "agari-frontend.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "agari-frontend.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "agari-frontend.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "agari-frontend.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + containers: + - name: frontend + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 3000 + name: http + readinessProbe: {{ toYaml .Values.readinessProbe | nindent 10 }} + livenessProbe: {{ toYaml .Values.livenessProbe | nindent 10 }} + resources: {{ toYaml .Values.resources | nindent 10 }} + nodeSelector: {{ toYaml .Values.nodeSelector | nindent 6 }} + tolerations: {{ toYaml .Values.tolerations | nindent 6 }} + affinity: {{ toYaml .Values.affinity | nindent 6 }} \ No newline at end of file diff --git a/helm/frontend/templates/ingress.yaml b/helm/frontend/templates/ingress.yaml new file mode 100644 index 0000000..b42e7d0 --- /dev/null +++ b/helm/frontend/templates/ingress.yaml @@ -0,0 +1,26 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "agari-frontend.fullname" . }} + annotations: {{ toYaml .Values.ingress.annotations | nindent 4 }} +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "agari-frontend.fullname" $root }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: {{ toYaml .Values.ingress.tls | nindent 2 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/frontend/templates/service.yaml b/helm/frontend/templates/service.yaml new file mode 100644 index 0000000..8309626 --- /dev/null +++ b/helm/frontend/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "agari-frontend.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "agari-frontend.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "agari-frontend.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/frontend/values.yaml b/helm/frontend/values.yaml new file mode 100644 index 0000000..eb01ce1 --- /dev/null +++ b/helm/frontend/values.yaml @@ -0,0 +1,52 @@ +replicaCount: 1 + +image: + repository: ghcr.io/methodlab/agari-frontend + tag: v0.0.1 + pullPolicy: Always + +service: + type: ClusterIP + port: 80 + targetPort: 3000 + +ingress: + enabled: true + className: private + annotations: {} + hosts: + - host: agari.openup.org.za + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + +resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + +readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + +livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 20 + failureThreshold: 5 + +nodeSelector: {} + +tolerations: [] + +affinity: {} \ No newline at end of file From 23c91b0a3ebb8996e47a20e68e80e4d228d60eab Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Thu, 2 Oct 2025 11:24:17 +0200 Subject: [PATCH 11/48] Use PhaseTwo Keycloak --- helm/keycloak/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index a02a8da..da3f0da 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -4,7 +4,7 @@ url: hostname: "keycloak-ilifu.openup.org.za" image: - repository: quay.io/keycloak/keycloak + repository: quay.io/phasetwo/phasetwo-keycloak pullPolicy: IfNotPresent tag: "26.2.5" From d14bc5a971d50ed6cf9e0c04a67daaf2a844c1aa Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Thu, 2 Oct 2025 11:24:30 +0200 Subject: [PATCH 12/48] Set KEYCLOAK_URL --- helm/folio/templates/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/folio/templates/deployment.yaml b/helm/folio/templates/deployment.yaml index 0a96e91..f8eefd6 100644 --- a/helm/folio/templates/deployment.yaml +++ b/helm/folio/templates/deployment.yaml @@ -48,6 +48,8 @@ spec: value: {{ .Values.folio.overture.score | quote }} - name: OVERTURE_SONG value: {{ .Values.folio.overture.song | quote }} + - name: KEYCLOAK_URL + value: {{ .Values.folio.auth.keycloak.host | quote }} livenessProbe: httpGet: path: /info/health From 05b57b593e355df3fd1d9ff03016843da8f6710f Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Wed, 8 Oct 2025 17:25:24 +0200 Subject: [PATCH 13/48] Set email provider for Keycloak --- helm/keycloak/values.yaml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index da3f0da..1d10461 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -44,16 +44,8 @@ extraEnv: value: "edge" - name: KC_CORS_ORIGINS value: "*" - - name: KC_HOSTNAME_URL - value: "https://keycloak-ilifu.openup.org.za" - - name: KC_HOSTNAME_ADMIN_URL - value: "https://keycloak-ilifu.openup.org.za" - - name: KC_HTTP_ENABLED - value: "true" - - name: KC_HTTPS_CERTIFICATE_FILE - value: "" - - name: KC_HTTPS_CERTIFICATE_KEY_FILE - value: "" + - name: KC_SPI_EMAIL_TEMPLATE_PROVIDER + value: freemarker probes: liveness: From d47b624f6b81f121913bfb2168e1dd12518bcb0d Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Thu, 9 Oct 2025 13:02:08 +0200 Subject: [PATCH 14/48] Add semi-private to project privacy options --- helm/folio-db/sql/init.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/folio-db/sql/init.sql b/helm/folio-db/sql/init.sql index 14155a5..3ebe960 100644 --- a/helm/folio-db/sql/init.sql +++ b/helm/folio-db/sql/init.sql @@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS projects ( organisation_id VARCHAR(255) NOT NULL DEFAULT 'default-org', -- Keycloak organisation ID user_id VARCHAR(255) NOT NULL, -- Keycloak user ID of creator pathogen_id UUID REFERENCES pathogens(id), - privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private')), + privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private', 'semi-private')), created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, deleted_at TIMESTAMP WITH TIME ZONE NULL @@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS studies ( name VARCHAR(255) NOT NULL, description TEXT, status VARCHAR(50) DEFAULT 'draft' CHECK (status IN ('draft', 'published')), - privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private')), + privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private', 'semi-private')), project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, From 46fd2868387dcc31f1d8aa1819b676c7d143d254 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Fri, 10 Oct 2025 15:08:34 +0200 Subject: [PATCH 15/48] Set folio db security context --- helm/folio-db/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/folio-db/values.yaml b/helm/folio-db/values.yaml index cf69752..c86c1a2 100644 --- a/helm/folio-db/values.yaml +++ b/helm/folio-db/values.yaml @@ -10,8 +10,8 @@ fullnameOverride: "folio-db" podAnnotations: {} -podSecurityContext: {} - # fsGroup: 2000 +podSecurityContext: + fsGroup: 999 securityContext: # capabilities: From 62edeae861c40d4c2d90acce9ed710c00e3cde87 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 21 Oct 2025 11:19:25 +0200 Subject: [PATCH 16/48] Update init.sql --- helm/folio-db/sql/init.sql | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/helm/folio-db/sql/init.sql b/helm/folio-db/sql/init.sql index 3ebe960..7af7b68 100644 --- a/helm/folio-db/sql/init.sql +++ b/helm/folio-db/sql/init.sql @@ -23,7 +23,19 @@ CREATE TABLE IF NOT EXISTS projects ( organisation_id VARCHAR(255) NOT NULL DEFAULT 'default-org', -- Keycloak organisation ID user_id VARCHAR(255) NOT NULL, -- Keycloak user ID of creator pathogen_id UUID REFERENCES pathogens(id), - privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private', 'semi-private')), + privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private')), + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE NULL +); + +-- Create organisations table +CREATE TABLE IF NOT EXISTS organisations ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + name VARCHAR(255) NOT NULL UNIQUE, + abbreviation VARCHAR(50), + url VARCHAR(255), + about TEXT, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, deleted_at TIMESTAMP WITH TIME ZONE NULL @@ -36,7 +48,7 @@ CREATE TABLE IF NOT EXISTS studies ( name VARCHAR(255) NOT NULL, description TEXT, status VARCHAR(50) DEFAULT 'draft' CHECK (status IN ('draft', 'published')), - privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private', 'semi-private')), + privacy VARCHAR(20) DEFAULT 'public' CHECK (privacy IN ('public', 'private')), project_id UUID NOT NULL REFERENCES projects(id) ON DELETE CASCADE, created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, @@ -51,6 +63,7 @@ CREATE INDEX IF NOT EXISTS idx_projects_privacy ON projects(privacy); CREATE INDEX IF NOT EXISTS idx_studies_project ON studies(project_id); CREATE INDEX IF NOT EXISTS idx_studies_study_id ON studies(study_id); CREATE INDEX IF NOT EXISTS idx_pathogens_name ON pathogens(name); +CREATE INDEX IF NOT EXISTS idx_organisations_name ON organisations(name); -- Create updated_at trigger function CREATE OR REPLACE FUNCTION update_updated_at_column() @@ -74,7 +87,9 @@ CREATE TRIGGER update_studies_updated_at BEFORE UPDATE ON studies FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); --- No sample data - clean slate for production use +CREATE TRIGGER update_organisations_updated_at + BEFORE UPDATE ON organisations + FOR EACH ROW EXECUTE FUNCTION update_updated_at_column(); -- Create views for easier querying CREATE OR REPLACE VIEW project_details AS @@ -118,8 +133,19 @@ JOIN projects p ON s.project_id = p.id AND p.deleted_at IS NULL LEFT JOIN pathogens pat ON p.pathogen_id = pat.id AND pat.deleted_at IS NULL WHERE s.deleted_at IS NULL; +CREATE OR REPLACE VIEW organisation_projects AS +SELECT + o.id as organisation_id, + o.name as organisation_name, + COUNT(p.id) as project_count +FROM organisations o +LEFT JOIN projects p ON o.id = p.organisation_id AND p.deleted_at IS NULL +WHERE o.deleted_at IS NULL +GROUP BY o.id, o.name; + COMMENT ON TABLE pathogens IS 'Reference table for pathogen information'; COMMENT ON TABLE projects IS 'Main projects table containing project metadata'; COMMENT ON TABLE studies IS 'Studies table containing study information linked to projects'; COMMENT ON VIEW project_details IS 'Denormalized view of projects with pathogen and study count information'; COMMENT ON VIEW study_details IS 'Denormalized view of studies with project and pathogen information'; +COMMENT ON TABLE organisations IS 'Table containing organisation information'; \ No newline at end of file From 2b7df275b48070f72e982356dab1b1633e488e7f Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 21 Oct 2025 14:00:33 +0200 Subject: [PATCH 17/48] KC cors --- helm/keycloak/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index 1d10461..788c524 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -22,6 +22,10 @@ ingress: nginx.ingress.kubernetes.io/proxy-buffer-size: "128k" nginx.ingress.kubernetes.io/proxy-buffering: "on" nginx.ingress.kubernetes.io/proxy-buffers-number: "20" + nginx.ingress.kubernetes.io/enable-cors: "true" + nginx.ingress.kubernetes.io/cors-allow-origin: "*" + nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" + nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" hosts: - host: keycloak-ilifu.openup.org.za paths: From d95957306ce524de8ca45649acad93c8447fe203 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Thu, 23 Oct 2025 16:10:53 +0200 Subject: [PATCH 18/48] Add Strimzi --- README.md | 9 ++- helm/kafka/Chart.lock | 6 ++ helm/kafka/Chart.yaml | 11 ++++ helm/kafka/templates/_helpers.tpl | 51 ++++++++++++++++ helm/kafka/templates/kafka.yaml | 56 ++++++++++++++++++ helm/kafka/templates/service.yaml | 18 ++++++ ...itnami.yaml => values-bitnami.yaml.backup} | 0 helm/kafka/values.yaml | 58 +++++++++++++++++++ 8 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 helm/kafka/Chart.lock create mode 100644 helm/kafka/Chart.yaml create mode 100644 helm/kafka/templates/_helpers.tpl create mode 100644 helm/kafka/templates/kafka.yaml create mode 100644 helm/kafka/templates/service.yaml rename helm/kafka/{values-bitnami.yaml => values-bitnami.yaml.backup} (100%) create mode 100644 helm/kafka/values.yaml diff --git a/README.md b/README.md index 9dc1e0a..716fc35 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,14 @@ kubectl port-forward -n agari service/minio 9000:9000 ### 3.2 Kafka Message Queue ```bash -helm repo add bitnami https://charts.bitnami.com/bitnami +# Add Strimzi repository for Kafka operator +helm repo add strimzi https://strimzi.io/charts/ -helm install kafka bitnami/kafka -f helm/kafka/values-bitnami.yaml -n agari +# Build chart dependencies (required for first-time deployment) +cd helm/kafka && helm dependency build && cd ../.. + +# Install Kafka using Strimzi operator +helm install kafka ./helm/kafka -n agari ``` ### 4. Setup Keycloak diff --git a/helm/kafka/Chart.lock b/helm/kafka/Chart.lock new file mode 100644 index 0000000..b92ec6b --- /dev/null +++ b/helm/kafka/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: strimzi-kafka-operator + repository: https://strimzi.io/charts/ + version: 0.39.0 +digest: sha256:609aad69bd2c1c1fb6e2d23fb158becb90021063e83e3cc8be15b263273204fe +generated: "2025-10-23T15:02:45.59200355+02:00" diff --git a/helm/kafka/Chart.yaml b/helm/kafka/Chart.yaml new file mode 100644 index 0000000..3dbae14 --- /dev/null +++ b/helm/kafka/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: kafka +description: Apache Kafka cluster using Strimzi operator +type: application +version: 0.1.0 +appVersion: "3.6.0" + +dependencies: + - name: strimzi-kafka-operator + version: 0.39.0 + repository: https://strimzi.io/charts/ \ No newline at end of file diff --git a/helm/kafka/templates/_helpers.tpl b/helm/kafka/templates/_helpers.tpl new file mode 100644 index 0000000..31608ea --- /dev/null +++ b/helm/kafka/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "kafka.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kafka.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kafka.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "kafka.labels" -}} +helm.sh/chart: {{ include "kafka.chart" . }} +{{ include "kafka.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kafka.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kafka.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/kafka/templates/kafka.yaml b/helm/kafka/templates/kafka.yaml new file mode 100644 index 0000000..eb2265b --- /dev/null +++ b/helm/kafka/templates/kafka.yaml @@ -0,0 +1,56 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: {{ include "kafka.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "kafka.labels" . | nindent 4 }} +spec: + kafka: + version: 3.6.0 + replicas: {{ .Values.kafka.replicas }} + listeners: + - name: plain + port: 9092 + type: internal + tls: false + - name: tls + port: 9093 + type: internal + tls: true + config: + offsets.topic.replication.factor: {{ .Values.kafka.config.offsetsTopicReplicationFactor }} + transaction.state.log.replication.factor: {{ .Values.kafka.config.transactionStateLogReplicationFactor }} + transaction.state.log.min.isr: {{ .Values.kafka.config.transactionStateLogMinIsr }} + default.replication.factor: {{ .Values.kafka.config.defaultReplicationFactor }} + min.insync.replicas: {{ .Values.kafka.config.minInsyncReplicas }} + inter.broker.protocol.version: "3.6" + storage: + {{- if .Values.kafka.storage.persistent }} + type: persistent-claim + size: {{ .Values.kafka.storage.size }} + {{- if .Values.kafka.storage.storageClass }} + class: {{ .Values.kafka.storage.storageClass }} + {{- end }} + {{- else }} + type: ephemeral + {{- end }} + resources: + {{- toYaml .Values.kafka.resources | nindent 6 }} + zookeeper: + replicas: {{ .Values.zookeeper.replicas }} + storage: + {{- if .Values.zookeeper.storage.persistent }} + type: persistent-claim + size: {{ .Values.zookeeper.storage.size }} + {{- if .Values.zookeeper.storage.storageClass }} + class: {{ .Values.zookeeper.storage.storageClass }} + {{- end }} + {{- else }} + type: ephemeral + {{- end }} + resources: + {{- toYaml .Values.zookeeper.resources | nindent 6 }} + entityOperator: + topicOperator: {} + userOperator: {} \ No newline at end of file diff --git a/helm/kafka/templates/service.yaml b/helm/kafka/templates/service.yaml new file mode 100644 index 0000000..e0ebcce --- /dev/null +++ b/helm/kafka/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: kafka + namespace: {{ .Release.Namespace }} + labels: + {{- include "kafka.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 9092 + targetPort: 9092 + protocol: TCP + name: kafka + selector: + strimzi.io/cluster: {{ include "kafka.fullname" . }} + strimzi.io/kind: Kafka + strimzi.io/name: {{ include "kafka.fullname" . }}-kafka \ No newline at end of file diff --git a/helm/kafka/values-bitnami.yaml b/helm/kafka/values-bitnami.yaml.backup similarity index 100% rename from helm/kafka/values-bitnami.yaml rename to helm/kafka/values-bitnami.yaml.backup diff --git a/helm/kafka/values.yaml b/helm/kafka/values.yaml new file mode 100644 index 0000000..1059c2d --- /dev/null +++ b/helm/kafka/values.yaml @@ -0,0 +1,58 @@ +# Strimzi Kafka configuration +# This replaces the Bitnami Kafka chart with Strimzi operator + +# Kafka cluster configuration +kafka: + replicas: 1 + + # Configuration for Kafka brokers + config: + # Replication factors for internal topics + offsetsTopicReplicationFactor: 1 + transactionStateLogReplicationFactor: 1 + transactionStateLogMinIsr: 1 + defaultReplicationFactor: 1 + minInsyncReplicas: 1 + + # Storage configuration + storage: + persistent: false # Set to false for development, true for production + size: 10Gi + # storageClass: "" # Uncomment and specify if needed + + # Resource limits + resources: + limits: + cpu: 1000m + memory: 2Gi + requests: + cpu: 500m + memory: 1Gi + +# Zookeeper configuration +zookeeper: + replicas: 1 + + # Storage configuration + storage: + persistent: false # Set to false for development, true for production + size: 5Gi + # storageClass: "" # Uncomment and specify if needed + + # Resource limits + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 200m + memory: 512Mi + +# Override default names if needed +nameOverride: "" +fullnameOverride: "kafka" + +# Strimzi operator will be installed as dependency +strimzi-kafka-operator: + # Enable the operator installation + enabled: true \ No newline at end of file From d2aaa80c37540cb5311ebcbce92ca7a59eb00a27 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Wed, 5 Nov 2025 12:05:16 +0200 Subject: [PATCH 19/48] Updage kafka deploy --- README.md | 2 +- helm/maestro/values.yaml | 8 ++++---- helm/song/values.yaml | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 716fc35..3c02926 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ helm repo add strimzi https://strimzi.io/charts/ cd helm/kafka && helm dependency build && cd ../.. # Install Kafka using Strimzi operator -helm install kafka ./helm/kafka -n agari +helm install kafka ./helm/kafka -n agari-kafka ``` ### 4. Setup Keycloak diff --git a/helm/maestro/values.yaml b/helm/maestro/values.yaml index 4ef798d..5b270d8 100644 --- a/helm/maestro/values.yaml +++ b/helm/maestro/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 fullnameOverride: "maestro" url: - hostname: "maestro-ilifu.openup.org.za" + hostname: "maestro-ilifu.openup.org.za" # ilifu/staging image: repository: ghcr.io/overture-stack/maestro @@ -67,10 +67,10 @@ maestro: stream: kafka: binder: - brokers: "kafka:9092" + brokers: "kafka-kafka-bootstrap.agari-kafka.svc:9092" bindings: songInput: - destination: "song-analysis" + destination: "song-analysis-prod" # prod/staging # Swagger Configuration springdoc: @@ -119,7 +119,7 @@ ingress: annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: maestro-ilifu.openup.org.za + - host: maestro-ilifu.openup.org.za # ilifu/staging paths: - path: / pathType: Prefix diff --git a/helm/song/values.yaml b/helm/song/values.yaml index 724ea1a..f3bca0f 100644 --- a/helm/song/values.yaml +++ b/helm/song/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 fullnameOverride: "song" url: - hostname: "song-ilifu.openup.org.za" + hostname: "song-staging.openup.org.za" #ilifu/staging image: repository: ghcr.io/overture-stack/song-server @@ -27,7 +27,7 @@ song: resourceserver: jwt: jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" - issuerUri: "http://keycloak-ilifu.openup.org.za/realms/agari" + issuerUri: "http://keycloak-staging.openup.org.za/realms/agari" #ilifu/staging client: study: @@ -60,7 +60,7 @@ song: server: provider: "keycloak" clientId: "dms" - clientSecret: "dms-secret" + clientSecret: "dms-secret" tokenName: "apiKey" keycloak: host: "http://keycloak:8080" @@ -74,14 +74,14 @@ song: resourceserver: jwt: jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" - issuerUri: "http://keycloak-ilifu.openup.org.za/realms/agari" + issuerUri: "http://keycloak-staging.openup.org.za/realms/agari" #ilifu/staging introspectionUri: "http://keycloak:8080/realms/agari/apikey/check_api_key/" # Kafka Variables (enabled) kafka: - bootstrapServers: "kafka:9092" + bootstrapServers: "kafka-kafka-bootstrap.agari-kafka.svc:9092" template: - defaultTopic: "song-analysis" + defaultTopic: "song-analysis-staging" #prod/staging # Swagger Variable swagger: @@ -118,7 +118,7 @@ ingress: nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" hosts: - - host: song-ilifu.openup.org.za + - host: song-staging.openup.org.za #ilifu/staging paths: - path: / pathType: Prefix From c69b2b899f73df5850135fe10063d06906daca4f Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 12:21:35 +0200 Subject: [PATCH 20/48] Elasticsearch and worker --- helm/elasticsearch/templates/configmap.yaml | 10 ++ helm/elasticsearch/templates/deployment.yaml | 34 ++++++- helm/elasticsearch/templates/pvc.yaml | 18 ++++ helm/elasticsearch/values.yaml | 75 ++------------- helm/folio-worker/Chart.yaml | 6 ++ helm/folio-worker/templates/_helpers.tpl | 51 ++++++++++ helm/folio-worker/templates/deployment.yaml | 99 ++++++++++++++++++++ helm/folio-worker/values.yaml | 57 +++++++++++ 8 files changed, 281 insertions(+), 69 deletions(-) create mode 100644 helm/elasticsearch/templates/configmap.yaml create mode 100644 helm/elasticsearch/templates/pvc.yaml create mode 100644 helm/folio-worker/Chart.yaml create mode 100644 helm/folio-worker/templates/_helpers.tpl create mode 100644 helm/folio-worker/templates/deployment.yaml create mode 100644 helm/folio-worker/values.yaml diff --git a/helm/elasticsearch/templates/configmap.yaml b/helm/elasticsearch/templates/configmap.yaml new file mode 100644 index 0000000..10d22f3 --- /dev/null +++ b/helm/elasticsearch/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.fullnameOverride | default (printf "%s-elasticsearch" .Release.Name) }}-mapping + labels: + app.kubernetes.io/name: {{ .Values.nameOverride | default "elasticsearch" }} + app.kubernetes.io/instance: {{ .Release.Name }} +data: + agari-index-mapping.json: | +{{ .Files.Get "configs/agari-index-mapping.json" | indent 4 }} \ No newline at end of file diff --git a/helm/elasticsearch/templates/deployment.yaml b/helm/elasticsearch/templates/deployment.yaml index bacc7bf..38bb39c 100644 --- a/helm/elasticsearch/templates/deployment.yaml +++ b/helm/elasticsearch/templates/deployment.yaml @@ -48,6 +48,29 @@ spec: # Bootstrap settings - name: bootstrap.memory_lock value: "false" + lifecycle: + postStart: + exec: + command: + - /bin/bash + - -c + - | + # Wait for Elasticsearch to be ready + until curl -s http://localhost:9200/_cluster/health | grep -q '"status":"yellow"\|"status":"green"'; do + echo "Waiting for Elasticsearch to be ready..." + sleep 10 + done + + # Check if agari-samples index exists + if curl -s -f -o /dev/null http://localhost:9200/agari-samples; then + echo "Index agari-samples already exists" + else + echo "Creating agari-samples index with mapping..." + curl -X PUT "http://localhost:9200/agari-samples" \ + -H 'Content-Type: application/json' \ + -d @/config/agari-index-mapping.json + echo "Index created successfully" + fi {{- if .Values.probes.liveness.enabled }} livenessProbe: {{- if .Values.probes.liveness.httpGet }} @@ -81,13 +104,18 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - - name: opensearch-data - mountPath: /usr/share/opensearch/data + - name: elasticsearch-data + mountPath: /usr/share/elasticsearch/data + - name: index-mapping + mountPath: /config volumes: - - name: opensearch-data + - name: elasticsearch-data {{- if .Values.persistence.enabled }} persistentVolumeClaim: claimName: {{ .Values.fullnameOverride | default (printf "%s-elasticsearch" .Release.Name) }}-data {{- else }} emptyDir: {} {{- end }} + - name: index-mapping + configMap: + name: {{ .Values.fullnameOverride | default (printf "%s-elasticsearch" .Release.Name) }}-mapping diff --git a/helm/elasticsearch/templates/pvc.yaml b/helm/elasticsearch/templates/pvc.yaml new file mode 100644 index 0000000..ec1b4b5 --- /dev/null +++ b/helm/elasticsearch/templates/pvc.yaml @@ -0,0 +1,18 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ .Values.fullnameOverride | default (printf "%s-elasticsearch" .Release.Name) }}-data + labels: + app.kubernetes.io/name: {{ .Values.nameOverride | default "elasticsearch" }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + accessModes: + - {{ .Values.persistence.accessMode }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} \ No newline at end of file diff --git a/helm/elasticsearch/values.yaml b/helm/elasticsearch/values.yaml index 11b1a72..91a0a3e 100644 --- a/helm/elasticsearch/values.yaml +++ b/helm/elasticsearch/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 fullnameOverride: "elasticsearch" url: - hostname: "es-ilifu.openup.org.za" + hostname: "elasticsearch.local" image: repository: docker.elastic.co/elasticsearch/elasticsearch @@ -32,6 +32,12 @@ elasticsearch: network: host: "0.0.0.0" +persistence: + enabled: true + storageClass: "" + size: 10Gi + accessMode: ReadWriteOnce + resources: limits: cpu: 300m @@ -40,7 +46,6 @@ resources: cpu: 100m memory: 300Mi -# Probe configuration probes: liveness: enabled: false @@ -55,77 +60,15 @@ probes: ingress: enabled: true - className: "private" + className: "nginx" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: es-ilifu.openup.org.za + - host: elasticsearch.local paths: - path: / pathType: Prefix -persistence: - enabled: false - - discovery: - type: "single-node" - - security: - enabled: false - - java: - opts: "-Xms256m -Xmx512m" - - network: - host: "0.0.0.0" - - compatibility: - override_main_response_version: true - -resources: - limits: - cpu: 500m - memory: 768Mi - ephemeral-storage: "4Gi" - requests: - cpu: 200m - memory: 512Mi - ephemeral-storage: "2Gi" - -ingress: - enabled: true - className: "private" - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "100m" - hosts: - - host: es-ilifu.openup.org.za - paths: - - path: / - pathType: Prefix - -persistence: - enabled: false - -probes: - liveness: - enabled: false - httpGet: - path: /_cluster/health - port: http - initialDelaySeconds: 300 - periodSeconds: 60 - timeoutSeconds: 10 - failureThreshold: 5 - - readiness: - enabled: true - tcpSocket: - port: 9200 - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - nodeSelector: {} tolerations: [] affinity: {} diff --git a/helm/folio-worker/Chart.yaml b/helm/folio-worker/Chart.yaml new file mode 100644 index 0000000..4bd1716 --- /dev/null +++ b/helm/folio-worker/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: folio-worker +description: A Helm chart for Folio Worker - AGARI Background Job Processor +type: application +version: 0.1.0 +appVersion: "0.1.0" \ No newline at end of file diff --git a/helm/folio-worker/templates/_helpers.tpl b/helm/folio-worker/templates/_helpers.tpl new file mode 100644 index 0000000..015fe6d --- /dev/null +++ b/helm/folio-worker/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "folio-worker.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "folio-worker.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "folio-worker.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "folio-worker.labels" -}} +helm.sh/chart: {{ include "folio-worker.chart" . }} +{{ include "folio-worker.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "folio-worker.selectorLabels" -}} +app.kubernetes.io/name: {{ include "folio-worker.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/folio-worker/templates/deployment.yaml b/helm/folio-worker/templates/deployment.yaml new file mode 100644 index 0000000..69ab51e --- /dev/null +++ b/helm/folio-worker/templates/deployment.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "folio-worker.fullname" . }} + labels: + {{- include "folio-worker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "folio-worker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "folio-worker.selectorLabels" . | nindent 8 }} + spec: + restartPolicy: {{ .Values.restartPolicy }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + + # Override the default command to run worker instead of web app + command: {{ .Values.worker.command | toJson }} + + env: + # Worker-specific environment variables + {{- if .Values.worker.env.POLL_INTERVAL }} + - name: POLL_INTERVAL + value: {{ .Values.worker.env.POLL_INTERVAL | quote }} + {{- end }} + {{- if .Values.worker.env.JOB_TIMEOUT }} + - name: JOB_TIMEOUT + value: {{ .Values.worker.env.JOB_TIMEOUT | quote }} + {{- end }} + + # Keycloak authentication configuration + - name: KEYCLOAK_HOST + value: {{ .Values.folio.auth.keycloak.host | quote }} + - name: KEYCLOAK_REALM + value: {{ .Values.folio.auth.keycloak.realm | quote }} + - name: KEYCLOAK_ISSUER + value: {{ .Values.folio.auth.keycloak.issuer | quote }} + - name: KEYCLOAK_CLIENT_ID + value: {{ .Values.folio.auth.keycloak.clientId | quote }} + - name: KEYCLOAK_CLIENT_SECRET + value: {{ .Values.folio.auth.keycloak.clientSecret | quote }} + + # Database configuration + - name: FOLIO_DB_HOST + value: {{ .Values.folio.database.host | quote }} + - name: FOLIO_DB_PORT + value: {{ .Values.folio.database.port | quote }} + - name: FOLIO_DB_NAME + value: {{ .Values.folio.database.name | quote }} + - name: FOLIO_DB_USER + value: {{ .Values.folio.database.user | quote }} + - name: FOLIO_DB_PASSWORD + value: {{ .Values.folio.database.password | quote }} + + # MinIO configuration + - name: MINIO_ENDPOINT + value: {{ .Values.folio.minio.endpoint | quote }} + - name: MINIO_ACCESS_KEY + value: {{ .Values.folio.minio.accessKey | quote }} + - name: MINIO_SECRET_KEY + value: {{ .Values.folio.minio.secretKey | quote }} + - name: MINIO_BUCKET + value: {{ .Values.folio.minio.bucket | quote }} + - name: MINIO_SECURE + value: {{ .Values.folio.minio.secure | quote }} + + # Elasticsearch configuration + - name: ELASTICSEARCH_URL + value: {{ .Values.folio.elasticsearch.url | quote }} + - name: ELASTICSEARCH_INDEX + value: {{ .Values.folio.elasticsearch.index | quote }} + + # Overture configuration + - name: OVERTURE_SCORE + value: {{ .Values.folio.overture.score | quote }} + - name: OVERTURE_SONG + value: {{ .Values.folio.overture.song | quote }} + + resources: + {{- toYaml .Values.resources | nindent 12 }} + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml new file mode 100644 index 0000000..d2c623f --- /dev/null +++ b/helm/folio-worker/values.yaml @@ -0,0 +1,57 @@ +replicaCount: 1 + +image: + repository: ghcr.io/openupsa/agari-folio + pullPolicy: "Always" + tag: "latest" + +worker: + command: ["python", "worker.py"] + + env: + POLL_INTERVAL: "1" + + JOB_TIMEOUT: "300" + +folio: + auth: + keycloak: + host: "http://keycloak:8080" + realm: "agari" + issuer: "http://keycloak.local/realms/agari" + clientId: "dms" + clientSecret: "dms-secret" + database: + host: "folio-db" + port: "5432" + name: "folio" + user: "admin" + password: "folio-db-pass-123" + + minio: + endpoint: "minio:9000" + accessKey: "minioadmin" + secretKey: "minioadmin" + bucket: "agari-data" + secure: false + + elasticsearch: + url: "http://elasticsearch:9200" + index: "agari-samples" + +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Restart policy for worker pods +restartPolicy: Always \ No newline at end of file From ef0d130d5b6d7437ecad49f57ccabfd4cdb5fd18 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 12:29:20 +0200 Subject: [PATCH 21/48] fixed score and song references --- helm/folio-worker/templates/deployment.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/helm/folio-worker/templates/deployment.yaml b/helm/folio-worker/templates/deployment.yaml index 69ab51e..27c3e07 100644 --- a/helm/folio-worker/templates/deployment.yaml +++ b/helm/folio-worker/templates/deployment.yaml @@ -76,11 +76,6 @@ spec: - name: ELASTICSEARCH_INDEX value: {{ .Values.folio.elasticsearch.index | quote }} - # Overture configuration - - name: OVERTURE_SCORE - value: {{ .Values.folio.overture.score | quote }} - - name: OVERTURE_SONG - value: {{ .Values.folio.overture.song | quote }} resources: {{- toYaml .Values.resources | nindent 12 }} From 937d5dcf2885cc14da18f8eb12214c9484ced148 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 12:47:21 +0200 Subject: [PATCH 22/48] Add security context to Elasticsearch deployment for permission fixes --- helm/elasticsearch/templates/deployment.yaml | 8 ++++++++ helm/elasticsearch/values.yaml | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/helm/elasticsearch/templates/deployment.yaml b/helm/elasticsearch/templates/deployment.yaml index 38bb39c..9112613 100644 --- a/helm/elasticsearch/templates/deployment.yaml +++ b/helm/elasticsearch/templates/deployment.yaml @@ -19,8 +19,16 @@ spec: app.kubernetes.io/name: {{ .Values.nameOverride | default "elasticsearch" }} app.kubernetes.io/instance: {{ .Release.Name }} spec: + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: elasticsearch + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: diff --git a/helm/elasticsearch/values.yaml b/helm/elasticsearch/values.yaml index 91a0a3e..0daf141 100644 --- a/helm/elasticsearch/values.yaml +++ b/helm/elasticsearch/values.yaml @@ -38,6 +38,15 @@ persistence: size: 10Gi accessMode: ReadWriteOnce +# Add security context to fix permissions +podSecurityContext: + fsGroup: 1000 + +securityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + resources: limits: cpu: 300m From 3cd2be100fd3cca7f89433f0e8dc87a67c8c615f Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 12:54:28 +0200 Subject: [PATCH 23/48] es fix --- helm/elasticsearch/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/elasticsearch/values.yaml b/helm/elasticsearch/values.yaml index 0daf141..f5885d6 100644 --- a/helm/elasticsearch/values.yaml +++ b/helm/elasticsearch/values.yaml @@ -34,7 +34,7 @@ elasticsearch: persistence: enabled: true - storageClass: "" + storageClass: "cinder-csi" size: 10Gi accessMode: ReadWriteOnce From 12921d75bad8a74fa09d6aa60b57417ac3ff39d7 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 13:16:06 +0200 Subject: [PATCH 24/48] worker branch update --- helm/folio-worker/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml index d2c623f..7db4e3d 100644 --- a/helm/folio-worker/values.yaml +++ b/helm/folio-worker/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: ghcr.io/openupsa/agari-folio pullPolicy: "Always" - tag: "latest" + tag: "staging" worker: command: ["python", "worker.py"] From b7d53752273a33b66eba4b1e44bcc31152b1bf93 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 13:38:27 +0200 Subject: [PATCH 25/48] upload size increase --- helm/folio/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index 3f7a8d7..3087d88 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -46,6 +46,7 @@ ingress: nginx.ingress.kubernetes.io/cors-allow-origin: "*" nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" + nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - host: folio-ilifu.openup.org.za paths: From 6939f5938fe85adc149f58da2f41f49ea6e757e6 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 15:21:05 +0200 Subject: [PATCH 26/48] new worker chart --- helm/folio-worker-old/Chart.yaml | 6 ++ helm/folio-worker-old/templates/_helpers.tpl | 51 ++++++++++ .../templates/deployment.yaml | 94 +++++++++++++++++++ helm/folio-worker-old/values.yaml | 57 +++++++++++ helm/folio-worker/Chart.yaml | 6 +- helm/folio-worker/templates/_helpers.tpl | 18 ++-- helm/folio-worker/templates/deployment.yaml | 90 ++++++------------ helm/folio-worker/templates/ingress.yaml | 41 ++++++++ helm/folio-worker/templates/service.yaml | 15 +++ helm/folio-worker/values.yaml | 48 ++++++---- 10 files changed, 337 insertions(+), 89 deletions(-) create mode 100644 helm/folio-worker-old/Chart.yaml create mode 100644 helm/folio-worker-old/templates/_helpers.tpl create mode 100644 helm/folio-worker-old/templates/deployment.yaml create mode 100644 helm/folio-worker-old/values.yaml create mode 100644 helm/folio-worker/templates/ingress.yaml create mode 100644 helm/folio-worker/templates/service.yaml diff --git a/helm/folio-worker-old/Chart.yaml b/helm/folio-worker-old/Chart.yaml new file mode 100644 index 0000000..4bd1716 --- /dev/null +++ b/helm/folio-worker-old/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: folio-worker +description: A Helm chart for Folio Worker - AGARI Background Job Processor +type: application +version: 0.1.0 +appVersion: "0.1.0" \ No newline at end of file diff --git a/helm/folio-worker-old/templates/_helpers.tpl b/helm/folio-worker-old/templates/_helpers.tpl new file mode 100644 index 0000000..015fe6d --- /dev/null +++ b/helm/folio-worker-old/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "folio-worker.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "folio-worker.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "folio-worker.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "folio-worker.labels" -}} +helm.sh/chart: {{ include "folio-worker.chart" . }} +{{ include "folio-worker.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "folio-worker.selectorLabels" -}} +app.kubernetes.io/name: {{ include "folio-worker.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/folio-worker-old/templates/deployment.yaml b/helm/folio-worker-old/templates/deployment.yaml new file mode 100644 index 0000000..27c3e07 --- /dev/null +++ b/helm/folio-worker-old/templates/deployment.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "folio-worker.fullname" . }} + labels: + {{- include "folio-worker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "folio-worker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "folio-worker.selectorLabels" . | nindent 8 }} + spec: + restartPolicy: {{ .Values.restartPolicy }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + + # Override the default command to run worker instead of web app + command: {{ .Values.worker.command | toJson }} + + env: + # Worker-specific environment variables + {{- if .Values.worker.env.POLL_INTERVAL }} + - name: POLL_INTERVAL + value: {{ .Values.worker.env.POLL_INTERVAL | quote }} + {{- end }} + {{- if .Values.worker.env.JOB_TIMEOUT }} + - name: JOB_TIMEOUT + value: {{ .Values.worker.env.JOB_TIMEOUT | quote }} + {{- end }} + + # Keycloak authentication configuration + - name: KEYCLOAK_HOST + value: {{ .Values.folio.auth.keycloak.host | quote }} + - name: KEYCLOAK_REALM + value: {{ .Values.folio.auth.keycloak.realm | quote }} + - name: KEYCLOAK_ISSUER + value: {{ .Values.folio.auth.keycloak.issuer | quote }} + - name: KEYCLOAK_CLIENT_ID + value: {{ .Values.folio.auth.keycloak.clientId | quote }} + - name: KEYCLOAK_CLIENT_SECRET + value: {{ .Values.folio.auth.keycloak.clientSecret | quote }} + + # Database configuration + - name: FOLIO_DB_HOST + value: {{ .Values.folio.database.host | quote }} + - name: FOLIO_DB_PORT + value: {{ .Values.folio.database.port | quote }} + - name: FOLIO_DB_NAME + value: {{ .Values.folio.database.name | quote }} + - name: FOLIO_DB_USER + value: {{ .Values.folio.database.user | quote }} + - name: FOLIO_DB_PASSWORD + value: {{ .Values.folio.database.password | quote }} + + # MinIO configuration + - name: MINIO_ENDPOINT + value: {{ .Values.folio.minio.endpoint | quote }} + - name: MINIO_ACCESS_KEY + value: {{ .Values.folio.minio.accessKey | quote }} + - name: MINIO_SECRET_KEY + value: {{ .Values.folio.minio.secretKey | quote }} + - name: MINIO_BUCKET + value: {{ .Values.folio.minio.bucket | quote }} + - name: MINIO_SECURE + value: {{ .Values.folio.minio.secure | quote }} + + # Elasticsearch configuration + - name: ELASTICSEARCH_URL + value: {{ .Values.folio.elasticsearch.url | quote }} + - name: ELASTICSEARCH_INDEX + value: {{ .Values.folio.elasticsearch.index | quote }} + + + resources: + {{- toYaml .Values.resources | nindent 12 }} + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/helm/folio-worker-old/values.yaml b/helm/folio-worker-old/values.yaml new file mode 100644 index 0000000..7db4e3d --- /dev/null +++ b/helm/folio-worker-old/values.yaml @@ -0,0 +1,57 @@ +replicaCount: 1 + +image: + repository: ghcr.io/openupsa/agari-folio + pullPolicy: "Always" + tag: "staging" + +worker: + command: ["python", "worker.py"] + + env: + POLL_INTERVAL: "1" + + JOB_TIMEOUT: "300" + +folio: + auth: + keycloak: + host: "http://keycloak:8080" + realm: "agari" + issuer: "http://keycloak.local/realms/agari" + clientId: "dms" + clientSecret: "dms-secret" + database: + host: "folio-db" + port: "5432" + name: "folio" + user: "admin" + password: "folio-db-pass-123" + + minio: + endpoint: "minio:9000" + accessKey: "minioadmin" + secretKey: "minioadmin" + bucket: "agari-data" + secure: false + + elasticsearch: + url: "http://elasticsearch:9200" + index: "agari-samples" + +resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Restart policy for worker pods +restartPolicy: Always \ No newline at end of file diff --git a/helm/folio-worker/Chart.yaml b/helm/folio-worker/Chart.yaml index 4bd1716..b27cad8 100644 --- a/helm/folio-worker/Chart.yaml +++ b/helm/folio-worker/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -name: folio-worker -description: A Helm chart for Folio Worker - AGARI Background Job Processor +name: folio +description: A Helm chart for Folio - AGARI Projects Service type: application version: 0.1.0 -appVersion: "0.1.0" \ No newline at end of file +appVersion: "0.1.0" diff --git a/helm/folio-worker/templates/_helpers.tpl b/helm/folio-worker/templates/_helpers.tpl index 015fe6d..3bbaa60 100644 --- a/helm/folio-worker/templates/_helpers.tpl +++ b/helm/folio-worker/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "folio-worker.name" -}} +{{- define "folio.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "folio-worker.fullname" -}} +{{- define "folio.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "folio-worker.chart" -}} +{{- define "folio.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "folio-worker.labels" -}} -helm.sh/chart: {{ include "folio-worker.chart" . }} -{{ include "folio-worker.selectorLabels" . }} +{{- define "folio.labels" -}} +helm.sh/chart: {{ include "folio.chart" . }} +{{ include "folio.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,7 +45,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "folio-worker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "folio-worker.name" . }} +{{- define "folio.selectorLabels" -}} +app.kubernetes.io/name: {{ include "folio.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/folio-worker/templates/deployment.yaml b/helm/folio-worker/templates/deployment.yaml index 27c3e07..f8eefd6 100644 --- a/helm/folio-worker/templates/deployment.yaml +++ b/helm/folio-worker/templates/deployment.yaml @@ -1,39 +1,28 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "folio-worker.fullname" . }} + name: {{ include "folio.fullname" . }} labels: - {{- include "folio-worker.labels" . | nindent 4 }} + {{- include "folio.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: - {{- include "folio-worker.selectorLabels" . | nindent 6 }} + {{- include "folio.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "folio-worker.selectorLabels" . | nindent 8 }} + {{- include "folio.selectorLabels" . | nindent 8 }} spec: - restartPolicy: {{ .Values.restartPolicy }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - - # Override the default command to run worker instead of web app - command: {{ .Values.worker.command | toJson }} - + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP env: - # Worker-specific environment variables - {{- if .Values.worker.env.POLL_INTERVAL }} - - name: POLL_INTERVAL - value: {{ .Values.worker.env.POLL_INTERVAL | quote }} - {{- end }} - {{- if .Values.worker.env.JOB_TIMEOUT }} - - name: JOB_TIMEOUT - value: {{ .Values.worker.env.JOB_TIMEOUT | quote }} - {{- end }} - # Keycloak authentication configuration - name: KEYCLOAK_HOST value: {{ .Values.folio.auth.keycloak.host | quote }} @@ -45,50 +34,33 @@ spec: value: {{ .Values.folio.auth.keycloak.clientId | quote }} - name: KEYCLOAK_CLIENT_SECRET value: {{ .Values.folio.auth.keycloak.clientSecret | quote }} - - # Database configuration - - name: FOLIO_DB_HOST + - name: DB_HOST value: {{ .Values.folio.database.host | quote }} - - name: FOLIO_DB_PORT + - name: DB_PORT value: {{ .Values.folio.database.port | quote }} - - name: FOLIO_DB_NAME + - name: DB_NAME value: {{ .Values.folio.database.name | quote }} - - name: FOLIO_DB_USER + - name: DB_USER value: {{ .Values.folio.database.user | quote }} - - name: FOLIO_DB_PASSWORD + - name: DB_PASSWORD value: {{ .Values.folio.database.password | quote }} - - # MinIO configuration - - name: MINIO_ENDPOINT - value: {{ .Values.folio.minio.endpoint | quote }} - - name: MINIO_ACCESS_KEY - value: {{ .Values.folio.minio.accessKey | quote }} - - name: MINIO_SECRET_KEY - value: {{ .Values.folio.minio.secretKey | quote }} - - name: MINIO_BUCKET - value: {{ .Values.folio.minio.bucket | quote }} - - name: MINIO_SECURE - value: {{ .Values.folio.minio.secure | quote }} - - # Elasticsearch configuration - - name: ELASTICSEARCH_URL - value: {{ .Values.folio.elasticsearch.url | quote }} - - name: ELASTICSEARCH_INDEX - value: {{ .Values.folio.elasticsearch.index | quote }} - - + - name: OVERTURE_SCORE + value: {{ .Values.folio.overture.score | quote }} + - name: OVERTURE_SONG + value: {{ .Values.folio.overture.song | quote }} + - name: KEYCLOAK_URL + value: {{ .Values.folio.auth.keycloak.host | quote }} + livenessProbe: + httpGet: + path: /info/health + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /info/health + port: http + initialDelaySeconds: 5 + periodSeconds: 5 resources: {{- toYaml .Values.resources | nindent 12 }} - - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file diff --git a/helm/folio-worker/templates/ingress.yaml b/helm/folio-worker/templates/ingress.yaml new file mode 100644 index 0000000..aa489ee --- /dev/null +++ b/helm/folio-worker/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "folio.fullname" . }} + labels: + {{- include "folio.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "folio.fullname" $ }} + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/folio-worker/templates/service.yaml b/helm/folio-worker/templates/service.yaml new file mode 100644 index 0000000..944be46 --- /dev/null +++ b/helm/folio-worker/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "folio.fullname" . }} + labels: + {{- include "folio.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "folio.selectorLabels" . | nindent 4 }} diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml index 7db4e3d..df6a9eb 100644 --- a/helm/folio-worker/values.yaml +++ b/helm/folio-worker/values.yaml @@ -3,14 +3,12 @@ replicaCount: 1 image: repository: ghcr.io/openupsa/agari-folio pullPolicy: "Always" - tag: "staging" + tag: "latest" worker: command: ["python", "worker.py"] - env: POLL_INTERVAL: "1" - JOB_TIMEOUT: "300" folio: @@ -22,29 +20,46 @@ folio: clientId: "dms" clientSecret: "dms-secret" database: + vendor: postgres host: "folio-db" port: "5432" name: "folio" user: "admin" password: "folio-db-pass-123" - - minio: - endpoint: "minio:9000" - accessKey: "minioadmin" - secretKey: "minioadmin" - bucket: "agari-data" - secure: false - - elasticsearch: - url: "http://elasticsearch:9200" - index: "agari-samples" + pool: + size: 10 + maxOverflow: 20 + recycle: 3600 + preping: true + timeout: 30 + options: "?connect_timeout=10&application_name=folio" + + overture: + score: "http://score:8087" + song: "http://song:8080" + +ingress: + enabled: true + className: private + annotations: + nginx.ingress.kubernetes.io/enable-cors: "true" + nginx.ingress.kubernetes.io/cors-allow-origin: "*" + nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" + nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" + nginx.ingress.kubernetes.io/proxy-body-size: "100m" + hosts: + - host: folio-ilifu.openup.org.za + paths: + - path: / + pathType: Prefix + tls: [] resources: limits: cpu: 500m memory: 512Mi requests: - cpu: 100m + cpu: 200m memory: 256Mi nodeSelector: {} @@ -52,6 +67,3 @@ nodeSelector: {} tolerations: [] affinity: {} - -# Restart policy for worker pods -restartPolicy: Always \ No newline at end of file From 179e825b117f870ea288c1ec8f50a1f781ed46c0 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 15:24:05 +0200 Subject: [PATCH 27/48] helm chart fix --- helm/folio-worker/Chart.yaml | 2 +- helm/folio-worker/templates/deployment.yaml | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/helm/folio-worker/Chart.yaml b/helm/folio-worker/Chart.yaml index b27cad8..f1bb626 100644 --- a/helm/folio-worker/Chart.yaml +++ b/helm/folio-worker/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: folio +name: folio-worker description: A Helm chart for Folio - AGARI Projects Service type: application version: 0.1.0 diff --git a/helm/folio-worker/templates/deployment.yaml b/helm/folio-worker/templates/deployment.yaml index f8eefd6..e802406 100644 --- a/helm/folio-worker/templates/deployment.yaml +++ b/helm/folio-worker/templates/deployment.yaml @@ -18,10 +18,7 @@ spec: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: {{ .Values.service.targetPort }} - protocol: TCP + env: # Keycloak authentication configuration - name: KEYCLOAK_HOST From b913b462010a00cc22668be134f6dbb2024d6c81 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 15:24:48 +0200 Subject: [PATCH 28/48] worker update --- helm/folio-worker/templates/ingress.yaml | 41 ------------------------ helm/folio-worker/templates/service.yaml | 15 --------- 2 files changed, 56 deletions(-) delete mode 100644 helm/folio-worker/templates/ingress.yaml delete mode 100644 helm/folio-worker/templates/service.yaml diff --git a/helm/folio-worker/templates/ingress.yaml b/helm/folio-worker/templates/ingress.yaml deleted file mode 100644 index aa489ee..0000000 --- a/helm/folio-worker/templates/ingress.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.ingress.enabled -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ include "folio.fullname" . }} - labels: - {{- include "folio.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.className }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - pathType: {{ .pathType }} - backend: - service: - name: {{ include "folio.fullname" $ }} - port: - number: {{ $.Values.service.port }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/folio-worker/templates/service.yaml b/helm/folio-worker/templates/service.yaml deleted file mode 100644 index 944be46..0000000 --- a/helm/folio-worker/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "folio.fullname" . }} - labels: - {{- include "folio.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "folio.selectorLabels" . | nindent 4 }} From 2071f7b83baa93d6ae13f13eea8aeb69058161bd Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 15:28:09 +0200 Subject: [PATCH 29/48] worker update --- helm/folio-worker/values.yaml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml index df6a9eb..252aeb3 100644 --- a/helm/folio-worker/values.yaml +++ b/helm/folio-worker/values.yaml @@ -34,25 +34,8 @@ folio: timeout: 30 options: "?connect_timeout=10&application_name=folio" - overture: - score: "http://score:8087" - song: "http://song:8080" -ingress: - enabled: true - className: private - annotations: - nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/cors-allow-origin: "*" - nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" - nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" - nginx.ingress.kubernetes.io/proxy-body-size: "100m" - hosts: - - host: folio-ilifu.openup.org.za - paths: - - path: / - pathType: Prefix - tls: [] + resources: limits: From 24d10221f1d731bed49f58cfcd78284d062199c5 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 15:29:21 +0200 Subject: [PATCH 30/48] remove overture environment variables from deployment configuration --- helm/folio-worker/templates/deployment.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/helm/folio-worker/templates/deployment.yaml b/helm/folio-worker/templates/deployment.yaml index e802406..24c2e3c 100644 --- a/helm/folio-worker/templates/deployment.yaml +++ b/helm/folio-worker/templates/deployment.yaml @@ -41,10 +41,6 @@ spec: value: {{ .Values.folio.database.user | quote }} - name: DB_PASSWORD value: {{ .Values.folio.database.password | quote }} - - name: OVERTURE_SCORE - value: {{ .Values.folio.overture.score | quote }} - - name: OVERTURE_SONG - value: {{ .Values.folio.overture.song | quote }} - name: KEYCLOAK_URL value: {{ .Values.folio.auth.keycloak.host | quote }} livenessProbe: From bb290228ae5f463477b5c85912244994bd35a68f Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 15:33:57 +0200 Subject: [PATCH 31/48] worker update --- helm/folio-worker/Chart.yaml | 2 +- helm/folio-worker/templates/_helpers.tpl | 16 ++++---- helm/folio-worker/templates/deployment.yaml | 45 +++++++++++++-------- helm/folio-worker/values.yaml | 11 +++-- 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/helm/folio-worker/Chart.yaml b/helm/folio-worker/Chart.yaml index f1bb626..b1a50df 100644 --- a/helm/folio-worker/Chart.yaml +++ b/helm/folio-worker/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: folio-worker -description: A Helm chart for Folio - AGARI Projects Service +description: A Helm chart for Folio Worker - AGARI Background Job Processor type: application version: 0.1.0 appVersion: "0.1.0" diff --git a/helm/folio-worker/templates/_helpers.tpl b/helm/folio-worker/templates/_helpers.tpl index 3bbaa60..ef3f725 100644 --- a/helm/folio-worker/templates/_helpers.tpl +++ b/helm/folio-worker/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{/* Expand the name of the chart. */}} -{{- define "folio.name" -}} +{{- define "folio-worker.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} @@ -10,7 +10,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "folio.fullname" -}} +{{- define "folio-worker.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} @@ -26,16 +26,16 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "folio.chart" -}} +{{- define "folio-worker.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} -{{- define "folio.labels" -}} -helm.sh/chart: {{ include "folio.chart" . }} -{{ include "folio.selectorLabels" . }} +{{- define "folio-worker.labels" -}} +helm.sh/chart: {{ include "folio-worker.chart" . }} +{{ include "folio-worker.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} @@ -45,7 +45,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{/* Selector labels */}} -{{- define "folio.selectorLabels" -}} -app.kubernetes.io/name: {{ include "folio.name" . }} +{{- define "folio-worker.selectorLabels" -}} +app.kubernetes.io/name: {{ include "folio-worker.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} diff --git a/helm/folio-worker/templates/deployment.yaml b/helm/folio-worker/templates/deployment.yaml index 24c2e3c..89a185b 100644 --- a/helm/folio-worker/templates/deployment.yaml +++ b/helm/folio-worker/templates/deployment.yaml @@ -1,24 +1,27 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "folio.fullname" . }} + name: {{ include "folio-worker.fullname" . }} labels: - {{- include "folio.labels" . | nindent 4 }} + {{- include "folio-worker.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: - {{- include "folio.selectorLabels" . | nindent 6 }} + {{- include "folio-worker.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "folio.selectorLabels" . | nindent 8 }} + {{- include "folio-worker.selectorLabels" . | nindent 8 }} spec: containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + # Override the default command to run worker instead of web app + command: {{ .Values.worker.command | toJson }} + env: # Keycloak authentication configuration - name: KEYCLOAK_HOST @@ -31,6 +34,8 @@ spec: value: {{ .Values.folio.auth.keycloak.clientId | quote }} - name: KEYCLOAK_CLIENT_SECRET value: {{ .Values.folio.auth.keycloak.clientSecret | quote }} + + # Database configuration (using same env vars as main folio service) - name: DB_HOST value: {{ .Values.folio.database.host | quote }} - name: DB_PORT @@ -41,19 +46,27 @@ spec: value: {{ .Values.folio.database.user | quote }} - name: DB_PASSWORD value: {{ .Values.folio.database.password | quote }} + + # Overture configuration + - name: OVERTURE_SCORE + value: {{ .Values.folio.overture.score | quote }} + - name: OVERTURE_SONG + value: {{ .Values.folio.overture.song | quote }} - name: KEYCLOAK_URL value: {{ .Values.folio.auth.keycloak.host | quote }} - livenessProbe: - httpGet: - path: /info/health - port: http - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /info/health - port: http - initialDelaySeconds: 5 - periodSeconds: 5 + resources: {{- toYaml .Values.resources | nindent 12 }} + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml index 252aeb3..c8071ec 100644 --- a/helm/folio-worker/values.yaml +++ b/helm/folio-worker/values.yaml @@ -7,10 +7,8 @@ image: worker: command: ["python", "worker.py"] - env: - POLL_INTERVAL: "1" - JOB_TIMEOUT: "300" +# Folio application configuration (same as main folio service) folio: auth: keycloak: @@ -32,10 +30,11 @@ folio: recycle: 3600 preping: true timeout: 30 - options: "?connect_timeout=10&application_name=folio" - - + options: "?connect_timeout=10&application_name=folio-worker" + overture: + score: "http://score:8087" + song: "http://song:8080" resources: limits: From 7af1278dd0536fd364cefd802b73d323f3f2a301 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 15:35:07 +0200 Subject: [PATCH 32/48] update image --- helm/folio/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index 3087d88..2db99ab 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: ghcr.io/openupsa/agari-folio pullPolicy: "Always" - tag: "latest" + tag: "staging" service: type: ClusterIP From b391a0776f037510fdaa0062166487d5cc89167b Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 15:39:43 +0200 Subject: [PATCH 33/48] worker update --- helm/folio-worker/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml index c8071ec..0cee911 100644 --- a/helm/folio-worker/values.yaml +++ b/helm/folio-worker/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: ghcr.io/openupsa/agari-folio pullPolicy: "Always" - tag: "latest" + tag: "staging" worker: command: ["python", "worker.py"] From 409ea56504f61c762dd2b8be1ad00e0a9065cf72 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 16:05:29 +0200 Subject: [PATCH 34/48] added elastic and minio urls --- helm/folio-worker/templates/deployment.yaml | 18 ++++++++++++++++++ helm/folio-worker/values.yaml | 11 +++++++++++ 2 files changed, 29 insertions(+) diff --git a/helm/folio-worker/templates/deployment.yaml b/helm/folio-worker/templates/deployment.yaml index 89a185b..053c77c 100644 --- a/helm/folio-worker/templates/deployment.yaml +++ b/helm/folio-worker/templates/deployment.yaml @@ -54,6 +54,24 @@ spec: value: {{ .Values.folio.overture.song | quote }} - name: KEYCLOAK_URL value: {{ .Values.folio.auth.keycloak.host | quote }} + + # MinIO configuration + - name: MINIO_ENDPOINT + value: {{ .Values.folio.minio.endpoint | quote }} + - name: MINIO_ACCESS_KEY + value: {{ .Values.folio.minio.accessKey | quote }} + - name: MINIO_SECRET_KEY + value: {{ .Values.folio.minio.secretKey | quote }} + - name: MINIO_BUCKET + value: {{ .Values.folio.minio.bucket | quote }} + - name: MINIO_SECURE + value: {{ .Values.folio.minio.secure | quote }} + + # Elasticsearch configuration + - name: ELASTICSEARCH_URL + value: {{ .Values.folio.elasticsearch.url | quote }} + - name: ELASTICSEARCH_INDEX + value: {{ .Values.folio.elasticsearch.index | quote }} resources: {{- toYaml .Values.resources | nindent 12 }} diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml index 0cee911..60a8485 100644 --- a/helm/folio-worker/values.yaml +++ b/helm/folio-worker/values.yaml @@ -36,6 +36,17 @@ folio: score: "http://score:8087" song: "http://song:8080" + minio: + endpoint: "minio:9000" + accessKey: "minioadmin" + secretKey: "minioadmin" + bucket: "agari-data" + secure: false + + elasticsearch: + url: "http://elasticsearch:9200" + index: "agari-samples" + resources: limits: cpu: 500m From 76288ea398422e24e1b3637dc2040f35a083d0c0 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Wed, 19 Nov 2025 16:22:12 +0200 Subject: [PATCH 35/48] worker update --- helm/folio-worker-old/Chart.yaml | 6 -- helm/folio-worker-old/templates/_helpers.tpl | 51 ---------- .../templates/deployment.yaml | 94 ------------------- helm/folio-worker-old/values.yaml | 57 ----------- helm/folio-worker/templates/deployment.yaml | 8 -- helm/folio-worker/values.yaml | 4 - 6 files changed, 220 deletions(-) delete mode 100644 helm/folio-worker-old/Chart.yaml delete mode 100644 helm/folio-worker-old/templates/_helpers.tpl delete mode 100644 helm/folio-worker-old/templates/deployment.yaml delete mode 100644 helm/folio-worker-old/values.yaml diff --git a/helm/folio-worker-old/Chart.yaml b/helm/folio-worker-old/Chart.yaml deleted file mode 100644 index 4bd1716..0000000 --- a/helm/folio-worker-old/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: folio-worker -description: A Helm chart for Folio Worker - AGARI Background Job Processor -type: application -version: 0.1.0 -appVersion: "0.1.0" \ No newline at end of file diff --git a/helm/folio-worker-old/templates/_helpers.tpl b/helm/folio-worker-old/templates/_helpers.tpl deleted file mode 100644 index 015fe6d..0000000 --- a/helm/folio-worker-old/templates/_helpers.tpl +++ /dev/null @@ -1,51 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "folio-worker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "folio-worker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "folio-worker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "folio-worker.labels" -}} -helm.sh/chart: {{ include "folio-worker.chart" . }} -{{ include "folio-worker.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "folio-worker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "folio-worker.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} \ No newline at end of file diff --git a/helm/folio-worker-old/templates/deployment.yaml b/helm/folio-worker-old/templates/deployment.yaml deleted file mode 100644 index 27c3e07..0000000 --- a/helm/folio-worker-old/templates/deployment.yaml +++ /dev/null @@ -1,94 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "folio-worker.fullname" . }} - labels: - {{- include "folio-worker.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "folio-worker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "folio-worker.selectorLabels" . | nindent 8 }} - spec: - restartPolicy: {{ .Values.restartPolicy }} - containers: - - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - - # Override the default command to run worker instead of web app - command: {{ .Values.worker.command | toJson }} - - env: - # Worker-specific environment variables - {{- if .Values.worker.env.POLL_INTERVAL }} - - name: POLL_INTERVAL - value: {{ .Values.worker.env.POLL_INTERVAL | quote }} - {{- end }} - {{- if .Values.worker.env.JOB_TIMEOUT }} - - name: JOB_TIMEOUT - value: {{ .Values.worker.env.JOB_TIMEOUT | quote }} - {{- end }} - - # Keycloak authentication configuration - - name: KEYCLOAK_HOST - value: {{ .Values.folio.auth.keycloak.host | quote }} - - name: KEYCLOAK_REALM - value: {{ .Values.folio.auth.keycloak.realm | quote }} - - name: KEYCLOAK_ISSUER - value: {{ .Values.folio.auth.keycloak.issuer | quote }} - - name: KEYCLOAK_CLIENT_ID - value: {{ .Values.folio.auth.keycloak.clientId | quote }} - - name: KEYCLOAK_CLIENT_SECRET - value: {{ .Values.folio.auth.keycloak.clientSecret | quote }} - - # Database configuration - - name: FOLIO_DB_HOST - value: {{ .Values.folio.database.host | quote }} - - name: FOLIO_DB_PORT - value: {{ .Values.folio.database.port | quote }} - - name: FOLIO_DB_NAME - value: {{ .Values.folio.database.name | quote }} - - name: FOLIO_DB_USER - value: {{ .Values.folio.database.user | quote }} - - name: FOLIO_DB_PASSWORD - value: {{ .Values.folio.database.password | quote }} - - # MinIO configuration - - name: MINIO_ENDPOINT - value: {{ .Values.folio.minio.endpoint | quote }} - - name: MINIO_ACCESS_KEY - value: {{ .Values.folio.minio.accessKey | quote }} - - name: MINIO_SECRET_KEY - value: {{ .Values.folio.minio.secretKey | quote }} - - name: MINIO_BUCKET - value: {{ .Values.folio.minio.bucket | quote }} - - name: MINIO_SECURE - value: {{ .Values.folio.minio.secure | quote }} - - # Elasticsearch configuration - - name: ELASTICSEARCH_URL - value: {{ .Values.folio.elasticsearch.url | quote }} - - name: ELASTICSEARCH_INDEX - value: {{ .Values.folio.elasticsearch.index | quote }} - - - resources: - {{- toYaml .Values.resources | nindent 12 }} - - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file diff --git a/helm/folio-worker-old/values.yaml b/helm/folio-worker-old/values.yaml deleted file mode 100644 index 7db4e3d..0000000 --- a/helm/folio-worker-old/values.yaml +++ /dev/null @@ -1,57 +0,0 @@ -replicaCount: 1 - -image: - repository: ghcr.io/openupsa/agari-folio - pullPolicy: "Always" - tag: "staging" - -worker: - command: ["python", "worker.py"] - - env: - POLL_INTERVAL: "1" - - JOB_TIMEOUT: "300" - -folio: - auth: - keycloak: - host: "http://keycloak:8080" - realm: "agari" - issuer: "http://keycloak.local/realms/agari" - clientId: "dms" - clientSecret: "dms-secret" - database: - host: "folio-db" - port: "5432" - name: "folio" - user: "admin" - password: "folio-db-pass-123" - - minio: - endpoint: "minio:9000" - accessKey: "minioadmin" - secretKey: "minioadmin" - bucket: "agari-data" - secure: false - - elasticsearch: - url: "http://elasticsearch:9200" - index: "agari-samples" - -resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 100m - memory: 256Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -# Restart policy for worker pods -restartPolicy: Always \ No newline at end of file diff --git a/helm/folio-worker/templates/deployment.yaml b/helm/folio-worker/templates/deployment.yaml index 053c77c..fa482f9 100644 --- a/helm/folio-worker/templates/deployment.yaml +++ b/helm/folio-worker/templates/deployment.yaml @@ -58,14 +58,6 @@ spec: # MinIO configuration - name: MINIO_ENDPOINT value: {{ .Values.folio.minio.endpoint | quote }} - - name: MINIO_ACCESS_KEY - value: {{ .Values.folio.minio.accessKey | quote }} - - name: MINIO_SECRET_KEY - value: {{ .Values.folio.minio.secretKey | quote }} - - name: MINIO_BUCKET - value: {{ .Values.folio.minio.bucket | quote }} - - name: MINIO_SECURE - value: {{ .Values.folio.minio.secure | quote }} # Elasticsearch configuration - name: ELASTICSEARCH_URL diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml index 60a8485..f264269 100644 --- a/helm/folio-worker/values.yaml +++ b/helm/folio-worker/values.yaml @@ -38,10 +38,6 @@ folio: minio: endpoint: "minio:9000" - accessKey: "minioadmin" - secretKey: "minioadmin" - bucket: "agari-data" - secure: false elasticsearch: url: "http://elasticsearch:9200" From 4f4a71af44544293009e427e6b54761beaab8c06 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Thu, 20 Nov 2025 16:25:53 +0200 Subject: [PATCH 36/48] frontend deploy fix --- helm/frontend/templates/deployment.yaml | 6 +++--- helm/frontend/templates/ingress.yaml | 1 + helm/frontend/values.yaml | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/helm/frontend/templates/deployment.yaml b/helm/frontend/templates/deployment.yaml index a0b5db0..f0d3806 100644 --- a/helm/frontend/templates/deployment.yaml +++ b/helm/frontend/templates/deployment.yaml @@ -27,6 +27,6 @@ spec: readinessProbe: {{ toYaml .Values.readinessProbe | nindent 10 }} livenessProbe: {{ toYaml .Values.livenessProbe | nindent 10 }} resources: {{ toYaml .Values.resources | nindent 10 }} - nodeSelector: {{ toYaml .Values.nodeSelector | nindent 6 }} - tolerations: {{ toYaml .Values.tolerations | nindent 6 }} - affinity: {{ toYaml .Values.affinity | nindent 6 }} \ No newline at end of file + nodeSelector: {{ toYaml .Values.nodeSelector | nindent 6 }} + tolerations: {{ toYaml .Values.tolerations | nindent 6 }} + affinity: {{ toYaml .Values.affinity | nindent 6 }} \ No newline at end of file diff --git a/helm/frontend/templates/ingress.yaml b/helm/frontend/templates/ingress.yaml index b42e7d0..bbc66b3 100644 --- a/helm/frontend/templates/ingress.yaml +++ b/helm/frontend/templates/ingress.yaml @@ -6,6 +6,7 @@ metadata: annotations: {{ toYaml .Values.ingress.annotations | nindent 4 }} spec: rules: + {{- $root := . }} {{- range .Values.ingress.hosts }} - host: {{ .host }} http: diff --git a/helm/frontend/values.yaml b/helm/frontend/values.yaml index eb01ce1..6503109 100644 --- a/helm/frontend/values.yaml +++ b/helm/frontend/values.yaml @@ -2,7 +2,7 @@ replicaCount: 1 image: repository: ghcr.io/methodlab/agari-frontend - tag: v0.0.1 + tag: v0.0.24 pullPolicy: Always service: From 9c3669667b545d04edb32fcc256f911f59816bf6 Mon Sep 17 00:00:00 2001 From: desafinadude Date: Fri, 21 Nov 2025 09:44:24 +0200 Subject: [PATCH 37/48] elastic --- .../configs/agari-index-mapping.json | 229 ++++++++++++------ helm/elasticsearch/values.yaml | 2 +- 2 files changed, 150 insertions(+), 81 deletions(-) diff --git a/helm/elasticsearch/configs/agari-index-mapping.json b/helm/elasticsearch/configs/agari-index-mapping.json index 490f83f..ce481c7 100644 --- a/helm/elasticsearch/configs/agari-index-mapping.json +++ b/helm/elasticsearch/configs/agari-index-mapping.json @@ -1,92 +1,161 @@ { "mappings": { "properties": { - "analysis": { + "id": {"type": "keyword"}, + "submission_id": {"type": "keyword"}, + "object_id": {"type": "keyword"}, + "isolate_id": {"type": "keyword"}, + "project_id": {"type": "keyword"}, + "pathogen_id": {"type": "keyword"}, + "created_at": {"type": "date"}, + "updated_at": {"type": "date"}, + "deleted_at": {"type": "date"}, + "tsv_row": {"type": "integer"}, + "error": {"type": "object"}, + "status": {"type": "keyword"}, + "seq_error": { "type": "object", "properties": { - "analysisId": {"type": "keyword"}, - "studyId": {"type": "keyword"}, - "analysisType": {"type": "keyword"}, - "analysisState": {"type": "keyword"}, - "createdAt": {"type": "date"}, - "updatedAt": {"type": "date"}, - "submittedAt": {"type": "date"}, - "publishedAt": {"type": "date"}, - "workflow": { - "type": "object", - "properties": { - "name": {"type": "keyword"}, - "version": {"type": "keyword"} - } - } + "row": {"type": "integer"}, + "seq_error": {"type": "text"} } }, - "file": { - "type": "object", - "properties": { - "objectId": {"type": "keyword"}, - "fileName": {"type": "text"}, - "fileType": {"type": "keyword"}, - "fileSize": {"type": "long"}, - "fileMd5sum": {"type": "keyword"}, - "fileAccess": {"type": "keyword"}, - "dataType": {"type": "keyword"}, - "info": { - "type": "object", - "enabled": true - } - } - }, - "study": { - "type": "object", - "properties": { - "studyId": {"type": "keyword"}, - "name": {"type": "text"}, - "description": {"type": "text"}, - "organization": {"type": "keyword"}, - "info": { - "type": "object", - "enabled": true - } - } - }, - "sample": { - "type": "object", - "properties": { - "sampleId": {"type": "keyword"}, - "sampleType": {"type": "keyword"}, - "specimen": { - "type": "object", - "properties": { - "specimenId": {"type": "keyword"}, - "specimenType": {"type": "keyword"}, - "tumourNormalDesignation": {"type": "keyword"} - } - }, - "donor": { - "type": "object", - "properties": { - "donorId": {"type": "keyword"}, - "submitterDonorId": {"type": "keyword"}, - "gender": {"type": "keyword"} - } - }, - "info": { - "type": "object", - "enabled": true - } - } - }, - "experiment": { + "isolate_data": { "type": "object", "properties": { - "platform": {"type": "keyword"}, - "experimentalStrategy": {"type": "keyword"}, - "libraryStrategy": {"type": "keyword"}, - "info": { - "type": "object", - "enabled": true - } + "study_id": {"type": "keyword"}, + "isolate_id": {"type": "keyword"}, + "specimen_collector_sample_id": {"type": "keyword"}, + "fasta_file_name": {"type": "keyword"}, + "fasta_header_name": {"type": "keyword"}, + "geo_loc_name_country": {"type": "keyword"}, + "geo_loc_name_state_province_territory": {"type": "keyword"}, + "geo_loc_name_city": {"type": "keyword"}, + "sample_collection_date": {"type": "date"}, + "specimen_source_material_category": {"type": "keyword"}, + "biospecimen": {"type": "keyword"}, + "other_biospecimen": {"type": "text"}, + "anatomical_part": {"type": "keyword"}, + "other_anatomical_part": {"type": "text"}, + "anatomical_material": {"type": "keyword"}, + "body_product": {"type": "keyword"}, + "other_body_product": {"type": "text"}, + "environmental_material": {"type": "keyword"}, + "environmental_site": {"type": "keyword"}, + "specimen_source_context": {"type": "text"}, + "sample_collected_by": {"type": "keyword"}, + "sample_receive_date": {"type": "date"}, + "sample_received_date": {"type": "date"}, + "purpose_of_sampling": {"type": "keyword"}, + "collection_device": {"type": "keyword"}, + "organism": {"type": "keyword"}, + "other_organism": {"type": "text"}, + "strain": {"type": "keyword"}, + "serotype": {"type": "keyword"}, + "serogroup": {"type": "keyword"}, + "K_type": {"type": "keyword"}, + "O_type": {"type": "keyword"}, + "wzi": {"type": "keyword"}, + "lineage_name": {"type": "keyword"}, + "lineage_clade_name": {"type": "keyword"}, + "other_lineage_clade_name": {"type": "text"}, + "genotype": {"type": "keyword"}, + "phenotype": {"type": "keyword"}, + "other_phenotype": {"type": "text"}, + "population": {"type": "keyword"}, + "host_scientific_name": {"type": "keyword"}, + "host_subject_id": {"type": "keyword"}, + "host_age": {"type": "keyword"}, + "host_age_unit": {"type": "keyword"}, + "host_age_bin": {"type": "keyword"}, + "host_sex": {"type": "keyword"}, + "host_gender": {"type": "keyword"}, + "other_host_sex": {"type": "text"}, + "other_host_gender": {"type": "text"}, + "subject_sex": {"type": "keyword"}, + "host_health_state": {"type": "keyword"}, + "host_disease": {"type": "keyword"}, + "other_host_disease": {"type": "text"}, + "host_disease_outcome": {"type": "keyword"}, + "host_health_outcome": {"type": "keyword"}, + "host_disease_stage": {"type": "keyword"}, + "subject_disease_outcome": {"type": "keyword"}, + "finding_by_cause": {"type": "keyword"}, + "infection_acquisition": {"type": "keyword"}, + "case_id": {"type": "keyword"}, + "case_identifier": {"type": "keyword"}, + "signs_and_symptoms": {"type": "keyword"}, + "complications": {"type": "keyword"}, + "symptom_onset_date": {"type": "date"}, + "antiviral_therapy": {"type": "text"}, + "vaccine_name": {"type": "keyword"}, + "vaccination_status": {"type": "keyword"}, + "vaccination_history": {"type": "text"}, + "host_vaccination_status": {"type": "keyword"}, + "date_of_last_vaccine_dose": {"type": "date"}, + "travel_history": {"type": "text"}, + "most_recent_departure_date": {"type": "date"}, + "infectious_agent": {"type": "keyword"}, + "second_host_scientific_name": {"type": "keyword"}, + "second_host_subject_id": {"type": "keyword"}, + "second_host_sex": {"type": "keyword"}, + "second_host_strain": {"type": "keyword"}, + "second_host_complex": {"type": "keyword"}, + "second_host_genotype": {"type": "keyword"}, + "second_host_phenotype": {"type": "keyword"}, + "insecticide": {"type": "keyword"}, + "insecticide_class": {"type": "keyword"}, + "attribute_vector": {"type": "keyword"}, + "breeding_habitat": {"type": "keyword"}, + "mosquito_density": {"type": "keyword"}, + "man_biting_rate": {"type": "keyword"}, + "depth": {"type": "keyword"}, + "altitude": {"type": "keyword"}, + "antimalarials": {"type": "keyword"}, + "other_antimalarials": {"type": "text"}, + "drug_resistance_type": {"type": "keyword"}, + "resistance_variant": {"type": "keyword"}, + "resistance_genes": {"type": "keyword"}, + "resistant_genes": {"type": "keyword"}, + "other_resistance_genes": {"type": "text"}, + "resistance_gene_symbol": {"type": "keyword"}, + "virulence_factor_genes": {"type": "keyword"}, + "other_virulence_factor_genes": {"type": "text"}, + "virulence_factor_gene": {"type": "keyword"}, + "plasmids_identified": {"type": "keyword"}, + "other_plasmids_identified": {"type": "text"}, + "sequenced_by": {"type": "keyword"}, + "sequencing_date": {"type": "date"}, + "sequencing_instrument": {"type": "keyword"}, + "sequencing_type": {"type": "keyword"}, + "sequencing_typing_method": {"type": "keyword"}, + "sequencing_assay_type": {"type": "keyword"}, + "purpose_of_sequencing": {"type": "keyword"}, + "sequencing_depth": {"type": "keyword"}, + "depth_of_coverage": {"type": "keyword"}, + "depth_of_coverage_value": {"type": "keyword"}, + "median_read_depth": {"type": "keyword"}, + "breadth_of_coverage_value": {"type": "keyword"}, + "assembly_method": {"type": "keyword"}, + "assembly_genome_size": {"type": "keyword"}, + "assembly_accession": {"type": "keyword"}, + "consensus_genome_length": {"type": "keyword"}, + "number_of_contigs": {"type": "keyword"}, + "n50": {"type": "keyword"}, + "%_gc": {"type": "keyword"}, + "reference_genome_accession": {"type": "keyword"}, + "bioinformatics_protocol": {"type": "text"}, + "specimen_processing": {"type": "keyword"}, + "biosample_accession": {"type": "keyword"}, + "sra_accession": {"type": "keyword"}, + "sequence_read_accession": {"type": "keyword"}, + "culture_collection": {"type": "keyword"}, + "study_site_id": {"type": "keyword"}, + "gene_name_1": {"type": "keyword"}, + "gene_symbol_1": {"type": "keyword"}, + "assay_target_name_1": {"type": "keyword"}, + "diagnostic_pcr_Ct_value_1": {"type": "keyword"}, + "diagnostic_pcr_protocol_1": {"type": "text"} } } } diff --git a/helm/elasticsearch/values.yaml b/helm/elasticsearch/values.yaml index f5885d6..717cb5a 100644 --- a/helm/elasticsearch/values.yaml +++ b/helm/elasticsearch/values.yaml @@ -34,7 +34,7 @@ elasticsearch: persistence: enabled: true - storageClass: "cinder-csi" + storageClass: "local-path" size: 10Gi accessMode: ReadWriteOnce From e75037b9700dc56f9320301cb504b5c2c1ddd11c Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Fri, 21 Nov 2025 11:17:40 +0200 Subject: [PATCH 38/48] Fix indentation --- helm/frontend/templates/deployment.yaml | 30 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/helm/frontend/templates/deployment.yaml b/helm/frontend/templates/deployment.yaml index f0d3806..d7d7564 100644 --- a/helm/frontend/templates/deployment.yaml +++ b/helm/frontend/templates/deployment.yaml @@ -24,9 +24,27 @@ spec: ports: - containerPort: 3000 name: http - readinessProbe: {{ toYaml .Values.readinessProbe | nindent 10 }} - livenessProbe: {{ toYaml .Values.livenessProbe | nindent 10 }} - resources: {{ toYaml .Values.resources | nindent 10 }} - nodeSelector: {{ toYaml .Values.nodeSelector | nindent 6 }} - tolerations: {{ toYaml .Values.tolerations | nindent 6 }} - affinity: {{ toYaml .Values.affinity | nindent 6 }} \ No newline at end of file + {{- if .Values.readinessProbe }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- end }} + {{- if .Values.livenessProbe }} + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: + {{- toYaml .Values.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: + {{- toYaml .Values.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} \ No newline at end of file From fd37e9dcb49df726f0fc06d3ff48b24885828475 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 24 Nov 2025 15:51:51 +0200 Subject: [PATCH 39/48] Add some Folio env config --- helm/folio/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index 2db99ab..cd1a480 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: ghcr.io/openupsa/agari-folio pullPolicy: "Always" - tag: "staging" + tag: "staging" #main/staging service: type: ClusterIP @@ -14,9 +14,9 @@ service: folio: auth: keycloak: - host: "http://keycloak:8080" + host: "https://keycloak-staging.openup.org.za" #ilifu/staging realm: "agari" - issuer: "http://keycloak.local/realms/agari" + issuer: "http://keycloak:8080/realms/agari" clientId: "dms" clientSecret: "dms-secret" database: @@ -48,7 +48,7 @@ ingress: nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" nginx.ingress.kubernetes.io/proxy-body-size: "100m" hosts: - - host: folio-ilifu.openup.org.za + - host: folio-staging.openup.org.za #ilifu/staging paths: - path: / pathType: Prefix From 98a30816895d0f7b391193941ee518d132d06ab3 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 24 Nov 2025 15:53:01 +0200 Subject: [PATCH 40/48] Frontend apply certificate --- helm/frontend/templates/ingress.yaml | 12 +++++++----- helm/frontend/values.yaml | 12 ++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/helm/frontend/templates/ingress.yaml b/helm/frontend/templates/ingress.yaml index bbc66b3..7c73397 100644 --- a/helm/frontend/templates/ingress.yaml +++ b/helm/frontend/templates/ingress.yaml @@ -5,6 +5,11 @@ metadata: name: {{ include "agari-frontend.fullname" . }} annotations: {{ toYaml .Values.ingress.annotations | nindent 4 }} spec: + ingressClassName: {{ .Values.ingress.className }} + {{- if .Values.ingress.tls }} + tls: + {{- toYaml .Values.ingress.tls | nindent 4 }} + {{- end }} rules: {{- $root := . }} {{- range .Values.ingress.hosts }} @@ -19,9 +24,6 @@ spec: name: {{ include "agari-frontend.fullname" $root }} port: number: {{ $.Values.service.port }} - {{- end }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: {{ toYaml .Values.ingress.tls | nindent 2 }} - {{- end }} + {{- end }} + {{- end }} {{- end -}} \ No newline at end of file diff --git a/helm/frontend/values.yaml b/helm/frontend/values.yaml index 6503109..caff02b 100644 --- a/helm/frontend/values.yaml +++ b/helm/frontend/values.yaml @@ -2,7 +2,7 @@ replicaCount: 1 image: repository: ghcr.io/methodlab/agari-frontend - tag: v0.0.24 + tag: v0.0.26 pullPolicy: Always service: @@ -13,13 +13,17 @@ service: ingress: enabled: true className: private - annotations: {} + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" # probably leave this as prod hosts: - - host: agari.openup.org.za + - host: agari-staging.openup.org.za # ilifu/staging paths: - path: / pathType: ImplementationSpecific - tls: [] + tls: + - secretName: agari-staging-tls-cert # try ilifu/staging + hosts: + - agari-staging.openup.org.za # try ilifu/staging resources: limits: From 2783293856af13e8209d1409631fad7692be18c7 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 25 Nov 2025 11:55:29 +0200 Subject: [PATCH 41/48] Add SSL certs --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 3c02926..1ae7748 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,13 @@ echo "127.0.0.1 song.local 127.0.0.1 folio.local" | sudo tee -a /etc/hosts ``` +## Set SSL cert secret + +``` +kubectl create secret tls folio-prod-tls-cert --cert=/path/to/tls.crt --key=/path/to/tls.key -n agari-prod +``` + + ## Service Access Services are available at these URLs: From 12cf16344a2f249b50d4d1825ff25fae99c13bb7 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 25 Nov 2025 11:56:50 +0200 Subject: [PATCH 42/48] Remove unused charts --- helm/arranger/Chart.yaml | 6 - helm/arranger/configs/base.json | 4 - helm/arranger/configs/extended.json | 324 ------------ helm/arranger/configs/facets.json | 56 --- helm/arranger/configs/matchbox.json | 5 - helm/arranger/configs/table.json | 600 ----------------------- helm/arranger/templates/deployment.yaml | 85 ---- helm/arranger/templates/ingress.yaml | 32 -- helm/arranger/templates/service.yaml | 17 - helm/arranger/values.yaml | 95 ---- helm/databases/Chart.yaml | 6 - helm/databases/templates/deployment.yaml | 122 ----- helm/databases/templates/pvc.yaml | 22 - helm/databases/templates/secret.yaml | 12 - helm/databases/templates/service.yaml | 17 - helm/databases/values.yaml | 84 ---- helm/kafka/Chart.lock | 6 - helm/kafka/Chart.yaml | 11 - helm/kafka/templates/_helpers.tpl | 51 -- helm/kafka/templates/kafka.yaml | 56 --- helm/kafka/templates/service.yaml | 18 - helm/kafka/values-bitnami.yaml.backup | 38 -- helm/kafka/values.yaml | 58 --- helm/maestro/Chart.yaml | 6 - helm/maestro/templates/deployment.yaml | 136 ----- helm/maestro/templates/ingress.yaml | 39 -- helm/maestro/templates/service.yaml | 17 - helm/maestro/values.yaml | 132 ----- helm/score/Chart.yaml | 6 - helm/score/templates/deployment.yaml | 150 ------ helm/score/templates/ingress.yaml | 39 -- helm/score/templates/service.yaml | 17 - helm/score/values.yaml | 121 ----- helm/song-db/Chart.yaml | 6 - helm/song-db/templates/deployment.yaml | 122 ----- helm/song-db/templates/pvc.yaml | 22 - helm/song-db/templates/secret.yaml | 12 - helm/song-db/templates/service.yaml | 17 - helm/song-db/values.yaml | 84 ---- helm/song/Chart.yaml | 6 - helm/song/templates/configmap.yaml | 0 helm/song/templates/deployment.yaml | 162 ------ helm/song/templates/ingress.yaml | 49 -- helm/song/templates/service.yaml | 17 - helm/song/values.yaml | 154 ------ 45 files changed, 3039 deletions(-) delete mode 100644 helm/arranger/Chart.yaml delete mode 100644 helm/arranger/configs/base.json delete mode 100644 helm/arranger/configs/extended.json delete mode 100644 helm/arranger/configs/facets.json delete mode 100644 helm/arranger/configs/matchbox.json delete mode 100644 helm/arranger/configs/table.json delete mode 100644 helm/arranger/templates/deployment.yaml delete mode 100644 helm/arranger/templates/ingress.yaml delete mode 100644 helm/arranger/templates/service.yaml delete mode 100644 helm/arranger/values.yaml delete mode 100644 helm/databases/Chart.yaml delete mode 100644 helm/databases/templates/deployment.yaml delete mode 100644 helm/databases/templates/pvc.yaml delete mode 100644 helm/databases/templates/secret.yaml delete mode 100644 helm/databases/templates/service.yaml delete mode 100644 helm/databases/values.yaml delete mode 100644 helm/kafka/Chart.lock delete mode 100644 helm/kafka/Chart.yaml delete mode 100644 helm/kafka/templates/_helpers.tpl delete mode 100644 helm/kafka/templates/kafka.yaml delete mode 100644 helm/kafka/templates/service.yaml delete mode 100644 helm/kafka/values-bitnami.yaml.backup delete mode 100644 helm/kafka/values.yaml delete mode 100644 helm/maestro/Chart.yaml delete mode 100644 helm/maestro/templates/deployment.yaml delete mode 100644 helm/maestro/templates/ingress.yaml delete mode 100644 helm/maestro/templates/service.yaml delete mode 100644 helm/maestro/values.yaml delete mode 100644 helm/score/Chart.yaml delete mode 100644 helm/score/templates/deployment.yaml delete mode 100644 helm/score/templates/ingress.yaml delete mode 100644 helm/score/templates/service.yaml delete mode 100644 helm/score/values.yaml delete mode 100644 helm/song-db/Chart.yaml delete mode 100644 helm/song-db/templates/deployment.yaml delete mode 100644 helm/song-db/templates/pvc.yaml delete mode 100644 helm/song-db/templates/secret.yaml delete mode 100644 helm/song-db/templates/service.yaml delete mode 100644 helm/song-db/values.yaml delete mode 100644 helm/song/Chart.yaml delete mode 100644 helm/song/templates/configmap.yaml delete mode 100644 helm/song/templates/deployment.yaml delete mode 100644 helm/song/templates/ingress.yaml delete mode 100644 helm/song/templates/service.yaml delete mode 100644 helm/song/values.yaml diff --git a/helm/arranger/Chart.yaml b/helm/arranger/Chart.yaml deleted file mode 100644 index 1f706ff..0000000 --- a/helm/arranger/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: arranger -description: A Helm chart for Arranger - Data portal and search interface for genomic data -type: application -version: 0.1.0 -appVersion: "3.0.0" diff --git a/helm/arranger/configs/base.json b/helm/arranger/configs/base.json deleted file mode 100644 index 6894e37..0000000 --- a/helm/arranger/configs/base.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "documentType": "file", - "index": "agari-index" -} \ No newline at end of file diff --git a/helm/arranger/configs/extended.json b/helm/arranger/configs/extended.json deleted file mode 100644 index 9dd88fe..0000000 --- a/helm/arranger/configs/extended.json +++ /dev/null @@ -1,324 +0,0 @@ -{ - "extended": [ - { - "displayName": "Object ID", - "fieldName": "object_id" - }, - { - "displayName": "Study", - "fieldName": "study_id" - }, - { - "displayName": "Data Type", - "fieldName": "data_type" - }, - { - "displayName": "Format", - "fieldName": "file_type" - }, - { - "displayName": "Access", - "fieldName": "file_access" - }, - { - "displayName": "Analysis Id", - "fieldName": "analysis.analysis_id" - }, - { - "displayName": "Analysis Type", - "fieldName": "analysis.analysis_type" - }, - { - "displayName": "Analysis Version", - "fieldName": "analysis.analysis_version" - }, - { - "displayName": "Analysis State", - "fieldName": "analysis.analysis_state" - }, - { - "displayName": "Last Updated", - "fieldName": "analysis.updated_at" - }, - { - "displayName": "First Published On", - "fieldName": "analysis.first_published_at" - }, - { - "displayName": "Last Published On", - "fieldName": "analysis.published_at" - }, - { - "displayName": "Strategy", - "fieldName": "analysis.experiment.experimentalStrategy" - }, - { - "displayName": "Model", - "fieldName": "analysis.experiment.model" - }, - { - "displayName": "Platform", - "fieldName": "analysis.experiment.platform" - }, - { - "displayName": "Sequencing Center", - "fieldName": "analysis.experiment.sequencingCenter" - }, - { - "displayName": "Sequencing Date", - "fieldName": "analysis.experiment.sequencingDate" - }, - { - "displayName": "Contact Email", - "fieldName": "analysis.collaborator.contactEmail" - }, - { - "displayName": "Contributor", - "fieldName": "analysis.collaborator.name" - }, - { - "displayName": "Created At", - "fieldName": "analysis.createdAt" - }, - { - "displayName": "Cause of Death", - "fieldName": "analysis.donor.causeOfDeath" - }, - { - "displayName": "Age At Diagnosis", - "fieldName": "analysis.donor.primaryDiagnosis.ageAtDiagnosis" - }, - { - "displayName": "Cancer Type Code", - "fieldName": "analysis.donor.primaryDiagnosis.cancerTypeCode" - }, - { - "displayName": "Stage", - "fieldName": "analysis.donor.primaryDiagnosis.clinicalStageGroup" - }, - { - "displayName": "Tumour Staging System", - "fieldName": "analysis.donor.primaryDiagnosis.clinicalTumourStagingSystem" - }, - { - "displayName": "Status At Follow Up", - "fieldName": "analysis.donor.primaryDiagnosis.followUp.diseaseStatusAtFollowUp" - }, - { - "displayName": "Interval Of Follow Up (Days)", - "fieldName": "analysis.donor.primaryDiagnosis.followUp.intervalOfFollowUp" - }, - { - "displayName": "Follow Up Id", - "fieldName": "analysis.donor.primaryDiagnosis.followUp.submitterFollowUpId" - }, - { - "displayName": "Follow-Up Treatment Id", - "fieldName": "analysis.donor.primaryDiagnosis.followUp.submitterTreatmentId" - }, - { - "displayName": "Donor Primary Diagnosis Id", - "fieldName": "analysis.donor.primaryDiagnosis.submitterPrimaryDiagnosisId" - }, - { - "displayName": "Chemotherapy Drug", - "fieldName": "analysis.donor.primaryDiagnosis.treatment.chemotherapy.drugName" - }, - { - "displayName": "Response to Treatment", - "fieldName": "analysis.donor.primaryDiagnosis.treatment.responseToTreatment" - }, - { - "displayName": "Treatment Id", - "fieldName": "analysis.donor.primaryDiagnosis.treatment.submitterTreatmentId" - }, - { - "displayName": "Treatment Duration (Days)", - "fieldName": "analysis.donor.primaryDiagnosis.treatment.treatmentDuration" - }, - { - "displayName": "Treatment Start Date", - "fieldName": "analysis.donor.primaryDiagnosis.treatment.treatmentStartInterval" - }, - { - "displayName": "Primary Site", - "fieldName": "analysis.donor.primarySite" - }, - { - "displayName": "Donor Id", - "fieldName": "analysis.donor.submitterDonorId" - }, - { - "displayName": "Survival Time (Days)", - "fieldName": "analysis.donor.survivalTime" - }, - { - "displayName": "Vital Status", - "fieldName": "analysis.donor.vitalStatus" - }, - { - "displayName": "DOI", - "fieldName": "analysis.publication.doi" - }, - { - "displayName": "Publication Status", - "fieldName": "analysis.publication.publication" - }, - { - "displayName": "Location", - "fieldName": "analysis.specimen.specimenAnatomicLocation" - }, - { - "displayName": "Specimen Primary Diagnosis Id", - "fieldName": "analysis.specimen.submitterPrimaryDiagnosisId" - }, - { - "displayName": "Specimen Id", - "fieldName": "analysis.specimen.submitterSpecimenId" - }, - { - "displayName": "Tumour Grade", - "fieldName": "analysis.specimen.tumourGrade" - }, - { - "displayName": "Grading System", - "fieldName": "analysis.specimen.tumourGradingSystem" - }, - { - "displayName": "Genome Build", - "fieldName": "analysis.workflow.genomeBuild" - }, - { - "displayName": "Workflow Analysis Type", - "fieldName": "analysis.workflow.inputs.analysisType" - }, - { - "displayName": "Workflow Normal Analysis Id", - "fieldName": "analysis.workflow.inputs.normalAnalysisId" - }, - { - "displayName": "Workflow Tumour Analysis Id", - "fieldName": "analysis.workflow.inputs.tumourAnalysisId" - }, - { - "displayName": "Workflow Run Id", - "fieldName": "analysis.workflow.runId" - }, - { - "displayName": "Workflow Session Id", - "fieldName": "analysis.workflow.sessionId" - }, - { - "displayName": "Workflow", - "fieldName": "analysis.workflow.workflowName" - }, - { - "displayName": "Workflow Short Name", - "fieldName": "analysis.workflow.workflowShortName" - }, - { - "displayName": "Workflow Version", - "fieldName": "analysis.workflow.workflowVersion" - }, - { - "displayName": "File Name", - "fieldName": "file.name" - }, - { - "displayName": "Data Type", - "fieldName": "file.data_type" - }, - { - "displayName": "Md5sum", - "fieldName": "file.md5sum" - }, - { - "displayName": "Size", - "fieldName": "file.size" - }, - { - "displayName": "Object Id", - "fieldName": "file.index_file.object_id" - }, - { - "displayName": "Index File Name", - "fieldName": "file.index_file.name" - }, - { - "displayName": "Index File Type", - "fieldName": "file.index_file.file_type" - }, - { - "displayName": "Index File Md5sum", - "fieldName": "file.index_file.md5sum" - }, - { - "displayName": "Index Data Type", - "fieldName": "file.index_file.data_type" - }, - { - "displayName": "Index Data Size", - "fieldName": "file.index_file.size" - }, - { - "displayName": "Song Donor Id", - "fieldName": "donors.donor_id" - }, - { - "displayName": "Donor Id", - "fieldName": "donors.submitter_donor_id" - }, - { - "displayName": "Reported Gender", - "fieldName": "donors.gender" - }, - { - "displayName": "Speciment Id", - "fieldName": "donors.specimens.specimen_id" - }, - { - "displayName": "Specimen Type", - "fieldName": "donors.specimens.specimen_type" - }, - { - "displayName": "Submitter Specimen Id", - "fieldName": "donors.specimens.submitter_specimen_id" - }, - { - "displayName": "Sample Id", - "fieldName": "donors.specimens.samples.sample_id" - }, - { - "displayName": "Submitter Sample Id", - "fieldName": "donors.specimens.samples.submitter_sample_id" - }, - { - "displayName": "Sample Type", - "fieldName": "donors.specimens.samples.sample_type" - }, - { - "displayName": "Matched Normal Id", - "fieldName": "donors.specimens.samples.matched_normal_submitter_sample_id" - }, - { - "displayName": "Tumour Normal Designation", - "fieldName": "donors.specimens.tumour_normal_designation" - }, - { - "displayName": "Specimen Tissue Source", - "fieldName": "donors.specimens.specimen_tissue_source" - }, - { - "displayName": "Data Category", - "fieldName": "dataCategory" - }, - { - "displayName": "JBrowse Coordinates", - "fieldName": "jbrowseCoordinates" - }, - { - "displayName": "Repository Code", - "fieldName": "repositories.code" - } - ] - } \ No newline at end of file diff --git a/helm/arranger/configs/facets.json b/helm/arranger/configs/facets.json deleted file mode 100644 index 6230fa5..0000000 --- a/helm/arranger/configs/facets.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "facets": { - "aggregations": [ - { - "active": true, - "fieldName": "analysis__collaborator__name", - "show": true - }, - { - "active": true, - "fieldName": "file_access", - "show": true - }, - { - "active": true, - "fieldName": "analysis__donor__primarySite", - "show": true - }, - { - "active": true, - "fieldName": "donors__gender", - "show": true - }, - { - "active": true, - "fieldName": "analysis__donor__primaryDiagnosis__ageAtDiagnosis", - "show": true - }, - { - "active": true, - "fieldName": "donors__specimens__specimen_tissue_source", - "show": true - }, - { - "active": true, - "fieldName": "file_type", - "show": true - }, - { - "active": true, - "fieldName": "data_type", - "show": true - }, - { - "active": true, - "fieldName": "analysis__workflow__workflowName", - "show": true - }, - { - "active": true, - "fieldName": "analysis__publication__publication", - "show": true - } - ] - } -} \ No newline at end of file diff --git a/helm/arranger/configs/matchbox.json b/helm/arranger/configs/matchbox.json deleted file mode 100644 index bba293b..0000000 --- a/helm/arranger/configs/matchbox.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "matchbox": [ - - ] - } \ No newline at end of file diff --git a/helm/arranger/configs/table.json b/helm/arranger/configs/table.json deleted file mode 100644 index c0f51a7..0000000 --- a/helm/arranger/configs/table.json +++ /dev/null @@ -1,600 +0,0 @@ -{ - "table": { - "columns": [ - { - "canChangeShow": true, - "fieldName": "donors.submitter_donor_id", - "jsonPath": "$.donors.hits.edges[*].node.submitter_donor_id", - "query": "donors { hits { edges { node { submitter_donor_id } } } }", - "show": true, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.collaborator.name", - "jsonPath": "$.analysis.collaborator.hits.edges[*].node.name", - "query": "analysis { collaborator { hits { edges { node { name } } } } }", - "show": true, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file_access", - "show": true, - "sortable": true - }, - - { - "canChangeShow": true, - "fieldName": "data_type", - "show": true, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file_type", - "show": true, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.experiment.experimentalStrategy", - "show": true, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.experiment.platform", - "show": true, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.analysis_id", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "object_id", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.analysis_type", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.analysis_version", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.analysis_state", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.updated_at", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "study_id", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.first_published_at", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.published_at", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.experiment.model", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.experiment.sequencingCenter", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.experiment.sequencingDate", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.collaborator.contactEmail", - "jsonPath": "$.analysis.collaborator.hits.edges[*].node.contactEmail", - "query": "analysis { collaborator { hits { edges { node { contactEmail } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.createdAt", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.causeOfDeath", - "jsonPath": "$.analysis.donor.causeOfDeath", - "query": "analysis { donor { causeOfDeath } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.ageAtDiagnosis", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.ageAtDiagnosis", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { ageAtDiagnosis } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.cancerTypeCode", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.cancerTypeCode", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { cancerTypeCode } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.clinicalStageGroup", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.clinicalStageGroup", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { clinicalStageGroup } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.clinicalTumourStagingSystem", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.clinicalTumourStagingSystem", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { clinicalTumourStagingSystem } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.followUp.diseaseStatusAtFollowUp", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.followUp.hits.edges[*].node.diseaseStatusAtFollowUp", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { followUp { hits { edges { node { diseaseStatusAtFollowUp } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.followUp.intervalOfFollowUp", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.followUp.hits.edges[*].node.intervalOfFollowUp", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { followUp { hits { edges { node { intervalOfFollowUp } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.followUp.submitterFollowUpId", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.followUp.hits.edges[*].node.submitterFollowUpId", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { followUp { hits { edges { node { submitterFollowUpId } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.followUp.submitterTreatmentId", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.followUp.hits.edges[*].node.submitterTreatmentId", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { followUp { hits { edges { node { submitterTreatmentId } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.submitterPrimaryDiagnosisId", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.submitterPrimaryDiagnosisId", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { submitterPrimaryDiagnosisId } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.treatment.chemotherapy.drugName", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.treatment.hits.edges[*].node.chemotherapy.hits.edges[*].node.drugName", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { treatment { hits { edges { node { chemotherapy { hits { edges { node { drugName } } } } } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.treatment.responseToTreatment", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.treatment.hits.edges[*].node.responseToTreatment", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { treatment { hits { edges { node { responseToTreatment } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.treatment.submitterTreatmentId", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.treatment.hits.edges[*].node.submitterTreatmentId", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { treatment { hits { edges { node { submitterTreatmentId } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.treatment.treatmentDuration", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.treatment.hits.edges[*].node.treatmentDuration", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { treatment { hits { edges { node { treatmentDuration } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primaryDiagnosis.treatment.treatmentStartInterval", - "jsonPath": "$.analysis.donor.primaryDiagnosis.hits.edges[*].node.treatment.hits.edges[*].node.treatmentStartInterval", - "query": "analysis { donor { primaryDiagnosis { hits { edges { node { treatment { hits { edges { node { treatmentStartInterval } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.primarySite", - "jsonPath": "$.analysis.donor.primarySite", - "query": "analysis { donor { primarySite } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.submitterDonorId", - "jsonPath": "$.analysis.donor.submitterDonorId", - "query": "analysis { donor { submitterDonorId } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.survivalTime", - "jsonPath": "$.analysis.donor.survivalTime", - "query": "analysis { donor { survivalTime } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.donor.vitalStatus", - "jsonPath": "$.analysis.donor.vitalStatus", - "query": "analysis { donor { vitalStatus } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.publication.doi", - "jsonPath": "$.analysis.publication.doi", - "query": "analysis { publication { doi } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.publication.publication", - "jsonPath": "$.analysis.publication.publication", - "query": "analysis { publication { publication } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.specimen.specimenAnatomicLocation", - "jsonPath": "$.analysis.specimen.specimenAnatomicLocation", - "query": "analysis { specimen { specimenAnatomicLocation } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.specimen.submitterPrimaryDiagnosisId", - "jsonPath": "$.analysis.specimen.submitterPrimaryDiagnosisId", - "query": "analysis { specimen { submitterPrimaryDiagnosisId } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.specimen.submitterSpecimenId", - "jsonPath": "$.analysis.specimen.submitterSpecimenId", - "query": "analysis { specimen { submitterSpecimenId } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.specimen.tumourGrade", - "jsonPath": "$.analysis.specimen.tumourGrade", - "query": "analysis { specimen { tumourGrade } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.specimen.tumourGradingSystem", - "jsonPath": "$.analysis.specimen.tumourGradingSystem", - "query": "analysis { specimen { tumourGradingSystem } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.genomeBuild", - "jsonPath": "$.analysis.workflow.genomeBuild", - "query": "analysis { workflow { genomeBuild } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.inputs.analysisType", - "jsonPath": "$.analysis.workflow.inputs.hits.edges[*].node.analysisType", - "query": "analysis { workflow { inputs { hits { edges { node { analysisType } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.inputs.normalAnalysisId", - "jsonPath": "$.analysis.workflow.inputs.hits.edges[*].node.normalAnalysisId", - "query": "analysis { workflow { inputs { hits { edges { node { normalAnalysisId } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.inputs.tumourAnalysisId", - "jsonPath": "$.analysis.workflow.inputs.hits.edges[*].node.tumourAnalysisId", - "query": "analysis { workflow { inputs { hits { edges { node { tumourAnalysisId } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.runId", - "jsonPath": "$.analysis.workflow.runId", - "query": "analysis { workflow { runId } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.sessionId", - "jsonPath": "$.analysis.workflow.sessionId", - "query": "analysis { workflow { sessionId } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.workflowName", - "jsonPath": "$.analysis.workflow.workflowName", - "query": "analysis { workflow { workflowName } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.workflowShortName", - "jsonPath": "$.analysis.workflow.workflowShortName", - "query": "analysis { workflow { workflowShortName } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "analysis.workflow.workflowVersion", - "jsonPath": "$.analysis.workflow.workflowVersion", - "query": "analysis { workflow { workflowVersion } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.name", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.data_type", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.md5sum", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "displayType": "bytes", - "fieldName": "file.size", - "show": true, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.index_file.object_id", - "jsonPath": "$.file.index_file.object_id", - "query": "file { index_file { object_id } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.index_file.name", - "jsonPath": "$.file.index_file.name", - "query": "file { index_file { name } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.index_file.file_type", - "jsonPath": "$.file.index_file.file_type", - "query": "file { index_file { file_type } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.index_file.md5sum", - "jsonPath": "$.file.index_file.md5sum", - "query": "file { index_file { md5sum } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.index_file.data_type", - "jsonPath": "$.file.index_file.data_type", - "query": "file { index_file { data_type } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "file.index_file.size", - "jsonPath": "$.file.index_file.size", - "query": "file { index_file { size } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.donor_id", - "jsonPath": "$.donors.hits.edges[*].node.donor_id", - "query": "donors { hits { edges { node { donor_id } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.gender", - "jsonPath": "$.donors.hits.edges[*].node.gender", - "query": "donors { hits { edges { node { gender } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.specimen_id", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.specimen_id", - "query": "donors { hits { edges { node { specimens { hits { edges { node { specimen_id } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.specimen_type", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.specimen_type", - "query": "donors { hits { edges { node { specimens { hits { edges { node { specimen_type } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.submitter_specimen_id", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.submitter_specimen_id", - "query": "donors { hits { edges { node { specimens { hits { edges { node { submitter_specimen_id } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.samples.sample_id", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.samples.hits.edges[*].node.sample_id", - "query": "donors { hits { edges { node { specimens { hits { edges { node { samples { hits { edges { node { sample_id } } } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.samples.submitter_sample_id", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.samples.hits.edges[*].node.submitter_sample_id", - "query": "donors { hits { edges { node { specimens { hits { edges { node { samples { hits { edges { node { submitter_sample_id } } } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.samples.sample_type", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.samples.hits.edges[*].node.sample_type", - "query": "donors { hits { edges { node { specimens { hits { edges { node { samples { hits { edges { node { sample_type } } } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.samples.matched_normal_submitter_sample_id", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.samples.hits.edges[*].node.matched_normal_submitter_sample_id", - "query": "donors { hits { edges { node { specimens { hits { edges { node { samples { hits { edges { node { matched_normal_submitter_sample_id } } } } } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.tumour_normal_designation", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.tumour_normal_designation", - "query": "donors { hits { edges { node { specimens { hits { edges { node { tumour_normal_designation } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "donors.specimens.specimen_tissue_source", - "jsonPath": "$.donors.hits.edges[*].node.specimens.hits.edges[*].node.specimen_tissue_source", - "query": "donors { hits { edges { node { specimens { hits { edges { node { specimen_tissue_source } } } } } } } }", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "dataCategory", - "show": false, - "sortable": true - }, - { - "canChangeShow": true, - "fieldName": "jbrowseCoordinates", - "show": false, - "sortable": true - }, - { - "canChangeShow": false, - "fieldName": "repositories.code", - "jsonPath": "$.repositories.hits.edges[*].node.code", - "query": "repositories { hits { edges { node { code } } } }", - "show": false, - "sortable": true - } - ] - } -} \ No newline at end of file diff --git a/helm/arranger/templates/deployment.yaml b/helm/arranger/templates/deployment.yaml deleted file mode 100644 index ec8b2dd..0000000 --- a/helm/arranger/templates/deployment.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.fullnameOverride | default "arranger" }} - labels: - app.kubernetes.io/name: arranger - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app.kubernetes.io/name: arranger - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: arranger - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - containers: - - name: arranger - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 5050 - protocol: TCP - env: - # Arranger Variables - - name: ENABLE_LOGS - value: "{{ .Values.arranger.enableLogs }}" - # Elasticsearch Variables - - name: ES_HOST - value: "{{ .Values.arranger.elasticsearch.host }}" - - name: ES_USER - value: "{{ .Values.arranger.elasticsearch.user }}" - - name: ES_PASS - value: "{{ .Values.arranger.elasticsearch.password }}" - # Stage Variables - - name: REACT_APP_BASE_URL - value: "{{ .Values.arranger.stage.baseUrl }}" - - name: REACT_APP_ARRANGER_ADMIN_ROOT - value: "{{ .Values.arranger.stage.adminRoot }}" - {{- if .Values.probes.readiness.enabled }} - readinessProbe: - httpGet: - path: {{ .Values.probes.readiness.path | default "/" }} - port: http - initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} - failureThreshold: {{ .Values.probes.readiness.failureThreshold }} - {{- end }} - {{- if .Values.probes.liveness.enabled }} - livenessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} - failureThreshold: {{ .Values.probes.liveness.failureThreshold }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: arranger-configs - mountPath: /app/modules/server/configs - readOnly: true - volumes: - - name: arranger-configs - configMap: - name: arranger-config - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/arranger/templates/ingress.yaml b/helm/arranger/templates/ingress.yaml deleted file mode 100644 index fdc6a74..0000000 --- a/helm/arranger/templates/ingress.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.ingress.enabled -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ .Values.fullnameOverride | default "arranger" }} - labels: - app.kubernetes.io/name: arranger - app.kubernetes.io/instance: {{ .Release.Name }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.className }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - pathType: {{ .pathType }} - backend: - service: - name: {{ $.Values.fullnameOverride | default "arranger" }} - port: - number: {{ $.Values.service.port }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/arranger/templates/service.yaml b/helm/arranger/templates/service.yaml deleted file mode 100644 index b21c925..0000000 --- a/helm/arranger/templates/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.fullnameOverride | default "arranger" }} - labels: - app.kubernetes.io/name: arranger - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: arranger - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/arranger/values.yaml b/helm/arranger/values.yaml deleted file mode 100644 index e804b14..0000000 --- a/helm/arranger/values.yaml +++ /dev/null @@ -1,95 +0,0 @@ -replicaCount: 1 - -fullnameOverride: "arranger" - -url: - hostname: "arranger-ilifu.openup.org.za" - -image: - repository: ghcr.io/overture-stack/arranger-server - pullPolicy: IfNotPresent - tag: "3.0.0-beta.33" - -nameOverride: "" - -service: - type: ClusterIP - port: 5050 - -arranger: - enableLogs: false - - elasticsearch: - host: "http://elasticsearch:9200" - user: "" - password: "" - compatibilityMode: true - - stage: - baseUrl: "http://arranger-ilifu.openup.org.za:3000" - adminRoot: "http://arranger-server:5050/graphql" - -resources: - limits: - cpu: 500m - memory: 1Gi - ephemeral-storage: "4Gi" - requests: - cpu: 200m - memory: 512Mi - ephemeral-storage: "2Gi" - -ingress: - enabled: true - className: "private" - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "50m" - hosts: - - host: arranger-ilifu.openup.org.za - paths: - - path: / - pathType: Prefix - -probes: - liveness: - enabled: false - readiness: - enabled: true - path: "/graphql" - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - elasticsearch: - host: "http://elasticsearch:9200" - user: "elastic" - password: "myelasticpassword" - - stage: - baseUrl: "http://arranger-ilifu.openup.org.za:3000" - adminRoot: "http://arranger-server:5050/graphql" - -resources: - limits: - cpu: 200m - memory: 1Gi - ephemeral-storage: 2Gi - requests: - cpu: 100m - memory: 256Mi - ephemeral-storage: 1Gi - -ingress: - enabled: true - className: "private" - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "50m" - hosts: - - host: arranger-ilifu.openup.org.za - paths: - - path: / - pathType: Prefix - -nodeSelector: {} -tolerations: [] -affinity: {} diff --git a/helm/databases/Chart.yaml b/helm/databases/Chart.yaml deleted file mode 100644 index d801b45..0000000 --- a/helm/databases/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: postgres -description: A Helm chart for PostgreSQL database -type: application -version: 0.1.0 -appVersion: "14" diff --git a/helm/databases/templates/deployment.yaml b/helm/databases/templates/deployment.yaml deleted file mode 100644 index 545c596..0000000 --- a/helm/databases/templates/deployment.yaml +++ /dev/null @@ -1,122 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - replicas: {{ .Values.replicaCount }} - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: postgresql - containerPort: 5432 - protocol: TCP - env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-secret - key: postgres-user - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-secret - key: postgres-password - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-secret - key: postgres-database - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - {{- if .Values.probes.liveness.enabled }} - livenessProbe: - {{- if .Values.probes.liveness.exec }} - exec: - command: - {{- range .Values.probes.liveness.exec.command }} - - {{ . | quote }} - {{- end }} - {{- else if .Values.probes.liveness.tcpSocket }} - tcpSocket: - port: {{ .Values.probes.liveness.tcpSocket.port }} - {{- end }} - initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} - successThreshold: {{ .Values.probes.liveness.successThreshold | default 1 }} - failureThreshold: {{ .Values.probes.liveness.failureThreshold }} - {{- end }} - {{- if .Values.probes.readiness.enabled }} - readinessProbe: - {{- if .Values.probes.readiness.exec }} - exec: - command: - {{- range .Values.probes.readiness.exec.command }} - - {{ . | quote }} - {{- end }} - {{- else if .Values.probes.readiness.tcpSocket }} - tcpSocket: - port: {{ .Values.probes.readiness.tcpSocket.port }} - {{- end }} - initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} - successThreshold: {{ .Values.probes.readiness.successThreshold | default 1 }} - failureThreshold: {{ .Values.probes.readiness.failureThreshold }} - {{- end }} - successThreshold: 1 - failureThreshold: 6 - {{- if .Values.persistence.enabled }} - volumeMounts: - - name: postgresql-data - mountPath: /var/lib/postgresql/data - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- if .Values.persistence.enabled }} - volumes: - - name: postgresql-data - persistentVolumeClaim: - claimName: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-pvc - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/databases/templates/pvc.yaml b/helm/databases/templates/pvc.yaml deleted file mode 100644 index 5b9ed53..0000000 --- a/helm/databases/templates/pvc.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if .Values.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-pvc - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - accessModes: - - {{ .Values.persistence.accessMode | quote }} - resources: - requests: - storage: {{ .Values.persistence.size | quote }} - {{- if .Values.persistence.storageClass }} - {{- if (eq "-" .Values.persistence.storageClass) }} - storageClassName: "" - {{- else }} - storageClassName: "{{ .Values.persistence.storageClass }}" - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/databases/templates/secret.yaml b/helm/databases/templates/secret.yaml deleted file mode 100644 index 4f2284a..0000000 --- a/helm/databases/templates/secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-secret - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} -type: Opaque -data: - postgres-user: {{ .Values.postgresql.postgresqlUsername | b64enc | quote }} - postgres-password: {{ .Values.postgresql.postgresqlPassword | b64enc | quote }} - postgres-database: {{ .Values.postgresql.postgresqlDatabase | b64enc | quote }} diff --git a/helm/databases/templates/service.yaml b/helm/databases/templates/service.yaml deleted file mode 100644 index 3bb3e60..0000000 --- a/helm/databases/templates/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: postgresql - protocol: TCP - name: postgresql - selector: - app.kubernetes.io/name: postgres - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/databases/values.yaml b/helm/databases/values.yaml deleted file mode 100644 index 3950285..0000000 --- a/helm/databases/values.yaml +++ /dev/null @@ -1,84 +0,0 @@ -replicaCount: 1 - -image: - repository: postgres - pullPolicy: IfNotPresent - tag: "14" - -nameOverride: "" -fullnameOverride: "" - -podAnnotations: {} - -podSecurityContext: - fsGroup: 999 - -securityContext: - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - runAsNonRoot: false - runAsUser: 999 - -service: - type: ClusterIP - port: 5432 - -persistence: - enabled: true - # storageClass: "" - accessMode: ReadWriteOnce - size: 8Gi - # existingClaim: "" - -postgresql: - postgresqlUsername: admin - postgresqlPassword: admin123 - postgresqlDatabase: keycloakDb - -databaseKey: "" # e.g., "keycloak", "song" - -probes: - liveness: - enabled: true - exec: - command: - - /bin/sh - - -c - - exec pg_isready -U "$(POSTGRES_USER)" -d "$(POSTGRES_DB)" -h 127.0.0.1 -p 5432 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 6 - - readiness: - enabled: true - exec: - command: - - /bin/sh - - -c - - -e - - | - exec pg_isready -U "$(POSTGRES_USER)" -d "$(POSTGRES_DB)" -h 127.0.0.1 -p 5432 - [ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ] - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - -resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 250m - memory: 256Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/helm/kafka/Chart.lock b/helm/kafka/Chart.lock deleted file mode 100644 index b92ec6b..0000000 --- a/helm/kafka/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: strimzi-kafka-operator - repository: https://strimzi.io/charts/ - version: 0.39.0 -digest: sha256:609aad69bd2c1c1fb6e2d23fb158becb90021063e83e3cc8be15b263273204fe -generated: "2025-10-23T15:02:45.59200355+02:00" diff --git a/helm/kafka/Chart.yaml b/helm/kafka/Chart.yaml deleted file mode 100644 index 3dbae14..0000000 --- a/helm/kafka/Chart.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v2 -name: kafka -description: Apache Kafka cluster using Strimzi operator -type: application -version: 0.1.0 -appVersion: "3.6.0" - -dependencies: - - name: strimzi-kafka-operator - version: 0.39.0 - repository: https://strimzi.io/charts/ \ No newline at end of file diff --git a/helm/kafka/templates/_helpers.tpl b/helm/kafka/templates/_helpers.tpl deleted file mode 100644 index 31608ea..0000000 --- a/helm/kafka/templates/_helpers.tpl +++ /dev/null @@ -1,51 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "kafka.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "kafka.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "kafka.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "kafka.labels" -}} -helm.sh/chart: {{ include "kafka.chart" . }} -{{ include "kafka.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "kafka.selectorLabels" -}} -app.kubernetes.io/name: {{ include "kafka.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} \ No newline at end of file diff --git a/helm/kafka/templates/kafka.yaml b/helm/kafka/templates/kafka.yaml deleted file mode 100644 index eb2265b..0000000 --- a/helm/kafka/templates/kafka.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: kafka.strimzi.io/v1beta2 -kind: Kafka -metadata: - name: {{ include "kafka.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - {{- include "kafka.labels" . | nindent 4 }} -spec: - kafka: - version: 3.6.0 - replicas: {{ .Values.kafka.replicas }} - listeners: - - name: plain - port: 9092 - type: internal - tls: false - - name: tls - port: 9093 - type: internal - tls: true - config: - offsets.topic.replication.factor: {{ .Values.kafka.config.offsetsTopicReplicationFactor }} - transaction.state.log.replication.factor: {{ .Values.kafka.config.transactionStateLogReplicationFactor }} - transaction.state.log.min.isr: {{ .Values.kafka.config.transactionStateLogMinIsr }} - default.replication.factor: {{ .Values.kafka.config.defaultReplicationFactor }} - min.insync.replicas: {{ .Values.kafka.config.minInsyncReplicas }} - inter.broker.protocol.version: "3.6" - storage: - {{- if .Values.kafka.storage.persistent }} - type: persistent-claim - size: {{ .Values.kafka.storage.size }} - {{- if .Values.kafka.storage.storageClass }} - class: {{ .Values.kafka.storage.storageClass }} - {{- end }} - {{- else }} - type: ephemeral - {{- end }} - resources: - {{- toYaml .Values.kafka.resources | nindent 6 }} - zookeeper: - replicas: {{ .Values.zookeeper.replicas }} - storage: - {{- if .Values.zookeeper.storage.persistent }} - type: persistent-claim - size: {{ .Values.zookeeper.storage.size }} - {{- if .Values.zookeeper.storage.storageClass }} - class: {{ .Values.zookeeper.storage.storageClass }} - {{- end }} - {{- else }} - type: ephemeral - {{- end }} - resources: - {{- toYaml .Values.zookeeper.resources | nindent 6 }} - entityOperator: - topicOperator: {} - userOperator: {} \ No newline at end of file diff --git a/helm/kafka/templates/service.yaml b/helm/kafka/templates/service.yaml deleted file mode 100644 index e0ebcce..0000000 --- a/helm/kafka/templates/service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kafka - namespace: {{ .Release.Namespace }} - labels: - {{- include "kafka.labels" . | nindent 4 }} -spec: - type: ClusterIP - ports: - - port: 9092 - targetPort: 9092 - protocol: TCP - name: kafka - selector: - strimzi.io/cluster: {{ include "kafka.fullname" . }} - strimzi.io/kind: Kafka - strimzi.io/name: {{ include "kafka.fullname" . }}-kafka \ No newline at end of file diff --git a/helm/kafka/values-bitnami.yaml.backup b/helm/kafka/values-bitnami.yaml.backup deleted file mode 100644 index 7df466a..0000000 --- a/helm/kafka/values-bitnami.yaml.backup +++ /dev/null @@ -1,38 +0,0 @@ -# Bitnami Kafka configuration -# This file provides configuration for the official Bitnami Kafka chart - -auth: - clientProtocol: plaintext - interBrokerProtocol: plaintext - -# Disable persistence for development -persistence: - enabled: false - -zookeeper: - persistence: - enabled: false - -listeners: - client: - protocol: PLAINTEXT - interbroker: - protocol: PLAINTEXT - -# Resource limits -resources: - limits: - cpu: 1000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - -zookeeper: - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 200m - memory: 512Mi diff --git a/helm/kafka/values.yaml b/helm/kafka/values.yaml deleted file mode 100644 index 1059c2d..0000000 --- a/helm/kafka/values.yaml +++ /dev/null @@ -1,58 +0,0 @@ -# Strimzi Kafka configuration -# This replaces the Bitnami Kafka chart with Strimzi operator - -# Kafka cluster configuration -kafka: - replicas: 1 - - # Configuration for Kafka brokers - config: - # Replication factors for internal topics - offsetsTopicReplicationFactor: 1 - transactionStateLogReplicationFactor: 1 - transactionStateLogMinIsr: 1 - defaultReplicationFactor: 1 - minInsyncReplicas: 1 - - # Storage configuration - storage: - persistent: false # Set to false for development, true for production - size: 10Gi - # storageClass: "" # Uncomment and specify if needed - - # Resource limits - resources: - limits: - cpu: 1000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - -# Zookeeper configuration -zookeeper: - replicas: 1 - - # Storage configuration - storage: - persistent: false # Set to false for development, true for production - size: 5Gi - # storageClass: "" # Uncomment and specify if needed - - # Resource limits - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 200m - memory: 512Mi - -# Override default names if needed -nameOverride: "" -fullnameOverride: "kafka" - -# Strimzi operator will be installed as dependency -strimzi-kafka-operator: - # Enable the operator installation - enabled: true \ No newline at end of file diff --git a/helm/maestro/Chart.yaml b/helm/maestro/Chart.yaml deleted file mode 100644 index 40602dd..0000000 --- a/helm/maestro/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: maestro -description: A Helm chart for Maestro - Workflow orchestration and data indexing service -type: application -version: 0.1.0 -appVersion: "4.3.0" diff --git a/helm/maestro/templates/deployment.yaml b/helm/maestro/templates/deployment.yaml deleted file mode 100644 index 8e5c083..0000000 --- a/helm/maestro/templates/deployment.yaml +++ /dev/null @@ -1,136 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-maestro" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "maestro" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - replicas: {{ .Values.replicaCount }} - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "maestro" }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "maestro" }} - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - containers: - - name: maestro - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 11235 - protocol: TCP - env: - # Maestro Variables - - name: MAESTRO_FAILURELOG_ENABLED - value: {{ .Values.maestro.failureLog.enabled | quote }} - - name: MAESTRO_FAILURELOG_DIR - value: {{ .Values.maestro.failureLog.dir | quote }} - - name: MAESTRO_LOGGING_LEVEL_ROOT - value: {{ .Values.maestro.logging.level.root | quote }} - - name: MAESTRO_NOTIFICATIONS_SLACK_ENABLED - value: {{ .Values.maestro.notifications.slack.enabled | quote }} - - # Song Repository Variables - {{- range $i, $repo := .Values.maestro.repositories }} - - name: MAESTRO_REPOSITORIES_{{ $i }}_CODE - value: {{ $repo.code | quote }} - - name: MAESTRO_REPOSITORIES_{{ $i }}_URL - value: {{ $repo.url | quote }} - - name: MAESTRO_REPOSITORIES_{{ $i }}_NAME - value: {{ $repo.name | quote }} - - name: MAESTRO_REPOSITORIES_{{ $i }}_ORGANIZATION - value: {{ $repo.organization | quote }} - - name: MAESTRO_REPOSITORIES_{{ $i }}_COUNTRY - value: {{ $repo.country | quote }} - {{- end }} - - # Elasticsearch Variables - - name: MAESTRO_ELASTICSEARCH_CLUSTER_NODES - value: {{ .Values.maestro.elasticsearch.cluster.nodes | quote }} - - name: MAESTRO_ELASTICSEARCH_CLIENT_BASICAUTH_ENABLED - value: {{ .Values.maestro.elasticsearch.client.basicAuth.enabled | quote }} - {{- if .Values.maestro.elasticsearch.client.basicAuth.enabled }} - - name: MAESTRO_ELASTICSEARCH_CLIENT_BASICAUTH_USER - value: {{ .Values.maestro.elasticsearch.client.basicAuth.user | quote }} - - name: MAESTRO_ELASTICSEARCH_CLIENT_BASICAUTH_PASSWORD - value: {{ .Values.maestro.elasticsearch.client.basicAuth.password | quote }} - {{- end }} - - name: MAESTRO_ELASTICSEARCH_CLIENT_TRUSTSELFSIGNCERT - value: {{ .Values.maestro.elasticsearch.client.trustSelfSignCert | quote }} - - name: MAESTRO_ELASTICSEARCH_INDEXES_ANALYSISCENTRIC_ENABLED - value: {{ .Values.maestro.elasticsearch.indexes.analysisCentric.enabled | quote }} - - name: MAESTRO_ELASTICSEARCH_INDEXES_FILECENTRIC_ENABLED - value: {{ .Values.maestro.elasticsearch.indexes.fileCentric.enabled | quote }} - - name: MAESTRO_ELASTICSEARCH_INDEXES_FILECENTRIC_NAME - value: {{ .Values.maestro.elasticsearch.indexes.fileCentric.name | quote }} - - name: MAESTRO_ELASTICSEARCH_INDEXES_FILECENTRIC_ALIAS - value: {{ .Values.maestro.elasticsearch.indexes.fileCentric.alias | quote }} - - name: MANAGEMENT_HEALTH_ELASTICSEARCH_ENABLED - value: {{ .Values.maestro.management.health.elasticsearch.enabled | quote }} - - # Spring Variables - - name: SPRING_MVC_ASYNC_REQUESTTIMEOUT - value: {{ .Values.maestro.spring.mvc.async.requestTimeout | quote }} - - name: SPRINGDOC_SWAGGERUI_PATH - value: {{ .Values.maestro.springdoc.swaggerui.path | quote }} - - # Kafka Variables - - name: SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS - value: {{ .Values.maestro.spring.cloud.stream.kafka.binder.brokers | quote }} - - name: SPRING_CLOUD_STREAM_BINDINGS_SONGINPUT_DESTINATION - value: {{ .Values.maestro.spring.cloud.stream.bindings.songInput.destination | quote }} - - {{- if .Values.probes.liveness.enabled }} - livenessProbe: - {{- if .Values.probes.liveness.httpGet }} - httpGet: - path: {{ .Values.probes.liveness.httpGet.path }} - port: {{ .Values.probes.liveness.httpGet.port }} - {{- else if .Values.probes.liveness.tcpSocket }} - tcpSocket: - port: {{ .Values.probes.liveness.tcpSocket.port }} - {{- end }} - initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} - failureThreshold: {{ .Values.probes.liveness.failureThreshold }} - {{- end }} - {{- if .Values.probes.readiness.enabled }} - readinessProbe: - {{- if .Values.probes.readiness.httpGet }} - httpGet: - path: {{ .Values.probes.readiness.httpGet.path }} - port: {{ .Values.probes.readiness.httpGet.port }} - {{- else if .Values.probes.readiness.tcpSocket }} - tcpSocket: - port: {{ .Values.probes.readiness.tcpSocket.port }} - {{- end }} - initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} - failureThreshold: {{ .Values.probes.readiness.failureThreshold }} - {{- end }} - - resources: - {{- toYaml .Values.resources | nindent 12 }} - - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/maestro/templates/ingress.yaml b/helm/maestro/templates/ingress.yaml deleted file mode 100644 index b4067c2..0000000 --- a/helm/maestro/templates/ingress.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- if .Values.ingress.enabled -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-maestro" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "maestro" }} - app.kubernetes.io/instance: {{ .Release.Name }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $.Values.fullnameOverride | default (printf "%s-maestro" $.Release.Name) }} - port: - number: {{ $.Values.service.port }} - {{- else }} - serviceName: {{ $.Values.fullnameOverride | default (printf "%s-maestro" $.Release.Name) }} - servicePort: {{ $.Values.service.port }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/maestro/templates/service.yaml b/helm/maestro/templates/service.yaml deleted file mode 100644 index 48a475c..0000000 --- a/helm/maestro/templates/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-maestro" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "maestro" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: {{ .Values.nameOverride | default "maestro" }} - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/maestro/values.yaml b/helm/maestro/values.yaml deleted file mode 100644 index 5b270d8..0000000 --- a/helm/maestro/values.yaml +++ /dev/null @@ -1,132 +0,0 @@ -replicaCount: 1 - -fullnameOverride: "maestro" - -url: - hostname: "maestro-ilifu.openup.org.za" # ilifu/staging - -image: - repository: ghcr.io/overture-stack/maestro - pullPolicy: IfNotPresent - tag: "4.3.0" - -nameOverride: "" - -service: - type: ClusterIP - port: 11235 - -maestro: - failureLog: - enabled: true - dir: "app/logs/maestro" - - logging: - level: - root: "INFO" - - notifications: - slack: - enabled: false - - repositories: - - code: "song.overture" - url: "http://song:8080" - name: "Overture" - organization: "Overture" - country: "CA" - auth: - type: "oauth2" - clientId: "dms" - clientSecret: "dms-secret" - tokenUrl: "http://keycloak:8080/realms/agari/protocol/openid-connect/token" - scope: "song.READ" - - elasticsearch: - cluster: - nodes: "http://elasticsearch:9200" - client: - basicAuth: - enabled: false - user: "" - password: "" - trustSelfSignCert: true - indexes: - analysisCentric: - enabled: false - fileCentric: - enabled: true - name: "agari-index" - alias: "file_centric" - - spring: - mvc: - async: - requestTimeout: -1 - cloud: - stream: - kafka: - binder: - brokers: "kafka-kafka-bootstrap.agari-kafka.svc:9092" - bindings: - songInput: - destination: "song-analysis-prod" # prod/staging - - # Swagger Configuration - springdoc: - swaggerui: - path: "/swagger-api" - - # Management Configuration - management: - health: - elasticsearch: - enabled: false - -resources: - limits: - cpu: 500m - memory: 1.5Gi - ephemeral-storage: 2Gi - requests: - cpu: 100m - memory: 512Mi - ephemeral-storage: 1Gi - -probes: - liveness: - enabled: false - httpGet: - path: /actuator/health - port: http - initialDelaySeconds: 300 - periodSeconds: 60 - timeoutSeconds: 10 - failureThreshold: 5 - - readiness: - enabled: true - tcpSocket: - port: 11235 - initialDelaySeconds: 90 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - -ingress: - enabled: true - className: "private" - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "100m" - hosts: - - host: maestro-ilifu.openup.org.za # ilifu/staging - paths: - - path: / - pathType: Prefix - -persistence: - enabled: false - -nodeSelector: {} -tolerations: [] -affinity: {} diff --git a/helm/score/Chart.yaml b/helm/score/Chart.yaml deleted file mode 100644 index 5112d92..0000000 --- a/helm/score/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: score -description: A Helm chart for Score - Object storage API for genomic data -type: application -version: 0.1.0 -appVersion: "5.11.0" diff --git a/helm/score/templates/deployment.yaml b/helm/score/templates/deployment.yaml deleted file mode 100644 index 10b4b50..0000000 --- a/helm/score/templates/deployment.yaml +++ /dev/null @@ -1,150 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-score" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "score" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "score" }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "score" }} - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - containers: - - name: score - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 8087 - protocol: TCP - env: - # Spring Configuration - - name: SPRING_PROFILES_ACTIVE - value: {{ .Values.score.spring.profiles.active | quote }} - - name: SERVER_PORT - value: {{ .Values.score.server.port | quote }} - # Song Variables - - name: METADATA_URL - value: {{ .Values.score.metadata.url | quote }} - {{- if .Values.score.metadata.ssl }} - - name: METADATA_SSL_ENABLED - value: {{ .Values.score.metadata.ssl.enabled | quote }} - {{- end }} - {{- if .Values.score.clientCredentials }} - # Client Credentials for Score to authenticate to SONG - - name: SCORE_CLIENTCREDENTIALS_ID - value: {{ .Values.score.clientCredentials.id | quote }} - - name: SCORE_CLIENTCREDENTIALS_SECRET - value: {{ .Values.score.clientCredentials.secret | quote }} - - name: SCORE_CLIENTCREDENTIALS_TOKENURL - value: {{ .Values.score.clientCredentials.tokenUrl | quote }} - - name: SCORE_CLIENTCREDENTIALS_SYSTEMSCOPE - value: {{ .Values.score.clientCredentials.systemScope | quote }} - {{- end }} - # Server Variables - - name: SERVER_SSL_ENABLED - value: {{ .Values.score.server.ssl.enabled | quote }} - # Object Storage Variables - - name: S3_ENDPOINT - value: {{ .Values.score.s3.endpoint | quote }} - - name: S3_ACCESSKEY - value: {{ .Values.score.s3.accesskey | quote }} - - name: S3_SECRETKEY - value: {{ .Values.score.s3.secretkey | quote }} - - name: S3_SIGV4ENABLED - value: {{ .Values.score.s3.sigv4enabled | quote }} - - name: S3_SECURED - value: {{ .Values.score.s3.secured | quote }} - - name: OBJECT_SENTINEL - value: {{ .Values.score.object.sentinel | quote }} - - name: BUCKET_NAME_OBJECT - value: {{ .Values.score.bucket.name.object | quote }} - - name: BUCKET_NAME_STATE - value: {{ .Values.score.bucket.name.state | quote }} - - name: UPLOAD_PARTSIZE - value: {{ .Values.score.upload.partsize | quote }} - - name: UPLOAD_CONNECTION_TIMEOUT - value: {{ .Values.score.upload.connection.timeout | quote }} - # Keycloak Variables - - name: AUTH_SERVER_PROVIDER - value: {{ .Values.score.auth.server.provider | quote }} - - name: AUTH_SERVER_CLIENTID - value: {{ .Values.score.auth.server.clientId | quote }} - - name: AUTH_SERVER_CLIENTSECRET - value: {{ .Values.score.auth.server.clientSecret | quote }} - - name: AUTH_SERVER_TOKENNAME - value: {{ .Values.score.auth.server.tokenName | quote }} - - name: AUTH_SERVER_KEYCLOAK_HOST - value: {{ .Values.score.auth.server.keycloak.host | quote }} - - name: AUTH_SERVER_KEYCLOAK_REALM - value: {{ .Values.score.auth.server.keycloak.realm | quote }} - - name: AUTH_SERVER_SCOPE_UPLOAD_STUDY_PREFIX - value: {{ .Values.score.auth.server.scope.upload.study.prefix | quote }} - - name: AUTH_SERVER_SCOPE_DOWNLOAD_STUDY_PREFIX - value: {{ .Values.score.auth.server.scope.download.study.prefix | quote }} - - name: AUTH_SERVER_SCOPE_DOWNLOAD_STUDY_SUFFIX - value: {{ .Values.score.auth.server.scope.download.study.suffix | quote }} - - name: AUTH_SERVER_SCOPE_UPLOAD_STUDY_SUFFIX - value: {{ .Values.score.auth.server.scope.upload.study.suffix | quote }} - - name: AUTH_SERVER_SCOPE_DOWNLOAD_SYSTEM - value: {{ .Values.score.auth.server.scope.download.system | quote }} - - name: AUTH_SERVER_SCOPE_UPLOAD_SYSTEM - value: {{ .Values.score.auth.server.scope.upload.system | quote }} - - name: AUTH_SERVER_URL - value: {{ .Values.score.auth.server.url | quote }} - - name: AUTH_JWT_PUBLICKEYURL - value: {{ .Values.score.auth.oauth2.resourceserver.jwt.jwkSetUri | quote }} - # Spring Security OAuth2 Resource Server JWT Configuration - {{- if .Values.score.auth.oauth2.resourceserver.jwt.jwkSetUri }} - - name: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI - value: {{ .Values.score.auth.oauth2.resourceserver.jwt.jwkSetUri | quote }} - {{- end }} - {{- if .Values.score.auth.oauth2.resourceserver.jwt.issuerUri }} - - name: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI - value: {{ .Values.score.auth.oauth2.resourceserver.jwt.issuerUri | quote }} - {{- end }} - # Force legacy public-key-location off so jwk/issuer are honored - - name: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_PUBLIC_KEY_LOCATION - value: "" - # Enable filter chain debug - - name: SPRING_SECURITY_FILTERCHAIN_DEBUG - value: "true" - - name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_SECURITY - value: DEBUG - - name: LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_SECURITY_OAUTH2 - value: DEBUG - # Duplicates with underscores to satisfy relaxed binding for clientId/clientSecret - - name: AUTH_SERVER_CLIENT_ID - value: {{ .Values.score.auth.server.clientId | quote }} - - name: AUTH_SERVER_CLIENT_SECRET - value: {{ .Values.score.auth.server.clientSecret | quote }} - - name: AUTH_SERVER_TOKEN_NAME - value: {{ .Values.score.auth.server.tokenName | quote }} - # Force Spring Security Resource Server to use JWKS + issuer and ignore legacy public-key-location - - name: SPRING_APPLICATION_JSON - value: >- - {"spring":{"security":{"oauth2":{"resourceserver":{"jwt":{"jwk-set-uri":"{{ .Values.score.auth.oauth2.resourceserver.jwt.jwkSetUri }}","issuer-uri":"{{ .Values.score.auth.oauth2.resourceserver.jwt.issuerUri }}","public-key-location":""}}}}}} - readinessProbe: - tcpSocket: - port: 8087 - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - livenessProbe: - tcpSocket: - port: 8087 - initialDelaySeconds: 120 - periodSeconds: 30 - timeoutSeconds: 10 - failureThreshold: 3 - resources: - {{- toYaml .Values.resources | nindent 12 }} diff --git a/helm/score/templates/ingress.yaml b/helm/score/templates/ingress.yaml deleted file mode 100644 index a95d549..0000000 --- a/helm/score/templates/ingress.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- if .Values.ingress.enabled -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-score" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "score" }} - app.kubernetes.io/instance: {{ .Release.Name }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - rules: - - host: {{ .Values.url.hostname | quote }} - http: - paths: - {{- range .Values.ingress.hosts }} - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $.Values.fullnameOverride | default (printf "%s-score" $.Release.Name) }} - port: - number: {{ $.Values.service.port }} - {{- else }} - serviceName: {{ $.Values.fullnameOverride | default (printf "%s-score" $.Release.Name) }} - servicePort: {{ $.Values.service.port }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/score/templates/service.yaml b/helm/score/templates/service.yaml deleted file mode 100644 index aa4c4e8..0000000 --- a/helm/score/templates/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-score" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "score" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: {{ .Values.nameOverride | default "score" }} - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/score/values.yaml b/helm/score/values.yaml deleted file mode 100644 index 1c024c2..0000000 --- a/helm/score/values.yaml +++ /dev/null @@ -1,121 +0,0 @@ -replicaCount: 1 - -fullnameOverride: "score" - -url: - hostname: "score-ilifu.openup.org.za" - -image: - repository: ghcr.io/overture-stack/score-server - pullPolicy: IfNotPresent - tag: "5.12.0" - -nameOverride: "" - -service: - type: ClusterIP - port: 8087 - -score: - # Spring Configuration - spring: - profiles: - active: "s3,prod,secure" - - # Server Configuration - server: - port: 8087 - ssl: - enabled: false - - # Song Variables - metadata: - url: "http://song:8080" - ssl: - enabled: false - - # Client credentials for Score to authenticate to SONG - clientCredentials: - id: "dms" - secret: "dms-secret" - tokenUrl: "http://keycloak:8080/realms/agari/protocol/openid-connect/token" - systemScope: "song.READ" - - s3: - endpoint: "https://minio-ilifu.openup.org.za" - accesskey: "admin" - secretkey: "admin123" - sigv4enabled: true - secured: false - - object: - sentinel: "heliograph" - - bucket: - name: - object: "object" - state: "state" - - upload: - partsize: "1073741824" - connection: - timeout: "1200000" - - auth: - server: - provider: "keycloak" - clientId: "dms" - clientSecret: "dms-secret" - tokenName: "apiKey" - keycloak: - host: "http://keycloak:8080" - realm: "agari" - scope: - download: - study: - prefix: "study" - suffix: ".READ" - system: "score.READ" - upload: - study: - prefix: "study" - suffix: ".WRITE" - system: "score.WRITE" - url: "http://keycloak:8080/realms/agari/apikey/check_api_key/" - oauth2: - resourceserver: - jwt: - jwkSetUri: "https://keycloak-ilifu.openup.org.za/realms/agari/protocol/openid-connect/certs" - issuerUri: "https://keycloak-ilifu.openup.org.za/realms/agari" - -ingress: - enabled: true - className: "private" - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "100m" - nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/cors-allow-origin: "*" - nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" - nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" - hosts: - - host: score-ilifu.openup.org.za - paths: - - path: / - pathType: Prefix - tls: [] - -resources: - limits: - cpu: 500m - memory: 1Gi - ephemeral-storage: 2Gi - requests: - cpu: 200m - memory: 512Mi - ephemeral-storage: 1Gi - -nodeSelector: {} -tolerations: [] -affinity: {} - - diff --git a/helm/song-db/Chart.yaml b/helm/song-db/Chart.yaml deleted file mode 100644 index f0c86af..0000000 --- a/helm/song-db/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: song-db -description: PostgreSQL database for Song -type: application -version: 0.1.0 -appVersion: "14" diff --git a/helm/song-db/templates/deployment.yaml b/helm/song-db/templates/deployment.yaml deleted file mode 100644 index 545c596..0000000 --- a/helm/song-db/templates/deployment.yaml +++ /dev/null @@ -1,122 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - replicas: {{ .Values.replicaCount }} - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: postgresql - containerPort: 5432 - protocol: TCP - env: - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-secret - key: postgres-user - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-secret - key: postgres-password - - name: POSTGRES_DB - valueFrom: - secretKeyRef: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-secret - key: postgres-database - - name: PGDATA - value: /var/lib/postgresql/data/pgdata - {{- if .Values.probes.liveness.enabled }} - livenessProbe: - {{- if .Values.probes.liveness.exec }} - exec: - command: - {{- range .Values.probes.liveness.exec.command }} - - {{ . | quote }} - {{- end }} - {{- else if .Values.probes.liveness.tcpSocket }} - tcpSocket: - port: {{ .Values.probes.liveness.tcpSocket.port }} - {{- end }} - initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} - successThreshold: {{ .Values.probes.liveness.successThreshold | default 1 }} - failureThreshold: {{ .Values.probes.liveness.failureThreshold }} - {{- end }} - {{- if .Values.probes.readiness.enabled }} - readinessProbe: - {{- if .Values.probes.readiness.exec }} - exec: - command: - {{- range .Values.probes.readiness.exec.command }} - - {{ . | quote }} - {{- end }} - {{- else if .Values.probes.readiness.tcpSocket }} - tcpSocket: - port: {{ .Values.probes.readiness.tcpSocket.port }} - {{- end }} - initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} - successThreshold: {{ .Values.probes.readiness.successThreshold | default 1 }} - failureThreshold: {{ .Values.probes.readiness.failureThreshold }} - {{- end }} - successThreshold: 1 - failureThreshold: 6 - {{- if .Values.persistence.enabled }} - volumeMounts: - - name: postgresql-data - mountPath: /var/lib/postgresql/data - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- if .Values.persistence.enabled }} - volumes: - - name: postgresql-data - persistentVolumeClaim: - claimName: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-pvc - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/song-db/templates/pvc.yaml b/helm/song-db/templates/pvc.yaml deleted file mode 100644 index 5b9ed53..0000000 --- a/helm/song-db/templates/pvc.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if .Values.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-pvc - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - accessModes: - - {{ .Values.persistence.accessMode | quote }} - resources: - requests: - storage: {{ .Values.persistence.size | quote }} - {{- if .Values.persistence.storageClass }} - {{- if (eq "-" .Values.persistence.storageClass) }} - storageClassName: "" - {{- else }} - storageClassName: "{{ .Values.persistence.storageClass }}" - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/song-db/templates/secret.yaml b/helm/song-db/templates/secret.yaml deleted file mode 100644 index 4f2284a..0000000 --- a/helm/song-db/templates/secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }}-secret - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} -type: Opaque -data: - postgres-user: {{ .Values.postgresql.postgresqlUsername | b64enc | quote }} - postgres-password: {{ .Values.postgresql.postgresqlPassword | b64enc | quote }} - postgres-database: {{ .Values.postgresql.postgresqlDatabase | b64enc | quote }} diff --git a/helm/song-db/templates/service.yaml b/helm/song-db/templates/service.yaml deleted file mode 100644 index 3bb3e60..0000000 --- a/helm/song-db/templates/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-postgres" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "postgres" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: postgresql - protocol: TCP - name: postgresql - selector: - app.kubernetes.io/name: postgres - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/song-db/values.yaml b/helm/song-db/values.yaml deleted file mode 100644 index cdd3bce..0000000 --- a/helm/song-db/values.yaml +++ /dev/null @@ -1,84 +0,0 @@ -replicaCount: 1 - -image: - repository: postgres - pullPolicy: IfNotPresent - tag: "14" - -nameOverride: "" -fullnameOverride: "song-db" - -podAnnotations: {} - -podSecurityContext: - fsGroup: 999 - -securityContext: - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - runAsNonRoot: false - runAsUser: 999 - -service: - type: ClusterIP - port: 5432 - -persistence: - enabled: true - # storageClass: "" - accessMode: ReadWriteOnce - size: 2Gi - # existingClaim: "" - -postgresql: - postgresqlUsername: admin - postgresqlPassword: song-db-pass-123 - postgresqlDatabase: songDb - -databaseKey: "song" - -probes: - liveness: - enabled: true - exec: - command: - - /bin/sh - - -c - - exec pg_isready -U "$(POSTGRES_USER)" -d "$(POSTGRES_DB)" -h 127.0.0.1 -p 5432 - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 6 - - readiness: - enabled: true - exec: - command: - - /bin/sh - - -c - - -e - - | - exec pg_isready -U "$(POSTGRES_USER)" -d "$(POSTGRES_DB)" -h 127.0.0.1 -p 5432 - [ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ] - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 3 - -resources: - limits: - cpu: 200m - memory: 256Mi - requests: - cpu: 100m - memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/helm/song/Chart.yaml b/helm/song/Chart.yaml deleted file mode 100644 index 60b90aa..0000000 --- a/helm/song/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: song -description: A Helm chart for Song metadata service -type: application -version: 0.1.0 -appVersion: "5.0.2" diff --git a/helm/song/templates/configmap.yaml b/helm/song/templates/configmap.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/helm/song/templates/deployment.yaml b/helm/song/templates/deployment.yaml deleted file mode 100644 index 82db2c4..0000000 --- a/helm/song/templates/deployment.yaml +++ /dev/null @@ -1,162 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-song" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "song" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - replicas: {{ .Values.replicaCount }} - strategy: - type: Recreate # Ensures only one pod at a time to avoid resource exhaustion - selector: - matchLabels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "song" }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "song" }} - app.kubernetes.io/instance: {{ .Release.Name }} - spec: - containers: - - name: song - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: http - containerPort: 8080 - protocol: TCP - env: - # Force auth configuration via JVM system properties (highest precedence) - - name: JAVA_OPTS - value: "-Dauth.server.provider={{ .Values.song.auth.server.provider }} -Dauth.server.clientID={{ .Values.song.auth.server.clientId }} -Dauth.server.keycloak.realm={{ .Values.song.auth.server.keycloak.realm }} -Dauth.server.keycloak.host={{ .Values.song.auth.server.keycloak.host }} -Dauth.server.scope.study.prefix={{ .Values.song.auth.server.scope.study.prefix }} -Dauth.server.introspectionUri={{ .Values.song.auth.oauth2.introspectionUri }}" - # Spring Run Profiles - - name: SPRING_PROFILES_ACTIVE - value: {{ .Values.song.spring.profiles.active | quote }} - # Flyway variables - - name: SPRING_FLYWAY_ENABLED - value: {{ .Values.song.spring.flyway.enabled | quote }} - # Song Variables - - name: ID_USELOCAL - value: {{ .Values.song.id.useLocal | quote }} - - name: SCHEMAS_ENFORCELATEST - value: {{ .Values.song.schemas.enforceLatest | quote }} - # Score Variables - - name: SCORE_URL - value: {{ .Values.song.score.url | quote }} - - name: SCORE_ACCESSTOKEN - value: {{ .Values.song.score.accessToken | quote }} - {{- if .Values.song.score.clientCredentials }} - # Score Client Credentials for SONG to authenticate to Score - - name: SCORE_CLIENTCREDENTIALS_ID - value: {{ .Values.song.score.clientCredentials.id | quote }} - - name: SCORE_CLIENTCREDENTIALS_SECRET - value: {{ .Values.song.score.clientCredentials.secret | quote }} - - name: SCORE_CLIENTCREDENTIALS_SYSTEMSCOPE - value: {{ .Values.song.score.clientCredentials.systemScope | quote }} - - name: SCORE_CLIENTCREDENTIALS_TOKENURL - value: {{ .Values.song.score.clientCredentials.tokenUrl | quote }} - {{- end }} - # Keycloak Variables - - name: AUTH_SERVER_PROVIDER - value: {{ .Values.song.auth.server.provider | quote }} - - name: AUTH_SERVER_CLIENTID - value: {{ .Values.song.auth.server.clientId | quote }} - - name: AUTH_SERVER_CLIENT_ID - value: {{ .Values.song.auth.server.clientId | quote }} - - name: AUTH_SERVER_KEYCLOAK_HOST - value: {{ .Values.song.auth.server.keycloak.host | quote }} - - name: AUTH_SERVER_KEYCLOAK_REALM - value: {{ .Values.song.auth.server.keycloak.realm | quote }} - # Alternative naming patterns for stubborn properties - - name: AUTH_SERVER_CLIENTSECRET - value: {{ .Values.song.auth.server.clientSecret | quote }} - - name: AUTH_SERVER_CLIENT_SECRET - value: {{ .Values.song.auth.server.clientSecret | quote }} - - name: AUTH_SERVER_SCOPE_STUDY_PREFIX - value: {{ .Values.song.auth.server.scope.study.prefix | quote }} - - name: AUTH_SERVER_SCOPE_STUDY_SUFFIX - value: {{ .Values.song.auth.server.scope.study.suffix | quote }} - - name: AUTH_SERVER_SCOPE_SYSTEM - value: {{ .Values.song.auth.server.scope.system | quote }} - # Environment variables using exact Spring Boot property naming - - name: AUTH_SERVER_PROVIDER - value: {{ .Values.song.auth.server.provider | quote }} - - name: AUTH_SERVER_CLIENTID - value: {{ .Values.song.auth.server.clientId | quote }} - - name: AUTH_SERVER_KEYCLOAK_HOST - value: {{ .Values.song.auth.server.keycloak.host | quote }} - - name: AUTH_SERVER_KEYCLOAK_REALM - value: {{ .Values.song.auth.server.keycloak.realm | quote }} - - name: AUTH_SERVER_SCOPE_STUDY_PREFIX - value: {{ .Values.song.auth.server.scope.study.prefix | quote }} - - name: AUTH_SERVER_INTROSPECTIONURI - value: {{ .Values.song.auth.oauth2.introspectionUri | quote }} - # OAuth2 JWT configuration - - name: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI - value: {{ .Values.song.auth.oauth2.resourceserver.jwt.jwkSetUri | quote }} - - name: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI - value: {{ .Values.song.auth.oauth2.resourceserver.jwt.issuerUri | quote }} - - name: AUTH_SERVER_INTROSPECTIONURI - value: {{ .Values.song.auth.oauth2.introspectionUri | quote }} - # Postgres Variables - - name: SPRING_DATASOURCE_URL - value: {{ .Values.song.datasource.url | quote }} - - name: SPRING_DATASOURCE_USERNAME - value: {{ .Values.song.datasource.username | quote }} - - name: SPRING_DATASOURCE_PASSWORD - value: {{ .Values.song.datasource.password | quote }} - # Kafka Variables - - name: SPRING_KAFKA_BOOTSTRAPSERVERS - value: {{ .Values.song.kafka.bootstrapServers | quote }} - - name: SPRING_KAFKA_TEMPLATE_DEFAULTTOPIC - value: {{ .Values.song.kafka.template.defaultTopic | quote }} - # Swagger Variable - - name: SWAGGER_ALTERNATEURL - value: {{ .Values.song.swagger.alternateUrl | quote }} - - {{- if .Values.probes.liveness.enabled }} - livenessProbe: - {{- if .Values.probes.liveness.httpGet }} - httpGet: - path: {{ .Values.probes.liveness.httpGet.path }} - port: {{ .Values.probes.liveness.httpGet.port }} - {{- else if .Values.probes.liveness.tcpSocket }} - tcpSocket: - port: {{ .Values.probes.liveness.tcpSocket.port }} - {{- end }} - initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} - failureThreshold: {{ .Values.probes.liveness.failureThreshold }} - {{- end }} - {{- if .Values.probes.readiness.enabled }} - readinessProbe: - {{- if .Values.probes.readiness.httpGet }} - httpGet: - path: {{ .Values.probes.readiness.httpGet.path }} - port: {{ .Values.probes.readiness.httpGet.port }} - {{- else if .Values.probes.readiness.tcpSocket }} - tcpSocket: - port: {{ .Values.probes.readiness.tcpSocket.port }} - {{- end }} - initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.probes.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} - failureThreshold: {{ .Values.probes.readiness.failureThreshold }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/song/templates/ingress.yaml b/helm/song/templates/ingress.yaml deleted file mode 100644 index b4fb9e0..0000000 --- a/helm/song/templates/ingress.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if .Values.ingress.enabled -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-song" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "song" }} - app.kubernetes.io/instance: {{ .Release.Name }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - - host: {{ .Values.url.hostname | quote }} - http: - paths: - {{- range .Values.ingress.hosts }} - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $.Values.fullnameOverride | default (printf "%s-song" $.Release.Name) }} - port: - number: {{ $.Values.service.port }} - {{- else }} - serviceName: {{ $.Values.fullnameOverride | default (printf "%s-song" $.Release.Name) }} - servicePort: {{ $.Values.service.port }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/song/templates/service.yaml b/helm/song/templates/service.yaml deleted file mode 100644 index 1822e79..0000000 --- a/helm/song/templates/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.fullnameOverride | default (printf "%s-song" .Release.Name) }} - labels: - app.kubernetes.io/name: {{ .Values.nameOverride | default "song" }} - app.kubernetes.io/instance: {{ .Release.Name }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: {{ .Values.nameOverride | default "song" }} - app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/song/values.yaml b/helm/song/values.yaml deleted file mode 100644 index f3bca0f..0000000 --- a/helm/song/values.yaml +++ /dev/null @@ -1,154 +0,0 @@ -replicaCount: 1 - -fullnameOverride: "song" - -url: - hostname: "song-staging.openup.org.za" #ilifu/staging - -image: - repository: ghcr.io/overture-stack/song-server - pullPolicy: IfNotPresent - tag: "edge" - -nameOverride: "" - -service: - type: ClusterIP - port: 8080 - -song: - spring: - profiles: - active: "prod,secure,kafka,s3,score-client-cred" - flyway: - enabled: true - security: - oauth2: - resourceserver: - jwt: - jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" - issuerUri: "http://keycloak-staging.openup.org.za/realms/agari" #ilifu/staging - - client: - study: - maxUploadFileSize: 10737418240 - maxNumberOfFilesPerUpload: 20 # Database Variables - datasource: - url: "jdbc:postgresql://song-db:5432/songDb?stringtype=unspecified" - username: "admin" - password: "song-db-pass-123" - - # Song Variables - id: - useLocal: true - schemas: - enforceLatest: true - - # Score Variables - score: - url: "http://score:8087" - accessToken: "" - # Client credentials for SONG to authenticate to Score - clientCredentials: - id: "dms" - secret: "dms-secret" - tokenUrl: "http://keycloak:8080/realms/agari/protocol/openid-connect/token" - systemScope: "openid" # this gets the RPT - - - auth: - server: - provider: "keycloak" - clientId: "dms" - clientSecret: "dms-secret" - tokenName: "apiKey" - keycloak: - host: "http://keycloak:8080" - realm: "agari" - scope: - study: - prefix: "STUDY." - suffix: ".WRITE" - system: "song.WRITE" - oauth2: - resourceserver: - jwt: - jwkSetUri: "http://keycloak:8080/realms/agari/protocol/openid-connect/certs" - issuerUri: "http://keycloak-staging.openup.org.za/realms/agari" #ilifu/staging - introspectionUri: "http://keycloak:8080/realms/agari/apikey/check_api_key/" - - # Kafka Variables (enabled) - kafka: - bootstrapServers: "kafka-kafka-bootstrap.agari-kafka.svc:9092" - template: - defaultTopic: "song-analysis-staging" #prod/staging - - # Swagger Variable - swagger: - alternateUrl: "/swagger-api" - -# Probe configuration -probes: - liveness: - enabled: false - httpGet: - path: /actuator/health - port: 8081 - initialDelaySeconds: 300 - periodSeconds: 60 - timeoutSeconds: 10 - failureThreshold: 5 - - readiness: - enabled: true - tcpSocket: - port: 8080 - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 3 - -ingress: - enabled: true - className: "private" - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "100m" - nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/cors-allow-origin: "*" - nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" - nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" - hosts: - - host: song-staging.openup.org.za #ilifu/staging - paths: - - path: / - pathType: Prefix - tls: [] - -# Resource limits -resources: - limits: - cpu: 500m - memory: 1.5Gi - ephemeral-storage: 2Gi - requests: - cpu: 150m - memory: 768Mi - ephemeral-storage: 1Gi - -# Node selection -nodeSelector: {} -tolerations: [] -affinity: {} - -# Pod security context -podSecurityContext: {} -securityContext: {} - -# Service account -serviceAccount: - create: false - annotations: {} - name: "" - -# Pod annotations -podAnnotations: {} From c5f0ae389c2289e6d022450c6c596077603ad3db Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 25 Nov 2025 12:00:39 +0200 Subject: [PATCH 43/48] Remove docs for unused charts --- README.md | 60 +++++-------------------------------------------------- 1 file changed, 5 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 1ae7748..e1e5051 100644 --- a/README.md +++ b/README.md @@ -45,20 +45,7 @@ helm install minio ./helm/minio -n agari kubectl port-forward -n agari service/minio 9000:9000 ``` -### 3.2 Kafka Message Queue - -```bash -# Add Strimzi repository for Kafka operator -helm repo add strimzi https://strimzi.io/charts/ - -# Build chart dependencies (required for first-time deployment) -cd helm/kafka && helm dependency build && cd ../.. - -# Install Kafka using Strimzi operator -helm install kafka ./helm/kafka -n agari-kafka -``` - -### 4. Setup Keycloak +#### 3.2 Setup Keycloak ```bash # Database @@ -72,23 +59,6 @@ Set up the **client** in Keycloak and copy the **secret** to **song**, **score** use `utils/update-secrets.sh` script to update the secrets in all services -### 5. Deploy Overture Stack - -#### 5.1 SONG - -```bash -# Database -helm install song-db ./helm/song-db -n agari - -# Song -helm install song ./helm/song -n agari -``` - -#### 5.2 SCORE -```bash -helm install score ./helm/score -n agari -``` - #### 5.3 ELASTICSEARCH ```bash # Elasticsearch @@ -100,20 +70,7 @@ curl -X PUT "http://elasticsearch.local/agari-index" \ -d @helm/elasticsearch/configs/agari-index-mapping.json ``` -#### 5.4 MAESTRO -```bash -helm install maestro ./helm/maestro -n agari -``` -#### 5.5 ARRANGER -```bash -# Set up Arranger configuration -kubectl create configmap arranger-config --from-file=helm/arranger/configs/ -n agari - -# Arranger -helm install arranger ./helm/arranger -n agari -``` - -#### 5.6 FOLIO Projects Service +#### 3.4 FOLIO Projects Service **Find Folio repo at [https://github.com/OpenUpSA/agari-folio](https://github.com/OpenUpSA/agari-folio)** @@ -133,11 +90,7 @@ helm install folio ./helm/folio -n agari For local development, you can use `/etc/hosts` to map the services: ```bash -echo "127.0.0.1 song.local -127.0.0.1 score.local -127.0.0.1 maestro.local -127.0.0.1 arranger.local -127.0.0.1 keycloak.local +echo "127.0.0.1 keycloak.local 127.0.0.1 elasticsearch.local 127.0.0.1 minio-console.local 127.0.0.1 folio.local" | sudo tee -a /etc/hosts @@ -154,9 +107,6 @@ kubectl create secret tls folio-prod-tls-cert --cert=/path/to/tls.crt --key=/pat Services are available at these URLs: -- **SONG API**: http://song.local/swagger-ui.html -- **Score API**: http://score.local/swagger-ui.html -- **Arranger GraphQL**: http://arranger.local/graphql - **Keycloak**: http://keycloak.local - **Elasticsearch**: http://elasticsearch.local - **MinIO Console**: http://minio-console.local @@ -205,7 +155,7 @@ curl -d "client_id=song-api" \ "http://keycloak.local/realms/agari/protocol/openid-connect/token" ``` -## Data Flow +## Data Flow - outdated 1. **Submit metadata** → SONG validates and stores in PostgreSQL 2. **Upload files** → Score stores in MinIO object storage @@ -213,7 +163,7 @@ curl -d "client_id=song-api" \ 4. **Index data** → Maestro processes and indexes in Elasticsearch 5. **Query data** → Arranger provides GraphQL API -## GraphQL Query Examples +## GraphQL Query Examples - outdated Visit http://arranger.local/graphql to access the GraphQL playground. Here are example queries you can copy and paste: From e712f72e7ea611f247b0c9c1c856836560460c57 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 25 Nov 2025 12:04:02 +0200 Subject: [PATCH 44/48] Adding SSL certs to charts --- helm/folio/values.yaml | 7 +++++-- helm/frontend/values.yaml | 10 ++++------ helm/keycloak/values.yaml | 17 ++++++++++------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index cd1a480..36db128 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: ghcr.io/openupsa/agari-folio pullPolicy: "Always" - tag: "staging" #main/staging + tag: "staging" #master/staging service: type: ClusterIP @@ -52,7 +52,10 @@ ingress: paths: - path: / pathType: Prefix - tls: [] + tls: + - secretName: folio-staging-tls-cert #ilifu/staging + hosts: + - folio-staging.openup.org.za #ilifu/staging resources: limits: diff --git a/helm/frontend/values.yaml b/helm/frontend/values.yaml index caff02b..c17e76b 100644 --- a/helm/frontend/values.yaml +++ b/helm/frontend/values.yaml @@ -2,7 +2,7 @@ replicaCount: 1 image: repository: ghcr.io/methodlab/agari-frontend - tag: v0.0.26 + tag: v0.0.29 pullPolicy: Always service: @@ -13,17 +13,15 @@ service: ingress: enabled: true className: private - annotations: - cert-manager.io/cluster-issuer: "letsencrypt-prod" # probably leave this as prod hosts: - - host: agari-staging.openup.org.za # ilifu/staging + - host: agari-staging.openup.org.za # agari/agari-staging paths: - path: / pathType: ImplementationSpecific tls: - - secretName: agari-staging-tls-cert # try ilifu/staging + - secretName: agari-staging-tls-cert # agari/agari-staging hosts: - - agari-staging.openup.org.za # try ilifu/staging + - agari-staging.openup.org.za # agari/agari-staging resources: limits: diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index 788c524..f47cf9a 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -1,7 +1,7 @@ replicaCount: 1 url: - hostname: "keycloak-ilifu.openup.org.za" + hostname: "keycloak-staging.openup.org.za" # ilifu/staging image: repository: quay.io/phasetwo/phasetwo-keycloak @@ -27,11 +27,14 @@ ingress: nginx.ingress.kubernetes.io/cors-allow-methods: "GET, POST, PUT, DELETE, OPTIONS" nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization" hosts: - - host: keycloak-ilifu.openup.org.za + - host: keycloak-staging.openup.org.za # ilifu/staging paths: - path: / pathType: Prefix - tls: [] + tls: + - secretName: keycloak-staging-tls-cert # ilifu/staging + hosts: + - keycloak-staging.openup.org.za # ilifu/staging extraEnv: - name: KC_FEATURES @@ -43,7 +46,7 @@ extraEnv: - name: JAVA_OPTS_APPEND value: "-Djava.awt.headless=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.net.preferIPv4Stack=true -server -Xms512m -Xmx768m" - name: KC_HOSTNAME - value: keycloak-ilifu.openup.org.za + value: keycloak-staging.openup.org.za # ilifu/staging - name: KC_PROXY value: "edge" - name: KC_CORS_ORIGINS @@ -78,11 +81,11 @@ keycloak: password: admin123 # Keycloak settings - hostname: keycloak-ilifu.openup.org.za + hostname: keycloak-staging.openup.org.za # ilifu/staging healthEnabled: true startupMode: start-dev - frontendUrl: "https://keycloak-ilifu.openup.org.za" - adminUrl: "https://keycloak-ilifu.openup.org.za" + frontendUrl: "https://keycloak-staging.openup.org.za" # ilifu/staging + adminUrl: "https://keycloak-staging.openup.org.za" # ilifu/staging # Resource limits - better configuration from values-keycloak.yaml resources: From d5d14d4cf9e2623f9c239f0dd3cc8670daca50bf Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 25 Nov 2025 12:05:49 +0200 Subject: [PATCH 45/48] Minio update Config and ingress changes to make downloads work Setting up buckets Increas resource availability Add SSL cert --- helm/minio/values.yaml | 98 ++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/helm/minio/values.yaml b/helm/minio/values.yaml index 465ed81..591b1cf 100644 --- a/helm/minio/values.yaml +++ b/helm/minio/values.yaml @@ -2,12 +2,25 @@ replicaCount: 1 fullnameOverride: "minio" +image: + repository: minio/minio + pullPolicy: IfNotPresent + tag: "RELEASE.2024-01-16T16-07-38Z" + +# Service configuration +service: + type: ClusterIP + port: 9000 + consolePort: 9001 + +# External URL configuration (required by template) url: - hostname: "minio-ilifu.openup.org.za" + hostname: "minio-staging.openup.org.za" # ilifu/staging +# Console configuration console: url: - hostname: "minio-console-ilifu.openup.org.za" + hostname: "minio-console-staging.openup.org.za" # ilifu/staging ingress: enabled: true className: "private" @@ -15,54 +28,61 @@ console: nginx.ingress.kubernetes.io/proxy-body-size: "1000m" tls: [] -image: - repository: minio/minio - pullPolicy: IfNotPresent - tag: "RELEASE.2024-01-16T16-07-38Z" - -nameOverride: "" - -service: - type: ClusterIP - port: 9000 - consolePort: 9001 - +# Main MinIO ingress - SIMPLIFIED VERSION ingress: enabled: true className: "private" annotations: nginx.ingress.kubernetes.io/proxy-body-size: "1000m" nginx.ingress.kubernetes.io/proxy-request-buffering: "off" + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" hosts: - - host: minio-ilifu.openup.org.za + - host: minio-staging.openup.org.za # ilifu/staging paths: - path: / pathType: Prefix - tls: [] + tls: + - secretName: minio-staging-tls-cert #ilifu/staging + hosts: + - minio-staging.openup.org.za #ilifu/staging + +# Environment variables for MinIO +extraEnv: + - name: MINIO_SERVER_URL + value: "https://minio-staging.openup.org.za" # ilifu/staging +# MinIO configuration minio: accessKey: admin secretKey: admin123 + # Buckets to create buckets: - name: state - name: object + - name: agari-data + # Enable bucket setup bucketSetup: enabled: true +# Resource limits resources: limits: - cpu: 300m - memory: 256Mi + cpu: 500m + memory: 512Mi requests: - cpu: 150m - memory: 128Mi + cpu: 250m + memory: 256Mi +# Persistence persistence: enabled: true - size: 5Gi + size: 10Gi + accessMode: ReadWriteOnce + storageClass: "" +# Health checks probes: liveness: enabled: true @@ -71,7 +91,7 @@ probes: port: api initialDelaySeconds: 30 periodSeconds: 30 - timeoutSeconds: 5 + timeoutSeconds: 10 failureThreshold: 3 readiness: @@ -81,38 +101,17 @@ probes: port: api initialDelaySeconds: 15 periodSeconds: 10 - timeoutSeconds: 3 + timeoutSeconds: 5 failureThreshold: 3 - enabled: true - -# Persistence -persistence: - enabled: true - size: 10Gi - accessMode: ReadWriteOnce - storageClass: "" - -# Resource limits -resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 250m - memory: 256Mi -# Node selection -nodeSelector: {} -tolerations: [] -affinity: {} - -# Pod security context +# Security context podSecurityContext: fsGroup: 1000 securityContext: runAsUser: 1000 runAsGroup: 1000 + runAsNonRoot: true # Service account serviceAccount: @@ -123,8 +122,13 @@ serviceAccount: # Pod annotations podAnnotations: {} -# MC (MinIO Client) for bucket setup +# MinIO Client for bucket setup mc: image: repository: minio/mc tag: "RELEASE.2024-01-13T08-44-48Z" + +# Node selection +nodeSelector: {} +tolerations: [] +affinity: {} From 713ea7bd6d8c69aecb8f899800896df925dc7f04 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Tue, 25 Nov 2025 13:36:30 +0200 Subject: [PATCH 46/48] Insatll folio worker --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e1e5051..83ef1c0 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,9 @@ helm install folio-db ./helm/folio-db -n agari # Folio helm install folio ./helm/folio -n agari +# Folio worker +helm install folio-worker ./helm/folio-worker -n agari + ``` From a55441cfcec61d07a4d2ad57882d2fdca797e47c Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Thu, 27 Nov 2025 15:39:19 +0200 Subject: [PATCH 47/48] Updates Increase resource limits Removing old env vars Adding frontend env var --- helm/folio-worker/values.yaml | 10 +++++----- helm/folio/templates/deployment.yaml | 4 ---- helm/folio/values.yaml | 14 +++++--------- helm/frontend/templates/deployment.yaml | 4 ++++ helm/frontend/values.yaml | 14 +++++++++----- helm/keycloak/values.yaml | 2 +- helm/minio/values.yaml | 4 ++-- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/helm/folio-worker/values.yaml b/helm/folio-worker/values.yaml index f264269..b003be7 100644 --- a/helm/folio-worker/values.yaml +++ b/helm/folio-worker/values.yaml @@ -3,7 +3,7 @@ replicaCount: 1 image: repository: ghcr.io/openupsa/agari-folio pullPolicy: "Always" - tag: "staging" + tag: "staging" # master/staging worker: command: ["python", "worker.py"] @@ -45,11 +45,11 @@ folio: resources: limits: - cpu: 500m - memory: 512Mi + cpu: 1500m + memory: 1Gi requests: - cpu: 200m - memory: 256Mi + cpu: 1000m + memory: 512Mi nodeSelector: {} diff --git a/helm/folio/templates/deployment.yaml b/helm/folio/templates/deployment.yaml index f8eefd6..9d380a8 100644 --- a/helm/folio/templates/deployment.yaml +++ b/helm/folio/templates/deployment.yaml @@ -44,10 +44,6 @@ spec: value: {{ .Values.folio.database.user | quote }} - name: DB_PASSWORD value: {{ .Values.folio.database.password | quote }} - - name: OVERTURE_SCORE - value: {{ .Values.folio.overture.score | quote }} - - name: OVERTURE_SONG - value: {{ .Values.folio.overture.song | quote }} - name: KEYCLOAK_URL value: {{ .Values.folio.auth.keycloak.host | quote }} livenessProbe: diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index 36db128..1a8e193 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -34,10 +34,6 @@ folio: timeout: 30 options: "?connect_timeout=10&application_name=folio" - overture: - score: "http://score:8087" - song: "http://song:8080" - ingress: enabled: true className: private @@ -53,17 +49,17 @@ ingress: - path: / pathType: Prefix tls: - - secretName: folio-staging-tls-cert #ilifu/staging + - secretName: folio-staging-tls-cert #prod/staging hosts: - folio-staging.openup.org.za #ilifu/staging resources: limits: - cpu: 500m - memory: 512Mi + cpu: 1500m + memory: 1Gi requests: - cpu: 200m - memory: 256Mi + cpu: 1000m + memory: 512Mi nodeSelector: {} diff --git a/helm/frontend/templates/deployment.yaml b/helm/frontend/templates/deployment.yaml index d7d7564..32398da 100644 --- a/helm/frontend/templates/deployment.yaml +++ b/helm/frontend/templates/deployment.yaml @@ -21,6 +21,10 @@ spec: - name: frontend image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.env }} + env: + {{- toYaml .Values.env | nindent 12 }} + {{- end }} ports: - containerPort: 3000 name: http diff --git a/helm/frontend/values.yaml b/helm/frontend/values.yaml index c17e76b..47469d7 100644 --- a/helm/frontend/values.yaml +++ b/helm/frontend/values.yaml @@ -2,7 +2,7 @@ replicaCount: 1 image: repository: ghcr.io/methodlab/agari-frontend - tag: v0.0.29 + tag: v0.0.37 pullPolicy: Always service: @@ -10,18 +10,22 @@ service: port: 80 targetPort: 3000 +env: + - name: AGARI_ENV + value: "production" # production/ + ingress: enabled: true className: private hosts: - - host: agari-staging.openup.org.za # agari/agari-staging + - host: agari.openup.org.za # agari/agari-staging paths: - path: / pathType: ImplementationSpecific tls: - - secretName: agari-staging-tls-cert # agari/agari-staging + - secretName: agari-tls-cert # agari/agari-staging hosts: - - agari-staging.openup.org.za # agari/agari-staging + - agari.openup.org.za # agari/agari-staging resources: limits: @@ -51,4 +55,4 @@ nodeSelector: {} tolerations: [] -affinity: {} \ No newline at end of file +affinity: {} diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml index f47cf9a..e37f35c 100644 --- a/helm/keycloak/values.yaml +++ b/helm/keycloak/values.yaml @@ -32,7 +32,7 @@ ingress: - path: / pathType: Prefix tls: - - secretName: keycloak-staging-tls-cert # ilifu/staging + - secretName: keycloak-staging-tls-cert # keycloak/keycloak-staging hosts: - keycloak-staging.openup.org.za # ilifu/staging diff --git a/helm/minio/values.yaml b/helm/minio/values.yaml index 591b1cf..3195a91 100644 --- a/helm/minio/values.yaml +++ b/helm/minio/values.yaml @@ -42,7 +42,7 @@ ingress: - path: / pathType: Prefix tls: - - secretName: minio-staging-tls-cert #ilifu/staging + - secretName: minio-staging-tls-cert #minio/minio-staging hosts: - minio-staging.openup.org.za #ilifu/staging @@ -78,7 +78,7 @@ resources: # Persistence persistence: enabled: true - size: 10Gi + size: 20Gi accessMode: ReadWriteOnce storageClass: "" From 19965340aef9e501b3322c369c560434910615c8 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 9 Feb 2026 15:53:16 +0200 Subject: [PATCH 48/48] Use secret store for db password --- helm/folio/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/helm/folio/values.yaml b/helm/folio/values.yaml index 1a8e193..49cf6a0 100644 --- a/helm/folio/values.yaml +++ b/helm/folio/values.yaml @@ -10,7 +10,6 @@ service: port: 80 targetPort: 8000 -# Folio application configuration folio: auth: keycloak: @@ -25,7 +24,7 @@ folio: port: "5432" name: "folio" user: "admin" - password: "folio-db-pass-123" + #password: "folio-db-pass-123" # get password from secret store pool: size: 10 maxOverflow: 20