From 7e5bb1199c0648084620248a74946730e56675fd Mon Sep 17 00:00:00 2001 From: Vladimir Antropov Date: Tue, 24 Mar 2026 17:29:53 +0100 Subject: [PATCH 01/42] feat: add migration support from single-server Platforma Add optional migration init containers to the Helm chart and corresponding CloudFormation parameters for migrating from an existing single-server Platforma installation. Migration flow: 1. Download database dump from user's S3 (aws-cli init container) 2. Restore database using platforma --restore-db (platforma init container) 3. Invalidate caches using platforma --invalidate-caches (platforma init container) 4. Sync primary storage from old S3 bucket to new (aws-cli init container) 5. Start Platforma normally All steps are idempotent via a marker file on the database PVC. Set migration.enabled: false after successful migration and redeploy. --- charts/platforma/templates/deployment.yaml | 195 ++++++++++++++++++ charts/platforma/values.yaml | 51 +++++ .../aws/cloudformation-eks-1-35.yaml | 121 ++++++++++- 3 files changed, 366 insertions(+), 1 deletion(-) diff --git a/charts/platforma/templates/deployment.yaml b/charts/platforma/templates/deployment.yaml index 1fba775..38c9274 100644 --- a/charts/platforma/templates/deployment.yaml +++ b/charts/platforma/templates/deployment.yaml @@ -75,6 +75,196 @@ spec: securityContext: fsGroup: 1010 runAsNonRoot: true + {{- if .Values.migration.enabled }} + initContainers: + # --------------------------------------------------------------- + # Init 1: Download database dump from S3 + # --------------------------------------------------------------- + {{- if .Values.migration.database.s3Uri }} + - name: download-dump + image: "{{ .Values.migration.awsCliImage.repository }}:{{ .Values.migration.awsCliImage.tag }}" + securityContext: + runAsUser: 1010 + runAsGroup: 1010 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -ec + - | + MARKER="/data/main/.migration-complete" + if [ -f "$MARKER" ]; then + echo "Migration already completed (marker exists), skipping download" + exit 0 + fi + echo "Downloading database dump from {{ .Values.migration.database.s3Uri }}..." + aws s3 cp "{{ .Values.migration.database.s3Uri }}" /tmp/migration/backup.gz + echo "Download complete" + {{- if .Values.migration.credentials.secretName }} + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.migration.credentials.secretName }} + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.migration.credentials.secretName }} + key: aws-secret-access-key + {{- end }} + volumeMounts: + - name: main-storage + mountPath: /data/main + - name: migration-tmp + mountPath: /tmp/migration + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + # --------------------------------------------------------------- + # Init 2: Restore database using Platforma --restore-db + # --------------------------------------------------------------- + - name: restore-database + image: {{ include "platforma.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + runAsUser: 1010 + runAsGroup: 1010 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -ec + - | + MARKER="/data/main/.migration-complete" + if [ -f "$MARKER" ]; then + echo "Migration already completed (marker exists), skipping restore" + exit 0 + fi + echo "Restoring database..." + platforma --restore-db=/tmp/migration/backup.gz --db-dir={{ include "platforma.path.database" . }} --force + echo "Database restored" + env: + - name: PL_LICENSE + valueFrom: + secretKeyRef: + name: {{ .Values.license.secretName }} + key: {{ .Values.license.secretKey }} + volumeMounts: + - name: database + mountPath: {{ include "platforma.path.database" . }} + - name: main-storage + mountPath: /data/main + - name: migration-tmp + mountPath: /tmp/migration + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + memory: 2Gi + # --------------------------------------------------------------- + # Init 3: Invalidate caches (required after restore) + # --------------------------------------------------------------- + - name: invalidate-caches + image: {{ include "platforma.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + runAsUser: 1010 + runAsGroup: 1010 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -ec + - | + MARKER="/data/main/.migration-complete" + if [ -f "$MARKER" ]; then + echo "Migration already completed (marker exists), skipping invalidation" + exit 0 + fi + echo "Invalidating caches..." + platforma --invalidate-caches --main-root=/data/main --db-dir={{ include "platforma.path.database" . }} + echo "Caches invalidated" + date -u > "$MARKER" + echo "Migration marker written" + env: + - name: PL_LICENSE + valueFrom: + secretKeyRef: + name: {{ .Values.license.secretName }} + key: {{ .Values.license.secretKey }} + volumeMounts: + - name: database + mountPath: {{ include "platforma.path.database" . }} + - name: main-storage + mountPath: /data/main + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + memory: 2Gi + {{- end }} + # --------------------------------------------------------------- + # Init 4: Sync primary storage from old S3 bucket + # --------------------------------------------------------------- + {{- if .Values.migration.storageSync.enabled }} + - name: sync-storage + image: "{{ .Values.migration.awsCliImage.repository }}:{{ .Values.migration.awsCliImage.tag }}" + securityContext: + runAsUser: 1010 + runAsGroup: 1010 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - /bin/sh + - -ec + - | + SOURCE="s3://{{ .Values.migration.storageSync.sourceBucket }}/{{ .Values.migration.storageSync.sourcePrefix }}" + DEST="s3://{{ .Values.storage.main.s3.bucket }}/{{ .Values.storage.main.s3.prefix }}" + SOURCE_REGION="{{ .Values.migration.storageSync.sourceRegion | default .Values.storage.main.s3.region }}" + DEST_REGION="{{ .Values.storage.main.s3.region }}" + + echo "Syncing primary storage..." + echo " Source: $SOURCE (region: $SOURCE_REGION)" + echo " Dest: $DEST (region: $DEST_REGION)" + + aws s3 sync "$SOURCE" "$DEST" \ + --source-region "$SOURCE_REGION" \ + --region "$DEST_REGION" \ + --no-progress + + echo "Storage sync complete" + env: + {{- if .Values.migration.credentials.secretName }} + # Source bucket credentials + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.migration.credentials.secretName }} + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.migration.credentials.secretName }} + key: aws-secret-access-key + {{- end }} + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + memory: 4Gi + {{- end }} + {{- end }} containers: - name: platforma image: {{ include "platforma.image" . }} @@ -387,6 +577,11 @@ spec: claimName: {{ include "platforma.workspacePvcName" . }} - name: main-storage emptyDir: {} + {{- if .Values.migration.enabled }} + - name: migration-tmp + emptyDir: + sizeLimit: 10Gi + {{- end }} - name: job-templates configMap: name: {{ include "platforma.fullname" . }}-job-template diff --git a/charts/platforma/values.yaml b/charts/platforma/values.yaml index 8d2ae54..bea6dd0 100644 --- a/charts/platforma/values.yaml +++ b/charts/platforma/values.yaml @@ -471,6 +471,57 @@ jobServiceAccount: name: "" annotations: {} +# ============================================================================= +# MIGRATION (from single-server installation) +# ============================================================================= +# One-time migration from an existing Platforma server to this EKS deployment. +# When enabled, init containers restore the database and sync primary storage +# before Platforma starts. +# +# Prerequisites: +# 1. Create database dump on source server (requires --debug-enabled): +# curl -s http://:9091/db/state_raw | gzip > backup.gz +# 2. Upload dump to an S3 bucket: +# aws s3 cp backup.gz s3:///backup.gz +# 3. Set migration values below and deploy +# 4. After successful start, set migration.enabled: false and redeploy +# ============================================================================= + +migration: + # -- Enable migration init containers + enabled: false + + # -- Image for S3 operations (download dump, sync storage) + awsCliImage: + repository: amazon/aws-cli + tag: "2.27.22" + + # -- Database restore from gzipped dump + database: + # -- S3 URI to database dump (created via /db/state_raw endpoint) + s3Uri: "" # e.g. s3://my-bucket/migration/backup.gz + + # -- Primary storage sync (copy data from old S3 bucket to new one) + storageSync: + # -- Enable storage sync (disable if you already copied data manually) + enabled: false + # -- Source S3 bucket name + sourceBucket: "" + # -- Source prefix within the bucket + sourcePrefix: "" + # -- Source bucket region (defaults to chart's storage.main.s3.region) + sourceRegion: "" + + # -- Credentials for accessing the source S3 bucket (database dump + old storage) + # These are used only during migration and can be removed after. + # Create the secret before deploying: + # kubectl create secret generic platforma-migration-creds \ + # --from-literal=aws-access-key-id=AKIA... \ + # --from-literal=aws-secret-access-key=... + credentials: + # -- Name of existing secret with aws-access-key-id and aws-secret-access-key keys + secretName: "" + # ============================================================================= # JOB CONFIGURATION # ============================================================================= diff --git a/infrastructure/aws/cloudformation-eks-1-35.yaml b/infrastructure/aws/cloudformation-eks-1-35.yaml index 35193f5..8ce1875 100644 --- a/infrastructure/aws/cloudformation-eks-1-35.yaml +++ b/infrastructure/aws/cloudformation-eks-1-35.yaml @@ -56,6 +56,15 @@ Metadata: default: Storage Parameters: - S3BucketName + - Label: + default: 'Migration (optional, one-time restore from existing server)' + Parameters: + - MigrationDatabaseS3Uri + - MigrationSourceBucket + - MigrationSourcePrefix + - MigrationSourceRegion + - MigrationAccessKey + - MigrationSecretKey - Label: default: 'Data libraries (optional, up to 3 external read-only S3 buckets)' Parameters: @@ -108,6 +117,18 @@ Metadata: default: 'Route53 hosted zone ID (e.g. Z0123456789ABCDEF)' AuthMethod: default: 'Authentication method (htpasswd or ldap)' + MigrationDatabaseS3Uri: + default: 'Database dump S3 URI (e.g. s3://bucket/backup.gz)' + MigrationSourceBucket: + default: 'Source primary storage S3 bucket name' + MigrationSourcePrefix: + default: 'Source bucket prefix (leave empty for root)' + MigrationSourceRegion: + default: 'Source bucket region (leave empty = cluster region)' + MigrationAccessKey: + default: 'AWS access key for source bucket' + MigrationSecretKey: + default: 'AWS secret key for source bucket' EnableDemoLibrary: default: 'Enable MiLaboratories demo data library' DataLibrary1Name: @@ -274,6 +295,53 @@ Parameters: NoEcho: true Description: Password for the LDAP search user. Required when LdapSearchUser is set. + # --- Migration (from existing server) --- + MigrationDatabaseS3Uri: + Type: String + Default: '' + Description: > + S3 URI to the database dump file (e.g. s3://my-bucket/backup.gz). + Created on the source server: curl -s http://localhost:9091/db/state_raw | gzip > backup.gz + Leave empty to skip migration (fresh install). + + MigrationSourceBucket: + Type: String + Default: '' + AllowedPattern: '^([a-z0-9]([a-z0-9.-]{1,61}[a-z0-9])?)?$' + ConstraintDescription: 'Lowercase letters, digits, hyphens, periods, 3-63 chars, or empty' + Description: > + Source S3 bucket for primary storage migration. Data will be synced + to the new bucket created by this stack. Leave empty to skip storage sync. + + MigrationSourcePrefix: + Type: String + Default: '' + Description: Prefix within the source bucket. Leave empty for the bucket root. + + MigrationSourceRegion: + Type: String + Default: '' + AllowedPattern: '^([a-z]{2}(-[a-z]+)+-\d{1,2})?$' + ConstraintDescription: 'Must be a valid AWS region or empty' + Description: > + Region of the source S3 bucket. Defaults to cluster region if empty. + + MigrationAccessKey: + Type: String + Default: '' + AllowedPattern: '^((?:AKIA|ASIA)[A-Z0-9]{16})?$' + ConstraintDescription: 'Must be a valid AWS access key ID or empty' + Description: > + AWS access key for the source S3 bucket (database dump and primary storage). + Required if the source bucket is in a different AWS account. + + MigrationSecretKey: + Type: String + Default: '' + NoEcho: true + Description: > + AWS secret key for the source S3 bucket. Required when MigrationAccessKey is set. + # --- Data libraries --- EnableDemoLibrary: Type: String @@ -2347,6 +2415,18 @@ Resources: Value: !Ref DataLibrary3AccessKey - Name: DATA_LIBRARY_3_SECRET_KEY Value: !Ref DataLibrary3SecretKey + - Name: MIGRATION_DATABASE_S3_URI + Value: !Ref MigrationDatabaseS3Uri + - Name: MIGRATION_SOURCE_BUCKET + Value: !Ref MigrationSourceBucket + - Name: MIGRATION_SOURCE_PREFIX + Value: !Ref MigrationSourcePrefix + - Name: MIGRATION_SOURCE_REGION + Value: !Ref MigrationSourceRegion + - Name: MIGRATION_ACCESS_KEY + Value: !Ref MigrationAccessKey + - Name: MIGRATION_SECRET_KEY + Value: !Ref MigrationSecretKey Tags: - Key: platforma Value: !Ref ClusterName @@ -2689,6 +2769,39 @@ Resources: fi fi + # --- Migration values --- + if [ -n "$MIGRATION_DATABASE_S3_URI" ]; then + # Create migration credentials secret + if [ -n "$MIGRATION_ACCESS_KEY" ]; then + kubectl create secret generic platforma-migration-creds \ + -n $NAMESPACE \ + --from-literal=aws-access-key-id="$MIGRATION_ACCESS_KEY" \ + --from-literal=aws-secret-access-key="$MIGRATION_SECRET_KEY" \ + --dry-run=client -o yaml | kubectl apply -f - + fi + cat >> /tmp/platforma-values.yaml << MIGRATION_BLOCK + migration: + enabled: true + database: + s3Uri: "${MIGRATION_DATABASE_S3_URI}" + MIGRATION_BLOCK + if [ -n "$MIGRATION_ACCESS_KEY" ]; then + cat >> /tmp/platforma-values.yaml << MIGRATION_CREDS + credentials: + secretName: platforma-migration-creds + MIGRATION_CREDS + fi + if [ -n "$MIGRATION_SOURCE_BUCKET" ]; then + cat >> /tmp/platforma-values.yaml << MIGRATION_SYNC + storageSync: + enabled: true + sourceBucket: "${MIGRATION_SOURCE_BUCKET}" + sourcePrefix: "${MIGRATION_SOURCE_PREFIX}" + sourceRegion: "${MIGRATION_SOURCE_REGION}" + MIGRATION_SYNC + fi + fi + if [ -n "$PLATFORMA_IMAGE" ]; then IMG_REPO=$(echo "$PLATFORMA_IMAGE" | sed 's/:.*//') IMG_TAG=$(echo "$PLATFORMA_IMAGE" | sed 's/.*://') @@ -2699,11 +2812,17 @@ Resources: IMAGE_BLOCK fi + # Increase timeout when migration is active (S3 sync can take hours) + HELM_TIMEOUT="15m" + if [ -n "$MIGRATION_DATABASE_S3_URI" ]; then + HELM_TIMEOUT="120m" + fi + helm upgrade --install platforma oci://ghcr.io/milaboratory/platforma-helm/platforma \ --version $PLATFORMA_VERSION \ -n $NAMESPACE \ -f /tmp/platforma-values.yaml \ - --atomic --timeout 15m + --atomic --timeout $HELM_TIMEOUT post_build: commands: - | From a6a6de1891806d92efe416a4feb397803adeec9c Mon Sep 17 00:00:00 2001 From: Vladimir Antropov Date: Wed, 25 Mar 2026 14:54:30 +0100 Subject: [PATCH 02/42] feat: rename migration to dataMigration, separate credentials, fix marker path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename migration → dataMigration in helm values and templates - Separate credentials for database dump and storage sync buckets - Use env var for database path instead of hardcoding in scripts - Increase migration-tmp emptyDir to 30Gi - Align CF template with renamed helm values key --- charts/platforma/templates/deployment.yaml | 154 ++++++++++-------- charts/platforma/values.yaml | 38 +++-- .../aws/cloudformation-eks-1-35.yaml | 90 +++++++--- 3 files changed, 170 insertions(+), 112 deletions(-) diff --git a/charts/platforma/templates/deployment.yaml b/charts/platforma/templates/deployment.yaml index 38c9274..db27801 100644 --- a/charts/platforma/templates/deployment.yaml +++ b/charts/platforma/templates/deployment.yaml @@ -75,14 +75,15 @@ spec: securityContext: fsGroup: 1010 runAsNonRoot: true - {{- if .Values.migration.enabled }} + {{- $dbPath := include "platforma.path.database" . -}} + {{- if and .Values.dataMigration.enabled (or .Values.dataMigration.database.s3Uri .Values.dataMigration.storageSync.enabled) }} initContainers: # --------------------------------------------------------------- # Init 1: Download database dump from S3 # --------------------------------------------------------------- - {{- if .Values.migration.database.s3Uri }} + {{- if .Values.dataMigration.database.s3Uri }} - name: download-dump - image: "{{ .Values.migration.awsCliImage.repository }}:{{ .Values.migration.awsCliImage.tag }}" + image: "{{ .Values.dataMigration.awsCliImage.repository }}:{{ .Values.dataMigration.awsCliImage.tag }}" securityContext: runAsUser: 1010 runAsGroup: 1010 @@ -93,30 +94,32 @@ spec: - /bin/sh - -ec - | - MARKER="/data/main/.migration-complete" + MARKER="${DB_DIR}/.migration-complete" if [ -f "$MARKER" ]; then - echo "Migration already completed (marker exists), skipping download" + echo "Import already completed (marker exists), skipping download" exit 0 fi - echo "Downloading database dump from {{ .Values.migration.database.s3Uri }}..." - aws s3 cp "{{ .Values.migration.database.s3Uri }}" /tmp/migration/backup.gz + echo "Downloading database dump from {{ .Values.dataMigration.database.s3Uri }}..." + aws s3 cp "{{ .Values.dataMigration.database.s3Uri }}" /tmp/migration/backup.gz echo "Download complete" - {{- if .Values.migration.credentials.secretName }} env: + - name: DB_DIR + value: {{ $dbPath }} + {{- if .Values.dataMigration.database.credentialsSecret }} - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: {{ .Values.migration.credentials.secretName }} + name: {{ .Values.dataMigration.database.credentialsSecret }} key: aws-access-key-id - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: {{ .Values.migration.credentials.secretName }} + name: {{ .Values.dataMigration.database.credentialsSecret }} key: aws-secret-access-key - {{- end }} + {{- end }} volumeMounts: - - name: main-storage - mountPath: /data/main + - name: database + mountPath: {{ $dbPath }} - name: migration-tmp mountPath: /tmp/migration resources: @@ -141,15 +144,17 @@ spec: - /bin/sh - -ec - | - MARKER="/data/main/.migration-complete" + MARKER="${DB_DIR}/.migration-complete" if [ -f "$MARKER" ]; then - echo "Migration already completed (marker exists), skipping restore" + echo "Import already completed (marker exists), skipping restore" exit 0 fi echo "Restoring database..." - platforma --restore-db=/tmp/migration/backup.gz --db-dir={{ include "platforma.path.database" . }} --force + platforma --restore-db=/tmp/migration/backup.gz --db-dir="${DB_DIR}" --force echo "Database restored" env: + - name: DB_DIR + value: {{ $dbPath }} - name: PL_LICENSE valueFrom: secretKeyRef: @@ -157,7 +162,7 @@ spec: key: {{ .Values.license.secretKey }} volumeMounts: - name: database - mountPath: {{ include "platforma.path.database" . }} + mountPath: {{ $dbPath }} - name: main-storage mountPath: /data/main - name: migration-tmp @@ -169,11 +174,12 @@ spec: limits: memory: 2Gi # --------------------------------------------------------------- - # Init 3: Invalidate caches (required after restore) + # Init 3: Sync primary storage from old S3 bucket + # (must complete before cache invalidation) # --------------------------------------------------------------- - - name: invalidate-caches - image: {{ include "platforma.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.dataMigration.storageSync.enabled }} + - name: sync-storage + image: "{{ .Values.dataMigration.awsCliImage.repository }}:{{ .Values.dataMigration.awsCliImage.tag }}" securityContext: runAsUser: 1010 runAsGroup: 1010 @@ -184,40 +190,58 @@ spec: - /bin/sh - -ec - | - MARKER="/data/main/.migration-complete" + MARKER="${DB_DIR}/.migration-complete" if [ -f "$MARKER" ]; then - echo "Migration already completed (marker exists), skipping invalidation" + echo "Import already completed (marker exists), skipping sync" exit 0 fi - echo "Invalidating caches..." - platforma --invalidate-caches --main-root=/data/main --db-dir={{ include "platforma.path.database" . }} - echo "Caches invalidated" - date -u > "$MARKER" - echo "Migration marker written" + + SOURCE="s3://{{ .Values.dataMigration.storageSync.sourceBucket }}/{{ .Values.dataMigration.storageSync.sourcePrefix }}" + DEST="s3://{{ .Values.storage.main.s3.bucket }}/{{ .Values.storage.main.s3.prefix }}" + SOURCE_REGION="{{ .Values.dataMigration.storageSync.sourceRegion | default .Values.storage.main.s3.region }}" + DEST_REGION="{{ .Values.storage.main.s3.region }}" + + echo "Syncing primary storage..." + echo " Source: $SOURCE (region: $SOURCE_REGION)" + echo " Dest: $DEST (region: $DEST_REGION)" + + aws s3 sync "$SOURCE" "$DEST" \ + --source-region "$SOURCE_REGION" \ + --region "$DEST_REGION" \ + --no-progress + + echo "Storage sync complete" env: - - name: PL_LICENSE + - name: DB_DIR + value: {{ $dbPath }} + {{- if .Values.dataMigration.storageSync.credentialsSecret }} + - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: {{ .Values.license.secretName }} - key: {{ .Values.license.secretKey }} + name: {{ .Values.dataMigration.storageSync.credentialsSecret }} + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.dataMigration.storageSync.credentialsSecret }} + key: aws-secret-access-key + {{- end }} volumeMounts: - name: database - mountPath: {{ include "platforma.path.database" . }} - - name: main-storage - mountPath: /data/main + mountPath: {{ $dbPath }} resources: requests: cpu: 500m memory: 512Mi limits: - memory: 2Gi + memory: 4Gi {{- end }} # --------------------------------------------------------------- - # Init 4: Sync primary storage from old S3 bucket + # Init 4: Invalidate caches (after storage sync, before start) # --------------------------------------------------------------- - {{- if .Values.migration.storageSync.enabled }} - - name: sync-storage - image: "{{ .Values.migration.awsCliImage.repository }}:{{ .Values.migration.awsCliImage.tag }}" + - name: invalidate-caches + image: {{ include "platforma.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: runAsUser: 1010 runAsGroup: 1010 @@ -228,41 +252,35 @@ spec: - /bin/sh - -ec - | - SOURCE="s3://{{ .Values.migration.storageSync.sourceBucket }}/{{ .Values.migration.storageSync.sourcePrefix }}" - DEST="s3://{{ .Values.storage.main.s3.bucket }}/{{ .Values.storage.main.s3.prefix }}" - SOURCE_REGION="{{ .Values.migration.storageSync.sourceRegion | default .Values.storage.main.s3.region }}" - DEST_REGION="{{ .Values.storage.main.s3.region }}" - - echo "Syncing primary storage..." - echo " Source: $SOURCE (region: $SOURCE_REGION)" - echo " Dest: $DEST (region: $DEST_REGION)" - - aws s3 sync "$SOURCE" "$DEST" \ - --source-region "$SOURCE_REGION" \ - --region "$DEST_REGION" \ - --no-progress - - echo "Storage sync complete" + MARKER="${DB_DIR}/.migration-complete" + if [ -f "$MARKER" ]; then + echo "Import already completed (marker exists), skipping invalidation" + exit 0 + fi + echo "Invalidating caches..." + platforma --invalidate-caches --main-root=/data/main --db-dir="${DB_DIR}" + echo "Caches invalidated" + date -u > "$MARKER" + echo "Import marker written" env: - {{- if .Values.migration.credentials.secretName }} - # Source bucket credentials - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ .Values.migration.credentials.secretName }} - key: aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY + - name: DB_DIR + value: {{ $dbPath }} + - name: PL_LICENSE valueFrom: secretKeyRef: - name: {{ .Values.migration.credentials.secretName }} - key: aws-secret-access-key - {{- end }} + name: {{ .Values.license.secretName }} + key: {{ .Values.license.secretKey }} + volumeMounts: + - name: database + mountPath: {{ $dbPath }} + - name: main-storage + mountPath: /data/main resources: requests: cpu: 500m memory: 512Mi limits: - memory: 4Gi + memory: 2Gi {{- end }} {{- end }} containers: @@ -577,10 +595,10 @@ spec: claimName: {{ include "platforma.workspacePvcName" . }} - name: main-storage emptyDir: {} - {{- if .Values.migration.enabled }} + {{- if and .Values.dataMigration.enabled .Values.dataMigration.database.s3Uri }} - name: migration-tmp emptyDir: - sizeLimit: 10Gi + sizeLimit: 30Gi {{- end }} - name: job-templates configMap: diff --git a/charts/platforma/values.yaml b/charts/platforma/values.yaml index bea6dd0..1fdefbe 100644 --- a/charts/platforma/values.yaml +++ b/charts/platforma/values.yaml @@ -472,23 +472,23 @@ jobServiceAccount: annotations: {} # ============================================================================= -# MIGRATION (from single-server installation) +# DATA IMPORT (from existing Platforma installation) # ============================================================================= -# One-time migration from an existing Platforma server to this EKS deployment. -# When enabled, init containers restore the database and sync primary storage -# before Platforma starts. +# One-time import from an existing Platforma server. When enabled, init +# containers restore the database and sync primary storage before Platforma +# starts. # # Prerequisites: # 1. Create database dump on source server (requires --debug-enabled): # curl -s http://:9091/db/state_raw | gzip > backup.gz # 2. Upload dump to an S3 bucket: # aws s3 cp backup.gz s3:///backup.gz -# 3. Set migration values below and deploy -# 4. After successful start, set migration.enabled: false and redeploy +# 3. Set dataMigration values below and deploy +# 4. After successful start, set dataMigration.enabled: false and redeploy # ============================================================================= -migration: - # -- Enable migration init containers +dataMigration: + # -- Enable data import init containers enabled: false # -- Image for S3 operations (download dump, sync storage) @@ -500,6 +500,12 @@ migration: database: # -- S3 URI to database dump (created via /db/state_raw endpoint) s3Uri: "" # e.g. s3://my-bucket/migration/backup.gz + # -- Credentials for the database dump S3 bucket + # Create the secret before deploying: + # kubectl create secret generic platforma-import-db-creds \ + # --from-literal=aws-access-key-id=AKIA... \ + # --from-literal=aws-secret-access-key=... + credentialsSecret: "" # -- Primary storage sync (copy data from old S3 bucket to new one) storageSync: @@ -511,16 +517,12 @@ migration: sourcePrefix: "" # -- Source bucket region (defaults to chart's storage.main.s3.region) sourceRegion: "" - - # -- Credentials for accessing the source S3 bucket (database dump + old storage) - # These are used only during migration and can be removed after. - # Create the secret before deploying: - # kubectl create secret generic platforma-migration-creds \ - # --from-literal=aws-access-key-id=AKIA... \ - # --from-literal=aws-secret-access-key=... - credentials: - # -- Name of existing secret with aws-access-key-id and aws-secret-access-key keys - secretName: "" + # -- Credentials for the source primary storage S3 bucket + # Create the secret before deploying: + # kubectl create secret generic platforma-import-storage-creds \ + # --from-literal=aws-access-key-id=AKIA... \ + # --from-literal=aws-secret-access-key=... + credentialsSecret: "" # ============================================================================= # JOB CONFIGURATION diff --git a/infrastructure/aws/cloudformation-eks-1-35.yaml b/infrastructure/aws/cloudformation-eks-1-35.yaml index 8ce1875..afc4f4a 100644 --- a/infrastructure/aws/cloudformation-eks-1-35.yaml +++ b/infrastructure/aws/cloudformation-eks-1-35.yaml @@ -60,11 +60,13 @@ Metadata: default: 'Migration (optional, one-time restore from existing server)' Parameters: - MigrationDatabaseS3Uri + - MigrationDbAccessKey + - MigrationDbSecretKey - MigrationSourceBucket - MigrationSourcePrefix - MigrationSourceRegion - - MigrationAccessKey - - MigrationSecretKey + - MigrationStorageAccessKey + - MigrationStorageSecretKey - Label: default: 'Data libraries (optional, up to 3 external read-only S3 buckets)' Parameters: @@ -125,10 +127,14 @@ Metadata: default: 'Source bucket prefix (leave empty for root)' MigrationSourceRegion: default: 'Source bucket region (leave empty = cluster region)' - MigrationAccessKey: - default: 'AWS access key for source bucket' - MigrationSecretKey: - default: 'AWS secret key for source bucket' + MigrationDbAccessKey: + default: 'AWS access key for database dump bucket' + MigrationDbSecretKey: + default: 'AWS secret key for database dump bucket' + MigrationStorageAccessKey: + default: 'AWS access key for source primary storage bucket' + MigrationStorageSecretKey: + default: 'AWS secret key for source primary storage bucket' EnableDemoLibrary: default: 'Enable MiLaboratories demo data library' DataLibrary1Name: @@ -326,21 +332,37 @@ Parameters: Description: > Region of the source S3 bucket. Defaults to cluster region if empty. - MigrationAccessKey: + MigrationDbAccessKey: Type: String Default: '' AllowedPattern: '^((?:AKIA|ASIA)[A-Z0-9]{16})?$' ConstraintDescription: 'Must be a valid AWS access key ID or empty' Description: > - AWS access key for the source S3 bucket (database dump and primary storage). - Required if the source bucket is in a different AWS account. + AWS access key for the database dump S3 bucket. + Required if the bucket is in a different AWS account. - MigrationSecretKey: + MigrationDbSecretKey: Type: String Default: '' NoEcho: true Description: > - AWS secret key for the source S3 bucket. Required when MigrationAccessKey is set. + AWS secret key for the database dump S3 bucket. + + MigrationStorageAccessKey: + Type: String + Default: '' + AllowedPattern: '^((?:AKIA|ASIA)[A-Z0-9]{16})?$' + ConstraintDescription: 'Must be a valid AWS access key ID or empty' + Description: > + AWS access key for the source primary storage S3 bucket. + Required if the bucket is in a different AWS account. + + MigrationStorageSecretKey: + Type: String + Default: '' + NoEcho: true + Description: > + AWS secret key for the source primary storage S3 bucket. # --- Data libraries --- EnableDemoLibrary: @@ -2423,10 +2445,14 @@ Resources: Value: !Ref MigrationSourcePrefix - Name: MIGRATION_SOURCE_REGION Value: !Ref MigrationSourceRegion - - Name: MIGRATION_ACCESS_KEY - Value: !Ref MigrationAccessKey - - Name: MIGRATION_SECRET_KEY - Value: !Ref MigrationSecretKey + - Name: MIGRATION_DB_ACCESS_KEY + Value: !Ref MigrationDbAccessKey + - Name: MIGRATION_DB_SECRET_KEY + Value: !Ref MigrationDbSecretKey + - Name: MIGRATION_STORAGE_ACCESS_KEY + Value: !Ref MigrationStorageAccessKey + - Name: MIGRATION_STORAGE_SECRET_KEY + Value: !Ref MigrationStorageSecretKey Tags: - Key: platforma Value: !Ref ClusterName @@ -2771,27 +2797,34 @@ Resources: # --- Migration values --- if [ -n "$MIGRATION_DATABASE_S3_URI" ]; then - # Create migration credentials secret - if [ -n "$MIGRATION_ACCESS_KEY" ]; then - kubectl create secret generic platforma-migration-creds \ + # Create database dump credentials secret + if [ -n "$MIGRATION_DB_ACCESS_KEY" ]; then + kubectl create secret generic platforma-migration-db-creds \ -n $NAMESPACE \ - --from-literal=aws-access-key-id="$MIGRATION_ACCESS_KEY" \ - --from-literal=aws-secret-access-key="$MIGRATION_SECRET_KEY" \ + --from-literal=aws-access-key-id="$MIGRATION_DB_ACCESS_KEY" \ + --from-literal=aws-secret-access-key="$MIGRATION_DB_SECRET_KEY" \ --dry-run=client -o yaml | kubectl apply -f - fi cat >> /tmp/platforma-values.yaml << MIGRATION_BLOCK - migration: + dataMigration: enabled: true database: s3Uri: "${MIGRATION_DATABASE_S3_URI}" MIGRATION_BLOCK - if [ -n "$MIGRATION_ACCESS_KEY" ]; then - cat >> /tmp/platforma-values.yaml << MIGRATION_CREDS - credentials: - secretName: platforma-migration-creds - MIGRATION_CREDS + if [ -n "$MIGRATION_DB_ACCESS_KEY" ]; then + cat >> /tmp/platforma-values.yaml << MIGRATION_DB_CREDS + credentialsSecret: platforma-migration-db-creds + MIGRATION_DB_CREDS fi if [ -n "$MIGRATION_SOURCE_BUCKET" ]; then + # Create storage credentials secret + if [ -n "$MIGRATION_STORAGE_ACCESS_KEY" ]; then + kubectl create secret generic platforma-migration-storage-creds \ + -n $NAMESPACE \ + --from-literal=aws-access-key-id="$MIGRATION_STORAGE_ACCESS_KEY" \ + --from-literal=aws-secret-access-key="$MIGRATION_STORAGE_SECRET_KEY" \ + --dry-run=client -o yaml | kubectl apply -f - + fi cat >> /tmp/platforma-values.yaml << MIGRATION_SYNC storageSync: enabled: true @@ -2799,6 +2832,11 @@ Resources: sourcePrefix: "${MIGRATION_SOURCE_PREFIX}" sourceRegion: "${MIGRATION_SOURCE_REGION}" MIGRATION_SYNC + if [ -n "$MIGRATION_STORAGE_ACCESS_KEY" ]; then + cat >> /tmp/platforma-values.yaml << MIGRATION_STORAGE_CREDS + credentialsSecret: platforma-migration-storage-creds + MIGRATION_STORAGE_CREDS + fi fi fi From d0ce15920dc209ee698b94d7adac4add374f0550 Mon Sep 17 00:00:00 2001 From: Vladimir Antropov Date: Wed, 25 Mar 2026 17:25:06 +0100 Subject: [PATCH 03/42] feat: add data migration to CF as pre-helm kubectl pods Migration runs as standalone kubectl pods in the CF buildspec before helm install. Four sequential steps: 1. Download database dump from S3 2. Restore database using platforma --restore-db 3. Sync primary storage between S3 buckets 4. Invalidate caches using platforma --invalidate-caches Supports cross-account via MigrationStorageUserArn bucket policy. Idempotent via marker file on the database PVC. --- charts/platforma/templates/deployment.yaml | 213 ---------------- charts/platforma/values.yaml | 53 ---- .../aws/cloudformation-eks-1-35.yaml | 232 +++++++++++++++--- 3 files changed, 193 insertions(+), 305 deletions(-) diff --git a/charts/platforma/templates/deployment.yaml b/charts/platforma/templates/deployment.yaml index db27801..1fba775 100644 --- a/charts/platforma/templates/deployment.yaml +++ b/charts/platforma/templates/deployment.yaml @@ -75,214 +75,6 @@ spec: securityContext: fsGroup: 1010 runAsNonRoot: true - {{- $dbPath := include "platforma.path.database" . -}} - {{- if and .Values.dataMigration.enabled (or .Values.dataMigration.database.s3Uri .Values.dataMigration.storageSync.enabled) }} - initContainers: - # --------------------------------------------------------------- - # Init 1: Download database dump from S3 - # --------------------------------------------------------------- - {{- if .Values.dataMigration.database.s3Uri }} - - name: download-dump - image: "{{ .Values.dataMigration.awsCliImage.repository }}:{{ .Values.dataMigration.awsCliImage.tag }}" - securityContext: - runAsUser: 1010 - runAsGroup: 1010 - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - command: - - /bin/sh - - -ec - - | - MARKER="${DB_DIR}/.migration-complete" - if [ -f "$MARKER" ]; then - echo "Import already completed (marker exists), skipping download" - exit 0 - fi - echo "Downloading database dump from {{ .Values.dataMigration.database.s3Uri }}..." - aws s3 cp "{{ .Values.dataMigration.database.s3Uri }}" /tmp/migration/backup.gz - echo "Download complete" - env: - - name: DB_DIR - value: {{ $dbPath }} - {{- if .Values.dataMigration.database.credentialsSecret }} - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ .Values.dataMigration.database.credentialsSecret }} - key: aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.dataMigration.database.credentialsSecret }} - key: aws-secret-access-key - {{- end }} - volumeMounts: - - name: database - mountPath: {{ $dbPath }} - - name: migration-tmp - mountPath: /tmp/migration - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 1Gi - # --------------------------------------------------------------- - # Init 2: Restore database using Platforma --restore-db - # --------------------------------------------------------------- - - name: restore-database - image: {{ include "platforma.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - runAsUser: 1010 - runAsGroup: 1010 - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - command: - - /bin/sh - - -ec - - | - MARKER="${DB_DIR}/.migration-complete" - if [ -f "$MARKER" ]; then - echo "Import already completed (marker exists), skipping restore" - exit 0 - fi - echo "Restoring database..." - platforma --restore-db=/tmp/migration/backup.gz --db-dir="${DB_DIR}" --force - echo "Database restored" - env: - - name: DB_DIR - value: {{ $dbPath }} - - name: PL_LICENSE - valueFrom: - secretKeyRef: - name: {{ .Values.license.secretName }} - key: {{ .Values.license.secretKey }} - volumeMounts: - - name: database - mountPath: {{ $dbPath }} - - name: main-storage - mountPath: /data/main - - name: migration-tmp - mountPath: /tmp/migration - resources: - requests: - cpu: 500m - memory: 512Mi - limits: - memory: 2Gi - # --------------------------------------------------------------- - # Init 3: Sync primary storage from old S3 bucket - # (must complete before cache invalidation) - # --------------------------------------------------------------- - {{- if .Values.dataMigration.storageSync.enabled }} - - name: sync-storage - image: "{{ .Values.dataMigration.awsCliImage.repository }}:{{ .Values.dataMigration.awsCliImage.tag }}" - securityContext: - runAsUser: 1010 - runAsGroup: 1010 - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - command: - - /bin/sh - - -ec - - | - MARKER="${DB_DIR}/.migration-complete" - if [ -f "$MARKER" ]; then - echo "Import already completed (marker exists), skipping sync" - exit 0 - fi - - SOURCE="s3://{{ .Values.dataMigration.storageSync.sourceBucket }}/{{ .Values.dataMigration.storageSync.sourcePrefix }}" - DEST="s3://{{ .Values.storage.main.s3.bucket }}/{{ .Values.storage.main.s3.prefix }}" - SOURCE_REGION="{{ .Values.dataMigration.storageSync.sourceRegion | default .Values.storage.main.s3.region }}" - DEST_REGION="{{ .Values.storage.main.s3.region }}" - - echo "Syncing primary storage..." - echo " Source: $SOURCE (region: $SOURCE_REGION)" - echo " Dest: $DEST (region: $DEST_REGION)" - - aws s3 sync "$SOURCE" "$DEST" \ - --source-region "$SOURCE_REGION" \ - --region "$DEST_REGION" \ - --no-progress - - echo "Storage sync complete" - env: - - name: DB_DIR - value: {{ $dbPath }} - {{- if .Values.dataMigration.storageSync.credentialsSecret }} - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ .Values.dataMigration.storageSync.credentialsSecret }} - key: aws-access-key-id - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.dataMigration.storageSync.credentialsSecret }} - key: aws-secret-access-key - {{- end }} - volumeMounts: - - name: database - mountPath: {{ $dbPath }} - resources: - requests: - cpu: 500m - memory: 512Mi - limits: - memory: 4Gi - {{- end }} - # --------------------------------------------------------------- - # Init 4: Invalidate caches (after storage sync, before start) - # --------------------------------------------------------------- - - name: invalidate-caches - image: {{ include "platforma.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - runAsUser: 1010 - runAsGroup: 1010 - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - command: - - /bin/sh - - -ec - - | - MARKER="${DB_DIR}/.migration-complete" - if [ -f "$MARKER" ]; then - echo "Import already completed (marker exists), skipping invalidation" - exit 0 - fi - echo "Invalidating caches..." - platforma --invalidate-caches --main-root=/data/main --db-dir="${DB_DIR}" - echo "Caches invalidated" - date -u > "$MARKER" - echo "Import marker written" - env: - - name: DB_DIR - value: {{ $dbPath }} - - name: PL_LICENSE - valueFrom: - secretKeyRef: - name: {{ .Values.license.secretName }} - key: {{ .Values.license.secretKey }} - volumeMounts: - - name: database - mountPath: {{ $dbPath }} - - name: main-storage - mountPath: /data/main - resources: - requests: - cpu: 500m - memory: 512Mi - limits: - memory: 2Gi - {{- end }} - {{- end }} containers: - name: platforma image: {{ include "platforma.image" . }} @@ -595,11 +387,6 @@ spec: claimName: {{ include "platforma.workspacePvcName" . }} - name: main-storage emptyDir: {} - {{- if and .Values.dataMigration.enabled .Values.dataMigration.database.s3Uri }} - - name: migration-tmp - emptyDir: - sizeLimit: 30Gi - {{- end }} - name: job-templates configMap: name: {{ include "platforma.fullname" . }}-job-template diff --git a/charts/platforma/values.yaml b/charts/platforma/values.yaml index 1fdefbe..8d2ae54 100644 --- a/charts/platforma/values.yaml +++ b/charts/platforma/values.yaml @@ -471,59 +471,6 @@ jobServiceAccount: name: "" annotations: {} -# ============================================================================= -# DATA IMPORT (from existing Platforma installation) -# ============================================================================= -# One-time import from an existing Platforma server. When enabled, init -# containers restore the database and sync primary storage before Platforma -# starts. -# -# Prerequisites: -# 1. Create database dump on source server (requires --debug-enabled): -# curl -s http://:9091/db/state_raw | gzip > backup.gz -# 2. Upload dump to an S3 bucket: -# aws s3 cp backup.gz s3:///backup.gz -# 3. Set dataMigration values below and deploy -# 4. After successful start, set dataMigration.enabled: false and redeploy -# ============================================================================= - -dataMigration: - # -- Enable data import init containers - enabled: false - - # -- Image for S3 operations (download dump, sync storage) - awsCliImage: - repository: amazon/aws-cli - tag: "2.27.22" - - # -- Database restore from gzipped dump - database: - # -- S3 URI to database dump (created via /db/state_raw endpoint) - s3Uri: "" # e.g. s3://my-bucket/migration/backup.gz - # -- Credentials for the database dump S3 bucket - # Create the secret before deploying: - # kubectl create secret generic platforma-import-db-creds \ - # --from-literal=aws-access-key-id=AKIA... \ - # --from-literal=aws-secret-access-key=... - credentialsSecret: "" - - # -- Primary storage sync (copy data from old S3 bucket to new one) - storageSync: - # -- Enable storage sync (disable if you already copied data manually) - enabled: false - # -- Source S3 bucket name - sourceBucket: "" - # -- Source prefix within the bucket - sourcePrefix: "" - # -- Source bucket region (defaults to chart's storage.main.s3.region) - sourceRegion: "" - # -- Credentials for the source primary storage S3 bucket - # Create the secret before deploying: - # kubectl create secret generic platforma-import-storage-creds \ - # --from-literal=aws-access-key-id=AKIA... \ - # --from-literal=aws-secret-access-key=... - credentialsSecret: "" - # ============================================================================= # JOB CONFIGURATION # ============================================================================= diff --git a/infrastructure/aws/cloudformation-eks-1-35.yaml b/infrastructure/aws/cloudformation-eks-1-35.yaml index afc4f4a..a0f5352 100644 --- a/infrastructure/aws/cloudformation-eks-1-35.yaml +++ b/infrastructure/aws/cloudformation-eks-1-35.yaml @@ -67,6 +67,7 @@ Metadata: - MigrationSourceRegion - MigrationStorageAccessKey - MigrationStorageSecretKey + - MigrationStorageUserArn - Label: default: 'Data libraries (optional, up to 3 external read-only S3 buckets)' Parameters: @@ -135,6 +136,8 @@ Metadata: default: 'AWS access key for source primary storage bucket' MigrationStorageSecretKey: default: 'AWS secret key for source primary storage bucket' + MigrationStorageUserArn: + default: 'IAM ARN for cross-account write access to destination bucket' EnableDemoLibrary: default: 'Enable MiLaboratories demo data library' DataLibrary1Name: @@ -364,6 +367,16 @@ Parameters: Description: > AWS secret key for the source primary storage S3 bucket. + MigrationStorageUserArn: + Type: String + Default: '' + AllowedPattern: '^(arn:aws:iam::\d{12}:(user|role)/[a-zA-Z0-9_./@-]+)?$' + ConstraintDescription: 'Must be a valid IAM user or role ARN, or empty' + Description: > + IAM ARN of the user/role that will sync data to the new S3 bucket. + Required for cross-account migration — grants write access on the + destination bucket. Example: arn:aws:iam::123456789012:user/migration-user + # --- Data libraries --- EnableDemoLibrary: Type: String @@ -676,6 +689,7 @@ Conditions: HasLibrary3Irsa: !And - !Not [!Equals [!Ref DataLibrary3Bucket, '']] - !Equals [!Ref DataLibrary3AccessKey, ''] + HasMigrationStorageUserArn: !Not [!Equals [!Ref MigrationStorageUserArn, '']] UseDefaultPlatformaVersion: !Equals [!Ref PlatformaVersion, ''] IsHtpasswd: !Equals [!Ref AuthMethod, 'htpasswd'] SystemAZa: !Equals [!Ref SystemNodeAZ, 'a'] @@ -2795,49 +2809,181 @@ Resources: fi fi - # --- Migration values --- + # --- Data migration (runs before helm install) --- if [ -n "$MIGRATION_DATABASE_S3_URI" ]; then - # Create database dump credentials secret + echo "=== Data migration: restoring from existing installation ===" + PVC_NAME="platforma-database" + DB_DIR="/data/database" + MARKER_FILE="${DB_DIR}/.migration-complete" + PLATFORMA_IMG="${PLATFORMA_IMAGE:-quay.io/milaboratories/platforma:${PLATFORMA_VERSION}}" + AWS_CLI_IMG="amazon/aws-cli:2.27.22" + + # Ensure the database PVC exists (helm hasn't run yet) + kubectl get pvc $PVC_NAME -n $NAMESPACE 2>/dev/null || \ + kubectl apply -n $NAMESPACE -f - <> /tmp/platforma-values.yaml << MIGRATION_BLOCK - dataMigration: - enabled: true - database: - s3Uri: "${MIGRATION_DATABASE_S3_URI}" - MIGRATION_BLOCK - if [ -n "$MIGRATION_DB_ACCESS_KEY" ]; then - cat >> /tmp/platforma-values.yaml << MIGRATION_DB_CREDS - credentialsSecret: platforma-migration-db-creds - MIGRATION_DB_CREDS + + STORAGE_ENV_ARGS="" + if [ -n "$MIGRATION_STORAGE_ACCESS_KEY" ]; then + STORAGE_ENV_ARGS="--env=AWS_ACCESS_KEY_ID=$MIGRATION_STORAGE_ACCESS_KEY --env=AWS_SECRET_ACCESS_KEY=$MIGRATION_STORAGE_SECRET_KEY" fi + + # Step 1: Download database dump + echo "--- Step 1/4: Downloading database dump ---" + kubectl run migration-download \ + -n $NAMESPACE \ + --image=$AWS_CLI_IMG \ + --restart=Never \ + --overrides='{ + "spec": { + "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, + "containers": [{ + "name": "migration-download", + "image": "'$AWS_CLI_IMG'", + "command": ["/bin/sh", "-ec", + "if [ -f '$MARKER_FILE' ]; then echo Migration already done; exit 0; fi; echo Downloading...; aws s3 cp '$MIGRATION_DATABASE_S3_URI' '$DB_DIR'/backup.gz; echo Done"], + "volumeMounts": [{"name": "database", "mountPath": "'$DB_DIR'"}] + '$( [ -n "$MIGRATION_DB_ACCESS_KEY" ] && echo ",\"env\": [{\"name\":\"AWS_ACCESS_KEY_ID\",\"value\":\"$MIGRATION_DB_ACCESS_KEY\"},{\"name\":\"AWS_SECRET_ACCESS_KEY\",\"value\":\"$MIGRATION_DB_SECRET_KEY\"}]" )' + }], + "volumes": [{"name": "database", "persistentVolumeClaim": {"claimName": "'$PVC_NAME'"}}], + "restartPolicy": "Never" + } + }' \ + -- /bin/sh -c "true" + kubectl wait pod/migration-download -n $NAMESPACE --for=condition=Ready --timeout=60s 2>/dev/null || true + kubectl logs -f migration-download -n $NAMESPACE + kubectl wait pod/migration-download -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=600s + kubectl delete pod migration-download -n $NAMESPACE + + # Step 2: Restore database + echo "--- Step 2/4: Restoring database ---" + kubectl run migration-restore \ + -n $NAMESPACE \ + --image=$PLATFORMA_IMG \ + --restart=Never \ + --overrides='{ + "spec": { + "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, + "containers": [{ + "name": "migration-restore", + "image": "'$PLATFORMA_IMG'", + "command": ["/bin/sh", "-ec", + "if [ -f '$MARKER_FILE' ]; then echo Migration already done; exit 0; fi; echo Restoring database...; platforma --restore-db='$DB_DIR'/backup.gz --db-dir='$DB_DIR' --force; echo Database restored"], + "env": [{"name": "PL_LICENSE", "valueFrom": {"secretKeyRef": {"name": "platforma-license", "key": "MI_LICENSE"}}}], + "volumeMounts": [{"name": "database", "mountPath": "'$DB_DIR'"}] + }], + "volumes": [{"name": "database", "persistentVolumeClaim": {"claimName": "'$PVC_NAME'"}}], + "restartPolicy": "Never" + } + }' \ + -- /bin/sh -c "true" + kubectl wait pod/migration-restore -n $NAMESPACE --for=condition=Ready --timeout=120s 2>/dev/null || true + kubectl logs -f migration-restore -n $NAMESPACE + kubectl wait pod/migration-restore -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=1800s + kubectl delete pod migration-restore -n $NAMESPACE + + # Step 3: Sync primary storage (if configured) if [ -n "$MIGRATION_SOURCE_BUCKET" ]; then - # Create storage credentials secret - if [ -n "$MIGRATION_STORAGE_ACCESS_KEY" ]; then - kubectl create secret generic platforma-migration-storage-creds \ - -n $NAMESPACE \ - --from-literal=aws-access-key-id="$MIGRATION_STORAGE_ACCESS_KEY" \ - --from-literal=aws-secret-access-key="$MIGRATION_STORAGE_SECRET_KEY" \ - --dry-run=client -o yaml | kubectl apply -f - - fi - cat >> /tmp/platforma-values.yaml << MIGRATION_SYNC - storageSync: - enabled: true - sourceBucket: "${MIGRATION_SOURCE_BUCKET}" - sourcePrefix: "${MIGRATION_SOURCE_PREFIX}" - sourceRegion: "${MIGRATION_SOURCE_REGION}" - MIGRATION_SYNC - if [ -n "$MIGRATION_STORAGE_ACCESS_KEY" ]; then - cat >> /tmp/platforma-values.yaml << MIGRATION_STORAGE_CREDS - credentialsSecret: platforma-migration-storage-creds - MIGRATION_STORAGE_CREDS - fi + echo "--- Step 3/4: Syncing primary storage ---" + SOURCE_REGION="${MIGRATION_SOURCE_REGION:-$REGION}" + kubectl run migration-sync \ + -n $NAMESPACE \ + --image=$AWS_CLI_IMG \ + --restart=Never \ + --overrides='{ + "spec": { + "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, + "containers": [{ + "name": "migration-sync", + "image": "'$AWS_CLI_IMG'", + "command": ["/bin/sh", "-ec", + "echo Syncing s3://'$MIGRATION_SOURCE_BUCKET'/'$MIGRATION_SOURCE_PREFIX' to s3://'$S3_BUCKET'/...; aws s3 sync s3://'$MIGRATION_SOURCE_BUCKET'/'$MIGRATION_SOURCE_PREFIX' s3://'$S3_BUCKET'/ --source-region '$SOURCE_REGION' --region '$REGION' --no-progress; echo Sync complete"] + '$( [ -n "$MIGRATION_STORAGE_ACCESS_KEY" ] && echo ",\"env\": [{\"name\":\"AWS_ACCESS_KEY_ID\",\"value\":\"$MIGRATION_STORAGE_ACCESS_KEY\"},{\"name\":\"AWS_SECRET_ACCESS_KEY\",\"value\":\"$MIGRATION_STORAGE_SECRET_KEY\"}]" )' + }], + "restartPolicy": "Never" + } + }' \ + -- /bin/sh -c "true" + kubectl wait pod/migration-sync -n $NAMESPACE --for=condition=Ready --timeout=60s 2>/dev/null || true + kubectl logs -f migration-sync -n $NAMESPACE + kubectl wait pod/migration-sync -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=86400s + kubectl delete pod migration-sync -n $NAMESPACE + else + echo "--- Step 3/4: Skipping storage sync (no source bucket) ---" fi + + # Step 4: Invalidate caches + echo "--- Step 4/4: Invalidating caches ---" + kubectl run migration-invalidate \ + -n $NAMESPACE \ + --image=$PLATFORMA_IMG \ + --restart=Never \ + --overrides='{ + "spec": { + "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, + "containers": [{ + "name": "migration-invalidate", + "image": "'$PLATFORMA_IMG'", + "command": ["/bin/sh", "-ec", + "if [ -f '$MARKER_FILE' ]; then echo Migration already done; exit 0; fi; echo Invalidating caches...; platforma --invalidate-caches --main-root=/data/main --db-dir='$DB_DIR'; date -u > '$MARKER_FILE'; echo Caches invalidated; echo Migration complete"], + "env": [{"name": "PL_LICENSE", "valueFrom": {"secretKeyRef": {"name": "platforma-license", "key": "MI_LICENSE"}}}], + "volumeMounts": [{"name": "database", "mountPath": "'$DB_DIR'"}] + }], + "volumes": [{"name": "database", "persistentVolumeClaim": {"claimName": "'$PVC_NAME'"}}], + "restartPolicy": "Never" + } + }' \ + -- /bin/sh -c "true" + kubectl wait pod/migration-invalidate -n $NAMESPACE --for=condition=Ready --timeout=120s 2>/dev/null || true + kubectl logs -f migration-invalidate -n $NAMESPACE + kubectl wait pod/migration-invalidate -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=1800s + kubectl delete pod migration-invalidate -n $NAMESPACE + + # Clean up dump file + echo "--- Cleaning up dump file ---" + kubectl run migration-cleanup \ + -n $NAMESPACE \ + --image=$AWS_CLI_IMG \ + --restart=Never \ + --overrides='{ + "spec": { + "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, + "containers": [{ + "name": "migration-cleanup", + "image": "'$AWS_CLI_IMG'", + "command": ["/bin/sh", "-ec", "rm -f '$DB_DIR'/backup.gz; echo Cleanup done"], + "volumeMounts": [{"name": "database", "mountPath": "'$DB_DIR'"}] + }], + "volumes": [{"name": "database", "persistentVolumeClaim": {"claimName": "'$PVC_NAME'"}}], + "restartPolicy": "Never" + } + }' \ + -- /bin/sh -c "true" + kubectl wait pod/migration-cleanup -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=120s + kubectl delete pod migration-cleanup -n $NAMESPACE + + echo "=== Data migration complete ===" fi if [ -n "$PLATFORMA_IMAGE" ]; then @@ -2850,17 +2996,11 @@ Resources: IMAGE_BLOCK fi - # Increase timeout when migration is active (S3 sync can take hours) - HELM_TIMEOUT="15m" - if [ -n "$MIGRATION_DATABASE_S3_URI" ]; then - HELM_TIMEOUT="120m" - fi - helm upgrade --install platforma oci://ghcr.io/milaboratory/platforma-helm/platforma \ --version $PLATFORMA_VERSION \ -n $NAMESPACE \ -f /tmp/platforma-values.yaml \ - --atomic --timeout $HELM_TIMEOUT + --atomic --timeout 15m post_build: commands: - | @@ -3145,6 +3285,20 @@ Resources: # the previous deny-all-except-role pattern because it locked out # account administrators from cleanup after stack deletion # (the IRSA role is deleted with the stack, leaving no one with access). + - !If + - HasMigrationStorageUserArn + - Sid: AllowMigrationUserWrite + Effect: Allow + Principal: + AWS: !Ref MigrationStorageUserArn + Action: + - s3:PutObject + - s3:GetObject + - s3:ListBucket + Resource: + - !GetAtt S3Bucket.Arn + - !Sub '${S3Bucket.Arn}/*' + - !Ref AWS::NoValue # --------------------------------------------------------------------------- # 8. ACM Certificate (DNS-validated via Route53) From a1e217ea97e2c39e2dd45c0865c789ea60a99372 Mon Sep 17 00:00:00 2001 From: Vladimir Antropov Date: Wed, 25 Mar 2026 19:13:32 +0100 Subject: [PATCH 04/42] fix: compact migration script to fit CodeBuild 25.6KB buildspec limit --- .../aws/cloudformation-eks-1-35.yaml | 178 +----------------- infrastructure/aws/migration.sh | 154 +++++++++++++++ 2 files changed, 158 insertions(+), 174 deletions(-) create mode 100644 infrastructure/aws/migration.sh diff --git a/infrastructure/aws/cloudformation-eks-1-35.yaml b/infrastructure/aws/cloudformation-eks-1-35.yaml index a0f5352..85c7792 100644 --- a/infrastructure/aws/cloudformation-eks-1-35.yaml +++ b/infrastructure/aws/cloudformation-eks-1-35.yaml @@ -2809,181 +2809,11 @@ Resources: fi fi - # --- Data migration (runs before helm install) --- + # --- Data migration (download script and run) --- if [ -n "$MIGRATION_DATABASE_S3_URI" ]; then - echo "=== Data migration: restoring from existing installation ===" - PVC_NAME="platforma-database" - DB_DIR="/data/database" - MARKER_FILE="${DB_DIR}/.migration-complete" - PLATFORMA_IMG="${PLATFORMA_IMAGE:-quay.io/milaboratories/platforma:${PLATFORMA_VERSION}}" - AWS_CLI_IMG="amazon/aws-cli:2.27.22" - - # Ensure the database PVC exists (helm hasn't run yet) - kubectl get pvc $PVC_NAME -n $NAMESPACE 2>/dev/null || \ - kubectl apply -n $NAMESPACE -f - </dev/null || true - kubectl logs -f migration-download -n $NAMESPACE - kubectl wait pod/migration-download -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=600s - kubectl delete pod migration-download -n $NAMESPACE - - # Step 2: Restore database - echo "--- Step 2/4: Restoring database ---" - kubectl run migration-restore \ - -n $NAMESPACE \ - --image=$PLATFORMA_IMG \ - --restart=Never \ - --overrides='{ - "spec": { - "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, - "containers": [{ - "name": "migration-restore", - "image": "'$PLATFORMA_IMG'", - "command": ["/bin/sh", "-ec", - "if [ -f '$MARKER_FILE' ]; then echo Migration already done; exit 0; fi; echo Restoring database...; platforma --restore-db='$DB_DIR'/backup.gz --db-dir='$DB_DIR' --force; echo Database restored"], - "env": [{"name": "PL_LICENSE", "valueFrom": {"secretKeyRef": {"name": "platforma-license", "key": "MI_LICENSE"}}}], - "volumeMounts": [{"name": "database", "mountPath": "'$DB_DIR'"}] - }], - "volumes": [{"name": "database", "persistentVolumeClaim": {"claimName": "'$PVC_NAME'"}}], - "restartPolicy": "Never" - } - }' \ - -- /bin/sh -c "true" - kubectl wait pod/migration-restore -n $NAMESPACE --for=condition=Ready --timeout=120s 2>/dev/null || true - kubectl logs -f migration-restore -n $NAMESPACE - kubectl wait pod/migration-restore -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=1800s - kubectl delete pod migration-restore -n $NAMESPACE - - # Step 3: Sync primary storage (if configured) - if [ -n "$MIGRATION_SOURCE_BUCKET" ]; then - echo "--- Step 3/4: Syncing primary storage ---" - SOURCE_REGION="${MIGRATION_SOURCE_REGION:-$REGION}" - kubectl run migration-sync \ - -n $NAMESPACE \ - --image=$AWS_CLI_IMG \ - --restart=Never \ - --overrides='{ - "spec": { - "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, - "containers": [{ - "name": "migration-sync", - "image": "'$AWS_CLI_IMG'", - "command": ["/bin/sh", "-ec", - "echo Syncing s3://'$MIGRATION_SOURCE_BUCKET'/'$MIGRATION_SOURCE_PREFIX' to s3://'$S3_BUCKET'/...; aws s3 sync s3://'$MIGRATION_SOURCE_BUCKET'/'$MIGRATION_SOURCE_PREFIX' s3://'$S3_BUCKET'/ --source-region '$SOURCE_REGION' --region '$REGION' --no-progress; echo Sync complete"] - '$( [ -n "$MIGRATION_STORAGE_ACCESS_KEY" ] && echo ",\"env\": [{\"name\":\"AWS_ACCESS_KEY_ID\",\"value\":\"$MIGRATION_STORAGE_ACCESS_KEY\"},{\"name\":\"AWS_SECRET_ACCESS_KEY\",\"value\":\"$MIGRATION_STORAGE_SECRET_KEY\"}]" )' - }], - "restartPolicy": "Never" - } - }' \ - -- /bin/sh -c "true" - kubectl wait pod/migration-sync -n $NAMESPACE --for=condition=Ready --timeout=60s 2>/dev/null || true - kubectl logs -f migration-sync -n $NAMESPACE - kubectl wait pod/migration-sync -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=86400s - kubectl delete pod migration-sync -n $NAMESPACE - else - echo "--- Step 3/4: Skipping storage sync (no source bucket) ---" - fi - - # Step 4: Invalidate caches - echo "--- Step 4/4: Invalidating caches ---" - kubectl run migration-invalidate \ - -n $NAMESPACE \ - --image=$PLATFORMA_IMG \ - --restart=Never \ - --overrides='{ - "spec": { - "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, - "containers": [{ - "name": "migration-invalidate", - "image": "'$PLATFORMA_IMG'", - "command": ["/bin/sh", "-ec", - "if [ -f '$MARKER_FILE' ]; then echo Migration already done; exit 0; fi; echo Invalidating caches...; platforma --invalidate-caches --main-root=/data/main --db-dir='$DB_DIR'; date -u > '$MARKER_FILE'; echo Caches invalidated; echo Migration complete"], - "env": [{"name": "PL_LICENSE", "valueFrom": {"secretKeyRef": {"name": "platforma-license", "key": "MI_LICENSE"}}}], - "volumeMounts": [{"name": "database", "mountPath": "'$DB_DIR'"}] - }], - "volumes": [{"name": "database", "persistentVolumeClaim": {"claimName": "'$PVC_NAME'"}}], - "restartPolicy": "Never" - } - }' \ - -- /bin/sh -c "true" - kubectl wait pod/migration-invalidate -n $NAMESPACE --for=condition=Ready --timeout=120s 2>/dev/null || true - kubectl logs -f migration-invalidate -n $NAMESPACE - kubectl wait pod/migration-invalidate -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=1800s - kubectl delete pod migration-invalidate -n $NAMESPACE - - # Clean up dump file - echo "--- Cleaning up dump file ---" - kubectl run migration-cleanup \ - -n $NAMESPACE \ - --image=$AWS_CLI_IMG \ - --restart=Never \ - --overrides='{ - "spec": { - "securityContext": {"fsGroup": 1010, "runAsUser": 1010, "runAsGroup": 1010}, - "containers": [{ - "name": "migration-cleanup", - "image": "'$AWS_CLI_IMG'", - "command": ["/bin/sh", "-ec", "rm -f '$DB_DIR'/backup.gz; echo Cleanup done"], - "volumeMounts": [{"name": "database", "mountPath": "'$DB_DIR'"}] - }], - "volumes": [{"name": "database", "persistentVolumeClaim": {"claimName": "'$PVC_NAME'"}}], - "restartPolicy": "Never" - } - }' \ - -- /bin/sh -c "true" - kubectl wait pod/migration-cleanup -n $NAMESPACE --for=jsonpath='{.status.phase}'=Succeeded --timeout=120s - kubectl delete pod migration-cleanup -n $NAMESPACE - - echo "=== Data migration complete ===" + curl -fsSL "https://platforma-cloudformation.s3.eu-central-1.amazonaws.com/migration.sh" -o /tmp/migration.sh + chmod +x /tmp/migration.sh + /tmp/migration.sh fi if [ -n "$PLATFORMA_IMAGE" ]; then diff --git a/infrastructure/aws/migration.sh b/infrastructure/aws/migration.sh new file mode 100644 index 0000000..96d6e98 --- /dev/null +++ b/infrastructure/aws/migration.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# ============================================================================= +# Platforma data migration script +# ============================================================================= +# Restores a Platforma database from a dump and optionally syncs primary +# storage from an existing S3 bucket. Runs as kubectl pods inside the +# target EKS cluster. +# +# Required environment variables: +# NAMESPACE - Kubernetes namespace +# REGION - AWS region +# S3_BUCKET - Destination S3 bucket (created by CF) +# PLATFORMA_VERSION - Platforma version (used for image tag) +# MIGRATION_DATABASE_S3_URI - S3 URI to database dump (e.g. s3://bucket/backup.gz) +# +# Optional environment variables: +# PLATFORMA_IMAGE - Custom platforma image (overrides default) +# MIGRATION_DB_ACCESS_KEY - AWS access key for database dump bucket +# MIGRATION_DB_SECRET_KEY - AWS secret key for database dump bucket +# MIGRATION_SOURCE_BUCKET - Source primary storage bucket (enables sync) +# MIGRATION_SOURCE_PREFIX - Prefix within source bucket +# MIGRATION_SOURCE_REGION - Source bucket region (defaults to REGION) +# MIGRATION_STORAGE_ACCESS_KEY - AWS access key for source storage bucket +# MIGRATION_STORAGE_SECRET_KEY - AWS secret key for source storage bucket +# ============================================================================= +set -euo pipefail + +if [ -z "${MIGRATION_DATABASE_S3_URI:-}" ]; then + echo "MIGRATION_DATABASE_S3_URI not set, skipping migration" + exit 0 +fi + +echo "=== Platforma data migration ===" + +# Configuration +PVC="platforma-database" +DB_DIR="/data/database" +MARKER="$DB_DIR/.migration-complete" +PL_IMG="${PLATFORMA_IMAGE:-quay.io/milaboratories/platforma:${PLATFORMA_VERSION}}" +AWS_IMG="amazon/aws-cli:2.27.22" +SEC_CTX='{"fsGroup":1010,"runAsUser":1010,"runAsGroup":1010}' +VOLS='[{"name":"db","persistentVolumeClaim":{"claimName":"'$PVC'"}}]' +MNTS='[{"name":"db","mountPath":"'$DB_DIR'"}]' + +# Helper: run a pod, stream logs, wait for success, cleanup +run_pod() { + local name="$1" image="$2" script="$3" env_json="$4" timeout="$5" + + echo " Starting pod: $name" + kubectl delete pod/"$name" -n "$NAMESPACE" --ignore-not-found 2>/dev/null + + cat </dev/null || true + kubectl logs -f "$name" -n "$NAMESPACE" || true + kubectl wait pod/"$name" -n "$NAMESPACE" --for=jsonpath='{.status.phase}'=Succeeded --timeout="${timeout}s" + kubectl delete pod/"$name" -n "$NAMESPACE" + echo " Pod $name completed" +} + +# Ensure database PVC exists (helm may not have run yet) +if ! kubectl get pvc "$PVC" -n "$NAMESPACE" 2>/dev/null; then + echo "Creating database PVC..." + cat < $MARKER +echo 'Caches invalidated'" \ + "$LIC_ENV" 1800 + +# Cleanup dump file +echo "--- Cleaning up dump file ---" +run_pod mgr-cleanup "$AWS_IMG" \ + "rm -f $DB_DIR/backup.gz; echo 'Cleanup done'" \ + "[]" 120 + +echo "=== Platforma data migration complete ===" From 5891e96536c2ec1b4e186b1a8be7be7b583101f4 Mon Sep 17 00:00:00 2001 From: Vladimir Antropov Date: Wed, 25 Mar 2026 19:49:24 +0100 Subject: [PATCH 05/42] fix: rewrite migration.sh with proper YAML and Helm PVC labels --- infrastructure/aws/migration.sh | 184 ++++++++++++++++++++++---------- 1 file changed, 130 insertions(+), 54 deletions(-) diff --git a/infrastructure/aws/migration.sh b/infrastructure/aws/migration.sh index 96d6e98..82d3956 100644 --- a/infrastructure/aws/migration.sh +++ b/infrastructure/aws/migration.sh @@ -11,17 +11,17 @@ # REGION - AWS region # S3_BUCKET - Destination S3 bucket (created by CF) # PLATFORMA_VERSION - Platforma version (used for image tag) -# MIGRATION_DATABASE_S3_URI - S3 URI to database dump (e.g. s3://bucket/backup.gz) +# MIGRATION_DATABASE_S3_URI - S3 URI to database dump # # Optional environment variables: -# PLATFORMA_IMAGE - Custom platforma image (overrides default) -# MIGRATION_DB_ACCESS_KEY - AWS access key for database dump bucket -# MIGRATION_DB_SECRET_KEY - AWS secret key for database dump bucket -# MIGRATION_SOURCE_BUCKET - Source primary storage bucket (enables sync) +# PLATFORMA_IMAGE - Custom platforma image +# MIGRATION_DB_ACCESS_KEY - AWS access key for dump bucket +# MIGRATION_DB_SECRET_KEY - AWS secret key for dump bucket +# MIGRATION_SOURCE_BUCKET - Source primary storage bucket # MIGRATION_SOURCE_PREFIX - Prefix within source bucket # MIGRATION_SOURCE_REGION - Source bucket region (defaults to REGION) -# MIGRATION_STORAGE_ACCESS_KEY - AWS access key for source storage bucket -# MIGRATION_STORAGE_SECRET_KEY - AWS secret key for source storage bucket +# MIGRATION_STORAGE_ACCESS_KEY - AWS access key for source storage +# MIGRATION_STORAGE_SECRET_KEY - AWS secret key for source storage # ============================================================================= set -euo pipefail @@ -32,42 +32,109 @@ fi echo "=== Platforma data migration ===" -# Configuration PVC="platforma-database" DB_DIR="/data/database" MARKER="$DB_DIR/.migration-complete" PL_IMG="${PLATFORMA_IMAGE:-quay.io/milaboratories/platforma:${PLATFORMA_VERSION}}" AWS_IMG="amazon/aws-cli:2.27.22" -SEC_CTX='{"fsGroup":1010,"runAsUser":1010,"runAsGroup":1010}' -VOLS='[{"name":"db","persistentVolumeClaim":{"claimName":"'$PVC'"}}]' -MNTS='[{"name":"db","mountPath":"'$DB_DIR'"}]' -# Helper: run a pod, stream logs, wait for success, cleanup +# Helper: run a pod with database PVC, stream logs, wait, cleanup run_pod() { - local name="$1" image="$2" script="$3" env_json="$4" timeout="$5" + local name="$1" image="$2" script="$3" timeout="$4" + shift 4 + # remaining args are optional: env var pairs NAME=VALUE echo " Starting pod: $name" kubectl delete pod/"$name" -n "$NAMESPACE" --ignore-not-found 2>/dev/null - cat </dev/null || true + kubectl logs -f "$name" -n "$NAMESPACE" || true + kubectl wait pod/"$name" -n "$NAMESPACE" --for=jsonpath='{.status.phase}'=Succeeded --timeout="${timeout}s" + kubectl delete pod/"$name" -n "$NAMESPACE" + echo " Pod $name completed" +} + +# Helper: run a pod that needs PL_LICENSE from secret +run_pl_pod() { + local name="$1" script="$2" timeout="$3" + + echo " Starting pod: $name" + kubectl delete pod/"$name" -n "$NAMESPACE" --ignore-not-found 2>/dev/null + + cat </dev/null || true kubectl logs -f "$name" -n "$NAMESPACE" || true @@ -76,7 +143,10 @@ EOF echo " Pod $name completed" } -# Ensure database PVC exists (helm may not have run yet) +# Ensure namespace exists +kubectl create namespace "$NAMESPACE" 2>/dev/null || true + +# Ensure database PVC exists with Helm labels (so helm install can adopt it) if ! kubectl get pvc "$PVC" -n "$NAMESPACE" 2>/dev/null; then echo "Creating database PVC..." cat < $MARKER echo 'Caches invalidated'" \ - "$LIC_ENV" 1800 + 1800 # Cleanup dump file echo "--- Cleaning up dump file ---" -run_pod mgr-cleanup "$AWS_IMG" \ - "rm -f $DB_DIR/backup.gz; echo 'Cleanup done'" \ - "[]" 120 +run_pod mgr-cleanup "$AWS_IMG" "rm -f $DB_DIR/backup.gz; echo 'Cleanup done'" 120 echo "=== Platforma data migration complete ===" From fe76f4a31d8d7e491688f94feefc63098f05edee Mon Sep 17 00:00:00 2001 From: Vladimir Antropov Date: Wed, 25 Mar 2026 19:58:21 +0100 Subject: [PATCH 06/42] fix: rewrite migration.sh with proper YAML generation via functions --- infrastructure/aws/migration.sh | 210 +++++++++++++++----------------- 1 file changed, 95 insertions(+), 115 deletions(-) diff --git a/infrastructure/aws/migration.sh b/infrastructure/aws/migration.sh index 82d3956..aabe469 100644 --- a/infrastructure/aws/migration.sh +++ b/infrastructure/aws/migration.sh @@ -2,27 +2,6 @@ # ============================================================================= # Platforma data migration script # ============================================================================= -# Restores a Platforma database from a dump and optionally syncs primary -# storage from an existing S3 bucket. Runs as kubectl pods inside the -# target EKS cluster. -# -# Required environment variables: -# NAMESPACE - Kubernetes namespace -# REGION - AWS region -# S3_BUCKET - Destination S3 bucket (created by CF) -# PLATFORMA_VERSION - Platforma version (used for image tag) -# MIGRATION_DATABASE_S3_URI - S3 URI to database dump -# -# Optional environment variables: -# PLATFORMA_IMAGE - Custom platforma image -# MIGRATION_DB_ACCESS_KEY - AWS access key for dump bucket -# MIGRATION_DB_SECRET_KEY - AWS secret key for dump bucket -# MIGRATION_SOURCE_BUCKET - Source primary storage bucket -# MIGRATION_SOURCE_PREFIX - Prefix within source bucket -# MIGRATION_SOURCE_REGION - Source bucket region (defaults to REGION) -# MIGRATION_STORAGE_ACCESS_KEY - AWS access key for source storage -# MIGRATION_STORAGE_SECRET_KEY - AWS secret key for source storage -# ============================================================================= set -euo pipefail if [ -z "${MIGRATION_DATABASE_S3_URI:-}" ]; then @@ -38,34 +17,38 @@ MARKER="$DB_DIR/.migration-complete" PL_IMG="${PLATFORMA_IMAGE:-quay.io/milaboratories/platforma:${PLATFORMA_VERSION}}" AWS_IMG="amazon/aws-cli:2.27.22" -# Helper: run a pod with database PVC, stream logs, wait, cleanup -run_pod() { - local name="$1" image="$2" script="$3" timeout="$4" - shift 4 - # remaining args are optional: env var pairs NAME=VALUE +# Helper: wait for pod completion, stream logs, cleanup +wait_pod() { + local name="$1" timeout="$2" + kubectl wait pod/"$name" -n "$NAMESPACE" --for=condition=Ready --timeout=120s 2>/dev/null || true + kubectl logs -f "$name" -n "$NAMESPACE" || true + kubectl wait pod/"$name" -n "$NAMESPACE" --for=jsonpath='{.status.phase}'=Succeeded --timeout="${timeout}s" + kubectl delete pod/"$name" -n "$NAMESPACE" + echo " Pod $name completed" +} - echo " Starting pod: $name" - kubectl delete pod/"$name" -n "$NAMESPACE" --ignore-not-found 2>/dev/null +# Helper: generate pod YAML +# Usage: make_pod