diff --git a/.github/workflows/publish_helm_charts.yml b/.github/workflows/publish_helm_charts.yml new file mode 100644 index 0000000..4e7d290 --- /dev/null +++ b/.github/workflows/publish_helm_charts.yml @@ -0,0 +1,34 @@ +name: Release Charts + +on: + push: + branches: + - master + +jobs: + release: + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + + - name: Install Helm + uses: azure/setup-helm@v4 + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.6.0 + with: + charts_dir: charts/metaflow + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/charts/metaflow/Chart.lock b/charts/metaflow/Chart.lock new file mode 100644 index 0000000..a6ebc0c --- /dev/null +++ b/charts/metaflow/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: metaflow-service + repository: "" + version: 0.2.0 +- name: metaflow-ui + repository: "" + version: 0.3.0 +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 14.3.3 +digest: sha256:c9397633f678654b4d610387e66930c1a08461e639d58c4cba2e460362a22dd7 +generated: "2024-12-17T14:23:51.728248-08:00" diff --git a/charts/metaflow/Chart.yaml b/charts/metaflow/Chart.yaml new file mode 100644 index 0000000..c267e67 --- /dev/null +++ b/charts/metaflow/Chart.yaml @@ -0,0 +1,28 @@ +apiVersion: v2 +appVersion: v2.4.13 +dependencies: + - name: metaflow-service + repository: "" + version: 0.2.0 + condition: metaflow-service.enabled + - name: metaflow-ui + repository: "" + version: 0.3.0 + condition: metaflow-ui.enabled + - name: postgresql + repository: https://charts.bitnami.com/bitnami + version: "^14" + condition: postgresql.enabled +description: Machine Learning platform +home: https://metaflow.org +keywords: + - mlops + - metaflow + - metadata service + - metaflow ui +name: metaflow +sources: + - https://github.com/Netflix/metaflow + - https://github.com/outerbounds/metaflow-tools +type: application +version: 0.0.2 diff --git a/k8s/helm/metaflow/LICENSE b/charts/metaflow/LICENSE similarity index 100% rename from k8s/helm/metaflow/LICENSE rename to charts/metaflow/LICENSE diff --git a/charts/metaflow/README.md b/charts/metaflow/README.md new file mode 100644 index 0000000..4b1c67f --- /dev/null +++ b/charts/metaflow/README.md @@ -0,0 +1,10 @@ +# Helm Chart for Metaflow Services + +This Helm chart provides a comprehensive deployment solution for Metaflow Services in a Kubernetes cluster. It includes three main components: + +* Metaflow Metadata Service - Manages workflow metadata and execution state +* Metaflow UI (Frontend and Backend) - Provides web-based visualization and monitoring +* PostgreSQL Database - Stores metadata for Metaflow services + +While the included PostgreSQL sub-chart enables quick evaluation and development setups, for production environments we strongly recommend using a managed PostgreSQL service (like AWS RDS). To use an external database, disable the PostgreSQL sub-chart by setting `postgresql.enabled: false`. + diff --git a/k8s/helm/metaflow/charts/metaflow-ui/.helmignore b/charts/metaflow/charts/metaflow-service/.helmignore similarity index 61% rename from k8s/helm/metaflow/charts/metaflow-ui/.helmignore rename to charts/metaflow/charts/metaflow-service/.helmignore index 0e8a0eb..04bf987 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/.helmignore +++ b/charts/metaflow/charts/metaflow-service/.helmignore @@ -2,6 +2,9 @@ # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. .DS_Store +Thumbs.db +ehthumbs.db + # Common VCS dirs .git/ .gitignore @@ -10,14 +13,40 @@ .hg/ .hgignore .svn/ + # Common backup files *.swp *.bak *.tmp *.orig *~ + # Various IDEs .project .idea/ *.tmproj .vscode/ +*.suo +*.user +*.userossc +*.sln.docstates + +# Logs +*.log + +# Node.js +node_modules/ +npm-debug.log +yarn-error.log + +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd +.Python +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ diff --git a/charts/metaflow/charts/metaflow-service/Chart.yaml b/charts/metaflow/charts/metaflow-service/Chart.yaml new file mode 100644 index 0000000..6286484 --- /dev/null +++ b/charts/metaflow/charts/metaflow-service/Chart.yaml @@ -0,0 +1,19 @@ +apiVersion: v2 +appVersion: v2.4.13 +description: A Helm chart to deploy Metadata Service for Metaflow +name: metaflow-service +type: application +version: 0.2.0 +maintainers: + - name: Savin Goyal + email: savin@outerbounds.co + - name: Oleg Avdeev + email: oleg@outerbounds.co +keywords: + - metaflow + - metadata + - service +sources: + - https://github.com/Netflix/metaflow-service + - https://github.com/outerbounds/metaflow-tools +icon: https://metaflow.org/images/metaflow.svg diff --git a/k8s/helm/metaflow/charts/metaflow-service/LICENSE b/charts/metaflow/charts/metaflow-service/LICENSE similarity index 100% rename from k8s/helm/metaflow/charts/metaflow-service/LICENSE rename to charts/metaflow/charts/metaflow-service/LICENSE diff --git a/k8s/helm/metaflow/charts/metaflow-service/templates/NOTES.txt b/charts/metaflow/charts/metaflow-service/templates/NOTES.txt similarity index 53% rename from k8s/helm/metaflow/charts/metaflow-service/templates/NOTES.txt rename to charts/metaflow/charts/metaflow-service/templates/NOTES.txt index be5330d..c54f8f8 100644 --- a/k8s/helm/metaflow/charts/metaflow-service/templates/NOTES.txt +++ b/charts/metaflow/charts/metaflow-service/templates/NOTES.txt @@ -1,22 +1,34 @@ 1. Get the application URL by running these commands: {{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} {{- end }} -{{- end }} {{- else if contains "NodePort" .Values.service.type }} export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "metaflow-service.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT + if [ -z "$NODE_PORT" ] || [ -z "$NODE_IP" ]; then + echo "Error: Unable to retrieve NodePort or Node IP." + else + echo http://$NODE_IP:$NODE_PORT + fi {{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "metaflow-service.fullname" . }}' + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "metaflow-service.fullname" . }}' export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "metaflow-service.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} + if [ -z "$SERVICE_IP" ]; then + echo "Error: Unable to retrieve LoadBalancer IP." + else + echo http://$SERVICE_IP:{{ .Values.service.port }} + fi {{- else if contains "ClusterIP" .Values.service.type }} export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "metaflow-service.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT + if [ -z "$POD_NAME" ] || [ -z "$CONTAINER_PORT" ]; then + echo "Error: Unable to retrieve Pod name or Container port." + else + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT + fi {{- end }} diff --git a/k8s/helm/metaflow/charts/metaflow-service/templates/_helpers.tpl b/charts/metaflow/charts/metaflow-service/templates/_helpers.tpl similarity index 100% rename from k8s/helm/metaflow/charts/metaflow-service/templates/_helpers.tpl rename to charts/metaflow/charts/metaflow-service/templates/_helpers.tpl diff --git a/k8s/helm/metaflow/charts/metaflow-service/templates/deployment.yaml b/charts/metaflow/charts/metaflow-service/templates/deployment.yaml similarity index 95% rename from k8s/helm/metaflow/charts/metaflow-service/templates/deployment.yaml rename to charts/metaflow/charts/metaflow-service/templates/deployment.yaml index 426dacf..0ac31e6 100644 --- a/k8s/helm/metaflow/charts/metaflow-service/templates/deployment.yaml +++ b/charts/metaflow/charts/metaflow-service/templates/deployment.yaml @@ -6,7 +6,7 @@ metadata: {{- include "metaflow-service.labels" . | nindent 4 }} spec: {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} + replicas: {{ .Values.replicaCount | default 1 }} {{- end }} selector: matchLabels: @@ -46,7 +46,7 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" command: ["/opt/latest/bin/python3", "-m", "services.metadata_service.server" ] - imagePullPolicy: {{ .Values.image.pullPolicy }} + imagePullPolicy: {{ .Values.image.pullPolicy | default "IfNotPresent" }} ports: - name: http containerPort: 8080 diff --git a/charts/metaflow/charts/metaflow-service/templates/hpa.yaml b/charts/metaflow/charts/metaflow-service/templates/hpa.yaml new file mode 100644 index 0000000..39a3b4c --- /dev/null +++ b/charts/metaflow/charts/metaflow-service/templates/hpa.yaml @@ -0,0 +1,27 @@ +{{ if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "metaflow-service.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "metaflow-service.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} +{{ end }} diff --git a/k8s/helm/metaflow/charts/metaflow-service/templates/ingress.yaml b/charts/metaflow/charts/metaflow-service/templates/ingress.yaml similarity index 79% rename from k8s/helm/metaflow/charts/metaflow-service/templates/ingress.yaml rename to charts/metaflow/charts/metaflow-service/templates/ingress.yaml index 2755c6a..1d31cb8 100644 --- a/k8s/helm/metaflow/charts/metaflow-service/templates/ingress.yaml +++ b/charts/metaflow/charts/metaflow-service/templates/ingress.yaml @@ -1,14 +1,15 @@ {{- if .Values.ingress.enabled -}} {{- $fullName := include "metaflow-service.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} +{{- $kubeVersion := default "1.14.0" (lookup "version" "") -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" $kubeVersion)) }} {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} {{- end }} {{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare ">=1.19-0" $kubeVersion -}} apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- else if semverCompare ">=1.14-0" $kubeVersion -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 @@ -23,7 +24,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" $kubeVersion) }} ingressClassName: {{ .Values.ingress.className }} {{- end }} {{- if .Values.ingress.tls }} @@ -43,11 +44,11 @@ spec: paths: {{- range .paths }} - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + {{- if and .pathType (semverCompare ">=1.18-0" $kubeVersion) }} pathType: {{ .pathType }} {{- end }} backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + {{- if semverCompare ">=1.19-0" $kubeVersion }} service: name: {{ $fullName }} port: diff --git a/k8s/helm/metaflow/charts/metaflow-service/templates/service.yaml b/charts/metaflow/charts/metaflow-service/templates/service.yaml similarity index 87% rename from k8s/helm/metaflow/charts/metaflow-service/templates/service.yaml rename to charts/metaflow/charts/metaflow-service/templates/service.yaml index 4d0fc44..14e0209 100644 --- a/k8s/helm/metaflow/charts/metaflow-service/templates/service.yaml +++ b/charts/metaflow/charts/metaflow-service/templates/service.yaml @@ -9,7 +9,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - type: {{ .Values.service.type }} + type: {{ .Values.service.type | default "ClusterIP" }} # Use default value ports: - name: metadata port: {{ .Values.service.port }} diff --git a/charts/metaflow/charts/metaflow-service/templates/serviceaccount.yaml b/charts/metaflow/charts/metaflow-service/templates/serviceaccount.yaml new file mode 100644 index 0000000..71f19c8 --- /dev/null +++ b/charts/metaflow/charts/metaflow-service/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{ if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "metaflow-service.serviceAccountName" . }} + labels: + {{ include "metaflow-service.labels" . | nindent 4 }} + {{ with .Values.serviceAccount.annotations }} + annotations: + {{ toYaml . | nindent 4 }} + {{ end }} +{{ end }} diff --git a/k8s/helm/metaflow/charts/metaflow-service/templates/tests/test-connection.yaml b/charts/metaflow/charts/metaflow-service/templates/tests/test-connection.yaml similarity index 63% rename from k8s/helm/metaflow/charts/metaflow-service/templates/tests/test-connection.yaml rename to charts/metaflow/charts/metaflow-service/templates/tests/test-connection.yaml index 5eb864f..35772c7 100644 --- a/k8s/helm/metaflow/charts/metaflow-service/templates/tests/test-connection.yaml +++ b/charts/metaflow/charts/metaflow-service/templates/tests/test-connection.yaml @@ -6,10 +6,20 @@ metadata: {{- include "metaflow-service.labels" . | nindent 4 }} annotations: "helm.sh/hook": test + "helm.sh/hook-weight": "0" spec: containers: - name: wget image: busybox command: ['wget'] args: ['{{ include "metaflow-service.fullname" . }}:{{ .Values.service.port }}'] + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" restartPolicy: Never + serviceAccountName: default + activeDeadlineSeconds: 60 diff --git a/k8s/helm/metaflow/charts/metaflow-service/values.yaml b/charts/metaflow/charts/metaflow-service/values.yaml similarity index 65% rename from k8s/helm/metaflow/charts/metaflow-service/values.yaml rename to charts/metaflow/charts/metaflow-service/values.yaml index b5914a3..b6457e4 100644 --- a/k8s/helm/metaflow/charts/metaflow-service/values.yaml +++ b/charts/metaflow/charts/metaflow-service/values.yaml @@ -2,62 +2,43 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 1 - -image: - repository: public.ecr.aws/outerbounds/metaflow_metadata_service - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} +affinity: {} -podSecurityContext: {} - # fsGroup: 2000 +autoscaling: + enabled: false + maxReplicas: 100 + minReplicas: 1 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 dbMigrations: - # Set to true to run migrations on every container start - runOnStart: true - # Set to true to only run them if db is empty. This setting has no effect is - # runOnStart is false. - # - # We recommend setting this to true for production environments with lots - # of runs, since running the migrations may lock the database for a while. - # For smaller/toy deployments you can set this to false onlyIfDbEmpty: true + runOnStart: true -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 +env: + - name: MF_METADATA_DB_SSL_MODE + value: prefer -service: - type: ClusterIP - port: 8080 - annotations: {} +envFrom: [] +# - configMapRef: +# name: example-configmap +# - secretRef: +# name: example-secret + +image: + pullPolicy: IfNotPresent + repository: public.ecr.aws/outerbounds/metaflow_metadata_service + tag: "" + +imagePullSecrets: [] +# - name: myRegistryKeySecret ingress: - enabled: false - className: "" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" + className: "" + enabled: false hosts: - host: chart-example.local paths: @@ -68,35 +49,82 @@ ingress: # hosts: # - chart-example.local +livenessProbe: + failureThreshold: 3 + httpGet: + path: /ping + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +metadatadb: + host: "" + name: "metaflow" + password: "" + port: 5432 + schema: "" + user: "metaflow" + +fullnameOverride: "" +nameOverride: "" + +# This section allows you to specify a node selector for the Kubernetes pods. +# Node selectors are key-value pairs used to constrain a pod to only be able to run on particular nodes. +# Example usage: +# nodeSelector: +# disktype: ssd +nodeSelector: {} + +podAnnotations: {} + # example-key: example-value + +podSecurityContext: {} + # fsGroup: 2000 + # runAsUser: 1000 + # runAsGroup: 3000 + +readinessProbe: + failureThreshold: 3 + httpGet: + path: /ping + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + +replicaCount: 1 + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: - # cpu: 100m - # memory: 128Mi + # cpu: 200m + # memory: 256Mi # requests: # cpu: 100m # memory: 128Mi -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 -affinity: {} +service: + annotations: {} + port: 8080 + type: ClusterIP -envFrom: [] +serviceAccount: + annotations: {} + create: true + name: "" -metadatadb: - port: 5432 - password: "" - host: "" - user: "" +tolerations: [] diff --git a/charts/metaflow/charts/metaflow-ui/.helmignore b/charts/metaflow/charts/metaflow-ui/.helmignore new file mode 100644 index 0000000..fdb6be2 --- /dev/null +++ b/charts/metaflow/charts/metaflow-ui/.helmignore @@ -0,0 +1,86 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. + +# macOS specific files +.DS_Store +.AppleDouble +.LSOverride + +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +.svnignore + +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +*.old +*.backup + +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ +*.code-workspace + +# Node.js dependencies +node_modules/ +npm-debug.log +yarn-error.log + +# Python specific +__pycache__/ +*.py[cod] +*.pyo +*.pyd +.Python +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Logs +*.log + +# Archives +*.zip +*.tar.gz +*.rar +*.7z +*.gz +*.bz2 +*.xz + +# Compiled files +*.o +*.obj +*.dll +*.class +*.exe +*.so +*.dylib +*.out +*.app +*.pdb +*.lib +*.a +*.la +*.lo +*.jar +*.war +*.ear +*.pyc +*.pyo +*.pyd +*.whl diff --git a/charts/metaflow/charts/metaflow-ui/Chart.yaml b/charts/metaflow/charts/metaflow-ui/Chart.yaml new file mode 100644 index 0000000..2901bfd --- /dev/null +++ b/charts/metaflow/charts/metaflow-ui/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: v2.4.13 +description: A Helm chart to deploy Metaflow UI components. +name: metaflow-ui +type: application +version: 0.3.0 +maintainers: + - name: Savin Goyal + email: savin@outerbounds.co + - name: Oleg Avdeev + email: oleg@outerbounds.co +keywords: + - metaflow ui +sources: + - https://github.com/Netflix/metaflow-service + - https://github.com/Netflix/metaflow-ui + - https://github.com/outerbounds/metaflow-tools +icon: https://metaflow.org/images/metaflow.svg diff --git a/k8s/helm/metaflow/charts/metaflow-ui/LICENSE b/charts/metaflow/charts/metaflow-ui/LICENSE similarity index 100% rename from k8s/helm/metaflow/charts/metaflow-ui/LICENSE rename to charts/metaflow/charts/metaflow-ui/LICENSE diff --git a/charts/metaflow/charts/metaflow-ui/README.md b/charts/metaflow/charts/metaflow-ui/README.md new file mode 100644 index 0000000..346fc8a --- /dev/null +++ b/charts/metaflow/charts/metaflow-ui/README.md @@ -0,0 +1,3 @@ +## Helm Chart for Metaflow UI + +This Helm chart facilitates the deployment of both the static and backend components of the Metaflow UI. It is designed to allow shared configurations for common settings such as `imagePullSecrets`, `tolerations`, and `affinity`. However, it also provides the flexibility to configure the static and backend components independently through the `uiStatic` and `uiBackend` sections, respectively. Additionally, the Helm chart includes options to configure ingress resources, which can be optionally deployed to serve the Metaflow UI. diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/NOTES.txt b/charts/metaflow/charts/metaflow-ui/templates/NOTES.txt similarity index 87% rename from k8s/helm/metaflow/charts/metaflow-ui/templates/NOTES.txt rename to charts/metaflow/charts/metaflow-ui/templates/NOTES.txt index b67d11c..5fa9510 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/NOTES.txt +++ b/charts/metaflow/charts/metaflow-ui/templates/NOTES.txt @@ -10,8 +10,8 @@ export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.uiBackend.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "metaflow-ui.fullname" . }}' + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "metaflow-ui.fullname" . }}' export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "metaflow-ui.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.uiBackend.service.port }} {{- else if contains "ClusterIP" .Values.uiBackend.service.type }} diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/_helpers.tpl b/charts/metaflow/charts/metaflow-ui/templates/_helpers.tpl similarity index 87% rename from k8s/helm/metaflow/charts/metaflow-ui/templates/_helpers.tpl rename to charts/metaflow/charts/metaflow-ui/templates/_helpers.tpl index 06e648d..54d3231 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/_helpers.tpl +++ b/charts/metaflow/charts/metaflow-ui/templates/_helpers.tpl @@ -6,7 +6,7 @@ Expand the name of the chart. {{- end }} {{- define "metaflow-ui.name-static" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}-static +{{- include "metaflow-ui.name" . }}-static {{- end }} {{/* @@ -36,7 +36,6 @@ If release name contains chart name it will be used as a full name. {{ include "metaflow-ui.fullname" . }}-static {{- end }} - {{/* Create chart name and version as used by the chart label. */}} @@ -56,11 +55,8 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} -{{/* -Common labels -*/}} {{- define "metaflow-ui.labelsStatic" -}} -helm.sh/chart: {{ include "metaflow-ui.chart" . }} +{{ include "metaflow-ui.labels" . }} {{ include "metaflow-ui.selectorLabelsStatic" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} @@ -76,9 +72,6 @@ app.kubernetes.io/name: {{ include "metaflow-ui.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} -{{/* -Selector labels -*/}} {{- define "metaflow-ui.selectorLabelsStatic" -}} app.kubernetes.io/name: {{ include "metaflow-ui.name-static" . }} app.kubernetes.io/instance: {{ .Release.Name }} @@ -95,6 +88,20 @@ Create the name of the service account to use {{- end }} {{- end }} +{{/* +Create the backendURL, which differs if we use an ingress or not. +*/}} +{{- define "metaflow-ui.staticUIBackendURL" -}} +{{- if .Values.ingress.enabled }} +{{- .Values.uiStatic.metaflowUIBackendURL | default "http://localhost/api/" }} +{{- else }} +{{- .Values.uiStatic.metaflowUIBackendURL | default "http://localhost:8083/api/" }} +{{- end }} +{{- end }} + +{{/* +Metadata DB Environment Variables +*/}} {{- define "metaflow-ui.metadatadbEnvVars" -}} - name: MF_METADATA_DB_NAME value: {{ .Values.uiBackend.metadatadb.name | quote }} @@ -110,9 +117,9 @@ Create the name of the service account to use {{- else }} - name: MF_METADATA_DB_HOST value: {{ .Release.Name }}-postgresql -{{- end -}} +{{- end }} {{- if .Values.uiBackend.metadatadb.schema }} - name: DB_SCHEMA_NAME value: {{ .Values.uiBackend.metadatadb.schema | quote }} {{- end }} -{{- end -}} +{{- end }} diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/backend_deployment.yaml b/charts/metaflow/charts/metaflow-ui/templates/backend_deployment.yaml similarity index 96% rename from k8s/helm/metaflow/charts/metaflow-ui/templates/backend_deployment.yaml rename to charts/metaflow/charts/metaflow-ui/templates/backend_deployment.yaml index fbe988c..65c2ddd 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/backend_deployment.yaml +++ b/charts/metaflow/charts/metaflow-ui/templates/backend_deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "metaflow-ui.labels" . | nindent 4 }} spec: - replicas: {{ .Values.uiBackend.replicaCount }} + replicas: {{ .Values.uiBackend.replicaCount | default 1 }} selector: matchLabels: {{- include "metaflow-ui.selectorLabels" . | nindent 6 }} @@ -30,8 +30,8 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.uiBackend.image.name }}:{{ .Values.uiBackend.image.tag | default .Chart.AppVersion }}" - command: ["/opt/latest/bin/python3", "-m", "services.ui_backend_service.ui_server" ] - imagePullPolicy: {{ .Values.uiBackend.image.pullPolicy }} + command: ["/opt/latest/bin/python3", "-m", "services.ui_backend_service.ui_server"] + imagePullPolicy: {{ .Values.uiBackend.image.pullPolicy | default "IfNotPresent" }} ports: - name: http containerPort: 8083 diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/backend_service.yaml b/charts/metaflow/charts/metaflow-ui/templates/backend_service.yaml similarity index 91% rename from k8s/helm/metaflow/charts/metaflow-ui/templates/backend_service.yaml rename to charts/metaflow/charts/metaflow-ui/templates/backend_service.yaml index 01281f1..50dd93b 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/backend_service.yaml +++ b/charts/metaflow/charts/metaflow-ui/templates/backend_service.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ include "metaflow-ui.fullname" . }} + namespace: {{ .Values.namespace }} labels: {{- include "metaflow-ui.labels" . | nindent 4 }} spec: diff --git a/charts/metaflow/charts/metaflow-ui/templates/ingress.yaml b/charts/metaflow/charts/metaflow-ui/templates/ingress.yaml new file mode 100644 index 0000000..43cb843 --- /dev/null +++ b/charts/metaflow/charts/metaflow-ui/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "metaflow-ui.fullname" . -}} +{{- $svcPort := .Values.uiBackend.service.port -}} +{{- $fullNameStatic := include "metaflow-ui.fullname-static" . -}} +{{- $svcPortStatic := .Values.uiStatic.service.port -}} +{{- $isK8sVersionAtLeast118 := semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion -}} +{{- $isK8sVersionAtLeast119 := semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} + +{{- if and .Values.ingress.className (not $isK8sVersionAtLeast118) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className }} + {{- end }} +{{- end }} + +apiVersion: {{ if $isK8sVersionAtLeast119 }}networking.k8s.io/v1{{ else if $isK8sVersionAtLeast118 }}networking.k8s.io/v1beta1{{ else }}extensions/v1beta1{{ end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "metaflow-ui.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className $isK8sVersionAtLeast118 }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range $path, $service := dict "/api" $fullName "/static" $fullNameStatic "/" $fullNameStatic }} + - path: {{ $path | quote }} + {{- if $isK8sVersionAtLeast118 }} + pathType: Prefix + {{- end }} + backend: + {{- if $isK8sVersionAtLeast119 }} + service: + name: {{ $service }} + port: + number: {{ if eq $service $fullName }}{{ $svcPort }}{{ else }}{{ $svcPortStatic }}{{ end }} + {{- else }} + serviceName: {{ $service }} + servicePort: {{ if eq $service $fullName }}{{ $svcPort }}{{ else }}{{ $svcPortStatic }}{{ end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/serviceaccount.yaml b/charts/metaflow/charts/metaflow-ui/templates/serviceaccount.yaml similarity index 92% rename from k8s/helm/metaflow/charts/metaflow-ui/templates/serviceaccount.yaml rename to charts/metaflow/charts/metaflow-ui/templates/serviceaccount.yaml index 4798c61..69b308e 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/serviceaccount.yaml +++ b/charts/metaflow/charts/metaflow-ui/templates/serviceaccount.yaml @@ -6,7 +6,9 @@ metadata: labels: {{- include "metaflow-ui.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} + {{- if . }} annotations: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} {{- end }} diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/static_deployment.yaml b/charts/metaflow/charts/metaflow-ui/templates/static_deployment.yaml similarity index 86% rename from k8s/helm/metaflow/charts/metaflow-ui/templates/static_deployment.yaml rename to charts/metaflow/charts/metaflow-ui/templates/static_deployment.yaml index 299fe0b..ee4233c 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/static_deployment.yaml +++ b/charts/metaflow/charts/metaflow-ui/templates/static_deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "metaflow-ui.labelsStatic" . | nindent 4 }} spec: - replicas: {{ .Values.uiStatic.replicaCount }} + replicas: {{ .Values.uiStatic.replicaCount | default 1 }} selector: matchLabels: {{- include "metaflow-ui.selectorLabelsStatic" . | nindent 6 }} @@ -22,15 +22,15 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "metaflow-ui.serviceAccountName" . }} + serviceAccountName: {{ include "metaflow-ui.serviceAccountName" . | default "default" }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: - name: "{{ .Chart.Name }}-static" securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.uiStatic.image.name }}:{{ .Values.uiStatic.image.tag }}" - imagePullPolicy: {{ .Values.uiStatic.image.pullPolicy }} + image: "{{ tpl .Values.uiStatic.image.name . }}:{{ tpl .Values.uiStatic.image.tag . }}" + imagePullPolicy: {{ .Values.uiStatic.image.pullPolicy | default "IfNotPresent" }} ports: - name: http containerPort: 3000 @@ -45,7 +45,7 @@ spec: port: http env: - name: METAFLOW_SERVICE - value: {{ .Values.uiStatic.metaflowUIBackendURL | quote }} + value: {{ include "metaflow-ui.staticUIBackendURL" . | quote }} {{- range .Values.env }} - name: {{ .name | quote }} value: {{ .value | quote }} diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/static_service.yaml b/charts/metaflow/charts/metaflow-ui/templates/static_service.yaml similarity index 75% rename from k8s/helm/metaflow/charts/metaflow-ui/templates/static_service.yaml rename to charts/metaflow/charts/metaflow-ui/templates/static_service.yaml index 3627ca9..425572e 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/static_service.yaml +++ b/charts/metaflow/charts/metaflow-ui/templates/static_service.yaml @@ -2,10 +2,12 @@ apiVersion: v1 kind: Service metadata: name: "{{ include "metaflow-ui.fullname-static" . }}" + namespace: {{ .Values.namespace | default "default" }} labels: {{- include "metaflow-ui.labelsStatic" . | nindent 4 }} spec: type: {{ .Values.uiStatic.service.type }} + loadBalancerIP: {{ .Values.uiStatic.service.loadBalancerIP | default "" }} ports: - port: {{ .Values.uiStatic.service.port }} targetPort: http diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/tests/test-connection.yaml b/charts/metaflow/charts/metaflow-ui/templates/tests/test-connection.yaml similarity index 58% rename from k8s/helm/metaflow/charts/metaflow-ui/templates/tests/test-connection.yaml rename to charts/metaflow/charts/metaflow-ui/templates/tests/test-connection.yaml index 8966050..fb9bbd8 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/tests/test-connection.yaml +++ b/charts/metaflow/charts/metaflow-ui/templates/tests/test-connection.yaml @@ -2,14 +2,24 @@ apiVersion: v1 kind: Pod metadata: name: "{{ include "metaflow-ui.fullname" . }}-test-connection" + namespace: {{ .Release.Namespace }} labels: {{- include "metaflow-ui.labels" . | nindent 4 }} annotations: "helm.sh/hook": test + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: containers: - name: wget image: busybox command: ['wget'] args: ['{{ include "metaflow-ui.fullname" . }}:{{ .Values.uiBackend.service.port }}'] + resources: + requests: + memory: "64Mi" + cpu: "250m" + limits: + memory: "128Mi" + cpu: "500m" restartPolicy: Never + activeDeadlineSeconds: 60 diff --git a/k8s/helm/metaflow/charts/metaflow-ui/values.yaml b/charts/metaflow/charts/metaflow-ui/values.yaml similarity index 66% rename from k8s/helm/metaflow/charts/metaflow-ui/values.yaml rename to charts/metaflow/charts/metaflow-ui/values.yaml index fccf446..498000a 100644 --- a/k8s/helm/metaflow/charts/metaflow-ui/values.yaml +++ b/charts/metaflow/charts/metaflow-ui/values.yaml @@ -6,13 +6,19 @@ uiBackend: tag: "" podAnnotations: {} - - env: [] + env: # Additional environment variables for metaflow-ui static deployment. Example config: # - name: METAFLOW_SERVICE # value: "http://localhost:8083/api/" + - name: MF_METADATA_DB_SSL_MODE + value: prefer envFrom: [] + # Example of loading environment variables from a ConfigMap and Secret: + # - configMapRef: + # name: my-config-map + # - secretRef: + # name: my-secret resources: {} # We usually recommend not to specify default resources and to leave this as a conscious @@ -34,9 +40,9 @@ uiBackend: metaflowDatastoreSysRootS3: "" # The root S3 bucket prefix that will be used by metaflow - + metaflowServiceURL: "http://localhost:8083/api/metadata" - # The metaflow metadata service URL + # The metaflow metadata service URL metaflowS3EndpointURL: "" # This should be set only when using an S3 compatible object storage like minio @@ -56,10 +62,9 @@ uiStatic: podAnnotations: {} resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # Default resources are not specified to allow flexibility and compatibility + # with various environments, including those with limited resources like Minikube. + # To specify resources, uncomment and adjust the following lines: # limits: # cpu: 200m # memory: 256Mi @@ -72,7 +77,8 @@ uiStatic: # - name: var_name # value: "var_value" - metaflowUIBackendURL: "http://localhost:8083/api/" + # This default assumes the backend service is running in the same + metaflowUIBackendURL: "" service: type: ClusterIP @@ -84,6 +90,8 @@ uiStatic: imagePullSecrets: [] podSecurityContext: {} # fsGroup: 2000 + # runAsUser: 1000 + # runAsGroup: 3000 securityContext: {} # capabilities: # drop: @@ -92,8 +100,28 @@ securityContext: {} # runAsNonRoot: true # runAsUser: 1000 nodeSelector: {} + # Example node selector to schedule pods on specific nodes: + # kubernetes.io/os: linux + # node-type: worker + tolerations: [] + # Example tolerations to allow scheduling on tainted nodes: + # - key: "dedicated" + # operator: "Equal" + # value: "analytics" + # effect: "NoSchedule" + affinity: {} + # Example pod affinity to co-locate related pods: + # podAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: app + # operator: In + # values: + # - metaflow-ui + # topologyKey: "kubernetes.io/hostname" # name overrides nameOverride: "" @@ -102,11 +130,13 @@ fullnameOverride: "" serviceAccount: # Specifies whether a service account should be created create: true - # Annotations to add to the service account - annotations: {} # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" + # Annotations to add to the service account + annotations: {} + # Example annotation for an IRSA role ARN: + # eks.amazonaws.com/role-arn: arn:aws:iam:::role/ ingress: enabled: false diff --git a/charts/metaflow/charts/postgresql-14.3.3.tgz b/charts/metaflow/charts/postgresql-14.3.3.tgz new file mode 100644 index 0000000..bf053d3 Binary files /dev/null and b/charts/metaflow/charts/postgresql-14.3.3.tgz differ diff --git a/charts/metaflow/values.yaml b/charts/metaflow/values.yaml new file mode 100644 index 0000000..b98ab60 --- /dev/null +++ b/charts/metaflow/values.yaml @@ -0,0 +1,34 @@ +metaflow-service: + metadatadb: + # WARNING: Default credentials - MUST be changed in production environments + password: metaflow # Change this in production using secrets management + name: metaflow + user: metaflow + +metaflow-ui: + # SECURITY NOTICE: + # The following credentials are default values for development only. + # For production deployments: + # 1. Use Kubernetes secrets + # 2. Use environment variables + # 3. Use external secrets management (e.g., HashiCorp Vault) + uiBackend: + metadatadb: + password: metaflow # Change this in production using secrets management + name: metaflow + user: metaflow + + ingress: + enabled: true + hosts: + - host: localhost # Change this to your production domain + +postgresql: + # WARNING: These are development defaults + # For production environments: + # - Use strong passwords + # - Consider using external managed database + enabled: true + postgresqlPassword: metaflow # Change this in production using secrets management + postgresqlDatabase: metaflow + postgresqlUsername: metaflow diff --git a/k8s/helm/metaflow/Chart.lock b/k8s/helm/metaflow/Chart.lock deleted file mode 100644 index 7edb663..0000000 --- a/k8s/helm/metaflow/Chart.lock +++ /dev/null @@ -1,12 +0,0 @@ -dependencies: -- name: metaflow-service - repository: "" - version: 0.1.0 -- name: metaflow-ui - repository: "" - version: 0.1.0 -- name: postgresql - repository: https://charts.bitnami.com/bitnami - version: 10.13.4 -digest: sha256:ed1d53f386d5f9a305d526737c854a92782fc6a2ef7725295de04853c0679ad4 -generated: "2021-11-05T16:00:15.003107-07:00" diff --git a/k8s/helm/metaflow/Chart.yaml b/k8s/helm/metaflow/Chart.yaml deleted file mode 100644 index e21bb71..0000000 --- a/k8s/helm/metaflow/Chart.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v2 -appVersion: 2.3.1 -dependencies: -- name: metaflow-service - repository: "" - version: 0.1.0 - condition: metaflow-service.enabled -- name: metaflow-ui - repository: "" - version: 0.2.0 - condition: metaflow-ui.enabled -- name: postgresql - repository: https://charts.bitnami.com/bitnami - version: ^10 - condition: postgresql.enabled -description: Machine Learning plaform -home: https://metaflow.org -keywords: -- mlops -name: metaflow -sources: -- https://github.com/Netflix/metaflow -- https://github.com/outerbounds/metaflow-tools -type: application -version: 0.0.1 diff --git a/k8s/helm/metaflow/README.md b/k8s/helm/metaflow/README.md deleted file mode 100644 index 0600497..0000000 --- a/k8s/helm/metaflow/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Helm Chart for Metaflow services - -This chart deploys Metaflow Services in a Kubernetes cluster. Specifically, it includes: - -* a sub-chart for Metaflow Metadata service -* a sub-chart for Metaflow UI, front end and back end -* PostgreSQL to store Metadata - -PostgreSQL subchart is included to make it easier to deploy Metaflow for evaluation or development purposes. In production, we recommend to use a managed PostgreSQL offering such as AWS RDS and disable PostgreSQL sub-chart by setting `postgresql.enabled` to `false`. - diff --git a/k8s/helm/metaflow/charts/metaflow-service/.helmignore b/k8s/helm/metaflow/charts/metaflow-service/.helmignore deleted file mode 100644 index 0e8a0eb..0000000 --- a/k8s/helm/metaflow/charts/metaflow-service/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/k8s/helm/metaflow/charts/metaflow-service/Chart.yaml b/k8s/helm/metaflow/charts/metaflow-service/Chart.yaml deleted file mode 100644 index 5550113..0000000 --- a/k8s/helm/metaflow/charts/metaflow-service/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -appVersion: v2.3.6 -description: A Helm chart to deploy Metadata Service for Metaflow -name: metaflow-service -type: application -version: 0.2.0 diff --git a/k8s/helm/metaflow/charts/metaflow-service/templates/serviceaccount.yaml b/k8s/helm/metaflow/charts/metaflow-service/templates/serviceaccount.yaml deleted file mode 100644 index 64b9fd5..0000000 --- a/k8s/helm/metaflow/charts/metaflow-service/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "metaflow-service.serviceAccountName" . }} - labels: - {{- include "metaflow-service.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/k8s/helm/metaflow/charts/metaflow-ui/Chart.yaml b/k8s/helm/metaflow/charts/metaflow-ui/Chart.yaml deleted file mode 100644 index 23a0506..0000000 --- a/k8s/helm/metaflow/charts/metaflow-ui/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -appVersion: v2.3.11 -description: A Helm chart for Kubernetes -name: metaflow-ui -type: application -version: 0.2.0 diff --git a/k8s/helm/metaflow/charts/metaflow-ui/README.md b/k8s/helm/metaflow/charts/metaflow-ui/README.md deleted file mode 100644 index 3aca66f..0000000 --- a/k8s/helm/metaflow/charts/metaflow-ui/README.md +++ /dev/null @@ -1,3 +0,0 @@ -## Helm Chart for Metaflow UI - -This helm chart deploys the static and backend components of the metaflow UI. It assumes that these deployments will mostly share similar configurations like `imagePullSecrets`, `tolerations`, and `affinity`. However, you can also configure the static and backend components separately via the `uiStatic` and `uiBackend` blocks respectively. Plus the helm chart also includes a way to configure the ingress resources that can optionally be deployed to server the metaflow UI. diff --git a/k8s/helm/metaflow/charts/metaflow-ui/templates/ingress.yaml b/k8s/helm/metaflow/charts/metaflow-ui/templates/ingress.yaml deleted file mode 100644 index 033c6dd..0000000 --- a/k8s/helm/metaflow/charts/metaflow-ui/templates/ingress.yaml +++ /dev/null @@ -1,90 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "metaflow-ui.fullname" . -}} -{{- $svcPort := .Values.uiBackend.service.port -}} -{{- $fullNameStatic := include "metaflow-ui.fullname-static" . -}} -{{- $svcPortStatic := .Values.uiStatic.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "metaflow-ui.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - - path: "/api" - {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} - pathType: Prefix - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - - path: "/" - {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} - pathType: Prefix - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullNameStatic }} - port: - number: {{ $svcPortStatic }} - {{- else }} - serviceName: {{ $fullNameStatic }} - servicePort: {{ $svcPortStatic }} - {{- end }} - - path: "/static" - {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} - pathType: Prefix - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullNameStatic }} - port: - number: {{ $svcPortStatic }} - {{- else }} - serviceName: {{ $fullNameStatic }} - servicePort: {{ $svcPortStatic }} - {{- end }} - - {{- end }} -{{- end }} diff --git a/k8s/helm/metaflow/charts/postgresql-10.13.4.tgz b/k8s/helm/metaflow/charts/postgresql-10.13.4.tgz deleted file mode 100644 index 3b5b997..0000000 Binary files a/k8s/helm/metaflow/charts/postgresql-10.13.4.tgz and /dev/null differ diff --git a/k8s/helm/metaflow/values.yaml b/k8s/helm/metaflow/values.yaml deleted file mode 100644 index 2bca1e8..0000000 --- a/k8s/helm/metaflow/values.yaml +++ /dev/null @@ -1,27 +0,0 @@ -metaflow-service: - metadatadb: - password: metaflow - name: metaflow - user: metaflow - -metaflow-ui: - # Note: for security reasons, these values should NOT be set in plain text. - # Better options of passing these values are using environment variables, - # kubernetes secret + values file, etc. - uiBackend: - metadatadb: - password: metaflow - name: metaflow - user: metaflow - - ingress: - enabled: true - hosts: - - host: localhost - -postgresql: - # postgresql.enabled -- Flag to install Postgresql - enabled: true - postgresqlPassword: metaflow - postgresqlDatabase: metaflow - postgresqlUsername: metaflow