From e4e546d4ef8d6e71ec9f42ac3533ccbdec977bec Mon Sep 17 00:00:00 2001 From: Anatole Date: Sun, 6 Aug 2023 15:49:34 +0200 Subject: [PATCH 01/16] terraform --- .github/workflows/ci.yaml | 175 +++++++++++++++++++++++++-------- terraform/deployment.tf | 200 ++++++++++++++++++++++++++++++++++++++ terraform/main.tf | 16 +++ terraform/providers.tf | 35 +++++++ terraform/variables.tf | 87 +++++++++++++++++ 5 files changed, 473 insertions(+), 40 deletions(-) create mode 100644 terraform/deployment.tf create mode 100644 terraform/main.tf create mode 100644 terraform/providers.tf create mode 100644 terraform/variables.tf diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f4884d11..62e4bb34 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,26 +14,27 @@ on: workflow_dispatch: inputs: deploy: - description: "Deploy location" + description: "Which environment to deploy to" required: true default: "none" type: choice options: - - production - - staging + - prod + - test - none concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + DEPLOY: ${{ (inputs.deploy != 'none' && inputs.deploy) || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'prod') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master') && 'prod') || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'test') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'dev') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'staged')) && 'test') || 'none' }} + jobs: ci: name: ImageProcessor Lint, Build, Test, Deploy runs-on: aws-runner env: - DEPLOY_PROD: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'production') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master') }} - DEPLOY_STAGE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'staging') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'dev') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'staged')) }} GOLANGCI_LINT_CACHE: /home/runner/.cache/golangci-lint concurrency: group: ${{ github.workflow }}-ci-${{ github.ref }} @@ -45,34 +46,28 @@ jobs: with: submodules: recursive - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} - uses: aws-actions/amazon-ecr-login@v1 - - name: Make build context - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + if: env.DEPLOY != 'none' run: | docker context create builders - name: Setup buildx uses: docker/setup-buildx-action@v2 - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + if: env.DEPLOY != 'none' with: install: true endpoint: builders + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Build docker image uses: docker/build-push-action@v3 - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + if: env.DEPLOY != 'none' with: context: . file: docker/full.Dockerfile @@ -81,29 +76,129 @@ jobs: cache-to: | type=gha,mode=max tags: | - ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/image-processor:latest - ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/image-processor:${{ github.sha }} + ghcr.io/seventv/image-processor:${{ env.DEPLOY }}-${{ github.sha }} + ghcr.io/seventv/image-processor:${{ env.DEPLOY }}-latest push: true - - name: Update deployment template - uses: danielr1996/envsubst-action@1.1.0 - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} - env: - IMAGE: ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/image-processor:${{ github.sha }} - with: - input: k8s/${{ (env.DEPLOY_PROD == 'true' && 'production') || 'staging' }}.template.yaml - output: k8s/deploy.yaml + validate: + name: API Deploy Validation + needs: ci + runs-on: ubuntu-latest + permissions: + pull-requests: write + defaults: + run: + working-directory: ./terraform - - name: Setup Kubectl - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} - uses: azure/setup-kubectl@v3.0 + steps: + - name: Checkout code + id: ok + if: env.DEPLOY != 'none' + uses: actions/checkout@v3 + + - name: "Setup Terraform" + if: steps.ok.outcome == 'success' + uses: hashicorp/setup-terraform@v1 + with: + cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} - - name: Deploy to k8s - if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }} + - name: "Terraform Init" + if: steps.ok.outcome == 'success' + id: init env: - KUBE_CONFIG_DATA: ${{ (env.DEPLOY_PROD == 'true' && secrets.KUBECONFIG) || secrets.KUBECONFIG_STAGE }} + TF_WORKSPACE: ${{ env.DEPLOY }} + run: terraform init + continue-on-error: true + + - name: "Terraform Workspace" + if: steps.ok.outcome == 'success' + run: terraform workspace select -or-create=true ${{ env.DEPLOY }} + + - name: Terraform fmt + if: steps.ok.outcome == 'success' + id: fmt + run: terraform fmt -check + continue-on-error: true + + - name: Terraform Validate + if: steps.ok.outcome == 'success' + id: validate + run: terraform validate -no-color + + - name: Terraform Variables + if: steps.ok.outcome == 'success' run: | - mkdir -p ~/.kube - (echo $KUBE_CONFIG_DATA | base64 -d) >> ~/.kube/config + cat < *.auto.tfvars + image_url="ghcr.io/seventv/image-processor:${{ env.DEPLOY }}-${{ github.sha }}" + image_pull_policy="IfNotPresent" + + EOF + + - name: "Terraform Plan" + if: steps.ok.outcome == 'success' + id: plan + run: terraform plan -no-color + + - uses: actions/github-script@v6 + if: steps.ok.outcome == 'success' && github.event_name == 'pull_request' + env: + PLAN: "terraform\n${{ steps.plan.outputs.stdout }}" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + // 1. Retrieve existing bot comments for the PR + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }) + const botComment = comments.find(comment => { + return comment.user.type === 'Bot' && comment.body.includes('Terraform Format and Style') + }) + + // 2. Prepare format of the comment + const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` + #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` + #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\` +
Validation Output + + \`\`\`\n + ${{ steps.validate.outputs.stdout }} + \`\`\` + +
+ + #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` + +
Show Plan + + \`\`\`\n + ${process.env.PLAN} + \`\`\` + +
+ + *Actor: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Workflow: \`${{ github.workflow }}\`*`; + + // 3. If we have a comment, update it, otherwise create a new one + if (botComment) { + github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + body: output + }) + } else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + } - kubectl apply -f k8s/deploy.yaml + - name: "Terraform Apply" + if: steps.ok.outcome == 'success' + id: apply + run: terraform apply -no-color -auto-approve + continue-on-error: true diff --git a/terraform/deployment.tf b/terraform/deployment.tf new file mode 100644 index 00000000..501d5237 --- /dev/null +++ b/terraform/deployment.tf @@ -0,0 +1,200 @@ +data "kubernetes_namespace" "app" { + metadata { + name = var.namespace + } +} + +resource "kubernetes_secret" "app" { + metadata { + name = "api" + namespace = var.namespace + } + + data = { + "config.yaml" = templatefile("${path.module}/config.template.yaml", { + rmq_uri = local.infra.rabbitmq_uri + }) + } +} + +resource "random_id" "jwt-secret" { + byte_length = 64 +} + +resource "kubernetes_deployment" "app" { + metadata { + name = "image-processor" + namespace = data.kubernetes_namespace.app.metadata[0].name + labels = { + app = "image-processor" + } + } + + lifecycle { + replace_triggered_by = [kubernetes_secret.app] + } + + timeouts { + create = "4m" + update = "2m" + delete = "2m" + } + + spec { + selector { + match_labels = { + app = "image-processor" + } + } + + replicas = 1 + + template { + metadata { + labels = { + app = "image-processor" + } + } + + spec { + container { + name = "image-processor" + image = local.image_url + + port { + name = "metrics" + container_port = 9100 + protocol = "TCP" + } + + port { + name = "health" + container_port = 9000 + protocol = "TCP" + } + + env { + name = "IMAGE_PROCESSOR_K8S_POD_NAME" + value_from { + field_ref { + field_path = "metadata.name" + } + } + } + + resources { + requests = { + cpu = var.production ? "6000m" : "3000m" + memory = var.production ? "7Gi" : "1Gi" + } + limits = { + cpu = var.production ? "6000m" : "3000m" + memory = var.production ? "7Gi" : "1Gi" + } + } + + volume_mount { + name = "config" + mount_path = "/app/config.yaml" + sub_path = "config.yaml" + } + + volume_mount { + name = "tempfs" + mount_path = "/tempfs" + } + + liveness_probe { + http_get { + port = "health" + path = "/" + } + initial_delay_seconds = 20 + timeout_seconds = 5 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 4 + } + + readiness_probe { + http_get { + port = "health" + path = "/" + } + initial_delay_seconds = 5 + timeout_seconds = 5 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 4 + } + + image_pull_policy = var.image_pull_policy + } + + volume { + name = "config" + secret { + secret_name = kubernetes_secret.app.metadata[0].name + } + } + + volume { + name = "tempfs" + empty_dir { + medium = "Memory" + } + } + } + } + } +} + +resource "kubernetes_service" "app" { + metadata { + name = "image-processor" + namespace = data.kubernetes_namespace.app.metadata[0].name + labels = { + app = "image-processor" + } + } + + spec { + selector = { + app = "image-processor" + } + + port { + name = "metrics" + port = 9000 + target_port = "metrics" + } + + port { + name = "health" + port = 9000 + target_port = "health" + } + } +} + +resource "kubectl_manifest" "app_monitor" { + depends_on = [kubernetes_deployment.app] + + yaml_body = < Date: Sun, 6 Aug 2023 16:15:38 +0200 Subject: [PATCH 02/16] missing kubectl provider --- .gitignore | 14 ++++++ terraform/.terraform.lock.hcl | 81 +++++++++++++++++++++++++++++++++++ terraform/providers.tf | 5 +++ 3 files changed, 100 insertions(+) create mode 100644 terraform/.terraform.lock.hcl diff --git a/.gitignore b/.gitignore index 98e82c08..5878fe26 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,17 @@ build/ out/ .cache *.log + +# Terraform local state files +**/.terraform/* +*.tfstate +*.tfstate.* +*.tfplan +crash.log +*.tfvars +override.tf +override.tf.json +*_override.tf +*_override.tf.json +.terraformrc +terraform.rc diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 00000000..adce0189 --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,81 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/gavinbunney/kubectl" { + version = "1.14.0" + constraints = "1.14.0" + hashes = [ + "h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=", + "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", + "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", + "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", + "zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4", + "zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5", + "zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5", + "zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf", + "zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93", + "zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.7.0" + constraints = "~> 5.7.0" + hashes = [ + "h1:A7p0npQ+UHlnapVuikOzhmgchAq8agtfZGkYiiEOnp0=", + "zh:03240d7fc041d5331db7fd5f2ca4fe031321d07d2a6ca27085c5020dae13f211", + "zh:0b5252b14c354636fe0348823195dd901b457de1a033015f4a7d11cfe998c766", + "zh:2bfb62325b0487be8d1850a964f09cca0d45148faec577459c2a24334ec9977b", + "zh:2f9e317ffc57d2b5117cfe8dc266f88aa139b760bc93d8adeed7ad533a78b5a3", + "zh:36512725c9d7c559927b98fead04be58494a3a997e5270b905a75a468e307427", + "zh:5483e696d3ea764f746d3fe439f7dcc49001c3c774122d7baa51ce01011f0075", + "zh:5967635cc14f969ea26622863a2e3f9d6a7ddd3e7d35a29a7275c5e10579ac8c", + "zh:7e63c94a64af5b7aeb36ea6e3719962f65a7c28074532c02549a67212d410bb8", + "zh:8a7d5f33b11a3f5c7281413b431fa85de149ed8493ec1eea73d50d2d80a475e6", + "zh:8e2ed2d986aaf590975a79a2f6b5e60e0dc7d804ab01a8c03ab181e41cfe9b0f", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c7b8ca1b17489f16a6d0f1fc2aa9c130978ea74c9c861d8435410567a0a888f", + "zh:a54385896a70524063f0c5420be26ff6f88909bd8e6902dd3e922577b21fd546", + "zh:aecd3a8fb70b938b58d93459bfb311540fd6aaf981924bf34abd48f953b4be0d", + "zh:f3de076fa3402768d27af0187c6a677777b47691d1f0f84c9b259ff66e65953e", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.18.1" + constraints = "2.18.1" + hashes = [ + "h1:h4ezMuMNyKRMRhlrErph7QOUToc77U+rVKdR48w6tr8=", + "zh:09d69d244f5e688d9b1582112aa5d151c5336278e43d39c88ae920c26536b753", + "zh:0df4c988056f7d84d9161c6c955ad7346364c261d100ef510a6cc7fa4a235197", + "zh:2d3d0cb2931b6153a7971ce8c6fae92722b1116e16f42abbaef115dba895c8d8", + "zh:47830e8fc1760860bfa4aaf418627ff3c6ffcac6cebbbc490e5e0e6b31287d80", + "zh:49467177b514bada0fb3b6982897a347498af8ef9ef8d9fd611fe21dfded2e25", + "zh:5c7eae2c51ba175822730a63ad59cf41604c76c46c5c97332506ab42023525ce", + "zh:6efae755f02df8ab65ce7a831f33bd4817359db205652fd4bc4b969302072b15", + "zh:7e6e97b79fecd25aaf0f4fb91da945a65c36fe2ba2a4313288a60ede55506aad", + "zh:b75f2c9dd24b355ffe73e7b2fcd3145fc32735068f0ec2eba2df63f792dd16e8", + "zh:dbef9698d842eb49a846db6d7694f159ae5154ffbb7a753a9d4cab88c462a6d4", + "zh:f1b1fd580d92eedd9c8224d463997ccff1a62851fea65106aac299efe9ab622a", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.5.1" + hashes = [ + "h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=", + "zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64", + "zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d", + "zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831", + "zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3", + "zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b", + "zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2", + "zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865", + "zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03", + "zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602", + "zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014", + ] +} diff --git a/terraform/providers.tf b/terraform/providers.tf index 81c50587..7f1264ef 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -13,6 +13,11 @@ terraform { random = { source = "hashicorp/random" } + + kubectl = { + source = "gavinbunney/kubectl" + version = "1.14.0" + } } } From e3f34afeb69c3758f69de59533186000fa87226a Mon Sep 17 00:00:00 2001 From: Anatole Date: Sun, 6 Aug 2023 16:19:06 +0200 Subject: [PATCH 03/16] config template --- .gitignore | 1 + terraform/config.template.yaml | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 terraform/config.template.yaml diff --git a/.gitignore b/.gitignore index 5878fe26..50ec2dba 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ tmp/ !.circleci/*.yaml !example.config.yaml !docker-compose.yaml +!terraform/config.template.yaml !.github/**/*.yaml packrd/ *-packr.go diff --git a/terraform/config.template.yaml b/terraform/config.template.yaml new file mode 100644 index 00000000..3c25ada4 --- /dev/null +++ b/terraform/config.template.yaml @@ -0,0 +1,19 @@ +level: debug + +worker: + temp_dir: /tempfs + threads_per_worker: 3 + jobs: 2 + +health: + enabled: true + bind: "0.0.0.0:9000" + +monitoring: + enabled: true + bind: "0.0.0.0.9100" + +message_queue: + rmq: + uri: ${rmq_uri} + max_reconnect_attempts: 10 From f44327d209be72921aaefa51c68c7276877dc080 Mon Sep 17 00:00:00 2001 From: Anatole Date: Sun, 6 Aug 2023 16:23:34 +0200 Subject: [PATCH 04/16] namespace --- terraform/deployment.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 501d5237..bb3d9bf9 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -185,7 +185,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: image-processor - namespace: ${kubernetes_namespace.app.metadata[0].name} + namespace: ${data.kubernetes_namespace.app.metadata[0].name} labels: app: image-processor spec: From c2c79e386f61dd34fc6b8e462120514fd1964eef Mon Sep 17 00:00:00 2001 From: Anatole Date: Sun, 6 Aug 2023 16:25:49 +0200 Subject: [PATCH 05/16] bad variables --- terraform/variables.tf | 49 ------------------------------------------ 1 file changed, 49 deletions(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 176f815d..a869c114 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -36,52 +36,3 @@ variable "image_pull_policy" { type = string default = "Always" } - -variable "cdn_url" { - type = string -} - -variable "http_addr" { - type = string - default = "0.0.0.0" -} - -variable "http_port_gql" { - type = number - default = 3000 -} - -variable "http_port_rest" { - type = number - default = 3100 -} - -variable "twitch_client_id" { - type = string - default = "" -} - -variable "twitch_client_secret" { - type = string - default = "" -} - -variable "twitch_redirect_uri" { - type = string - default = "" -} - -variable "discord_client_id" { - type = string - default = "" -} - -variable "discord_client_secret" { - type = string - default = "" -} - -variable "discord_redirect_uri" { - type = string - default = "" -} From 19b783b6f4c5e38ec0f53594b7fb44af120ba24f Mon Sep 17 00:00:00 2001 From: Anatole Date: Sun, 6 Aug 2023 16:30:30 +0200 Subject: [PATCH 06/16] typos --- terraform/deployment.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/deployment.tf b/terraform/deployment.tf index bb3d9bf9..8b0567fe 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -6,7 +6,7 @@ data "kubernetes_namespace" "app" { resource "kubernetes_secret" "app" { metadata { - name = "api" + name = "image-processor" namespace = var.namespace } @@ -165,7 +165,7 @@ resource "kubernetes_service" "app" { port { name = "metrics" - port = 9000 + port = 9100 target_port = "metrics" } From d11dcc078fdc18de4db0858cc8869049c2388e04 Mon Sep 17 00:00:00 2001 From: Anatole Date: Sun, 6 Aug 2023 16:36:41 +0200 Subject: [PATCH 07/16] adjust scale --- terraform/config.template.yaml | 2 +- terraform/deployment.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/config.template.yaml b/terraform/config.template.yaml index 3c25ada4..c07a2268 100644 --- a/terraform/config.template.yaml +++ b/terraform/config.template.yaml @@ -2,7 +2,7 @@ level: debug worker: temp_dir: /tempfs - threads_per_worker: 3 + threads_per_worker: 1 jobs: 2 health: diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 8b0567fe..d3b6cfb5 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -84,11 +84,11 @@ resource "kubernetes_deployment" "app" { resources { requests = { - cpu = var.production ? "6000m" : "3000m" + cpu = var.production ? "6000m" : "1500m" memory = var.production ? "7Gi" : "1Gi" } limits = { - cpu = var.production ? "6000m" : "3000m" + cpu = var.production ? "6000m" : "1500m" memory = var.production ? "7Gi" : "1Gi" } } From eccded2d3c6863252fe10bb72b27186fd4522c2b Mon Sep 17 00:00:00 2001 From: Anatole Date: Sun, 6 Aug 2023 16:42:38 +0200 Subject: [PATCH 08/16] typo and worker scale --- terraform/config.template.yaml | 6 +++--- terraform/deployment.tf | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/terraform/config.template.yaml b/terraform/config.template.yaml index c07a2268..c285e90e 100644 --- a/terraform/config.template.yaml +++ b/terraform/config.template.yaml @@ -2,8 +2,8 @@ level: debug worker: temp_dir: /tempfs - threads_per_worker: 1 - jobs: 2 + threads_per_worker: ${worker_threads} + jobs: ${worker_jobs} health: enabled: true @@ -11,7 +11,7 @@ health: monitoring: enabled: true - bind: "0.0.0.0.9100" + bind: "0.0.0.0:9100" message_queue: rmq: diff --git a/terraform/deployment.tf b/terraform/deployment.tf index d3b6cfb5..4b1c7b75 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -13,6 +13,8 @@ resource "kubernetes_secret" "app" { data = { "config.yaml" = templatefile("${path.module}/config.template.yaml", { rmq_uri = local.infra.rabbitmq_uri + worker_threads = var.production ? 3 : 1 + worker_jobs = var.production ? 2 : 1 }) } } From 5f6fff481036ca594b96b557d5a990e9dffb45de Mon Sep 17 00:00:00 2001 From: Anatole Date: Sun, 6 Aug 2023 23:13:08 +0200 Subject: [PATCH 09/16] s3 --- Makefile | 6 ++++++ terraform/config.template.yaml | 7 +++++++ terraform/deployment.tf | 3 +++ terraform/providers.tf | 7 +++++++ 4 files changed, 23 insertions(+) diff --git a/Makefile b/Makefile index e49f4350..c88c18c1 100644 --- a/Makefile +++ b/Makefile @@ -26,3 +26,9 @@ dev_deps: yarn $(MAKE) -C cpp dev_deps $(MAKE) -C go dev_deps + +terraform: + terraform -chdir=./terraform init + +deploy: + terraform -chdir=./terraform apply -auto-approve diff --git a/terraform/config.template.yaml b/terraform/config.template.yaml index c285e90e..3dfb4d6c 100644 --- a/terraform/config.template.yaml +++ b/terraform/config.template.yaml @@ -14,6 +14,13 @@ monitoring: bind: "0.0.0.0:9100" message_queue: + mode: "RMQ" + jobs_queue: "seventv_image_processor_jobs" rmq: uri: ${rmq_uri} max_reconnect_attempts: 10 + +s3: + region: ${s3_region} + access_token: "${s3_access_key}" + secret_key: "${s3_secret_key}" diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 4b1c7b75..97f36fdd 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -15,6 +15,9 @@ resource "kubernetes_secret" "app" { rmq_uri = local.infra.rabbitmq_uri worker_threads = var.production ? 3 : 1 worker_jobs = var.production ? 2 : 1 + s3_region = local.infra.region + s3_access_key = local.infra.s3_access_key.id + s3_secret_key = local.infra.s3_access_key.secret }) } } diff --git a/terraform/providers.tf b/terraform/providers.tf index 7f1264ef..e1128f94 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -31,6 +31,13 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) } +provider "kubectl" { + host = data.aws_eks_cluster.cluster.endpoint + token = data.aws_eks_cluster_auth.cluster.token + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + load_config_file = false +} + data "aws_eks_cluster" "cluster" { name = local.infra_workspace_name } From bf005cf7682a3070a79f758a53469687da9815f2 Mon Sep 17 00:00:00 2001 From: Anatole Date: Sat, 12 Aug 2023 04:03:05 +0200 Subject: [PATCH 10/16] custom s3 --- terraform/config.template.yaml | 1 + terraform/deployment.tf | 11 ++++++----- terraform/main.tf | 5 +++++ terraform/variables.tf | 13 +++++++++++++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/terraform/config.template.yaml b/terraform/config.template.yaml index 3dfb4d6c..baff59ce 100644 --- a/terraform/config.template.yaml +++ b/terraform/config.template.yaml @@ -24,3 +24,4 @@ s3: region: ${s3_region} access_token: "${s3_access_key}" secret_key: "${s3_secret_key}" + endpoint: "${s3_endpoint}" \ No newline at end of file diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 97f36fdd..82081ec6 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -12,12 +12,13 @@ resource "kubernetes_secret" "app" { data = { "config.yaml" = templatefile("${path.module}/config.template.yaml", { - rmq_uri = local.infra.rabbitmq_uri + rmq_uri = local.infra.rabbitmq_uri worker_threads = var.production ? 3 : 1 - worker_jobs = var.production ? 2 : 1 - s3_region = local.infra.region - s3_access_key = local.infra.s3_access_key.id - s3_secret_key = local.infra.s3_access_key.secret + worker_jobs = var.production ? 2 : 1 + s3_region = local.s3.region + s3_access_key = local.s3.ak + s3_secret_key = local.s3.sk + s3_endpoint = local.s3.endpoint != null ? local.s3.endpoint : "" }) } } diff --git a/terraform/main.tf b/terraform/main.tf index 960c425f..02377f55 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -13,4 +13,9 @@ locals { infra_workspace_name = replace(terraform.workspace, "image-processor", "infra") infra = data.terraform_remote_state.infra.outputs image_url = var.image_url != null ? var.image_url : format("ghcr.io/seventv/image-processor:%s-latest", trimprefix(terraform.workspace, "seventv-image-processor-")) + s3 = var.s3 != null ? var.s3 : { + region = local.infra.region + ak = local.infra.s3_access_key.id + sk = local.infra.s3_access_key.secret + } } diff --git a/terraform/variables.tf b/terraform/variables.tf index a869c114..81bf120d 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -36,3 +36,16 @@ variable "image_pull_policy" { type = string default = "Always" } + +variable "s3" { + type = object({ + endpoint = string + region = string + ak = string + sk = string + internal_bucket = string + public_bucket = string + }) + nullable = true + default = null +} From 8cf22fc88a9f8f4a1f79b4ed053f8db0207ce2e7 Mon Sep 17 00:00:00 2001 From: Melonify Date: Mon, 18 Sep 2023 16:04:11 -0400 Subject: [PATCH 11/16] Bump libwebp to version 1.3.2 --- cpp/third-party/libwebp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/third-party/libwebp b/cpp/third-party/libwebp index 404c1622..ca332209 160000 --- a/cpp/third-party/libwebp +++ b/cpp/third-party/libwebp @@ -1 +1 @@ -Subproject commit 404c1622f89f8def606eb571e17aca1bda39be19 +Subproject commit ca332209cb5567c9b249c86788cb2dbf8847e760 From 61dc67d2842d277658a275b131c3d57865988785 Mon Sep 17 00:00:00 2001 From: Melonify Date: Mon, 18 Sep 2023 19:41:39 -0400 Subject: [PATCH 12/16] Remove sub package checks for WebP in cmake --- cpp/apps/convert_png/CMakeLists.txt | 2 +- cpp/apps/dump_png/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/apps/convert_png/CMakeLists.txt b/cpp/apps/convert_png/CMakeLists.txt index 7cd472ed..ffb7e72a 100644 --- a/cpp/apps/convert_png/CMakeLists.txt +++ b/cpp/apps/convert_png/CMakeLists.txt @@ -1,7 +1,7 @@ project(convert_png) find_package(Gifski REQUIRED) -find_package(WebP REQUIRED webpmux webp) +find_package(WebP REQUIRED) find_package(OpenCV REQUIRED) find_package(libavif REQUIRED) diff --git a/cpp/apps/dump_png/CMakeLists.txt b/cpp/apps/dump_png/CMakeLists.txt index 6babefb4..5653bbc0 100644 --- a/cpp/apps/dump_png/CMakeLists.txt +++ b/cpp/apps/dump_png/CMakeLists.txt @@ -1,7 +1,7 @@ project(webp_dump) find_package(OpenCV REQUIRED) -find_package(WebP REQUIRED webp webpdemux) +find_package(WebP REQUIRED) find_package(libavif REQUIRED) add_executable(dump_png dump_png.cpp) From 8dc13262cb29c2076ed6ca6e30f4f4aa3853b9f7 Mon Sep 17 00:00:00 2001 From: Melonify Date: Tue, 19 Sep 2023 01:52:44 -0400 Subject: [PATCH 13/16] Fix docker build --- cpp/third-party/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/third-party/Makefile b/cpp/third-party/Makefile index bc4164b1..aab77e2e 100644 --- a/cpp/third-party/Makefile +++ b/cpp/third-party/Makefile @@ -39,6 +39,7 @@ _libavif: _build _aom -DCMAKE_PREFIX_PATH=$$(realpath $$(pwd)/../../../out) \ -D_AOM_INCLUDEDIR=$$(realpath $$(pwd)/../../../out/include) \ -D_AOM_LIBDIR=$$(realpath $$(pwd)/../../../../out/lib) \ + -DCMAKE_INSTALL_PREFIX=$$(realpath $$(pwd)/../../../out/) \ -DCMAKE_INSTALL_BINDIR=$$(realpath $$(pwd)/../../../out/bin) \ -DCMAKE_INSTALL_LIBDIR=$$(realpath $$(pwd)/../../../out/lib) \ -DCMAKE_INSTALL_INCLUDEDIR=$$(realpath $$(pwd)/../../../out/include) && \ From 4d1d31a1f4bad28d576f9a59b08ea08d20d7993f Mon Sep 17 00:00:00 2001 From: bread <75878656+broadeditz@users.noreply.github.com> Date: Tue, 28 Nov 2023 02:33:56 +0100 Subject: [PATCH 14/16] bump gifski to version 1.31.1 --- cpp/third-party/gifski | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/third-party/gifski b/cpp/third-party/gifski index 47e71bc8..7531529b 160000 --- a/cpp/third-party/gifski +++ b/cpp/third-party/gifski @@ -1 +1 @@ -Subproject commit 47e71bc89ed58e4d731c55dac65c6bdd3963950c +Subproject commit 7531529ba771507721c638a7ec653ea899ca17da From d6368623418ea3053a46615694b3277e791fd27f Mon Sep 17 00:00:00 2001 From: bread <75878656+broadeditz@users.noreply.github.com> Date: Fri, 1 Dec 2023 20:32:54 +0100 Subject: [PATCH 15/16] bump message-queue version --- go/go.mod | 3 ++- go/go.sum | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go/go.mod b/go/go.mod index db0b0e24..05fedb73 100644 --- a/go/go.mod +++ b/go/go.mod @@ -10,7 +10,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/prometheus/client_golang v1.12.2 github.com/seventv/common v0.0.0-20220930061340-588faaebd0d7 - github.com/seventv/message-queue/go v0.0.0-20220623223012-800919900c0d + github.com/seventv/message-queue/go v0.0.0-20231201171845-1bb9d5db6881 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 github.com/valyala/fasthttp v1.38.0 @@ -42,6 +42,7 @@ require ( github.com/prometheus/common v0.36.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rabbitmq/amqp091-go v1.3.4 // indirect + github.com/seventv/message-queue v0.0.0-20231201171845-1bb9d5db6881 // indirect github.com/spf13/afero v1.8.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect diff --git a/go/go.sum b/go/go.sum index 2bdce881..cd087d99 100644 --- a/go/go.sum +++ b/go/go.sum @@ -253,8 +253,12 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/seventv/common v0.0.0-20220930061340-588faaebd0d7 h1:8IfV57rVEsAca8Yj5iqKzvueNCMPDw5J/ynWkTrAibI= github.com/seventv/common v0.0.0-20220930061340-588faaebd0d7/go.mod h1:jHHFe3uNMyzb/ReDqMvaI/A1euvV/PW/G+2PhcWQqWw= +github.com/seventv/message-queue v0.0.0-20231201171845-1bb9d5db6881 h1:u6/W1jwypoxJ2gEiA2QEnTfLyMkgbTbuFrF2YvOP0LU= +github.com/seventv/message-queue v0.0.0-20231201171845-1bb9d5db6881/go.mod h1:iGzjTPvD36tzY1zqz/xYjG1D1NnmlzbSUZU+7TX8FkI= github.com/seventv/message-queue/go v0.0.0-20220623223012-800919900c0d h1:A3LaacuowEN3oKAkQZakz4eK90bafCSurxMSp4zUnHk= github.com/seventv/message-queue/go v0.0.0-20220623223012-800919900c0d/go.mod h1:L1iYDSmltUnxlVGX9RayCoVi3e8aNtFrrLJL6Jv+mrM= +github.com/seventv/message-queue/go v0.0.0-20231201171845-1bb9d5db6881 h1:hftRpO0JO4o+EKJFwTMK5jW2ayMR3PLgCJm9tBKp5tI= +github.com/seventv/message-queue/go v0.0.0-20231201171845-1bb9d5db6881/go.mod h1:L1iYDSmltUnxlVGX9RayCoVi3e8aNtFrrLJL6Jv+mrM= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= From 807c1c11545e041f56f5e886ed3670bdce80ca59 Mon Sep 17 00:00:00 2001 From: bread <75878656+broadeditz@users.noreply.github.com> Date: Fri, 1 Dec 2023 20:33:03 +0100 Subject: [PATCH 16/16] increase resources --- terraform/deployment.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/deployment.tf b/terraform/deployment.tf index 82081ec6..65def096 100644 --- a/terraform/deployment.tf +++ b/terraform/deployment.tf @@ -13,8 +13,8 @@ resource "kubernetes_secret" "app" { data = { "config.yaml" = templatefile("${path.module}/config.template.yaml", { rmq_uri = local.infra.rabbitmq_uri - worker_threads = var.production ? 3 : 1 - worker_jobs = var.production ? 2 : 1 + worker_threads = 3 + worker_jobs = 2 s3_region = local.s3.region s3_access_key = local.s3.ak s3_secret_key = local.s3.sk @@ -90,12 +90,12 @@ resource "kubernetes_deployment" "app" { resources { requests = { - cpu = var.production ? "6000m" : "1500m" - memory = var.production ? "7Gi" : "1Gi" + cpu = "6000m" + memory = "7Gi" } limits = { - cpu = var.production ? "6000m" : "1500m" - memory = var.production ? "7Gi" : "1Gi" + cpu = "6000m" + memory = "7Gi" } }