terraform

.github/workflows/ci.yaml

@@ -14,26 +14,27 @@ on:
   workflow_dispatch:
     inputs:
       deploy:
-        description: "Deploy location"
+        description: "Which environment to deploy to"
         required: true
         default: "none"
         type: choice
         options:
-          - production
-          - staging
+          - prod
+          - test
           - none
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  DEPLOY: ${{ (inputs.deploy != 'none' && inputs.deploy) || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'prod') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master') && 'prod') || ((github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'test') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'dev') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'staged')) && 'test') || 'none' }}
+
 jobs:
   ci:
     name: ImageProcessor Lint, Build, Test, Deploy
     runs-on: aws-runner
     env:
-      DEPLOY_PROD: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'production') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'master') }}
-      DEPLOY_STAGE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'staging') || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'dev') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'staged')) }}
       GOLANGCI_LINT_CACHE: /home/runner/.cache/golangci-lint
     concurrency:
       group: ${{ github.workflow }}-ci-${{ github.ref }}
@@ -45,34 +46,28 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }}
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ secrets.AWS_REGION }}
-
-      - name: Login to Amazon ECR
-        id: login-ecr
-        if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }}
-        uses: aws-actions/amazon-ecr-login@v1
-
       - name: Make build context
-        if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }}
+        if: env.DEPLOY != 'none'
         run: |
           docker context create builders
 
       - name: Setup buildx
         uses: docker/setup-buildx-action@v2
-        if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }}
+        if: env.DEPLOY != 'none'
         with:
           install: true
           endpoint: builders
 
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
       - name: Build docker image
         uses: docker/build-push-action@v3
-        if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }}
+        if: env.DEPLOY != 'none'
         with:
           context: .
           file: docker/full.Dockerfile
@@ -81,29 +76,129 @@ jobs:
           cache-to: |
             type=gha,mode=max
           tags: |
-            ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/image-processor:latest
-            ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/image-processor:${{ github.sha }}
+            ghcr.io/seventv/image-processor:${{ env.DEPLOY }}-${{ github.sha }}
+            ghcr.io/seventv/image-processor:${{ env.DEPLOY }}-latest
           push: true
 
-      - name: Update deployment template
-        uses: danielr1996/envsubst-action@1.1.0
-        if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }}
-        env:
-          IMAGE: ${{ steps.login-ecr.outputs.registry }}/${{ (env.DEPLOY_PROD == 'true' && '7tv') || '7tv-stage' }}/image-processor:${{ github.sha }}
-        with:
-          input: k8s/${{ (env.DEPLOY_PROD == 'true' && 'production') || 'staging' }}.template.yaml
-          output: k8s/deploy.yaml
+  validate:
+    name: API Deploy Validation
+    needs: ci
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    defaults:
+      run:
+        working-directory: ./terraform
 
-      - name: Setup Kubectl
-        if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }}
-        uses: azure/setup-kubectl@v3.0
+    steps:
+      - name: Checkout code
+        id: ok
+        if: env.DEPLOY != 'none'
+        uses: actions/checkout@v3
+
+      - name: "Setup Terraform"
+        if: steps.ok.outcome == 'success'
+        uses: hashicorp/setup-terraform@v1
+        with:
+          cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }}
 
-      - name: Deploy to k8s
-        if: ${{ env.DEPLOY_PROD == 'true' || env.DEPLOY_STAGE == 'true' }}
+      - name: "Terraform Init"
+        if: steps.ok.outcome == 'success'
+        id: init
         env:
-          KUBE_CONFIG_DATA: ${{ (env.DEPLOY_PROD == 'true' && secrets.KUBECONFIG) || secrets.KUBECONFIG_STAGE }}
+          TF_WORKSPACE: ${{ env.DEPLOY }}
+        run: terraform init
+        continue-on-error: true
+
+      - name: "Terraform Workspace"
+        if: steps.ok.outcome == 'success'
+        run: terraform workspace select -or-create=true ${{ env.DEPLOY }}
+
+      - name: Terraform fmt
+        if: steps.ok.outcome == 'success'
+        id: fmt
+        run: terraform fmt -check
+        continue-on-error: true
+
+      - name: Terraform Validate
+        if: steps.ok.outcome == 'success'
+        id: validate
+        run: terraform validate -no-color
+
+      - name: Terraform Variables
+        if: steps.ok.outcome == 'success'
         run: |
-          mkdir -p ~/.kube
-          (echo $KUBE_CONFIG_DATA | base64 -d) >> ~/.kube/config
+          cat <<EOF > *.auto.tfvars
+            image_url="ghcr.io/seventv/image-processor:${{ env.DEPLOY }}-${{ github.sha }}"
+            image_pull_policy="IfNotPresent"
+
+          EOF
+
+      - name: "Terraform Plan"
+        if: steps.ok.outcome == 'success'
+        id: plan
+        run: terraform plan -no-color
+
+      - uses: actions/github-script@v6
+        if: steps.ok.outcome == 'success' && github.event_name == 'pull_request'
+        env:
+          PLAN: "terraform\n${{ steps.plan.outputs.stdout }}"
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // 1. Retrieve existing bot comments for the PR
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            })
+            const botComment = comments.find(comment => {
+              return comment.user.type === 'Bot' && comment.body.includes('Terraform Format and Style')
+            })
+
+            // 2. Prepare format of the comment
+            const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+            #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+            #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
+            <details><summary>Validation Output</summary>
+
+            \`\`\`\n
+            ${{ steps.validate.outputs.stdout }}
+            \`\`\`
+
+            </details>
+
+            #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+
+            <details><summary>Show Plan</summary>
+
+            \`\`\`\n
+            ${process.env.PLAN}
+            \`\`\`
+
+            </details>
+
+            *Actor: @${{ github.actor }}, Action: \`${{ github.event_name }}\`, Workflow: \`${{ github.workflow }}\`*`;
+
+            // 3. If we have a comment, update it, otherwise create a new one
+            if (botComment) {
+              github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: output
+              })
+            } else {
+              github.rest.issues.createComment({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body: output
+              })
+            }
 
-          kubectl apply -f k8s/deploy.yaml
+      - name: "Terraform Apply"
+        if: steps.ok.outcome == 'success'
+        id: apply
+        run: terraform apply -no-color -auto-approve
+        continue-on-error: true

.gitignore

@@ -39,6 +39,7 @@ tmp/
 !.circleci/*.yaml
 !example.config.yaml
 !docker-compose.yaml
+!terraform/config.template.yaml
 !.github/**/*.yaml
 packrd/
 *-packr.go
@@ -49,3 +50,17 @@ build/
 out/
 .cache
 *.log
+
+# Terraform local state files
+**/.terraform/*
+*.tfstate
+*.tfstate.*
+*.tfplan
+crash.log
+*.tfvars
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+.terraformrc
+terraform.rc

Makefile

@@ -26,3 +26,9 @@ dev_deps:
 	yarn
 	$(MAKE) -C cpp dev_deps
 	$(MAKE) -C go dev_deps
+
+terraform:
+	terraform -chdir=./terraform init
+
+deploy:
+	terraform -chdir=./terraform apply -auto-approve

cpp/apps/convert_png/CMakeLists.txt

@@ -1,7 +1,7 @@
 project(convert_png)
 
 find_package(Gifski REQUIRED)
-find_package(WebP REQUIRED webpmux webp)
+find_package(WebP REQUIRED)
 find_package(OpenCV REQUIRED)
 find_package(libavif REQUIRED)
 

cpp/apps/dump_png/CMakeLists.txt

@@ -1,7 +1,7 @@
 project(webp_dump)
 
 find_package(OpenCV REQUIRED)
-find_package(WebP REQUIRED webp webpdemux)
+find_package(WebP REQUIRED)
 find_package(libavif REQUIRED)
 
 add_executable(dump_png dump_png.cpp)

cpp/third-party/Makefile

@@ -39,6 +39,7 @@ _libavif: _build _aom
 			-DCMAKE_PREFIX_PATH=$$(realpath $$(pwd)/../../../out) \
 			-D_AOM_INCLUDEDIR=$$(realpath $$(pwd)/../../../out/include) \
 			-D_AOM_LIBDIR=$$(realpath $$(pwd)/../../../../out/lib) \
+			-DCMAKE_INSTALL_PREFIX=$$(realpath $$(pwd)/../../../out/) \
 			-DCMAKE_INSTALL_BINDIR=$$(realpath $$(pwd)/../../../out/bin) \
 			-DCMAKE_INSTALL_LIBDIR=$$(realpath $$(pwd)/../../../out/lib) \
 			-DCMAKE_INSTALL_INCLUDEDIR=$$(realpath $$(pwd)/../../../out/include) && \

go/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/h2non/filetype v1.1.3
 	github.com/prometheus/client_golang v1.12.2
 	github.com/seventv/common v0.0.0-20220930061340-588faaebd0d7
-	github.com/seventv/message-queue/go v0.0.0-20220623223012-800919900c0d
+	github.com/seventv/message-queue/go v0.0.0-20231201171845-1bb9d5db6881
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.12.0
 	github.com/valyala/fasthttp v1.38.0
@@ -42,6 +42,7 @@ require (
 	github.com/prometheus/common v0.36.0 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rabbitmq/amqp091-go v1.3.4 // indirect
+	github.com/seventv/message-queue v0.0.0-20231201171845-1bb9d5db6881 // indirect
 	github.com/spf13/afero v1.8.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect

go/go.sum

@@ -253,8 +253,12 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/seventv/common v0.0.0-20220930061340-588faaebd0d7 h1:8IfV57rVEsAca8Yj5iqKzvueNCMPDw5J/ynWkTrAibI=
 github.com/seventv/common v0.0.0-20220930061340-588faaebd0d7/go.mod h1:jHHFe3uNMyzb/ReDqMvaI/A1euvV/PW/G+2PhcWQqWw=
+github.com/seventv/message-queue v0.0.0-20231201171845-1bb9d5db6881 h1:u6/W1jwypoxJ2gEiA2QEnTfLyMkgbTbuFrF2YvOP0LU=
+github.com/seventv/message-queue v0.0.0-20231201171845-1bb9d5db6881/go.mod h1:iGzjTPvD36tzY1zqz/xYjG1D1NnmlzbSUZU+7TX8FkI=
 github.com/seventv/message-queue/go v0.0.0-20220623223012-800919900c0d h1:A3LaacuowEN3oKAkQZakz4eK90bafCSurxMSp4zUnHk=
 github.com/seventv/message-queue/go v0.0.0-20220623223012-800919900c0d/go.mod h1:L1iYDSmltUnxlVGX9RayCoVi3e8aNtFrrLJL6Jv+mrM=
+github.com/seventv/message-queue/go v0.0.0-20231201171845-1bb9d5db6881 h1:hftRpO0JO4o+EKJFwTMK5jW2ayMR3PLgCJm9tBKp5tI=
+github.com/seventv/message-queue/go v0.0.0-20231201171845-1bb9d5db6881/go.mod h1:L1iYDSmltUnxlVGX9RayCoVi3e8aNtFrrLJL6Jv+mrM=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=