diff --git a/.github/workflows/build-newman.yaml b/.github/workflows/build-newman.yaml index de304b2e..a44e371b 100644 --- a/.github/workflows/build-newman.yaml +++ b/.github/workflows/build-newman.yaml @@ -13,6 +13,7 @@ on: pull_request: paths: - 'docker/**' + - 'helm/**' # push: # branches: # - master @@ -43,8 +44,8 @@ jobs: run: | # Get the latest tag from ECR (or set to default if no tags) LATEST_TAG=$(aws ecr describe-images --repository-name $ECR_REPOSITORY \ - --query 'sort_by(imageDetails, &imagePushedAt)[-1].imageTags[0]' \ - --output text 2>/dev/null | awk '{print ($0 == "None" || $0 == "" ? "0.1.8" : $0)}') + --query 'sort_by(imageDetails, &imagePushedAt)[-1].imageTags[0]' \ + --output text 2>/dev/null | awk '{print ($0 == "None" || $0 == "" ? "0.1.8" : $0)}') # Increment the patch version by 1 IFS='.' read -r -a TAG_PARTS <<< "$LATEST_TAG" @@ -55,21 +56,21 @@ jobs: echo "image_tag=$NEW_TAG" >> $GITHUB_ENV - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 - - name: Build, tag, and push image to Amazon ECR - id: build-image - uses: docker/build-push-action@v6 - timeout-minutes: 15 - with: - platforms: | - linux/amd64 - context: './docker' - file: './docker/Dockerfile.k8s' - push: true - tags: | - ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ env.image_tag }} + # - name: Build, tag, and push image to Amazon ECR + # id: build-image + # uses: docker/build-push-action@v6 + # timeout-minutes: 15 + # with: + # platforms: | + # linux/amd64 + # context: './docker' + # file: './docker/Dockerfile.k8s' + # push: true + # tags: | + # ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}:${{ env.image_tag }} # an event for calling a listening workflow # - name: Update helm chart @@ -78,3 +79,25 @@ jobs: # event-type: helm-chart-update # client-payload: '{"application-image-tag": "${{ env.image_tag }}"}' + check-updated-charts: + runs-on: ubuntu-22.04 + outputs: + chart-list: ${{ steps.get-changed-files.outputs.chart-list }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: get list of changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + since_last_remote_commit: true + - name: Get of changed helm charts changed files + id: get-changed-files + run: | + set -xe + for CHART in ${{ steps.changed-files.outputs.all_changed_files }}; do + [[ $(echo ${CHART} | grep 'helm/.*/') ]] && echo ${CHART} | awk -F / '{print $2}' >> changed_charts.txt + done + cat changed_charts.txt | uniq | jq --raw-input --slurp --compact-output 'split("\n") | map(select(length > 0))' > result.txt + cat result.txt + echo "chart-list=$(cat result.txt)" >> $GITHUB_OUTPUT diff --git a/.gitignore b/.gitignore index 459724d2..6074f814 100644 --- a/.gitignore +++ b/.gitignore @@ -129,6 +129,9 @@ crashlytics-build.properties *.war *.ear +# MongoDB data +docker/data/* + # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* @@ -268,4 +271,10 @@ tests-logs/ logs elm-stuff *.keystore -Main.js \ No newline at end of file +Main.js + +# Ignore helm charts +*/*.tgz + +# Ignore passhprases +*/ecrpass.* diff --git a/docker/Dockerfile b/docker/Dockerfile index 7f43c173..dcf381c2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,10 +8,10 @@ ARG gid=1001 RUN apt-get update -y RUN apt-get install coreutils wget -y -ENV MAVEN_VERSION 3.6.3 -ENV MAVEN_FILE apache-maven-${MAVEN_VERSION}-bin.tar.gz -ENV NODE_VERSION 22.11.0 -ENV NODE_FILE node-v${NODE_VERSION}-linux-x64.tar.gz +ENV MAVEN_VERSION=3.6.3 +ENV MAVEN_FILE=apache-maven-${MAVEN_VERSION}-bin.tar.gz +ENV NODE_VERSION=22.11.0 +ENV NODE_FILE=node-v${NODE_VERSION}-linux-x64.tar.gz # Jenkins is run with user `newman`, uid = 1000 # If you bind mount a volume from the host or a data container, @@ -21,7 +21,7 @@ RUN groupadd -g ${gid} ${group} \ # Install Java. RUN apt-get install openjdk-8-jdk -y -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ # Install Maven RUN wget --no-verbose -O /tmp/${MAVEN_FILE} \ @@ -31,7 +31,6 @@ RUN wget --no-verbose -O /tmp/${MAVEN_FILE} \ RUN echo "9792c717f5845d952907d5144b8253c3 /tmp/${MAVEN_FILE}" | \ md5sum -c - # install in /opt/maven RUN mkdir -p /opt/maven @@ -65,4 +64,13 @@ ENV PATH=$JAVA_HOME/bin:$PATH USER root RUN apt-get install sudo curl -y netbase +RUN mkdir /.m2 +COPY . newman +WORKDIR /newman +RUN ls -lah +RUN mvn clean install -DskipTests -Dmaven.repo.local=~/.m2/repository +RUN mkdir -p /newman-server/bin \ + && cp /newman/newman-server/target/newman-server-1.0.jar /newman-server/bin/newman-server.jar +RUN chown -R ${user}:${group} /newman-server +RUN rm -rf /newman USER ${user} diff --git a/docker/Dockerfile.k8s b/docker/Dockerfile.k8s new file mode 100644 index 00000000..00408ad4 --- /dev/null +++ b/docker/Dockerfile.k8s @@ -0,0 +1,78 @@ +FROM ubuntu:24.04 + +ARG user=newman +ARG group=newman +ARG uid=1001 +ARG gid=1001 + +RUN apt-get update -y +RUN apt-get install coreutils wget -y + +ENV MAVEN_VERSION=3.6.3 +ENV MAVEN_FILE=apache-maven-${MAVEN_VERSION}-bin.tar.gz +ENV NODE_VERSION=22.11.0 +ENV NODE_FILE=node-v${NODE_VERSION}-linux-x64.tar.gz + +# Jenkins is run with user `newman`, uid = 1000 +# If you bind mount a volume from the host or a data container, +# ensure you use the same uid +RUN groupadd -g ${gid} ${group} \ + && useradd -d "/home/${user}" -u ${uid} -g ${gid} -m -s /bin/bash ${user} + +# Install Java. +RUN apt-get update && apt-get install -y openjdk-8-jdk-headless unzip +ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ + +# Install Maven +RUN wget --no-verbose -O /tmp/${MAVEN_FILE} \ + http://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${MAVEN_FILE} + +# stop building if md5sum does not match +RUN echo "9792c717f5845d952907d5144b8253c3 /tmp/${MAVEN_FILE}" | \ + md5sum -c + +# install in /opt/maven +RUN mkdir -p /opt/maven + +RUN tar xzf /tmp/${MAVEN_FILE} --strip-components=1 \ + -C /opt/maven + +RUN ln -s /opt/maven/bin/mvn /usr/local/bin +RUN rm -f /tmp/${MAVEN_FILE} + +# get node +RUN echo 'get node tar.gz' +RUN wget --no-verbose -O /tmp/${NODE_FILE} \ + https://nodejs.org/dist/v${NODE_VERSION}/${NODE_FILE} + +RUN mkdir /opt/node + +# unpack node tar.gz +RUN echo 'unpacking node tar.gz' && \ + tar zxvf /tmp/${NODE_FILE} --strip-components=1 \ + -C /opt/node && \ + rm -f /tmp/${NODE_FILE} + +RUN ln -s /opt/node/bin/node /usr/local/bin && ln -s /opt/node/bin/npm /usr/local/bin + +RUN node --version && \ + npm --version + +VOLUME /data/ +ENV PATH=$JAVA_HOME/bin:$PATH + +USER root +RUN apt-get install sudo curl -y netbase + +RUN mkdir /.m2 && chown ${user}:${group} /.m2 +#COPY --chown=${user}:${group} --from=newman . /newman +RUN wget https://github.com/giga-dev/newman/archive/refs/heads/master.zip -O /tmp/master.zip \ + && unzip /tmp/master.zip -d /tmp \ + && mv /tmp/newman-master /newman \ + && rm /tmp/master.zip +RUN chown -R ${user}:${group} /newman +USER ${user} +WORKDIR /newman +RUN mvn clean install +WORKDIR / +# ENTRYPOINT [ "sh", "-c", "/newman", "/newman/newman-server/bin/newman-server.sh"] diff --git a/docker/docker-build.sh b/docker/docker-build.sh index 007e375a..5291f1a4 100755 --- a/docker/docker-build.sh +++ b/docker/docker-build.sh @@ -1,6 +1,6 @@ #!/bin/bash DIRNAME=`cd $(dirname ${BASH_SOURCE[0]}) && pwd` -docker build --build-arg user=$USER --build-arg uid=$(id -u) --build-arg gid=$(id -g) -t newman ${DIRNAME} +docker build --build-arg user=$USER --build-arg uid=$(id -u) --build-arg gid=$(id -g) -t newman/newman-server:latest ${DIRNAME} -docker pull mongo \ No newline at end of file +#docker pull mongo:4.0.27 diff --git a/docker/server-run.sh b/docker/server-run.sh index 71bc5427..1dfbbe17 100755 --- a/docker/server-run.sh +++ b/docker/server-run.sh @@ -1,6 +1,6 @@ #!/bin/bash DIRNAME=`cd $(dirname ${BASH_SOURCE[0]}) && pwd` - +echo $DIRNAME MODE="-d" if [[ -n "$1" ]]; then if [[ "$1" == "-iii" ]]; then @@ -10,7 +10,6 @@ if [[ -n "$1" ]]; then fi fi - docker stop newman-server docker rm newman-server @@ -21,4 +20,4 @@ while [[ -z "$(${cmd})" ]]; do sleep 5s done -docker run ${MODE} --link mongo-server --rm -v "${DIRNAME}/..":"/newman" --name newman-server --user $(id -u) -p 8443:8443 newman "/newman/newman-server/bin/newman-server.sh" \ No newline at end of file +docker run ${MODE} --link mongo-server --rm -v "${DIRNAME}/..":"/newman" --name newman-server --user $(id -u) -p 8443:8443 newman "/newman/newman-server/bin/newman-server.sh" diff --git a/helm/mongo-server/.helmignore b/helm/mongo-server/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/helm/mongo-server/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/mongo-server/Chart.yaml b/helm/mongo-server/Chart.yaml new file mode 100644 index 00000000..9bd061ee --- /dev/null +++ b/helm/mongo-server/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: mongo-server +description: A Helm chart for deploying MongoDB +type: application +version: 0.1.10 +appVersion: "4.0.27" diff --git a/helm/mongo-server/templates/configmap.yaml b/helm/mongo-server/templates/configmap.yaml new file mode 100644 index 00000000..30ab4011 --- /dev/null +++ b/helm/mongo-server/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: "{{ .Release.Name }}-config" + namespace: {{ .Release.Namespace }} + labels: + app: mongo-server +data: + mongodb.conf: | + storage: + dbPath: /data/db + net: + bindIp: 0.0.0.0 + port: 27017 + security: + authorization: enabled \ No newline at end of file diff --git a/helm/mongo-server/templates/pvc.yaml b/helm/mongo-server/templates/pvc.yaml new file mode 100644 index 00000000..62baa5f4 --- /dev/null +++ b/helm/mongo-server/templates/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "{{ .Release.Name }}-pvc" + namespace: {{ .Release.Namespace }} + labels: + app: mongo-server +spec: + accessModes: {{ .Values.persistence.accessModes | toJson }} + resources: + requests: + storage: {{ .Values.persistence.size }} + storageClassName: {{ .Values.persistence.storageClass | quote }} \ No newline at end of file diff --git a/helm/mongo-server/templates/service.yaml b/helm/mongo-server/templates/service.yaml new file mode 100644 index 00000000..82dee09c --- /dev/null +++ b/helm/mongo-server/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: mongo-server +spec: + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.port }} + selector: + app: mongo-server diff --git a/helm/mongo-server/templates/statefulset.yaml b/helm/mongo-server/templates/statefulset.yaml new file mode 100644 index 00000000..b4bbe094 --- /dev/null +++ b/helm/mongo-server/templates/statefulset.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: mongo-server +spec: + serviceName: mongo-server-headless + replicas: 1 + selector: + matchLabels: + app: mongo-server + template: + metadata: + labels: + app: mongo-server + spec: + initContainers: + - name: init-mongo-config + image: busybox + command: + - sh + - -c + - | + cp /config/mongodb.conf /etc/mongo/mongodb.conf + volumeMounts: + - name: config-volume + mountPath: /config + - name: mongo-config + mountPath: /etc/mongo + containers: + - name: mongo-server + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: {{ .Values.service.port }} + env: + - name: MONGODB_ROOT_PASSWORD + value: {{ .Values.mongodb.password | quote }} + - name: MONGODB_USERNAME + value: {{ .Values.mongodb.username | quote }} + - name: MONGODB_PASSWORD + value: {{ .Values.mongodb.password | quote }} + - name: MONGODB_DATABASE + value: {{ .Values.mongodb.database | quote }} + command: + - mongod + - "--config" + - /etc/mongo/mongodb.conf + volumeMounts: + - name: mongo-data + mountPath: "/data/db" + - name: mongo-config + mountPath: "/etc/mongo/" + volumes: + - name: config-volume + configMap: + name: "{{ .Release.Name }}-config" + securityContext: + fsGroup: 1001 + volumeClaimTemplates: + - metadata: + name: mongo-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi + - metadata: + name: mongo-config + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/helm/mongo-server/values.yaml b/helm/mongo-server/values.yaml new file mode 100644 index 00000000..28c43b0d --- /dev/null +++ b/helm/mongo-server/values.yaml @@ -0,0 +1,30 @@ +image: + repository: bitnami/mongodb + tag: 4.0.27-debian-9-r118 + pullPolicy: IfNotPresent + +service: + type: ClusterIP + port: 27017 + +resources: + limits: + cpu: 500m + memory: 1024Mi + requests: + cpu: 250m + memory: 256Mi + +mongodb: + username: newman + password: newman + database: newman-db + +persistence: + enabled: true + accessModes: + - ReadWriteOnce + size: 8Gi + storageClass: gp3 + +tolerations: [] # Define tolerations if needed for running on specific nodes diff --git a/helm/newman-server/.helmignore b/helm/newman-server/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/helm/newman-server/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/newman-server/Chart.yaml b/helm/newman-server/Chart.yaml new file mode 100644 index 00000000..f3144ae9 --- /dev/null +++ b/helm/newman-server/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: newman +description: Helm chart for the Newman Server application +type: application +version: 0.1.16 +appVersion: 0.1.16 \ No newline at end of file diff --git a/helm/newman-server/templates/ingress.yaml b/helm/newman-server/templates/ingress.yaml new file mode 100644 index 00000000..f54672b8 --- /dev/null +++ b/helm/newman-server/templates/ingress.yaml @@ -0,0 +1,37 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: newman-server-ingress + namespace: newman + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: newman-server + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/newman-server/templates/pvc.yaml b/helm/newman-server/templates/pvc.yaml new file mode 100644 index 00000000..1da38bbc --- /dev/null +++ b/helm/newman-server/templates/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "{{ .Release.Name }}-pvc" + namespace: newman + labels: + app: newman-server +spec: + accessModes: {{ .Values.persistence.accessModes | toJson }} + resources: + requests: + storage: {{ .Values.persistence.size }} + storageClassName: {{ .Values.persistence.storageClass | quote }} diff --git a/helm/newman-server/templates/service.yaml b/helm/newman-server/templates/service.yaml new file mode 100644 index 00000000..704bdce5 --- /dev/null +++ b/helm/newman-server/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: newman-server +spec: + type: NodePort + selector: + app: newman-server + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + nodePort: 30080 \ No newline at end of file diff --git a/helm/newman-server/templates/statefulset.yaml b/helm/newman-server/templates/statefulset.yaml new file mode 100644 index 00000000..88389b1f --- /dev/null +++ b/helm/newman-server/templates/statefulset.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: newman-server +spec: + serviceName: newman-server-headless + replicas: 1 + selector: + matchLabels: + app: newman-server + template: + metadata: + labels: + app: newman + spec: + containers: + - name: newman-server + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: {{ .Values.service.targetPort }} + command: + - /bin/sh + - -c + - | + /newman/newman-server/bin/newman-server.sh + env: + - name: MONGODB_HOST + value: {{ .Values.mongodb.host | quote }} + - name: MONGODB_PORT + value: {{ .Values.mongodb.port | quote }} + - name: MONGODB_DATABASE + value: {{ .Values.mongodb.database | quote }} + - name: MONGODB_USERNAME + value: {{ .Values.mongodb.username | quote }} + - name: MONGODB_PASSWORD + value: {{ .Values.mongodb.password | quote }} + volumeMounts: + - name: data-volume + mountPath: /data/newman-server + volumeClaimTemplates: + - metadata: + name: data-volume + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/helm/newman-server/values.yaml b/helm/newman-server/values.yaml new file mode 100644 index 00000000..66c78561 --- /dev/null +++ b/helm/newman-server/values.yaml @@ -0,0 +1,35 @@ +replicaCount: 1 + +image: + repository: 573366771204.dkr.ecr.us-east-1.amazonaws.com/xap-newman/server + tag: 0.1.16 + pullPolicy: IfNotPresent + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + +ingress: + enabled: true + annotations: {} + hosts: + - host: newman-server.gigaspaces.net + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + +persistence: + enabled: true + size: 10Gi + accessModes: + - ReadWriteOnce + storageClass: gp3 + +mongodb: + host: mongo-server + port: 27017 + username: newman + password: newman + database: newman-db \ No newline at end of file diff --git a/helm/server/.helmignore b/helm/server/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/helm/server/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/server/Chart.yaml b/helm/server/Chart.yaml new file mode 100644 index 00000000..86f18f0c --- /dev/null +++ b/helm/server/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: server +description: Helm chart for the Newman Server application +type: application +version: 0.1.17 +appVersion: 0.1.17 \ No newline at end of file diff --git a/helm/server/templates/ingress.yaml b/helm/server/templates/ingress.yaml new file mode 100644 index 00000000..f54672b8 --- /dev/null +++ b/helm/server/templates/ingress.yaml @@ -0,0 +1,37 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: newman-server-ingress + namespace: newman + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: newman-server + port: + number: {{ $.Values.service.port }} + {{- end }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/server/templates/pvc.yaml b/helm/server/templates/pvc.yaml new file mode 100644 index 00000000..1da38bbc --- /dev/null +++ b/helm/server/templates/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "{{ .Release.Name }}-pvc" + namespace: newman + labels: + app: newman-server +spec: + accessModes: {{ .Values.persistence.accessModes | toJson }} + resources: + requests: + storage: {{ .Values.persistence.size }} + storageClassName: {{ .Values.persistence.storageClass | quote }} diff --git a/helm/server/templates/service.yaml b/helm/server/templates/service.yaml new file mode 100644 index 00000000..704bdce5 --- /dev/null +++ b/helm/server/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: newman-server +spec: + type: NodePort + selector: + app: newman-server + ports: + - port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + nodePort: 30080 \ No newline at end of file diff --git a/helm/server/templates/statefulset.yaml b/helm/server/templates/statefulset.yaml new file mode 100644 index 00000000..9673772b --- /dev/null +++ b/helm/server/templates/statefulset.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + app: server +spec: + serviceName: newman-server-headless + replicas: 1 + selector: + matchLabels: + app: server + template: + metadata: + labels: + app: server + spec: + containers: + - name: newman-server + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: {{ .Values.service.targetPort }} + command: + - /bin/sh + - -c + - | + /newman/newman-server/bin/newman-server.sh + env: + - name: MONGODB_HOST + value: {{ .Values.mongodb.host | quote }} + - name: MONGODB_PORT + value: {{ .Values.mongodb.port | quote }} + - name: MONGODB_DATABASE + value: {{ .Values.mongodb.database | quote }} + - name: MONGODB_USERNAME + value: {{ .Values.mongodb.username | quote }} + - name: MONGODB_PASSWORD + value: {{ .Values.mongodb.password | quote }} + volumeMounts: + - name: data-volume + mountPath: /data/newman-server + volumeClaimTemplates: + - metadata: + name: data-volume + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/helm/server/values.yaml b/helm/server/values.yaml new file mode 100644 index 00000000..d58b1e58 --- /dev/null +++ b/helm/server/values.yaml @@ -0,0 +1,35 @@ +replicaCount: 1 + +image: + repository: 573366771204.dkr.ecr.us-east-1.amazonaws.com/xap-newman/server + tag: 0.1.17 + pullPolicy: IfNotPresent + +service: + type: ClusterIP + port: 8080 + targetPort: 8080 + +ingress: + enabled: true + annotations: {} + hosts: + - host: newman-server.gigaspaces.net + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + +persistence: + enabled: true + size: 10Gi + accessModes: + - ReadWriteOnce + storageClass: gp3 + +mongodb: + host: mongo-server + port: 27017 + username: newman + password: newman + database: newman-db \ No newline at end of file diff --git a/minikube/cronjob.yaml b/minikube/cronjob.yaml new file mode 100644 index 00000000..c2bfe269 --- /dev/null +++ b/minikube/cronjob.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: refresh-ecr-token + namespace: default +spec: + schedule: "0 */12 * * *" # Runs every 12 hours + jobTemplate: + spec: + template: + spec: + containers: + - name: ecr-token-refresh + image: amazon/aws-cli:latest + args: + - /bin/bash + - -c + - > + aws ecr get-login-password --region us-east-1 | + kubectl create secret docker-registry ecr-secret + --docker-server=573366771204.dkr.ecr.us-east-1.amazonaws.com + --docker-username=AWS + --docker-password=$(aws ecr get-login-password --region us-east-1) + --dry-run=client -o yaml | + kubectl apply -f - + restartPolicy: OnFailure diff --git a/minikube/role.yaml b/minikube/role.yaml new file mode 100644 index 00000000..38fb7b89 --- /dev/null +++ b/minikube/role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ecr-secret-manager + namespace: newman +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete"] diff --git a/minikube/rolebinding.yaml b/minikube/rolebinding.yaml new file mode 100644 index 00000000..c447e4c0 --- /dev/null +++ b/minikube/rolebinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ecr-secret-manager-binding + namespace: newman +subjects: + - kind: ServiceAccount + name: ecr-refresh-serviceaccount + namespace: newman +roleRef: + kind: Role + name: ecr-secret-manager + apiGroup: rbac.authorization.k8s.io diff --git a/minikube/secret.yaml b/minikube/secret.yaml new file mode 100644 index 00000000..d3c87d05 --- /dev/null +++ b/minikube/secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: aws-credentials + namespace: newman +type: Opaque +data: + aws_access_key_id: + aws_secret_access_key: \ No newline at end of file diff --git a/minikube/serviceaccount.yaml b/minikube/serviceaccount.yaml new file mode 100644 index 00000000..36ede263 --- /dev/null +++ b/minikube/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ecr-refresh-serviceaccount + namespace: newman diff --git a/newman-server/bin/build.sh b/newman-server/bin/build.sh index 9b913dae..9422eef4 100755 --- a/newman-server/bin/build.sh +++ b/newman-server/bin/build.sh @@ -7,6 +7,6 @@ cd ${DIRNAME}/../ #echo "running elm make" #./elm-make.sh -echo "Running mvn package " +echo "Running mvn package" cd ${DIRNAME}/../../ mvn clean install \ No newline at end of file diff --git a/newman-server/bin/newman-server.sh b/newman-server/bin/newman-server.sh index 46cebb58..4f7110f9 100755 --- a/newman-server/bin/newman-server.sh +++ b/newman-server/bin/newman-server.sh @@ -49,10 +49,6 @@ _EOF # System environment variables # Mongo host address to connect to from newman server # remote mongo db - xap-builder.gspaces.com:27017 -export NEWMAN_MONGO_DB_HOST=${NEWMAN_MONGO_DB_HOST="mongo-server"} - -# Mongo db name to access in database -export NEWMAN_MONGO_DB_NAME=${NEWMAN_MONGO_DB_NAME="newman-db"} export NEWMAN_SERVER_SPOTINST_TOKEN=${NEWMAN_SERVER_SPOTINST_TOKEN=""} export NEWMAN_SERVER_SPOTINST_ACCOUNT_ID=${NEWMAN_SERVER_SPOTINST_ACCOUNT_ID=""} @@ -68,8 +64,10 @@ export NEWMAN_SERVER_SPOTINST_ACCOUNT_ID=${NEWMAN_SERVER_SPOTINST_ACCOUNT_ID=""} java -Dproduction=true \ -Dnewman.server.spotinst.token="${NEWMAN_SERVER_SPOTINST_TOKEN}" \ -Dnewman.server.spotinst.accountId="${NEWMAN_SERVER_SPOTINST_ACCOUNT_ID}" \ - -Dnewman.mongo.db.host=${NEWMAN_MONGO_DB_HOST} \ - -Dnewman.mongo.db.name=${NEWMAN_MONGO_DB_NAME} \ + -Dnewman.mongo.db.host=${MONGODB_HOST} \ + -Dnewman.mongo.db.name=${MONGODB_DATABASE} \ + -Dnewman.mongo.db.username=${MONGODB_USERNAME} \ + -Dnewman.mongo.db.password=${MONGODB_PASSWORD} \ -Dnewman.server.realm-config-path=../config/realm.properties \ -Dnewman.keys-folder-path=../keys/server.keystore \ -Dnewman.certificate=../certs/keystore.p12 \ diff --git a/newman-server/config/realm.properties b/newman-server/config/realm.properties new file mode 100644 index 00000000..a396dfbc --- /dev/null +++ b/newman-server/config/realm.properties @@ -0,0 +1,28 @@ +ester=ester, user +yuval=yuval, user +evgeny=evgeny, user +moran=moran, user +root=root, admin +support=support, user +mishel=mishel, user +sagiv=sagiv, user +yonatan=yonatan, user +michael=michael, user +irena=irena, user +tomer=tomer, user +anton=anton, user +oleksii=oleksii, user +alesia=alesia, user +olha=olha, user +michaelg=michaelg, user +sapir=sapir, user +shai=shai, user +davyd=davyd, user +esubotin=esubotin, user +inbal=inbal, user +tolik=tolik, user +rami=rami, user +ariel=ariel, user +chani=chani, user +krystyna=krystyna, user +user=user, user \ No newline at end of file diff --git a/newman-server/pom.xml b/newman-server/pom.xml index 4dbf433e..1101e5a3 100644 --- a/newman-server/pom.xml +++ b/newman-server/pom.xml @@ -19,8 +19,6 @@ default - - ${project.groupId} @@ -47,6 +45,11 @@ org.glassfish.jersey.core jersey-server + + org.glassfish.jersey.media + jersey-media-multipart + + org.glassfish.jersey.containers jersey-container-servlet @@ -61,7 +64,8 @@ org.mongodb - mongo-java-driver + mongodb-driver-sync + ${mongodb.version} org.mongodb.morphia diff --git a/newman-server/src/main/java/com/gigaspaces/newman/NewmanResource.java b/newman-server/src/main/java/com/gigaspaces/newman/NewmanResource.java index b9ade19a..31149466 100644 --- a/newman-server/src/main/java/com/gigaspaces/newman/NewmanResource.java +++ b/newman-server/src/main/java/com/gigaspaces/newman/NewmanResource.java @@ -11,6 +11,7 @@ import com.mongodb.MongoClient; import com.mongodb.MongoInterruptedException; import com.mongodb.client.DistinctIterable; +import com.mongodb.client.MongoClients; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.Filters; @@ -122,7 +123,7 @@ public class NewmanResource { private ServerStatus serverStatus = new ServerStatus(ServerStatus.Status.RUNNING); private Thread serverSuspendThread; - public NewmanResource(@Context ServletContext servletContext) { + public NewmanResource(@Context ServletContext servletContext) throws UnknownHostException { this.config = Config.fromString(servletContext.getInitParameter("config")); //noinspection SpellCheckingInspection mongoClient = new MongoClient(config.getMongo().getHost()); @@ -140,7 +141,7 @@ public NewmanResource(@Context ServletContext servletContext) { jobConfigDAO = new JobConfigDAO(morphia, mongoClient, config.getMongo().getDb()); prioritizedJobDAO = new PrioritizedJobDAO(morphia, mongoClient, config.getMongo().getDb()); - MongoDatabase db = mongoClient.getDatabase(config.getMongo().getDb()); + MongoDatabase db = MongoClients.create().getDatabase(config.getMongo().getDb()); MongoCollection testCollection = db.getCollection("Test"); distinctTestsByAssignedAgentFilter = testCollection.distinct("assignedAgent", String.class); @@ -2429,7 +2430,7 @@ public Build getBuild(final @PathParam("id") String id) { @Path("db") @Produces(MediaType.APPLICATION_JSON) public Response deleteCollections() { - MongoDatabase db = mongoClient.getDatabase(config.getMongo().getDb()); + MongoDatabase db = MongoClients.create().getDatabase(config.getMongo().getDb()); List deleted = new ArrayList<>(); for (String name : db.listCollectionNames()) { if (!"system.indexes".equals(name)) { @@ -2587,7 +2588,7 @@ public Response deleteOfflineAgent(final @PathParam("agentName") String agentNam @Path("db/{collectionName}") @Produces(MediaType.APPLICATION_JSON) public Response deleteCollection(final @PathParam("collectionName") String collectionName) { - MongoDatabase db = mongoClient.getDatabase(config.getMongo().getDb()); + MongoDatabase db = MongoClients.create().getDatabase(config.getMongo().getDb()); MongoCollection myCollection = db.getCollection(collectionName); if (myCollection != null) { myCollection.drop(); @@ -2625,7 +2626,7 @@ public UserPrefs getCurrentUser(@Context SecurityContext sc) { @Path("db") @Produces(MediaType.APPLICATION_JSON) public Response getCollections() { - MongoDatabase db = mongoClient.getDatabase(config.getMongo().getDb()); + MongoDatabase db = MongoClients.create().getDatabase(config.getMongo().getDb()); List res = new ArrayList<>(); for (String name : db.listCollectionNames()) { if (!"system.indexes".equals(name)) { diff --git a/newman-server/src/main/java/com/gigaspaces/newman/SpotinstResource.java b/newman-server/src/main/java/com/gigaspaces/newman/SpotinstResource.java index 2cf80fb9..44e408f7 100644 --- a/newman-server/src/main/java/com/gigaspaces/newman/SpotinstResource.java +++ b/newman-server/src/main/java/com/gigaspaces/newman/SpotinstResource.java @@ -5,6 +5,7 @@ import com.gigaspaces.newman.dao.AgentDAO; import com.gigaspaces.newman.spotinst.ElasticGroup; import com.mongodb.MongoClient; +import com.mongodb.client.MongoClients; import org.mongodb.morphia.Morphia; import org.mongodb.morphia.query.Query; import org.mongodb.morphia.query.QueryResults; @@ -17,6 +18,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.io.IOException; +import java.net.UnknownHostException; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -31,7 +33,7 @@ public class SpotinstResource { private final AgentDAO agentDAO; private final Config config; - public SpotinstResource(@Context ServletContext servletContext) { + public SpotinstResource(@Context ServletContext servletContext) throws UnknownHostException { this.config = Config.fromString(servletContext.getInitParameter("config")); spotinstClient = new SpotinstClient(); mongoClient = new MongoClient(config.getMongo().getHost()); diff --git a/pom.xml b/pom.xml index 60eab7b0..5b942aa9 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ UTF-8 9.4.11.v20180605 2.18 - 3.0.0 + 5.1.0 0.111 1.3.2 1.3.2 @@ -150,7 +150,7 @@ org.mongodb - mongo-java-driver + mongo-driver-sync ${mongodb.version}