Init agent integration deb

Makefile

@@ -374,7 +374,7 @@ amazon-linux-sources.tgz:
 
 amazon-linux-rpm-integrated: .amazon-linux-rpm-integrated-done
 
-.generic-rpm-integrated-done:
+.generic-rpm-integrated-done: get-cni-sources
 	./scripts/update-version.sh
 	cp packaging/generic-rpm-integrated/amazon-ecs-init.spec amazon-ecs-init.spec
 	cp packaging/generic-rpm-integrated/ecs.service ecs.service
@@ -388,7 +388,40 @@ amazon-linux-rpm-integrated: .amazon-linux-rpm-integrated-done
 
 generic-rpm-integrated: .generic-rpm-integrated-done
 
-.generic-rpm-done:
+VERSION = $(shell cat ecs-init/ECSVERSION)
+
+.generic-deb-integrated-done: get-cni-sources
+	mkdir -p BUILDROOT
+	./scripts/update-version.sh
+	tar -czf ./amazon-ecs-init_${VERSION}.orig.tar.gz ecs-init scripts README.md
+	cp -r packaging/generic-deb-integrated/debian ecs-init scripts misc agent agent-container amazon-ecs-cni-plugins amazon-vpc-cni-plugins README.md VERSION BUILDROOT
+	cd BUILDROOT && debuild -uc -us --lintian-opts --suppress-tags bad-distribution-in-changes-file,file-in-unusual-dir
+	touch .generic-deb-integrated-done
+
+generic-deb-integrated: .generic-deb-integrated-done
+
+ARCH:=$(shell uname -m)
+ifeq (${ARCH},x86_64)
+	AGENT_FILENAME=ecs-agent-v${VERSION}.tar
+else ifeq (${ARCH},aarch64)
+	AGENT_FILENAME=ecs-agent-arm64-v${VERSION}.tar
+# osx M1 instances
+else ifeq (${ARCH},arm64)
+	AGENT_FILENAME=ecs-agent-arm64-v${VERSION}.tar
+endif
+
+BUILDROOT/ecs-agent.tar:
+	mkdir -p BUILDROOT
+	curl -o BUILDROOT/ecs-agent.tar https://s3.amazonaws.com/amazon-ecs-agent/${AGENT_FILENAME}
+
+${AGENT_FILENAME}: BUILDROOT/ecs-agent.tar
+	cp BUILDROOT/ecs-agent.tar ${AGENT_FILENAME}
+
+rpm-in-docker: ${AGENT_FILENAME}
+	docker build -t "amazon/amazon-ecs-init:build" -f "scripts/dockerfiles/build.dockerfile" .
+	docker run -u "$(shell id -u)" --tmpfs /.cache -v "$(shell pwd):/workspace/amazon-ecs-init" "amazon/amazon-ecs-init:build"
+
+.generic-rpm-done: ${AGENT_FILENAME}
 	./scripts/update-version.sh
 	cp packaging/generic-rpm/amazon-ecs-init.spec amazon-ecs-init.spec
 	cp packaging/generic-rpm/ecs.service ecs.service
@@ -409,26 +442,17 @@ generic-rpm: .generic-rpm-done
 	cd BUILDROOT && debuild -uc -us --lintian-opts --suppress-tags bad-distribution-in-changes-file,file-in-unusual-dir
 	touch .deb-done
 
+.PHONY: deb
 deb: .deb-done
 
 clean:
-	# ensure docker is running and we can talk to it, abort if not:
-	docker ps > /dev/null
-	-docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild:make"
-	-docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild-windows:make"
 	rm -f misc/certs/host-certs.crt &> /dev/null
 	rm -rf misc/pause-container/image/
 	rm -rf misc/pause-container/rootfs/
 	rm -rf misc/plugins/
 	rm -rf out/
 	rm -rf rootfs/
 	-$(MAKE) -C $(ECS_CNI_REPOSITORY_SRC_DIR) clean
-	-$(MAKE) -C misc/netkitten $(MFLAGS) clean
-	-$(MAKE) -C misc/volumes-test $(MFLAGS) clean
-	-$(MAKE) -C misc/exec-command-agent-test $(MFLAGS) clean
-	-$(MAKE) -C misc/gremlin $(MFLAGS) clean
-	-$(MAKE) -C misc/image-cleanup-test-images $(MFLAGS) clean
-	-$(MAKE) -C misc/container-health $(MFLAGS) clean
 	-rm -f .get-deps-stamp
 	-rm -f .builder-image-stamp
 	-rm -f .out-stamp
@@ -460,3 +484,16 @@ clean:
 	-rm -f .amazon-linux-rpm-integrated-done
 	-rm -f .generic-rpm-integrated-done
 	-rm -f amazon-ecs-volume-plugin
+
+clean-all: clean
+	# for our dockerfree builds, we likely don't have docker
+	# ensure docker is running and we can talk to it, abort if not:
+	docker ps > /dev/null
+	-docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild:make"
+	-docker rmi $(BUILDER_IMAGE) "amazon/amazon-ecs-agent-cleanbuild-windows:make"
+	-$(MAKE) -C misc/netkitten $(MFLAGS) clean
+	-$(MAKE) -C misc/volumes-test $(MFLAGS) clean
+	-$(MAKE) -C misc/exec-command-agent-test $(MFLAGS) clean
+	-$(MAKE) -C misc/gremlin $(MFLAGS) clean
+	-$(MAKE) -C misc/image-cleanup-test-images $(MFLAGS) clean
+	-$(MAKE) -C misc/container-health $(MFLAGS) clean

buildspecs/pr-build-ubuntu.yml

@@ -0,0 +1,63 @@
+version: 0.2
+
+env:
+  variables:
+    # Github username of the forked repo on which to make builds
+    GITHUBUSERNAME: prateekchaudhry
+
+phases:
+  install:
+    commands:
+      - architecture=""
+      # Same buildspec for different architectures - detect the architecture here and rename the artifacts accordingly
+      - case $(uname -m) in
+           x86_64)
+             architecture="amd64"
+           ;;
+           aarch64)
+             architecture="arm64"
+           ;;
+        esac
+
+      # Need to install GOLANG using apt for making deb
+      # Remove existing go installation (goenv utility)
+      - rm -rf /root/.goenv/
+
+  build:
+    commands:
+      - echo "build_id = $CODEBUILD_LOG_PATH" 2>&1 | tee -a $BUILD_LOG
+      - echo Building agent image 2>&1 | tee -a $BUILD_LOG
+      - AGENT_VERSION=$(cat VERSION)
+      - ECS_AGENT_DEB="amazon-ecs-init_${AGENT_VERSION}-1_${architecture}.deb"
+      - ECS_AGENT_DEB_TAR="amazon-ecs-init_${AGENT_VERSION}-1.debian.tar.xz"
+      - echo $(pwd) 2>&1 | tee -a $BUILD_LOG
+
+      # Path readjustment for codebuild testing with fork and setting GOPATH appropriately
+      - cd ../../../..
+      - export GOPATH=$GOPATH:$(pwd)
+      - cd src/github.com
+      - mv $GITHUBUSERNAME aws
+      - cd aws/amazon-ecs-agent
+
+      # Building agent deb
+      - GO111MODULE=auto
+      - sudo apt-get update -y | tee -a $BUILD_LOG
+      - sudo apt install -y make | tee -a $BUILD_LOG
+      - sudo apt install -y dpkg-dev | tee -a $BUILD_LOG
+      - sudo apt install -y devscripts | tee -a $BUILD_LOG
+      - sudo apt install -y debhelper | tee -a $BUILD_LOG
+      - sudo apt install -y golang | tee -a $BUILD_LOG
+      - which go
+      - go version
+      - make generic-deb-integrated 2>&1 | tee -a $BUILD_LOG
+      - ls | tee -a $BUILD_LOG
+
+  post_build:
+    commands:
+
+artifacts:
+  files:
+    - $ECS_AGENT_DEB
+    - $ECS_AGENT_DEB_TAR
+    - $BUILD_LOG
+  name: $CODEBUILD_RESOLVED_SOURCE_VERSION

buildspecs/pr-build.yml

@@ -3,7 +3,7 @@ version: 0.2
 env:
   variables:
     # Github username of the forked repo on which to make builds
-    GITHUBUSERNAME: aws
+    GITHUBUSERNAME: prateekchaudhry
 
 phases:
   install:
@@ -19,13 +19,13 @@ phases:
            ;;
         esac
 
-      # Set up proper go version using goenv utility (pre-installed in CodeBuild). Need to use this because default images come with 1.14.x
+      # Set up proper go version
       - GOVERSION="$(cat GO_VERSION)"
       - BUILD_LOG="build_${architecture}.log"
       - GOLANG_TAR="go${GOVERSION}.linux-${architecture}.tar.gz"
 
       # Need to install GOLANG explicitly as required versions do not come preinstalled
-      # Remove existing go installation and install downloaded binaries
+      # Remove existing go installation (goenv utility) and install downloaded binaries
       - rm -rf /root/.goenv/
       - wget -O /tmp/${GOLANG_TAR} https://storage.googleapis.com/golang/${GOLANG_TAR} | tee $BUILD_LOG
       - tar -C /usr/local -xzf /tmp/${GOLANG_TAR} | tee -a $BUILD_LOG
@@ -45,6 +45,7 @@ phases:
       - echo Building agent image 2>&1 | tee -a $BUILD_LOG
       - AGENT_VERSION=$(cat VERSION)
       - ECS_AGENT_TAR="ecs-agent-v${AGENT_VERSION}.tar"
+      - ECS_AGENT_RPM="amazon-ecs-init-1.60.0-1.x86_64.rpm"
       - echo $(pwd) 2>&1 | tee -a $BUILD_LOG
 
       # Path readjustment for codebuild testing with fork and setting GOPATH appropriately
@@ -57,10 +58,13 @@ phases:
       # Building agent tars
       - GO111MODULE=auto
       - make dockerfree-agent-image 2>&1 | tee -a $BUILD_LOG
+      - make generic-rpm-integrated 2>&1 | tee -a $BUILD_LOG
+      - ls
       # Rename artifacts for architecture
       - |
         if [[ $architecture == "arm64" ]] ; then
           mv $ECS_AGENT_TAR "ecs-agent-arm64-v${AGENT_VERSION}.tar"
+          ECS_AGENT_RPM="amazon-ecs-init-${AGENT_VERSION}-1.aarch64.rpm"
           ECS_AGENT_TAR="ecs-agent-arm64-v${AGENT_VERSION}.tar"
         fi
 
@@ -70,6 +74,7 @@ phases:
 artifacts:
   files:
     - $ECS_AGENT_TAR
+    - $ECS_AGENT_RPM
     - $BUILD_LOG
   name: $CODEBUILD_RESOLVED_SOURCE_VERSION
 

packaging/generic-deb-integrated/debian/rules

@@ -1,22 +1,27 @@
 #!/usr/bin/make -f
 # -*- makefile -*-
+#  version is the "init version"
 VERSION := $(shell dpkg-parsechangelog -S Version)
+AGENT_VERSION := $(word 1, $(subst -, ,$(VERSION)))
 
-# Uncomment this to turn on verbose mode.
 export DH_VERBOSE=1
 
 %:
 	dh $@
 
 override_dh_auto_build:
+	./scripts/get-host-certs
+	./scripts/build-cni-plugins
+	./scripts/build-integrated true "" false true
+	./scripts/build-agent-image
 	./scripts/gobuild.sh debian
 
 clean:
 	dh $@
 	rm -f amazon-ecs-init
 
 override_dh_auto_install:
-	cp ecs-agent.tar debian/amazon-ecs-init/var/cache/ecs/ecs-agent-v${VERSION}.tar
+	cp ./ecs-agent-v${AGENT_VERSION}.tar debian/amazon-ecs-init/var/cache/ecs/ecs-agent-v${VERSION}.tar
 	echo "2" >debian/amazon-ecs-init/var/cache/ecs/state
 	ln -s "/var/cache/ecs/ecs-agent-v${VERSION}.tar" debian/amazon-ecs-init/var/cache/ecs/ecs-agent.tar
 	dh_installsystemd --no-start --no-enable --name=ecs

packaging/generic-deb-integrated/debian/source/include-binaries

@@ -1 +1,2 @@
-ecs-agent.tar
+misc/pause-container/pause-image-tar-files/amazon-ecs-pause-arm64.tar
+misc/pause-container/pause-image-tar-files/amazon-ecs-pause-amd64.tar

packaging/generic-deb-integrated/debian/source/options

@@ -0,0 +1,4 @@
+# Automatically add upstream changes to the quilt overlay.
+# http://manpages.ubuntu.com/manpages/trusty/man1/dpkg-source.1.html
+# This supports reusing the orig.tar.gz for debian increments.
+auto-commit

packaging/generic-rpm-integrated/amazon-ecs-init.spec

@@ -31,7 +31,6 @@ Source2:        amazon-ecs-volume-plugin.service
 Source3:        amazon-ecs-volume-plugin.socket
 
 BuildRequires:  systemd
-BuildRequires:	glibc-static
 Requires:       systemd
 Requires:       iptables
 Requires:       procps
@@ -45,7 +44,6 @@ required routes among its preparation steps.
 %setup -c
 
 %build
-./scripts/build-pause
 ./scripts/get-host-certs
 ./scripts/build-cni-plugins
 ./scripts/build-integrated true "" false true

scripts/gobuild.sh

@@ -17,6 +17,7 @@ set -e
 export TOPWD="$(pwd)"
 export BUILDDIR="$(mktemp -d)"
 export SRCPATH="${BUILDDIR}/src/github.com/aws/amazon-ecs-agent"
+export GOPATH="${TOPWD}:${BUILDDIR}"
 export GO111MODULE="auto"
 
 if [ -d "${TOPWD}/.git" ]; then