Optimize Server to build faster from noodle

Jenkinsfile

@@ -7,13 +7,13 @@ pipeline {
         }
     }
     parameters {
-        choice(name: 'HTTP_SERVER', choices: ['nginx-1.8', 'apache-2.2', 'apache-2.4'], description: 'HTTP Server')
+        choice(name: 'HTTP_SERVER', choices: ['apache-2.4', 'apache-2.2', 'nginx-1.8'], description: 'HTTP Server')
         choice(name: 'PHP_VERSION', choices: ['7.2', '7.1', '7.0', '5.6'], description: 'PHP Version')
         choice(name: 'MAGENTO_VERSION', choices: ['2.3.0', '2.2.7', '2.2.6', '2.1.16'], description: 'Magento Version')
-        choice(name: 'GITHUB_REPO', choices: ['magestore-shark/pos-pro', 'Magestore/pos-standard', 'Magestore/pos-pro', 'Magestore/pos-enterprise'], description: 'Github repository')
+        choice(name: 'GITHUB_REPO', choices: ['Magestore-Noodle/pos-enterprise', 'Magestore-Noodle/pos-pro', 'magestore-shark/pos-pro', 'Magestore/pos-standard', 'Magestore/pos-pro', 'Magestore/pos-enterprise'], description: 'Github repository')
         string(name: 'GITHUB_BRANCH', defaultValue: '4-develop', description: 'Github branch or pull request. Example: 3-develop, pull/3')
         choice(name: 'TIME_TO_LIVE', choices: ['1h', '2h', '4h', '1d', '7d'], description: 'Server living time')
-        credentials(name: 'GITHUB_USER', description: 'Github username and password', defaultValue: 'c005e544-9ad8-48be-ba44-a0f6d519a2ec', credentialType: "Username with password", required: true)
+        credentials(name: 'GITHUB_USER', description: 'Github username and password', defaultValue: 'd79f2198-246f-431c-94c5-3d2824ca8e31', credentialType: "Username with password", required: true)
     }
     environment {
         CI = 'true'

bin/apache-entrypoint

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Start apache2
+apache2-foreground

bin/build.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env sh
+echo "This step running on node $NODE_NAME"
 
 if [[ ! -z "${JENKINS_DATA}" ]]; then
     cd $JENKINS_DATA/workspace/$JOB_BASE_NAME
@@ -34,7 +35,7 @@ else
     git fetch --depth 1 origin +refs/$GITHUB_BRANCH/merge
 fi
 git checkout FETCH_HEAD
-
+rm -rf .git
 if [ $? -ne 0 ]; then
     exit 1
 fi
@@ -51,32 +52,99 @@ cp -Rf client/pos/build server/app/code/Magestore/Webpos/build/apps/pos
 cp ../$COMPOSE_FILE docker-compose.yml
 COMPOSE_HTTP_TIMEOUT=200 docker-compose up -d
 
-PORT=`docker-compose port --protocol=tcp magento 80 | sed 's/0.0.0.0://'`
-MAGENTO_URL="http://$NODE_IP:$PORT"
+# check db container is run correctly
+WHILE_LIMIT=10 # timeout 360 seconds
+while ! DBISUP=`docker-compose ps | grep 3306 | grep Up`
+do
+    if [ ! -z "$DBISUP" ]; then
+        break
+    else
+        docker-compose rm db # remove stopped container
+        COMPOSE_HTTP_TIMEOUT=200 docker-compose up -d
+        if [ $WHILE_LIMIT -lt 1 ]; then
+            break
+        fi
+    fi
+    WHILE_LIMIT=$(( WHILE_LIMIT - 1 ))
+    sleep 3
+done
+
+echo "Wait for mysql work"
+COMPOSE_HTTP_TIMEOUT=200 docker-compose exec -T magento php mysql.php
+
+# Install magento
+echo "Install magento"
+set +x
+MAGENTO_CMD='php bin/magento setup:install --use-rewrites=1 \
+--db-host=db \
+--db-name=magento \
+--db-password=magento \
+--db-prefix=m_ \
+--admin-firstname=Admin \
+--admin-lastname=MFTF \
+--admin-email=admin@localhost.com \
+--admin-user=admin \
+--admin-password=admin123 \
+--base-url=$BASE_URL \
+--backend-frontname=admin \
+--admin-use-security-key=0 \
+--key=8f1e9249ca82c072122ae8d08bc0b0cf '
+set -x
+docker-compose exec -u www-data -T magento bash -c "$MAGENTO_CMD"
 
-# Check magento installation
-COUNT_LIMIT=120 # timeout 600 seconds
-while ! RESPONSE=`docker-compose exec -T magento curl -s https://localhost.com/magento_version`
+#check maintenance flag
+! `docker-compose exec -T magento [ -f var/.maintenance.flag ]` || sh -c "\
+    echo 'Magento is maintenance mode try to restart container'; \
+    COMPOSE_HTTP_TIMEOUT=200 docker-compose restart magento "
+
+sleep 3
+echo "Check magento installation"
+COUNT_LIMIT=10 # timeout 600 seconds
+while ! RESPONSE=`docker-compose exec -T magento curl -s localhost/magento_version`
 do
     if [ $COUNT_LIMIT -lt 1 ]; then
         break
     fi
     COUNT_LIMIT=$(( COUNT_LIMIT - 1 ))
-    sleep 5
+    sleep 3
 done
 
-if [[ ${RESPONSE:0:8} != "Magento/" ]]; then
-    docker-compose restart magento
-    PORT=`docker-compose port --protocol=tcp magento 80 | sed 's/0.0.0.0://'`
-    MAGENTO_URL="http://$NODE_IP:$PORT"
-    while ! RESPONSE=`docker-compose exec -T magento curl -s https://localhost.com/magento_version`
-    do
-        sleep 5
-    done
+if [[ "${RESPONSE:0:8}" != "Magento/" ]]; then
+    echo "Cannot setup magento"
+    exit 1
 fi
 
+# Upgrade module (if needed)
+# install POS
+echo "Install POS modules:"
+#sed -i 's/#AUTO_ADD_VOLUME_server_app_code_Magestore/- \.\/server\/app\/code\/Magestore:\/var\/www\/html\/app\/code\/Magestore/g' docker-compose.yml
+#docker-compose up -d magento
+CONTAINER_ID=`docker-compose ps -q magento`
+docker exec -i -u www-data $CONTAINER_ID mkdir -p /var/www/html/app/code
+docker cp ./server/app/code/Magestore $CONTAINER_ID:/var/www/html/app/code
+docker exec -i $CONTAINER_ID chown -R www-data:www-data /var/www/html/app/code/Magestore
+
+echo "Wait for mysql work"
+COMPOSE_HTTP_TIMEOUT=200 docker-compose exec -T magento php mysql.php
+
+PORT=`docker-compose port --protocol=tcp magento 80 | sed 's/0.0.0.0://'`
+MAGENTO_URL="http://$NODE_IP:$PORT"
+
 # Correct magento url
 docker-compose exec -u www-data -T magento bash -c \
     "php bin/magento setup:store-config:set \
     --admin-use-security-key=0 \
     --base-url=$MAGENTO_URL/ "
+
+docker-compose exec -u www-data -T magento bash -c "php bin/magento setup:upgrade"
+docker-compose exec -u www-data -T magento bash -c "php bin/magento webpos:deploy"
+docker-compose exec -u www-data -T magento bash -c "php bin/magento indexer:reindex"
+docker-compose exec -u www-data -T magento bash -c "composer require zendframework/zend-barcode"
+# Update config for testing
+MAGENTO_CMD='php bin/magento config:set cms/wysiwyg/enabled disabled ; \
+php bin/magento config:set admin/security/admin_account_sharing 1 ; \
+php bin/magento config:set admin/captcha/enable 0 '
+docker-compose exec -u www-data -T magento bash -c "$MAGENTO_CMD"
+
+# clear magento cache
+docker-compose exec -u www-data -T magento bash -c "php bin/magento cache:clean"

bin/finish.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env sh
-
+echo "This step running on node $NODE_NAME"
 set +x
 
 if [[ ! -z "${JENKINS_DATA}" ]]; then

bin/nginx-entrypoint

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Start php-fpm
+php-fpm -y /usr/local/etc/php-fpm.conf -D
+# Start nginx
+nginx -g "daemon off;"

bin/run.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env sh
+echo "This step running on node $NODE_NAME"
 
 if [[ ! -z "${JENKINS_DATA}" ]]; then
     cd $JENKINS_DATA/workspace/$JOB_BASE_NAME
@@ -12,8 +13,13 @@ if [[ -z "$PORT" ]]; then
     echo "Server is not running..."
     exit 1
 fi
+PORTS=`docker-compose port --protocol=tcp magento 443 | sed 's/0.0.0.0://'`
+if [[ -z "$PORTS" ]]; then
+    echo "Port 443 is not open or server is not running..."
+fi
 
 MAGENTO_URL="http://$NODE_IP:$PORT"
+MAGENTO_SECURE_URL="https://$NODE_IP:$PORTS"
 
 PORT=`docker-compose port --protocol=tcp phpmyadmin 80 | sed 's/0.0.0.0://'`
 PHPMYADMIN_URL="http://$NODE_IP:$PORT"
@@ -27,11 +33,30 @@ echo "Server Info: $HTTP_SERVER php-$PHP_VERSION Magento-$MAGENTO_VERSION"
 echo "Built from: $GITHUB_REPO $GITHUB_BRANCH"
 echo ""
 echo "Magento: $MAGENTO_URL/admin"
+[ -z "$PORTS" ] || echo "         $MAGENTO_SECURE_URL/admin"
+echo "POS:     $MAGENTO_URL/pub/apps/pos/"
+[ -z "$PORTS" ] || echo "         $MAGENTO_SECURE_URL/pub/apps/pos/"
 echo "Admin: admin/admin123"
 echo "PHPMyAdmin: $PHPMYADMIN_URL"
-echo "EMAIL: $EMAIL_URL"
+echo "MAIL BOX: $EMAIL_URL"
 echo ""
 
+# Slack hook
+INFO="\n"
+INFO="${INFO}Server Info: $HTTP_SERVER php-$PHP_VERSION Magento-$MAGENTO_VERSION \n"
+INFO="${INFO}Built from: $GITHUB_REPO $GITHUB_BRANCH \n"
+INFO="${INFO}\n"
+INFO="${INFO}Magento: $MAGENTO_URL/admin \n"
+[ -z "$PORTS" ] || INFO="${INFO}         $MAGENTO_SECURE_URL/admin \n"
+INFO="${INFO}POS:     $MAGENTO_URL/pub/apps/pos/ \n"
+[ -z "$PORTS" ] || INFO="${INFO}         $MAGENTO_SECURE_URL/pub/apps/pos/ \n"
+INFO="${INFO}Admin: admin/admin123 \n"
+INFO="${INFO}PHPMyAdmin: $PHPMYADMIN_URL \n"
+INFO="${INFO}MAIL BOX: $EMAIL_URL \n"
+INFO="${INFO}"
+
+curl -X POST -s --data-urlencode "payload={\"text\": \"[RUNNING] <$RUN_DISPLAY_URL|$BUILD_DISPLAY_NAME> $INFO \"}" ${SLACK_HOOKS_POS4:-localhost}
+
 # Living time
 set -x
 sleep $TIME_TO_LIVE

db/Dockerfile

@@ -0,0 +1,4 @@
+FROM percona/percona-server:5.7
+
+# Copy data to image
+COPY --chown=mysql:mysql mysql /var/lib/mysql

db/conf.d/my.cnf

@@ -0,0 +1,15 @@
+[mysqld]
+innodb_buffer_pool_size = 2048M
+innodb_buffer_pool_instances = 2
+innodb_read_io_threads = 8
+innodb_write_io_threads = 8
+thread_cache_size = 20
+key_buffer_size = 128M
+query_cache_limit = 10M
+query_cache_size = 10M
+innodb_log_file_size = 512M
+innodb_log_files_in_group = 1
+innodb_log_buffer_size = 64M
+innodb_flush_method=O_DIRECT
+innodb_flush_log_at_trx_commit=0
+innodb_file_per_table

magento-2.3.0/apache-2.4/docker-compose.php-7.1.yml

@@ -1,22 +1,35 @@
-version: '3'
+version: '2.1'
 services:
   magento:
     image: magestore/mftf:2.3.0-apache-php7.1.25
     volumes:
-      - ./server/app/code/Magestore:/var/www/html/app/code/Magestore
+      # - ./server/app/code/Magestore:/var/www/html/app/code/Magestore
+      - ../bin/apache-entrypoint:/usr/local/bin/entrypoint
     ports:
       - 8080-8089:80
+      - 8180-8189:443
     networks:
       webnet:
         aliases:
           - localhost.com
+    environment:
+      BASE_URL: http://localhost/
+      MAGENTO_DEV_MODE: developer
+    mem_limit: 3GB
+    memswap_limit: 0
+    entrypoint:
+      - entrypoint
 
   db:
-    image: magestore/mftf-db:percona-5.7
+    image: magestore/mftf-db:percona-5.7-lessdata
     environment:
       MYSQL_ROOT_PASSWORD: magento
+    volumes:
+      - ./../db/conf.d:/etc/my.cnf.d
     networks:
       - webnet
+    mem_limit: 3GB
+    memswap_limit: 0
 
   phpmyadmin:
     image: phpmyadmin/phpmyadmin

magento-2.3.0/apache-2.4/docker-compose.php-7.2.yml

@@ -0,0 +1,53 @@
+version: '2.1'
+services:
+  magento:
+    image: magestore/mftf:2.3.0-apache-php7.2.13
+    volumes:
+      #- ./server/app/code/Magestore:/var/www/html/app/code/Magestore
+      - ../bin/apache-entrypoint:/usr/local/bin/entrypoint
+    ports:
+      - 8080-8089:80
+      - 8180-8189:443
+    networks:
+      webnet:
+        aliases:
+          - localhost.com
+    environment:
+      BASE_URL: http://localhost/
+      MAGENTO_DEV_MODE: developer
+    mem_limit: 3GB
+    memswap_limit: 0
+    entrypoint:
+      - entrypoint
+
+  db:
+    image: magestore/mftf-db:percona-5.7-lessdata
+    environment:
+      MYSQL_ROOT_PASSWORD: magento
+    volumes:
+      - ./../db/conf.d:/etc/my.cnf.d
+    networks:
+      - webnet
+    mem_limit: 3GB
+    memswap_limit: 0
+
+  phpmyadmin:
+    image: phpmyadmin/phpmyadmin
+    ports:
+      - 8060-8069:80
+    environment:
+      PMA_HOST: db
+      PMA_USER: root
+      PMA_PASSWORD: magento
+    networks:
+      - webnet
+
+  mailhog:
+    image: mailhog/mailhog
+    ports:
+      - 8020-8029:8025
+    networks:
+      - webnet
+
+networks:
+  webnet:

magento-2.3.0/nginx-1.8/docker-compose.php-7.1.yml

@@ -0,0 +1,53 @@
+version: '2.1'
+services:
+  magento:
+    image: magestore/mftf:2.3.0-nginx-php7.1.25
+    volumes:
+      #- ./server/app/code/Magestore:/var/www/html/app/code/Magestore
+      - ../bin/nginx-entrypoint:/usr/local/bin/nginx-entrypoint
+    ports:
+      - 8080-8089:80
+      - 8180-8189:443
+    networks:
+      webnet:
+        aliases:
+          - localhost.com
+    environment:
+      BASE_URL: http://localhost/
+      MAGENTO_DEV_MODE: developer
+    mem_limit: 3GB
+    memswap_limit: 0
+    entrypoint:
+      - nginx-entrypoint
+
+  db:
+    image: magestore/mftf-db:percona-5.7-lessdata
+    environment:
+      MYSQL_ROOT_PASSWORD: magento
+    volumes:
+      - ./../db/conf.d:/etc/my.cnf.d
+    networks:
+      - webnet
+    mem_limit: 3GB
+    memswap_limit: 0
+
+  phpmyadmin:
+    image: phpmyadmin/phpmyadmin
+    ports:
+      - 8060-8069:80
+    environment:
+      PMA_HOST: db
+      PMA_USER: root
+      PMA_PASSWORD: magento
+    networks:
+      - webnet
+
+  mailhog:
+    image: mailhog/mailhog
+    ports:
+      - 8020-8029:8025
+    networks:
+      - webnet
+
+networks:
+  webnet: