WIP: Update everything for StartOS 0.4.0

.dockerignore

@@ -1 +1,2 @@
-**/.git
+.git
+.gitmodules

.github/workflows/build.yml

@@ -0,0 +1,20 @@
+name: Build
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths-ignore: ['*.md']
+    branches: ['master']
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    if: github.event.pull_request.draft == false
+    uses: start9labs/shared-workflows/.github/workflows/build.yml@master
+    # with:
+    #   FREE_DISK_SPACE: true
+    secrets:
+      DEV_KEY: ${{ secrets.DEV_KEY }}

.github/workflows/release.yml

@@ -0,0 +1,20 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*.*'
+
+jobs:
+  release:
+    uses: start9labs/shared-workflows/.github/workflows/release.yml@master
+    with:
+      # FREE_DISK_SPACE: true
+      RELEASE_REGISTRY: ${{ vars.RELEASE_REGISTRY }}
+      S3_S9PKS_BASE_URL: ${{ vars.S3_S9PKS_BASE_URL }}
+    secrets:
+      DEV_KEY: ${{ secrets.DEV_KEY }}
+      S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+      S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+    permissions:
+      contents: write

.github/workflows/tagAndRelease.yml

@@ -0,0 +1,25 @@
+name: Tag and Release
+
+on:
+  push:
+    branches: ['master']
+    paths-ignore: ['*.md']
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tag:
+    uses: start9labs/shared-workflows/.github/workflows/tagAndRelease.yml@master
+    with:
+      REFERENCE_REGISTRY: ${{ vars.REFERENCE_REGISTRY }}
+      # FREE_DISK_SPACE: true
+      RELEASE_REGISTRY: ${{ vars.RELEASE_REGISTRY }}
+      S3_S9PKS_BASE_URL: ${{ vars.S3_S9PKS_BASE_URL }}
+    secrets:
+      DEV_KEY: ${{ secrets.DEV_KEY }}
+      S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
+      S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}
+    permissions:
+      contents: write

.gitignore

@@ -1,5 +1,9 @@
 *.s9pk
-scripts/*.js
+startos/*.js
+node_modules/
 .DS_Store
 .vscode/
 docker-images
+javascript
+ncc-cache
+package-lock.json

Dockerfile

@@ -137,14 +137,18 @@ COPY ./nginx/*.conf /etc/nginx/sites-available/
 RUN mkdir /etc/nginx/sites-enabled && \
     ln -sf /etc/nginx/sites-available/mainnet.conf /etc/nginx/sites-enabled/dojo.conf
 
-### Docker entrypoint
+### Daemon scripts
 
 COPY ./config.env /usr/local/bin/config.env
-COPY --chmod=755 ./docker_entrypoint.sh /usr/local/bin/
+COPY --chmod=755 ./db-entrypoint.sh /usr/local/bin/
+COPY --chmod=755 ./soroban-entrypoint.sh /usr/local/bin/
+COPY --chmod=755 ./backend-entrypoint.sh /usr/local/bin/
 COPY --chmod=755 ./check-synced.sh /usr/local/bin/
 COPY --chmod=755 ./check-api.sh /usr/local/bin/
 COPY --chmod=755 ./check-mysql.sh /usr/local/bin/
 COPY --chmod=755 ./check-pushtx.sh /usr/local/bin/
 COPY --chmod=755 ./check-soroban.sh /usr/local/bin/
 COPY --chmod=755 ./functions.sh /usr/local/bin/
 COPY --chmod=755 ./samourai-dojo/docker/my-dojo/soroban/restart.sh /usr/local/bin/soroban-restart.sh
+
+RUN mkdir -p /run/secrets

Makefile

@@ -1,63 +1,10 @@
-PKG_ID := $(shell yq e ".id" < manifest.yaml)
-PKG_VERSION := $(shell yq e ".version" < manifest.yaml)
-TS_FILES := $(shell find ./ -name \*.ts)
-
-# delete the target of a rule if it has changed and its recipe exits with a nonzero exit status
-.DELETE_ON_ERROR:
-
-all: verify
-
-verify: $(PKG_ID).s9pk
-	@start-sdk verify s9pk $(PKG_ID).s9pk
-	@echo " Done!"
-	@echo "   Filesize: $(shell du -h $(PKG_ID).s9pk) is ready"
-
-install:
-	@if [ ! -f ~/.embassy/config.yaml ]; then echo "You must define \"host: http://server-name.local\" in ~/.embassy/config.yaml config file first."; exit 1; fi
-	@echo "\nInstalling to $$(grep -v '^#' ~/.embassy/config.yaml | cut -d'/' -f3) ...\n"
-	@[ -f $(PKG_ID).s9pk ] || ( $(MAKE) && echo "\nInstalling to $$(grep -v '^#' ~/.embassy/config.yaml | cut -d'/' -f3) ...\n" )
-	@start-cli package install $(PKG_ID).s9pk
-
-clean:
-	rm -rf docker-images
-	rm -f $(PKG_ID).s9pk
-	rm -f scripts/*.js
-
-clean-manifest:
-	@sed -i '' '/^[[:blank:]]*#/d' manifest.yaml
-	@echo; echo "Comments successfully removed from manifest.yaml file."; echo
-
-scripts/embassy.js: $(TS_FILES)
-	deno run --allow-read --allow-write --allow-env --allow-net scripts/bundle.ts
-
-arm:
-	@rm -f docker-images/aarch64.tar
-	ARCH=aarch64 $(MAKE)
-
-x86:
-	@rm -f docker-images/x86_64.tar
-	ARCH=x86_64 $(MAKE)
-
-docker-images/aarch64.tar: Dockerfile docker_entrypoint.sh config.env nginx/mainnet.conf nginx/testnet.conf samourai-dojo check-api.sh check-mysql.sh check-pushtx.sh check-synced.sh check-soroban.sh functions.sh
-ifeq ($(ARCH),x86_64)
-else
-	mkdir -p docker-images
-	docker buildx build --tag start9/$(PKG_ID)/main:$(PKG_VERSION) --build-arg ARCH=aarch64 --platform=linux/arm64 -o type=docker,dest=docker-images/aarch64.tar .
-endif
-
-docker-images/x86_64.tar: Dockerfile docker_entrypoint.sh config.env nginx/mainnet.conf nginx/testnet.conf samourai-dojo check-api.sh check-mysql.sh check-pushtx.sh check-synced.sh check-soroban.sh functions.sh
-ifeq ($(ARCH),aarch64)
-else
-	mkdir -p docker-images
-	docker buildx build --tag start9/$(PKG_ID)/main:$(PKG_VERSION) --build-arg ARCH=x86_64 --platform=linux/amd64 -o type=docker,dest=docker-images/x86_64.tar .
-endif
-
-$(PKG_ID).s9pk: manifest.yaml instructions.md icon.png LICENSE.md scripts/embassy.js docker-images/aarch64.tar docker-images/x86_64.tar
-ifeq ($(ARCH),aarch64)
-	@echo "start-sdk: Preparing aarch64 package ..."
-else ifeq ($(ARCH),x86_64)
-	@echo "start-sdk: Preparing x86_64 package ..."
-else
-	@echo "start-sdk: Preparing Universal Package ..."
-endif
-	@start-sdk pack
+# ARCHES := x86 arm
+ARCHES := x86
+# overrides to s9pk.mk must precede the include statement
+include s9pk.mk
+
+assets: icon.png instructions.md LICENSE.md
+	@mkdir -p assets
+	@cp icon.png assets/
+	@cp instructions.md assets/
+	@cp LICENSE.md assets/

backend-entrypoint.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -ea
+
+source /usr/local/bin/config.env
+
+echo "[i] Dojo Tor address: $TOR_ADDRESS"
+mkdir -p /var/lib/tor/hsv3dojo
+echo "$TOR_ADDRESS" > /var/lib/tor/hsv3dojo/hostname
+
+if [ "$COMMON_BTC_NETWORK" = "testnet" ]; then
+	PAIRING_URL="http://$TOR_ADDRESS/test/v2"
+	EXPLORER_ENDPOINT="mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/testnet4"
+	echo "[i] Running on TESTNET"
+else
+	PAIRING_URL="http://$TOR_ADDRESS/v2"
+	EXPLORER_ENDPOINT="mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion"
+	echo "[i] Running on MAINNET"
+fi
+
+echo "[i] Pairing URL: $PAIRING_URL"
+
+# Set dojo config corresponding to current network
+if [ "$COMMON_BTC_NETWORK" = "testnet" ]; then
+	cp /home/node/app/static/admin/conf/index-testnet.js /home/node/app/static/admin/conf/index.js
+	ln -sf /etc/nginx/sites-available/testnet.conf /etc/nginx/sites-enabled/dojo.conf
+else
+	cp /home/node/app/static/admin/conf/index-mainnet.js /home/node/app/static/admin/conf/index.js
+	ln -sf /etc/nginx/sites-available/mainnet.conf /etc/nginx/sites-enabled/dojo.conf
+fi
+
+mkdir -p /var/lib/tor/hsv3explorer
+echo -n "$EXPLORER_ENDPOINT" > /var/lib/tor/hsv3explorer/hostname
+
+cat << EOF > /root/stats.json
+{
+  "pairingCode": '{"pairing":{"type":"dojo.api","version":"$DOJO_VERSION_TAG","apikey":"$NODE_API_KEY","url":"$PAIRING_URL"},"explorer":{"type":"explorer.btc_rpc_explorer","url":"http://$EXPLORER_ENDPOINT"}}',
+  "adminKey": "$NODE_ADMIN_KEY"
+}
+EOF
+
+# Start dojo
+exec pm2-runtime -u node --raw /home/node/app/pm2.config.cjs

check-api.sh

@@ -4,7 +4,7 @@
 source /usr/local/bin/config.env
 source /usr/local/bin/functions.sh
 
-admin_key=$(yq -e '.admin-key' /root/start9/config.yaml 2>/dev/null)
+admin_key=$(jq -r '.dojo.adminKey' /root/store.json 2>/dev/null)
 access_token=$(cat /run/secrets/access_token 2>/dev/null)
 
 if [ -z "$access_token" ] || ! check_token "$access_token"; then

check-pushtx.sh

@@ -4,7 +4,7 @@
 source /usr/local/bin/config.env
 source /usr/local/bin/functions.sh
 
-admin_key=$(yq -e '.admin-key' /root/start9/config.yaml 2>/dev/null)
+admin_key=$(jq -r '.dojo.adminKey' /root/store.json 2>/dev/null)
 access_token=$(cat /run/secrets/access_token 2>/dev/null)
 
 if [ -z "$access_token" ] || ! check_token "$access_token"; then

check-synced.sh

@@ -6,7 +6,7 @@ source /usr/local/bin/functions.sh
 
 bci_result=$(curl -sS --user "$BITCOIND_RPC_USER:$BITCOIND_RPC_PASSWORD" --data-binary '{"jsonrpc": "1.0", "id": "gbci", "method": "getblockchaininfo", "params": []}' -H 'content-type: text/plain;' http://$BITCOIND_IP:$BITCOIND_RPC_PORT/ 2>&1)
 bci_return=$?
-bci_error=$(echo "$bci_result" | yq -e '.message' 2>/dev/null)
+bci_error=$(echo "$bci_result" | jq -r '.message // "null"' 2>/dev/null)
 
 if [[ $bci_return -ne 0 ]]; then
   echo "Error contacting Bitcoin RPC: $bci_result" >&2
@@ -16,19 +16,19 @@ elif [ "$bci_error" != "null" ]; then
   exit 61
 fi
 
-bci_block_count=$(echo "$bci_result" | yq -e '.result.blocks' 2>/dev/null)
-bci_block_ibd=$(echo "$bci_result" | yq -e '.result.initialblockdownload' 2>/dev/null)
+bci_block_count=$(echo "$bci_result" | jq -r '.result.blocks // "null"' 2>/dev/null)
+bci_block_ibd=$(echo "$bci_result" | jq -r '.result.initialblockdownload // "null"' 2>/dev/null)
 if [ "$bci_block_count" = "null" ]; then
   echo "Error ascertaining Bitcoin blockchain status: $bci_error" >&2
   exit 61
 elif [ "$bci_block_ibd" != "false" ] ; then
-  bci_block_headers=$(echo "$bci_result" | yq -e '.result.headers' 2>/dev/null)
+  bci_block_headers=$(echo "$bci_result" | jq -r '.result.headers // "null"' 2>/dev/null)
   echo -n "Bitcoin blockchain is not fully synced yet: $bci_block_count downloaded of $bci_block_headers blocks" >&2
   echo " ($(expr ${bci_block_count}00 / $bci_block_headers)%)" >&2
   exit 61
 fi
 
-admin_key=$(yq -e '.admin-key' /root/start9/config.yaml 2>/dev/null)
+admin_key=$(jq -r '.dojo.adminKey' /root/store.json 2>/dev/null)
 access_token=$(cat /run/secrets/access_token 2>/dev/null)
 
 if [ -z "$access_token" ] || ! check_token "$access_token"; then
@@ -42,8 +42,8 @@ fi
 
 account_status=$(get_account_status "$access_token")
 pushtx_status=$(get_pushtx_status "$access_token")
-synced_blocks=$(echo "$account_status" | yq -e '.blocks' 2>/dev/null)
-bitcoind_blocks=$(echo "$pushtx_status" | yq -e '.data.bitcoind.blocks' 2>/dev/null)
+synced_blocks=$(echo "$account_status" | jq -r '.blocks // "null"' 2>/dev/null)
+bitcoind_blocks=$(echo "$pushtx_status" | jq -r '.data.bitcoind.blocks // "null"' 2>/dev/null)
 if [[ $synced_blocks -eq $bitcoind_blocks ]]; then
   exit 0
 else