Refactor setup scripts

.github/workflows/CI.yml

@@ -28,6 +28,14 @@ jobs:
       - name: Check out the project
         uses: actions/checkout@v3
 
+      - name: Install Rust 1.88
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.88.0
+          override: true
+          profile: minimal
+          components: clippy
+
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -37,9 +45,7 @@ jobs:
         run: |
           sudo apt update
           sudo apt install -y python3-pip nodejs
-          curl --proto '=https' --tlsv1.3 -sSf https://sh.rustup.rs | sh -s -- -y
-          source $HOME/.cargo/env
-          pip install python_jwt
+          pip install jwcrypto cbor2
           git clone https://github.com/iden3/circom.git
           cd circom
           git checkout v2.1.6
@@ -48,7 +54,6 @@ jobs:
           export PATH=$PATH:~/.cargo/bin
           cd ..
           git submodule update --init --recursive
-          pip install git+https://github.com/peppelinux/pyMDOC-CBOR.git
 
       - name: Run circuit setup for rs256
         run: |
@@ -96,85 +101,85 @@ jobs:
       - name: Run ZKSetup for rs256
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace zksetup --name rs256
+          cargo run --bin crescent-cli --release --features print-trace zksetup --name rs256
 
       - name: Run Prove for rs256
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace prove --name rs256
+          cargo run --bin crescent-cli --release --features print-trace prove --name rs256
 
       - name: Run Show for rs256
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace show --name rs256
+          cargo run --bin crescent-cli --release --features print-trace show --name rs256
 
       - name: Run Verify for rs256
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace verify --name rs256
+          cargo run --bin crescent-cli --release --features print-trace verify --name rs256
 
 # RS256-sd Commands
       - name: Run ZKSetup for rs256-sd
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace zksetup --name rs256-sd
+          cargo run --bin crescent-cli --release --features print-trace zksetup --name rs256-sd
 
       - name: Run Prove for rs256-sd
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace prove --name rs256-sd
+          cargo run --bin crescent-cli --release --features print-trace prove --name rs256-sd
 
       - name: Run Show for rs256-sd
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace show --name rs256-sd
+          cargo run --bin crescent-cli --release --features print-trace show --name rs256-sd
 
       - name: Run Verify for rs256-sd
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace verify --name rs256-sd
+          cargo run --bin crescent-cli --release --features print-trace verify --name rs256-sd
 
 # RS256-db Commands
       - name: Run ZKSetup for rs256-db
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace zksetup --name rs256-db
+          cargo run --bin crescent-cli --release --features print-trace zksetup --name rs256-db
 
       - name: Run Prove for rs256-db
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace prove --name rs256-db
+          cargo run --bin crescent-cli --release --features print-trace prove --name rs256-db
 
       - name: Run Show for rs256-db
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace show --name rs256-db
+          cargo run --bin crescent-cli --release --features print-trace show --name rs256-db
 
       - name: Run Verify for rs256-db
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace verify --name rs256-db
+          cargo run --bin crescent-cli --release --features print-trace verify --name rs256-db
 
 # mDL Commands
       - name: Run ZKSetup for mDL
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace zksetup --name mdl1
+          cargo run --bin crescent-cli --release --features print-trace zksetup --name mdl1
 
       - name: Run Prove for mDL
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace prove --name mdl1
+          cargo run --bin crescent-cli --release --features print-trace prove --name mdl1
 
       - name: Run Show for mDL
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace show --name mdl1
+          cargo run --bin crescent-cli --release --features print-trace show --name mdl1
 
       - name: Run Verify for mDL
         run: |
           cd creds
-          cargo run --bin crescent --release --features print-trace verify --name mdl1
+          cargo run --bin crescent-cli --release --features print-trace verify --name mdl1
 
 # Build sample
       - name: Run the sample setup script

.gitignore

@@ -1,23 +1,12 @@
 .vscode/
 target/
 Cargo.lock
+bin/
 
-# ignore test vectors, but not the README
-creds/test-vectors
-!creds/test-vectors/README
+# sample output folder created by docker build
+crescent-extension
 
-
-circuit_setup/circuits-mdl/circomlib
-circuit_setup/inputs/*/**
-!circuit_setup/inputs/*/*.json
-
-# sample files
-sample/issuer/.well-known/
-sample/issuer/keys/
-sample/client/node_modules
-sample/client/mdl.json
-sample/client/dist
-sample/client_helper/data
-sample/verifier/data
-
-crescent-extension
+# There are additional sub-project .gitignore files:
+#  creds/.gitignore
+#  circuit_setup/.gitignore
+#  sample/.gitignore

Cargo.toml

@@ -0,0 +1,28 @@
+[workspace]
+resolver = "2"
+
+members = [
+    "creds",
+    "sample/client_helper", 
+    "sample/issuer",
+    "sample/verifier",
+    "sample/setup_service",
+    "circuit_setup/mdl-tools"
+]
+
+# project profiles go here and not in child Cargo.toml files
+
+[profile.dev.package."crescent-sample-client-helper"]
+
+[profile.dev.package."crescent-sample-issuer"]
+
+[profile.dev.package."crescent-sample-verifier"]
+
+[profile.dev.package.crescent]
+
+[profile.dev.package.wasmer-vm]
+opt-level = 3
+debug-assertions = false    # We need this to work around a bug in Wasmer
+
+[profile.dev]
+opt-level = 3

build_all.sh

@@ -1,50 +1,122 @@
-#!/bin/bash
+#!/usr/bin/bash
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+#
+
+set -e
+shopt -s extglob
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+readonly CRESCENT_ENV=${CRESCENT_ENV:-release}
+[[ "$CRESCENT_ENV" =~ ^(release|debug)$ ]] || { echo "Invalid CRESCENT_ENV: $CRESCENT_ENV" >&2; exit 1; }
+RELEASE_FLAG=$([[ "$CRESCENT_ENV" == "debug" ]] && echo "" || echo "--release")
+BIN=$(pwd)/target/${CRESCENT_ENV}
+
+
+# Check for required shell commands
+check_prereqs() {
+  for cmd in "$@"; do
+    if ! command -v "$cmd" &> /dev/null; then
+      echo -e "\033[0;31m❌ Error: '$cmd' is required but not installed or not in PATH.\033[0m" >&2
+      missing=true
+    fi
+  done
+}
+
+# Check for required Python packages
+check_pip_pkgs() {
+  for pkg in "$@"; do
+    if ! python -c "import $pkg" &>/dev/null; then
+      echo -e "\033[0;31m❌ Error: Python package '$pkg' is not installed.\033[0m" >&2
+      missing=true
+    fi
+  done
+}
+
+echo "🔍 Checking prerequisites..."
+check_prereqs node npm python3 circom rustc cargo ssh
+check_pip_pkgs jwcrypto cbor2
+
+
+# halt if any prerequisites are missing
+if [ "${missing:-false}" = true ]; then
+  echo -e "\033[0;31m❌ Some prerequisites are missing. See circuit_setup\README.md for help with dependencies.\033[0m" >&2
+  exit 1
+fi
+
+SECONDS=0
+
+# Check for "trim" argument to have script clean extraneous artifacts
+do_trim=false; for arg in "$@"; do [[ "$arg" == "trim" ]] && do_trim=true && break; done
 
-RELEASE_FLAG="--release"
 
 git submodule update --init --recursive
 
-#
+
+# Build everything from the sub-projects top level workspace at once
+cargo build $RELEASE_FLAG --features print-trace
+
+
 # Circuit setup
-#
-cd circuit_setup/scripts
-./run_setup.sh rs256
-./run_setup.sh rs256-sd
-./run_setup.sh rs256-db
-./run_setup.sh mdl1
+# Generates circom circuits and artifacts in circuit_setup/generated_files/
+# Final output is copied to creds/test-vectors/[mdl1, rs256, rs256-sd, rs256-db]
+# The setup scripts are run in parallel for each circuit type to take advantage of multiple CPU cores
+#   as circuit generation is CPU intensive but single-threaded.
+pushd circuit_setup/scripts > /dev/null
+./run_setup.sh mdl1 &
+./run_setup.sh rs256 &
+./run_setup.sh rs256-sd &
+./run_setup.sh rs256-db &
+wait
+popd > /dev/null
 
-cd ../../creds
+# Ensure the output directories exist
+pushd creds > /dev/null
 for d in test-vectors/rs256 test-vectors/rs256-sd test-vectors/rs256-db test-vectors/mdl1; do
   if [ ! -d "$d" ]; then
-    echo "❌ Error: Missing directory creds/'$d'" >&2
+    echo "❌ Error: Missing directory creds/$d" >&2
     exit 1
   fi
 done
 
-cargo run --bin crescent $RELEASE_FLAG --features print-trace zksetup --name rs256
-cargo run --bin crescent $RELEASE_FLAG --features print-trace prove --name rs256
-cargo run --bin crescent $RELEASE_FLAG --features print-trace show --name rs256
-cargo run --bin crescent $RELEASE_FLAG --features print-trace verify --name rs256
+if [ "$do_trim" = true ]; then
+  echo "Cleaning up intermediate artifacts..."
+  rm -rf ../circuit_setup/generated_files/!(README.md)
+fi
+
 
-cargo run --bin crescent $RELEASE_FLAG --features print-trace zksetup --name rs256-sd
-cargo run --bin crescent $RELEASE_FLAG --features print-trace prove --name rs256-sd
-cargo run --bin crescent $RELEASE_FLAG --features print-trace show --name rs256-sd
-cargo run --bin crescent $RELEASE_FLAG --features print-trace verify --name rs256-sd
+crescent="${BIN}/crescent-cli"
 
-cargo run --bin crescent $RELEASE_FLAG --features print-trace zksetup --name rs256-db
-cargo run --bin crescent $RELEASE_FLAG --features print-trace prove --name rs256-db
-cargo run --bin crescent $RELEASE_FLAG --features print-trace show --name rs256-db
-cargo run --bin crescent $RELEASE_FLAG --features print-trace verify --name rs256-db
+if [ "$do_trim" = true ]; then
+  echo "Cleaning up build artifacts..."
+  cargo clean
+fi
 
-cargo run --bin crescent $RELEASE_FLAG --features print-trace zksetup --name mdl1
-cargo run --bin crescent $RELEASE_FLAG --features print-trace prove --name mdl1
-cargo run --bin crescent $RELEASE_FLAG --features print-trace show --name mdl1
-cargo run --bin crescent $RELEASE_FLAG --features print-trace verify --name mdl1
-cd ..
+declare -A LABEL_COLORS=(
+  [rs256]=$'\033[0;35m'
+  [rs256-sd]=$'\033[0;36m'
+  [rs256-db]=$'\033[1;33m'
+  [mdl1]=$'\033[1;34m'
+)
+
+RESET=$'\033[0m'
+
+for name in "${!LABEL_COLORS[@]}"; do
+  color="${LABEL_COLORS[$name]}"
+  {
+    $crescent zksetup --name "$name"
+    $crescent prove   --name "$name"
+    $crescent show    --name "$name"
+    $crescent verify  --name "$name"
+  } 2>&1 | sed "s/^/\\${color}[${name}]\\${RESET} /" &
+done
+
+wait
 
 #
 # Sample setup
 #
-cd sample
-# Node must be available for the .js scripts to be executed
- ./setup-sample.sh
+../sample/setup-sample.sh
+
+echo -e "\033[0;32mBuild-all completed in $SECONDS seconds\033[0m"

circuit_setup/.gitignore

@@ -1,4 +1,7 @@
-generated_files/*
+generated_files/*/**
 !generated_files/README.md
-logs
+
+inputs/*/**
+!inputs/*/*.json
+
 scripts/__pycache__/*

circuit_setup/README.md

@@ -30,7 +30,7 @@ curl --proto '=https' --tlsv1.3 -sSf https://sh.rustup.rs | sh
 3. Install required Python modules
 
 ```bash
-pip install python_jwt
+pip install jwcrypto cbor2
 ```
 
 4. Install [Circom](https://github.com/iden3/circom)
@@ -53,21 +53,6 @@ Either clone this repo with the option `--recurse-submodules`, or for existing r
 git submodule update --init --recursive
 ```
 
-6. For mDL credentials, the [pyMDOC-CBOR](https://github.com/IdentityPython/pyMDOC-CBOR) Python module must be installed, with the command
-
-```bash
-# Linux
-pip install git+https://github.com/peppelinux/pyMDOC-CBOR.git
-```
-
-```bash
-# Windows
-# pyMDL-MDOC has a bug in the setup we need to fix for proper installation on Windows
-git clone https://github.com/peppelinux/pyMDL-MDOC.git
-cd pyMDL-MDOC
-sed -i "s|i.replace(f'{_pkg_name}/', '')|i.replace(f'{_pkg_name}\\\\\\\\', '')|g" setup.py
-pip install .
-```
 ## Sample JWT and mDL
 
 To work with Crescent, the prover and verifier both need the issuer's public key, and the prover needs a JWT.