cleanup: make consensus more stable after restart #178

Workflow file for this run

.github/workflows/integ-tests.yml at 9023401

	name: Integration Tests

	on:
	pull_request:
	push:
	branches: [main]

	jobs:
	build:
	runs-on: blacksmith-16vcpu-ubuntu-2204
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: useblacksmith/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}

	- name: Set up Rust toolchain
	run: \|
	rustup update 1.87
	rustup default 1.87

	- name: Install dependencies
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang

	- name: Docker meta
	id: meta
	uses: docker/metadata-action@v5
	with:
	images: trieve/threshold
	tags: \|
	type=raw,value=latest
	type=ref,event=pr
	type=sha

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3

	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Build and push Docker image
	uses: useblacksmith/build-push-action@v1
	with:
	push: true
	tags: ${{ steps.meta.outputs.tags }}

	dkg_test-12:
	runs-on: blacksmith-16vcpu-ubuntu-2204
	needs: build
	env:
	MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
	MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
	IS_TESTNET: "true"
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: useblacksmith/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}

	- name: Set up Rust toolchain
	run: \|
	rustup update 1.87
	rustup default 1.87

	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Pull Docker image
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	docker pull "trieve/threshold:pr-${{ github.event.pull_request.number }}"
	else
	docker pull trieve/threshold:latest
	fi

	- name: Install dependencies
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang

	- name: Create NodeCluster (3 nodes)
	run: ./scripts/clear_dkg_keys.sh

	- name: Start 12-node stack (docker-compose)
	run: \|
	sed -i "s/trieve\/threshold:latest/trieve\/threshold:pr-${{ github.event.pull_request.number }}/g" docker-compose-12.yaml
	docker compose -f docker-compose-12.yaml up -d

	- name: Wait for gRPC endpoints
	run: \|
	echo "Waiting for nodes to expose ports 50057-50068..."
	all_ready=false
	for i in {1..30}; do
	all_ready=true
	for port in {50057..50068}; do
	if ! nc -z localhost $port; then
	all_ready=false
	break
	fi
	done
	if $all_ready; then
	echo "All ports 50057-50068 are open."
	break
	fi
	echo "Ports not ready yet – retry #$i"
	sleep 5
	done

	- name: Run end-to-end integration tests
	run: cargo run --bin integration-tests check-dkg --port-range 50057-50068

	dkg_test_5:
	runs-on: blacksmith-16vcpu-ubuntu-2204
	needs: build
	env:
	MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
	MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
	IS_TESTNET: "true"
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: useblacksmith/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}

	- name: Set up Rust toolchain
	run: \|
	rustup update 1.87
	rustup default 1.87

	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Pull Docker image
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	docker pull "trieve/threshold:pr-${{ github.event.pull_request.number }}"
	else
	docker pull trieve/threshold:latest
	fi

	- name: Install dependencies
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang

	- name: Create NodeCluster (5 nodes)
	run: ./scripts/clear_dkg_keys.sh

	- name: Start 5-node stack (docker-compose)
	run: \|
	sed -i "s/trieve\/threshold:latest/trieve\/threshold:pr-${{ github.event.pull_request.number }}/g" docker-compose.yaml
	docker compose -f docker-compose.yaml up -d

	- name: Wait for gRPC endpoint
	run: \|
	echo "Waiting for node to expose port 50051-50055..."
	for i in {1..30}; do
	if nc -z localhost {50051..50055}; then
	echo "Port 50057-50061 is open."
	break
	fi
	echo "Port not ready yet – retry #$i"
	sleep 5
	done

	- name: Run end-to-end integration tests
	run: cargo run --bin integration-tests check-dkg --port-range 50051-50055

	end-to-end-deposit-withdrawl:
	runs-on: blacksmith-16vcpu-ubuntu-2204
	needs: build
	env:
	MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
	MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
	IS_TESTNET: "true"
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: useblacksmith/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}

	- name: Set up Rust toolchain
	run: \|
	rustup update 1.87
	rustup default 1.87

	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Pull Docker image
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	docker pull "trieve/threshold:pr-${{ github.event.pull_request.number }}"
	else
	docker pull trieve/threshold:latest
	fi

	- name: Clean cargo
	run: cargo clean

	- name: Install dependencies
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang

	- name: Start 5-node stack (docker-compose)
	run: \|
	sed -i "s/trieve\/threshold:latest/trieve\/threshold:pr-${{ github.event.pull_request.number }}/g" docker-compose.yaml
	docker compose -f docker-compose.yaml up -d

	- name: Wait for gRPC endpoint
	run: \|
	echo "Waiting for node to expose port 50051-50055..."
	for i in {1..30}; do
	if nc -z localhost {50051..50055}; then
	echo "Port 50051-50055 is open."
	break
	fi
	echo "Port not ready yet – retry #$i"
	sleep 5
	done

	- name: Run end-to-end integration tests
	run: cargo run --bin integration-tests test --port-range 50051-50055

	- name: Shutdown stack & print logs
	run: \|
	docker compose -f docker-compose.yaml logs --no-color \|\| true
	docker compose -f docker-compose.yaml down --volumes --remove-orphans \|\| true

	dkg_test_chaos_network:
	runs-on: blacksmith-16vcpu-ubuntu-2204
	needs: build
	env:
	MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
	MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
	IS_TESTNET: "true"
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: useblacksmith/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}

	- name: Set up Rust toolchain
	run: \|
	rustup update 1.87
	rustup default 1.87

	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Pull Docker image
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	docker pull "trieve/threshold:pr-${{ github.event.pull_request.number }}"
	else
	docker pull trieve/threshold:latest
	fi

	- name: Clean cargo
	run: cargo clean

	- name: Install dependencies
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang

	- name: Create NodeCluster (3 nodes)
	run: ./scripts/clear_dkg_keys.sh

	- name: Start 5-node stack with network chaos
	run: \|
	sed -i "s/trieve\/threshold:latest/trieve\/threshold:pr-${{ github.event.pull_request.number }}/g" docker-compose.yaml
	docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml up -d

	- name: Wait for gRPC endpoint
	run: \|
	echo "Waiting for node to expose port 50051-50055..."
	for i in {1..30}; do
	if nc -z localhost {50051..50055}; then
	echo "Port 50051-50055 is open."
	break
	fi
	echo "Port not ready yet – retry #$i"
	sleep 5
	done

	- name: Run end-to-end integration tests with network chaos
	run: cargo run --bin integration-tests check-dkg --port-range 50051-50055

	- name: Shutdown stack & print logs
	run: \|
	docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml logs --no-color \|\| true
	docker compose -f docker-compose.yaml -f docker-compose.chaos-network.yaml down --volumes --remove-orphans \|\| true

	dkg_test_chaos_nodes:
	runs-on: blacksmith-16vcpu-ubuntu-2204
	needs: build
	env:
	MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
	MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
	IS_TESTNET: "true"
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: useblacksmith/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}

	- name: Set up Rust toolchain
	run: \|
	rustup update 1.87
	rustup default 1.87

	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Pull Docker image
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	docker pull "trieve/threshold:pr-${{ github.event.pull_request.number }}"
	else
	docker pull trieve/threshold:latest
	fi

	- name: Install dependencies
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang

	- name: Create NodeCluster (3 nodes)
	run: ./scripts/clear_dkg_keys.sh

	- name: Start 5-node stack with node chaos
	run: \|
	sed -i "s/trieve\/threshold:latest/trieve\/threshold:pr-${{ github.event.pull_request.number }}/g" docker-compose.yaml
	docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml up -d

	- name: Wait for gRPC endpoint
	run: \|
	echo "Waiting for node to expose port 50051-50055..."
	for i in {1..30}; do
	if nc -z localhost {50051..50055}; then
	echo "Port 50051-50055 is open."
	break
	fi
	echo "Port not ready yet – retry #$i"
	sleep 5
	done

	- name: Run end-to-end integration tests with node chaos
	run: cargo run --bin integration-tests check-dkg --port-range 50051-50055

	- name: Shutdown stack & print logs
	run: \|
	docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml logs --no-color \|\| true
	docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml down --volumes --remove-orphans \|\| true

	end-to-end-deposit-withdrawl-chaos-network:
	runs-on: blacksmith-16vcpu-ubuntu-2204
	needs: build
	env:
	MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
	MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
	IS_TESTNET: "true"
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: useblacksmith/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}

	- name: Set up Rust toolchain
	run: \|
	rustup update 1.87
	rustup default 1.87

	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Pull Docker image
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	docker pull "trieve/threshold:pr-${{ github.event.pull_request.number }}"
	else
	docker pull trieve/threshold:latest
	fi

	- name: Clean cargo
	run: cargo clean

	- name: Install dependencies
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang

	- name: Start 5-node stack with network chaos
	run: \|
	# Modify docker-compose.yaml to the correct tag
	sed -i "s/trieve\/threshold:latest/trieve\/threshold:pr-${{ github.event.pull_request.number }}/g" docker-compose.yaml
	docker compose -f docker-compose.yaml up -d

	- name: Wait for gRPC endpoint
	run: \|
	echo "Waiting for node to expose port 50051-50055..."
	for i in {1..30}; do
	if nc -z localhost {50051..50055}; then
	echo "Port 50051-50055 is open."
	break
	fi
	echo "Port not ready yet – retry #$i"
	sleep 5
	done

	- name: Run end-to-end integration tests with network chaos
	run: cargo run --bin integration-tests test --port-range 50051-50055

	- name: Shutdown stack & print logs
	run: \|
	docker compose -f docker-compose.yaml logs --no-color \|\| true
	docker compose -f docker-compose.yaml down --volumes --remove-orphans \|\| true

	end-to-end-deposit-withdrawl-chaos-nodes:
	runs-on: blacksmith-16vcpu-ubuntu-2204
	needs: build
	env:
	MNEMONIC: ${{ secrets.TEST_MNEMONIC }}
	MNEMONIC_TO: ${{ secrets.TEST_MNEMONIC_TO }}
	IS_TESTNET: "true"
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: useblacksmith/cache@v5
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-e2e-${{ hashFiles('**/Cargo.toml') }}

	- name: Set up Rust toolchain
	run: \|
	rustup update 1.87
	rustup default 1.87

	- name: Login to Docker Hub
	uses: docker/login-action@v3
	with:
	username: ${{ secrets.DOCKER_USERNAME }}
	password: ${{ secrets.DOCKER_PASSWORD }}

	- name: Pull Docker image
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	docker pull "trieve/threshold:pr-${{ github.event.pull_request.number }}"
	else
	docker pull trieve/threshold:latest
	fi

	- name: Install dependencies
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libclang-dev clang

	- name: Create NodeCluster (3 nodes)
	run: ./scripts/clear_dkg_keys.sh

	- name: Start 5-node stack with node chaos
	run: \|
	docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml up -d

	- name: Wait for gRPC endpoint
	run: \|
	echo "Waiting for node to expose port 50051-50055..."
	for i in {1..30}; do
	if nc -z localhost {50051..50055}; then
	echo "Port 50051-50055 is open."
	break
	fi
	echo "Port not ready yet – retry #$i"
	sleep 5
	done

	- name: Run end-to-end integration tests with node chaos
	run: cargo run --bin integration-tests test --port-range 50051-50055

	- name: Shutdown stack & print logs
	run: \|
	docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml logs --no-color \|\| true
	docker compose -f docker-compose.yaml -f docker-compose.chaos-nodes.yaml down --volumes --remove-orphans \|\| true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

cleanup: make consensus more stable after restart #178

Workflow file

cleanup: make consensus more stable after restart #178

Uh oh!

Workflow file for this run