diff --git a/Cargo.lock b/Cargo.lock index cfaabbbe15..78c5a2460f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14608,7 +14608,6 @@ dependencies = [ "async-trait", "bitcoin", "bitcoind-async-client", - "borsh", "format_serde_error", "http 1.4.0", "jsonrpsee", @@ -15547,6 +15546,7 @@ dependencies = [ "strata-ol-chain-types-new", "strata-ol-chainstate-types", "strata-ol-state-types", + "strata-paas", "strata-primitives", "strata-state", "strata-test-utils", @@ -15590,10 +15590,14 @@ dependencies = [ name = "strata-dbtool" version = "0.3.0-alpha.1" dependencies = [ + "alpen-ee-common", + "alpen-ee-database", "argh", "hex", "serde", "serde_json", + "sled", + "strata-acct-types", "strata-asm-logs", "strata-asm-proto-checkpoint-types", "strata-checkpoint-types", @@ -15605,8 +15609,11 @@ dependencies = [ "strata-identifiers", "strata-ol-chain-types", "strata-ol-chain-types-new", + "strata-paas", "strata-primitives", "tracing-subscriber 0.3.23", + "typed-sled", + "zkaleido", ] [[package]] diff --git a/bin/strata-dbtool/Cargo.toml b/bin/strata-dbtool/Cargo.toml index e19bf412be..c1b8e1d80b 100644 --- a/bin/strata-dbtool/Cargo.toml +++ b/bin/strata-dbtool/Cargo.toml @@ -15,8 +15,14 @@ argh.workspace = true hex.workspace = true serde.workspace = true serde_json.workspace = true +sled.workspace = true tracing-subscriber.workspace = true +typed-sled.workspace = true +zkaleido.workspace = true +alpen-ee-common.workspace = true +alpen-ee-database.workspace = true +strata-acct-types.workspace = true strata-asm-logs.workspace = true strata-asm-proto-checkpoint-types.workspace = true strata-checkpoint-types.workspace = true @@ -28,4 +34,5 @@ strata-db-types.workspace = true strata-identifiers.workspace = true strata-ol-chain-types.workspace = true strata-ol-chain-types-new.workspace = true +strata-paas.workspace = true strata-primitives.workspace = true diff --git a/bin/strata-dbtool/README.md b/bin/strata-dbtool/README.md index 6789796a14..a15cd392be 100644 --- a/bin/strata-dbtool/README.md +++ b/bin/strata-dbtool/README.md @@ -32,6 +32,7 @@ strata-dbtool [OPTIONS] ### Global Options - `-d, --datadir ` - Node data directory (default: `data`) +- `--ee-datadir ` - Alpen-client data directory. Required for any `ee-*` subcommand; points at the alpen-client's `--datadir`, not the strata node's. ## Commands @@ -340,6 +341,214 @@ Execute revert with block deletion: strata-dbtool revert-ol-state -f -d 858c390aaaabd7c457cb24c955d06fb9de0f6666d0b692e3b1a01b426705885b --l1-reorg-safe-depth 6 ``` +## Prover Task Admin + +> [!WARNING] +> +> These commands mutate the prover-task store and the checkpoint-proof receipt +> store. Stop the node before using them — concurrent writes from a running +> prover will conflict with these edits and may corrupt state. + +Every mutating subcommand is a **dry run** unless `-f, --force` is passed — +the same UX as `revert-ol-state`. Without `--force`, the command prints +what *would* happen and a `Use --force to execute these changes.` hint; +with `--force`, the mutation actually lands. + +### Semantics — `abandon` vs `reset` vs `delete` + +| Verb | Final status | When to use | +|------------|-------------------------------------------------|-----------------------------------------------------------| +| `abandon` | `PermanentFailure { error: "abandoned via dbtool" }` | Stop the recovery scanner from respawning a stuck task while keeping an audit trail. | +| `reset` | `Pending` (retry-after cleared) | Force a fresh prove attempt — drops accumulated retry count. | +| `delete` | row removed | Prefer `abandon` unless you really want no trace left. | +| `backfill` | `Pending` (newly inserted) | Queue a proof request "from outside" — e.g. for an epoch the node never picked up. | + +### `get-prover-task` +Fetch a single prover task record by its hex-encoded key. + +```bash +strata-dbtool get-prover-task [OPTIONS] +``` + +### `get-prover-tasks-summary` +Aggregate counts by status, plus a bounded slice of matching entries. + +```bash +strata-dbtool get-prover-tasks-summary [--status ] [--limit ] [OPTIONS] +``` + +**Options:** +- `--status ` — one of `all` (default), `pending`, `proving`, `completed`, `transient-failure`, `permanent-failure`, `unfinished`, `terminal` +- `--limit ` — maximum entries to include in the output (default: 20) + +### `abandon-prover-task` +Mark a single task as `PermanentFailure { error: "abandoned via dbtool" }`. + +```bash +strata-dbtool abandon-prover-task --force +``` + +### `abandon-prover-tasks` +Bulk-abandon every `Pending` or `Proving` task. Without `--force`, prints +the change set as a dry run. + +```bash +strata-dbtool abandon-prover-tasks --all-unfinished --force +``` + +### `reset-prover-task` +Flip a task back to `Pending` and clear its retry-after timestamp. + +```bash +strata-dbtool reset-prover-task --force +``` + +### `delete-prover-task` +Hard-delete a task row. + +```bash +strata-dbtool delete-prover-task --force +``` + +### `backfill-checkpoint-proof-task` +Queue a fresh `Pending` checkpoint-proof task for an epoch. Resolves the +canonical commitment at the epoch and constructs the task key via the shared +`CheckpointProofTask` encoding, so the running node will pick the task up on +its next startup-recovery pass. + +```bash +strata-dbtool backfill-checkpoint-proof-task --force +``` + +### `backfill-prover-task-raw` +Insert a `Pending` task record under a caller-provided raw key. Escape hatch +for proof kinds without a typed helper. + +```bash +strata-dbtool backfill-prover-task-raw --force +``` + +### `get-checkpoint-proof` +Fetch the stored proof receipt for an OL checkpoint epoch. + +```bash +strata-dbtool get-checkpoint-proof [OPTIONS] +``` + +### `delete-checkpoint-proof` +Delete the stored proof receipt for an epoch. Operates on the canonical +commitment at that epoch. Use case: force a re-prove after a guest-program +upgrade. + +```bash +strata-dbtool delete-checkpoint-proof --force +``` + +## EE Prover Task & Receipt Admin + +> [!WARNING] +> +> These commands mutate the EE prover store under the **alpen-client** +> datadir (not the strata node's). Stop the alpen-client before using +> them — concurrent writes from a running chunk/acct prover will conflict +> with these edits and may corrupt state. + +The alpen-client maintains a separate sled instance for prover-side +persistence — shared task tree (chunk + acct), chunk-receipt store, and +typed acct-proof store. All `ee-*` subcommands require `--ee-datadir`, +which points at the alpen-client's `--datadir`. + +### Which surface to use + +| Concern | Lives in | Subcommand prefix | +|----------------------|-----------------------|--------------------| +| OL checkpoint proofs | strata node datadir | (no prefix) | +| EE chunk proofs | alpen-client datadir | `ee-*` (`--kind chunk`) | +| EE acct/batch proofs | alpen-client datadir | `ee-*` (`--kind acct`) | + +Chunk and acct tasks share one tree, disambiguated by a single-byte +**kind tag** at the start of every task key (`b'c'` for chunk, `b'a'` +for acct). The `--kind` filter on the summary and bulk-abandon commands +selects on that tag; single-key commands operate on opaque keys, so the +kind comes from whatever the key starts with. + +### `ee-get-prover-task` +Fetch a single EE prover task record by its hex-encoded key. + +```bash +strata-dbtool --ee-datadir ee-get-prover-task [OPTIONS] +``` + +### `ee-get-prover-tasks-summary` +Aggregate counts by status, plus a bounded slice of matching entries. + +```bash +strata-dbtool --ee-datadir ee-get-prover-tasks-summary [--status ] [--kind ] [--limit ] [OPTIONS] +``` + +**Options:** +- `--status ` — same set as the OL summary command (`all`, `pending`, …, `terminal`). +- `--kind ` — one of `all` (default), `chunk`, `acct`. + +### `ee-abandon-prover-task` +Mark a single EE task as `PermanentFailure { error: "abandoned via dbtool" }`. + +```bash +strata-dbtool --ee-datadir ee-abandon-prover-task --force +``` + +### `ee-abandon-prover-tasks` +Bulk-abandon every `Pending`/`Proving` EE task, optionally restricted by +kind. Without `--force`, prints the change set as a dry run. + +```bash +strata-dbtool --ee-datadir ee-abandon-prover-tasks --all-unfinished [--kind ] --force +``` + +### `ee-reset-prover-task` +Flip an EE task back to `Pending` and clear its retry-after timestamp. + +```bash +strata-dbtool --ee-datadir ee-reset-prover-task --force +``` + +### `ee-delete-prover-task` +Hard-delete an EE task record. + +```bash +strata-dbtool --ee-datadir ee-delete-prover-task --force +``` + +### `ee-backfill-prover-task-raw` +Insert a `Pending` EE task record under a caller-provided raw key. EE +task keys come from the chunk/acct spec encodings; raw is the only +supported backfill path (no typed equivalent of `backfill-checkpoint-proof-task`). + +```bash +strata-dbtool --ee-datadir ee-backfill-prover-task-raw --force +``` + +### `ee-get-chunk-receipt` / `ee-delete-chunk-receipt` +Inspect or remove a stored chunk-proof receipt by its task key. Use +case: drop a stale receipt after a guest-program upgrade so the chunk +prover re-proves it. + +```bash +strata-dbtool --ee-datadir ee-get-chunk-receipt [OPTIONS] +strata-dbtool --ee-datadir ee-delete-chunk-receipt --force +``` + +### `ee-get-acct-proof` / `ee-delete-acct-proof` +Inspect or remove a stored acct/batch proof. The batch id is passed as +`:` (each 32 bytes), matching `BatchId`'s +`Display` format — copy directly from the alpen-client's logs. Delete +also clears the secondary `ProofId → BatchId` index. + +```bash +strata-dbtool --ee-datadir ee-get-acct-proof : [OPTIONS] +strata-dbtool --ee-datadir ee-delete-acct-proof : --force +``` + ## Output Formats ### Porcelain Format (Default) diff --git a/bin/strata-dbtool/src/cli.rs b/bin/strata-dbtool/src/cli.rs index f458a2b364..8c82f10ee4 100644 --- a/bin/strata-dbtool/src/cli.rs +++ b/bin/strata-dbtool/src/cli.rs @@ -9,10 +9,24 @@ use argh::FromArgs; use crate::cmd::{ broadcaster::{GetBroadcasterSummaryArgs, GetBroadcasterTxArgs}, checkpoint::{GetCheckpointArgs, GetCheckpointsSummaryArgs, GetEpochSummaryArgs}, + checkpoint_proof::{DeleteCheckpointProofArgs, GetCheckpointProofArgs}, client_state::GetClientStateUpdateArgs, + ee_prover_task::{ + EeAbandonProverTaskArgs, EeAbandonProverTasksArgs, EeBackfillProverTaskRawArgs, + EeDeleteProverTaskArgs, EeGetProverTaskArgs, EeGetProverTasksSummaryArgs, + EeResetProverTaskArgs, + }, + ee_receipts::{ + EeDeleteAcctProofArgs, EeDeleteChunkReceiptArgs, EeGetAcctProofArgs, EeGetChunkReceiptArgs, + }, l1::{GetL1BlockArgs, GetL1SummaryArgs}, ol::{GetOLBlockArgs, GetOLSummaryArgs}, ol_state::{GetOLStateArgs, RevertOLStateArgs}, + prover_task::{ + AbandonProverTaskArgs, AbandonProverTasksArgs, BackfillCheckpointProofTaskArgs, + BackfillProverTaskRawArgs, DeleteProverTaskArgs, GetProverTaskArgs, + GetProverTasksSummaryArgs, ResetProverTaskArgs, + }, syncinfo::GetSyncinfoArgs, writer::{GetWriterPayloadArgs, GetWriterSummaryArgs}, }; @@ -25,6 +39,11 @@ pub(crate) struct Cli { #[argh(option, short = 'd', default = "PathBuf::from(\"data\")")] pub(crate) datadir: PathBuf, + /// alpen-client data directory — required for any `ee-*` subcommand. + /// Points at the alpen-client's `--datadir`, not the strata node's. + #[argh(option)] + pub(crate) ee_datadir: Option, + #[argh(subcommand)] pub(crate) cmd: Command, } @@ -48,6 +67,27 @@ pub(crate) enum Command { GetSyncinfo(GetSyncinfoArgs), GetOLState(GetOLStateArgs), RevertOLState(RevertOLStateArgs), + GetProverTask(GetProverTaskArgs), + GetProverTasksSummary(GetProverTasksSummaryArgs), + AbandonProverTask(AbandonProverTaskArgs), + AbandonProverTasks(AbandonProverTasksArgs), + ResetProverTask(ResetProverTaskArgs), + DeleteProverTask(DeleteProverTaskArgs), + GetCheckpointProof(GetCheckpointProofArgs), + DeleteCheckpointProof(DeleteCheckpointProofArgs), + BackfillCheckpointProofTask(BackfillCheckpointProofTaskArgs), + BackfillProverTaskRaw(BackfillProverTaskRawArgs), + EeGetProverTask(EeGetProverTaskArgs), + EeGetProverTasksSummary(EeGetProverTasksSummaryArgs), + EeAbandonProverTask(EeAbandonProverTaskArgs), + EeAbandonProverTasks(EeAbandonProverTasksArgs), + EeResetProverTask(EeResetProverTaskArgs), + EeDeleteProverTask(EeDeleteProverTaskArgs), + EeBackfillProverTaskRaw(EeBackfillProverTaskRawArgs), + EeGetChunkReceipt(EeGetChunkReceiptArgs), + EeDeleteChunkReceipt(EeDeleteChunkReceiptArgs), + EeGetAcctProof(EeGetAcctProofArgs), + EeDeleteAcctProof(EeDeleteAcctProofArgs), } /// Output format diff --git a/bin/strata-dbtool/src/cmd/checkpoint_proof.rs b/bin/strata-dbtool/src/cmd/checkpoint_proof.rs new file mode 100644 index 0000000000..1624ce54d6 --- /dev/null +++ b/bin/strata-dbtool/src/cmd/checkpoint_proof.rs @@ -0,0 +1,124 @@ +//! Admin commands operating on the checkpoint-proof receipt store. +//! +//! Pairs with `get-checkpoint` / `get-checkpoints-summary` which surface +//! checkpoint payloads; the receipts themselves live in a separate tree +//! (`CheckpointProofSchema`) keyed by [`strata_identifiers::EpochCommitment`]. + +use argh::FromArgs; +use strata_cli_common::errors::{DisplayableError, DisplayedError}; +use strata_db_types::traits::{CheckpointProofDatabase, DatabaseBackend}; +use strata_identifiers::Epoch; + +use crate::{ + cli::OutputFormat, + cmd::{checkpoint::get_canonical_epoch_commitment_at, prover_task_common::print_force_hint}, + output::{ + checkpoint_proof::{CheckpointProofInfo, DeletedCheckpointProofInfo}, + output, + }, +}; + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "get-checkpoint-proof")] +/// Fetch the stored proof receipt for an OL checkpoint epoch. +pub(crate) struct GetCheckpointProofArgs { + /// checkpoint epoch + #[argh(positional)] + pub(crate) epoch: Epoch, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "delete-checkpoint-proof")] +/// Delete a stored checkpoint proof receipt for an epoch. +/// +/// Operates on the canonical commitment at the given epoch. Use case: +/// force a re-prove after a guest-program upgrade or to drop a stale +/// receipt from a broken run. Dry-run unless `--force` is passed. +pub(crate) struct DeleteCheckpointProofArgs { + /// checkpoint epoch + #[argh(positional)] + pub(crate) epoch: Epoch, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +/// Fetch the proof receipt for a checkpoint epoch. +pub(crate) fn get_checkpoint_proof( + db: &impl DatabaseBackend, + args: GetCheckpointProofArgs, +) -> Result<(), DisplayedError> { + let commitment = get_canonical_epoch_commitment_at(db, args.epoch)?.ok_or_else(|| { + DisplayedError::UserError( + "No canonical checkpoint commitment at epoch".to_string(), + Box::new(args.epoch), + ) + })?; + + let receipt = db + .checkpoint_proof_db() + .get_proof(commitment) + .internal_error("Failed to read checkpoint proof")? + .ok_or_else(|| { + DisplayedError::UserError( + "No checkpoint proof stored for epoch".to_string(), + Box::new(args.epoch), + ) + })?; + + let info = CheckpointProofInfo::from_receipt(args.epoch, *commitment.last_blkid(), &receipt); + output(&info, args.output_format) +} + +/// Delete the proof receipt for a checkpoint epoch. +pub(crate) fn delete_checkpoint_proof( + db: &impl DatabaseBackend, + args: DeleteCheckpointProofArgs, +) -> Result<(), DisplayedError> { + let commitment = get_canonical_epoch_commitment_at(db, args.epoch)?.ok_or_else(|| { + DisplayedError::UserError( + "No canonical checkpoint commitment at epoch".to_string(), + Box::new(args.epoch), + ) + })?; + + // Resolve existence up front so the dry run still emits the same + // structured `existed` field operators check against. + let existed = db + .checkpoint_proof_db() + .get_proof(commitment) + .internal_error("Failed to read checkpoint proof")? + .is_some(); + + if !args.force { + let ack = DeletedCheckpointProofInfo { + epoch: args.epoch, + terminal_blkid: *commitment.last_blkid(), + existed, + }; + output(&ack, args.output_format)?; + print_force_hint(); + return Ok(()); + } + + let actually_existed = db + .checkpoint_proof_db() + .del_proof(commitment) + .internal_error("Failed to delete checkpoint proof")?; + + let ack = DeletedCheckpointProofInfo { + epoch: args.epoch, + terminal_blkid: *commitment.last_blkid(), + existed: actually_existed, + }; + output(&ack, args.output_format) +} diff --git a/bin/strata-dbtool/src/cmd/ee_prover_task.rs b/bin/strata-dbtool/src/cmd/ee_prover_task.rs new file mode 100644 index 0000000000..1bd3d40696 --- /dev/null +++ b/bin/strata-dbtool/src/cmd/ee_prover_task.rs @@ -0,0 +1,456 @@ +//! Admin commands operating on the EE prover task store. +//! +//! These talk directly to [`alpen_ee_database::EeProverDbSled`]'s +//! [`strata_db_types::traits::ProverTaskDatabase`] impl. Same DB contract +//! as the OL surface, but the underlying store lives in a separate sled +//! instance under the alpen-client datadir (`/sled`), so +//! mutations here can't race with OL writers. +//! +//! Chunk and acct tasks share one tree, disambiguated by a single-byte +//! kind tag at the start of the key (`b'c'` / `b'a'`). The `--kind` +//! filter on the summary and bulk-abandon commands selects on that tag. +//! +//! Every mutating verb follows the `revert-ol-state` UX: without +//! `-f/--force` the command is a dry run; with `--force` the mutation +//! actually lands. + +use std::{fmt, str::FromStr}; + +use alpen_ee_database::EeProverDbSled; +use argh::FromArgs; +use strata_cli_common::errors::{DisplayableError, DisplayedError}; +use strata_db_types::traits::ProverTaskDatabase; +use strata_paas::{TaskRecordData, TaskStatus}; + +use crate::{ + cli::OutputFormat, + cmd::prover_task_common::{parse_task_key, print_force_hint, StatusFilter, ABANDONED_REASON}, + output::{ + output, + prover_task::{ProverTaskInfo, ProverTasksSummaryInfo}, + }, +}; + +/// EE task kind filter. Matches on the kind tag carried by the key's +/// first byte — the same convention used by the alpen-client's prover +/// builders. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum KindFilter { + All, + Chunk, + Acct, +} + +impl KindFilter { + fn matches(&self, key: &[u8]) -> bool { + match self { + Self::All => true, + Self::Chunk => key.first().copied() == Some(b'c'), + Self::Acct => key.first().copied() == Some(b'a'), + } + } +} + +#[derive(Debug)] +pub(crate) struct UnsupportedKindFilter; + +impl fmt::Display for UnsupportedKindFilter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "must be one of: all, chunk, acct") + } +} + +impl FromStr for KindFilter { + type Err = UnsupportedKindFilter; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "all" => Ok(Self::All), + "chunk" => Ok(Self::Chunk), + "acct" => Ok(Self::Acct), + _ => Err(UnsupportedKindFilter), + } + } +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-get-prover-task")] +/// Fetch a single EE prover task record by its hex-encoded key. +pub(crate) struct EeGetProverTaskArgs { + /// hex-encoded task key (as stored by `EeProverDbSled`) + #[argh(positional)] + pub(crate) key_hex: String, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-get-prover-tasks-summary")] +/// Summarize EE prover tasks by status and kind, with a bounded slice +/// of entries. +pub(crate) struct EeGetProverTasksSummaryArgs { + /// status filter: all (default), pending, proving, completed, + /// transient-failure, permanent-failure, unfinished, terminal + #[argh(option, default = "StatusFilter::All")] + pub(crate) status: StatusFilter, + + /// kind filter: all (default), chunk, acct + #[argh(option, default = "KindFilter::All")] + pub(crate) kind: KindFilter, + + /// max number of matching entries to include in the output (default 20) + #[argh(option, default = "20")] + pub(crate) limit: usize, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-abandon-prover-task")] +/// Mark a single EE prover task as `PermanentFailure { error: "abandoned via dbtool" }`. +/// +/// Leaves the record in the DB for audit; recovery will not respawn it. +/// Dry-run unless `--force` is passed. +pub(crate) struct EeAbandonProverTaskArgs { + /// hex-encoded task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-abandon-prover-tasks")] +/// Bulk-abandon every Pending/Proving EE prover task, optionally +/// restricted by kind. Dry-run unless `--force` is passed. +pub(crate) struct EeAbandonProverTasksArgs { + /// only consider Pending/Proving tasks (currently the only supported + /// selector — kept explicit so future selectors can be added) + #[argh(switch)] + pub(crate) all_unfinished: bool, + + /// kind filter: all (default), chunk, acct + #[argh(option, default = "KindFilter::All")] + pub(crate) kind: KindFilter, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-reset-prover-task")] +/// Reset an EE prover task to `Pending` and clear its retry-after timestamp. +/// Dry-run unless `--force` is passed. +pub(crate) struct EeResetProverTaskArgs { + /// hex-encoded task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-delete-prover-task")] +/// Hard-delete an EE prover task record. +/// +/// Prefer `ee-abandon-prover-task` unless you really want the row gone. +/// Dry-run unless `--force` is passed. +pub(crate) struct EeDeleteProverTaskArgs { + /// hex-encoded task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-backfill-prover-task-raw")] +/// Insert a `Pending` EE task record under a raw hex-encoded key. +/// +/// EE task keys are produced by the chunk/acct spec encodings — they're +/// not easily reconstructible offline, so this raw escape hatch is the +/// only supported backfill path. Dry-run unless `--force` is passed. +pub(crate) struct EeBackfillProverTaskRawArgs { + /// hex-encoded task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +pub(crate) fn ee_get_prover_task( + db: &EeProverDbSled, + args: EeGetProverTaskArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + let record = db + .get_task(key.clone()) + .internal_error("Failed to read EE prover task record")? + .ok_or_else(|| { + DisplayedError::UserError( + "No EE prover task found for key".to_string(), + Box::new(args.key_hex.clone()), + ) + })?; + + let info = ProverTaskInfo::from_ee_record(&key, &record); + output(&info, args.output_format) +} + +pub(crate) fn ee_get_prover_tasks_summary( + db: &EeProverDbSled, + args: EeGetProverTasksSummaryArgs, +) -> Result<(), DisplayedError> { + let all = db + .list_all_tasks() + .internal_error("Failed to list EE prover tasks")?; + + let mut pending = 0usize; + let mut proving = 0usize; + let mut completed = 0usize; + let mut transient_failure = 0usize; + let mut permanent_failure = 0usize; + let mut matched = 0usize; + let mut entries: Vec = Vec::new(); + + for (key, record) in &all { + // The aggregate counters always reflect the full set — kind + + // status filters only affect what lands in `matched` / `entries`, + // so an operator can still see how many rows live in the store. + match record.status() { + TaskStatus::Pending => pending += 1, + TaskStatus::Proving { .. } => proving += 1, + TaskStatus::Completed => completed += 1, + TaskStatus::TransientFailure { .. } => transient_failure += 1, + TaskStatus::PermanentFailure { .. } => permanent_failure += 1, + } + if args.status.matches(record.status()) && args.kind.matches(key) { + matched += 1; + if entries.len() < args.limit { + entries.push(ProverTaskInfo::from_ee_record(key, record)); + } + } + } + + let summary = ProverTasksSummaryInfo { + total: all.len(), + pending, + proving, + completed, + transient_failure, + permanent_failure, + matched, + returned: entries.len(), + entries, + }; + + output(&summary, args.output_format) +} + +pub(crate) fn ee_abandon_prover_task( + db: &EeProverDbSled, + args: EeAbandonProverTaskArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + + let mut record = db + .get_task(key.clone()) + .internal_error("Failed to read EE prover task record")? + .ok_or_else(|| { + DisplayedError::UserError( + "No EE prover task found for key".to_string(), + Box::new(args.key_hex.clone()), + ) + })?; + + if record.status().is_terminal() { + return Err(DisplayedError::UserError( + "Task is already in a terminal state".to_string(), + Box::new(args.key_hex), + )); + } + + if !args.force { + println!("would abandon: {}", args.key_hex); + print_force_hint(); + return Ok(()); + } + + record.set_status(TaskStatus::PermanentFailure { + error: ABANDONED_REASON.to_string(), + }); + db.put_task(key, record) + .internal_error("Failed to persist abandoned EE task")?; + + println!("abandoned: {}", args.key_hex); + Ok(()) +} + +pub(crate) fn ee_abandon_prover_tasks( + db: &EeProverDbSled, + args: EeAbandonProverTasksArgs, +) -> Result<(), DisplayedError> { + if !args.all_unfinished { + return Err(DisplayedError::UserError( + "--all-unfinished is the only currently supported selector".to_string(), + Box::new(()), + )); + } + + let unfinished = db + .list_unfinished() + .internal_error("Failed to list unfinished EE prover tasks")?; + + let mut abandoned = 0usize; + for (key, mut record) in unfinished { + if !args.kind.matches(&key) { + continue; + } + let key_hex = hex::encode(&key); + if args.force { + record.set_status(TaskStatus::PermanentFailure { + error: ABANDONED_REASON.to_string(), + }); + db.put_task(key, record) + .internal_error("Failed to persist abandoned EE task")?; + println!("abandoned: {key_hex}"); + } else { + println!("would abandon: {key_hex}"); + } + abandoned += 1; + } + + let verb = if args.force { + "abandoned" + } else { + "would abandon" + }; + println!("{verb} {abandoned} task(s)"); + if !args.force { + print_force_hint(); + } + Ok(()) +} + +pub(crate) fn ee_reset_prover_task( + db: &EeProverDbSled, + args: EeResetProverTaskArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + + let mut record = db + .get_task(key.clone()) + .internal_error("Failed to read EE prover task record")? + .ok_or_else(|| { + DisplayedError::UserError( + "No EE prover task found for key".to_string(), + Box::new(args.key_hex.clone()), + ) + })?; + + if !args.force { + println!("would reset: {}", args.key_hex); + print_force_hint(); + return Ok(()); + } + + record.set_status(TaskStatus::Pending); + record.set_retry_after_secs(None); + db.put_task(key, record) + .internal_error("Failed to persist reset EE task")?; + + println!("reset: {}", args.key_hex); + Ok(()) +} + +pub(crate) fn ee_delete_prover_task( + db: &EeProverDbSled, + args: EeDeleteProverTaskArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + + // Resolve existence up front so the dry run can surface a clear + // error rather than silently "previewing" a no-op delete. + let exists = db + .get_task(key.clone()) + .internal_error("Failed to read EE prover task record")? + .is_some(); + if !exists { + return Err(DisplayedError::UserError( + "No EE prover task found for key".to_string(), + Box::new(args.key_hex), + )); + } + + if !args.force { + println!("would delete: {}", args.key_hex); + print_force_hint(); + return Ok(()); + } + + db.delete_task(key) + .internal_error("Failed to delete EE prover task")?; + + println!("deleted: {}", args.key_hex); + Ok(()) +} + +pub(crate) fn ee_backfill_prover_task_raw( + db: &EeProverDbSled, + args: EeBackfillProverTaskRawArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + + if !args.force { + println!("would backfill EE prover task: {}", args.key_hex); + print_force_hint(); + return Ok(()); + } + + let record = TaskRecordData::new(TaskStatus::Pending); + db.insert_task(key, record) + .internal_error("Failed to insert EE prover task")?; + + println!("backfilled EE prover task: {}", args.key_hex); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kind_filter_matches_chunk_acct_and_all() { + assert!(KindFilter::All.matches(b"anything")); + assert!(KindFilter::All.matches(&[])); + + assert!(KindFilter::Chunk.matches(b"c-foo")); + assert!(!KindFilter::Chunk.matches(b"a-foo")); + assert!(!KindFilter::Chunk.matches(&[])); + + assert!(KindFilter::Acct.matches(b"a-foo")); + assert!(!KindFilter::Acct.matches(b"c-foo")); + assert!(!KindFilter::Acct.matches(&[])); + } + + #[test] + fn kind_filter_from_str_accepts_three_canonical_values() { + assert_eq!("all".parse::().unwrap(), KindFilter::All); + assert_eq!("CHUNK".parse::().unwrap(), KindFilter::Chunk); + assert_eq!("acct".parse::().unwrap(), KindFilter::Acct); + assert!("bogus".parse::().is_err()); + } +} diff --git a/bin/strata-dbtool/src/cmd/ee_receipts.rs b/bin/strata-dbtool/src/cmd/ee_receipts.rs new file mode 100644 index 0000000000..a6b4fde768 --- /dev/null +++ b/bin/strata-dbtool/src/cmd/ee_receipts.rs @@ -0,0 +1,281 @@ +//! Admin commands operating on the EE chunk-receipt and acct-proof trees. +//! +//! Chunk receipts are keyed by the opaque chunk-task key (`Vec`), +//! matching the `paas::ReceiptStore` shape. Acct proofs are keyed by +//! [`alpen_ee_common::BatchId`], which the CLI parses from a +//! `prev_block_hex:last_block_hex` literal — the same shape the +//! `BatchId::Display` impl emits, so operators can copy directly from +//! logs. + +use alpen_ee_common::BatchId; +use alpen_ee_database::EeProverDbSled; +use argh::FromArgs; +use strata_acct_types::Hash; +use strata_cli_common::errors::{DisplayableError, DisplayedError}; +use strata_primitives::buf::Buf32; + +use crate::{ + cli::OutputFormat, + cmd::prover_task_common::{parse_task_key, print_force_hint}, + output::{ + ee_receipts::{DeletedEeReceiptInfo, EeReceiptInfo}, + output, + }, +}; + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-get-chunk-receipt")] +/// Fetch a stored chunk-proof receipt by its task key. +pub(crate) struct EeGetChunkReceiptArgs { + /// hex-encoded chunk task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-delete-chunk-receipt")] +/// Delete a stored chunk-proof receipt. +/// +/// Use case: drop a stale receipt after a guest-program upgrade so the +/// next chunk-prover run re-proves it. Dry-run unless `--force` is passed. +pub(crate) struct EeDeleteChunkReceiptArgs { + /// hex-encoded chunk task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-get-acct-proof")] +/// Fetch the stored acct/batch proof for a [`BatchId`]. +pub(crate) struct EeGetAcctProofArgs { + /// batch id as ":" (each 32 bytes) + #[argh(positional)] + pub(crate) batch_id: String, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "ee-delete-acct-proof")] +/// Delete the stored acct/batch proof for a [`BatchId`]. +/// +/// Also clears the secondary `ProofId → BatchId` index so future +/// `get_proof_by_id` lookups miss instead of dangling. Dry-run unless +/// `--force` is passed. +pub(crate) struct EeDeleteAcctProofArgs { + /// batch id as ":" (each 32 bytes) + #[argh(positional)] + pub(crate) batch_id: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +/// Parses `":"` into a [`BatchId`]. +/// +/// Each hex half is exactly 32 bytes (64 hex chars). Both parts are +/// required — there is no fallback to a single-hash form because the +/// underlying type literally is a pair of hashes. +pub(crate) fn parse_batch_id(s: &str) -> Result { + let (prev, last) = s.split_once(':').ok_or_else(|| { + DisplayedError::UserError( + "Expected batch id as :".to_string(), + Box::new(s.to_string()), + ) + })?; + let prev_hash = parse_hash(prev)?; + let last_hash = parse_hash(last)?; + Ok(BatchId::from_parts(prev_hash, last_hash)) +} + +fn parse_hash(s: &str) -> Result { + let trimmed = s.strip_prefix("0x").unwrap_or(s); + let bytes = hex::decode(trimmed).map_err(|e| { + DisplayedError::UserError("Invalid hex-encoded block hash".to_string(), Box::new(e)) + })?; + let arr: [u8; 32] = bytes.as_slice().try_into().map_err(|_| { + DisplayedError::UserError( + format!( + "Block hash must be exactly 32 bytes (got {} bytes)", + bytes.len() + ), + Box::new(s.to_string()), + ) + })?; + Ok(Buf32(arr)) +} + +pub(crate) fn ee_get_chunk_receipt( + db: &EeProverDbSled, + args: EeGetChunkReceiptArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + let receipt = db + .get_chunk_receipt(&key) + .internal_error("Failed to read chunk receipt")? + .ok_or_else(|| { + DisplayedError::UserError( + "No chunk receipt stored for task key".to_string(), + Box::new(args.key_hex.clone()), + ) + })?; + + let info = EeReceiptInfo::from_receipt(args.key_hex, "chunk", &receipt); + output(&info, args.output_format) +} + +pub(crate) fn ee_delete_chunk_receipt( + db: &EeProverDbSled, + args: EeDeleteChunkReceiptArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + + // Resolve existence up front so the dry run still emits the same + // structured `existed` field operators check against. + let existed = db + .get_chunk_receipt(&key) + .internal_error("Failed to read chunk receipt")? + .is_some(); + + if !args.force { + let ack = DeletedEeReceiptInfo { + address: args.key_hex, + kind: "chunk", + existed, + }; + output(&ack, args.output_format)?; + print_force_hint(); + return Ok(()); + } + + let actually_existed = db + .delete_chunk_receipt(&key) + .internal_error("Failed to delete chunk receipt")?; + + let ack = DeletedEeReceiptInfo { + address: args.key_hex, + kind: "chunk", + existed: actually_existed, + }; + output(&ack, args.output_format) +} + +pub(crate) fn ee_get_acct_proof( + db: &EeProverDbSled, + args: EeGetAcctProofArgs, +) -> Result<(), DisplayedError> { + let batch_id = parse_batch_id(&args.batch_id)?; + let receipt = db + .get_acct_proof(batch_id) + .internal_error("Failed to read acct proof")? + .ok_or_else(|| { + DisplayedError::UserError( + "No acct proof stored for batch id".to_string(), + Box::new(args.batch_id.clone()), + ) + })?; + + let info = EeReceiptInfo::from_receipt(args.batch_id, "acct", &receipt); + output(&info, args.output_format) +} + +pub(crate) fn ee_delete_acct_proof( + db: &EeProverDbSled, + args: EeDeleteAcctProofArgs, +) -> Result<(), DisplayedError> { + let batch_id = parse_batch_id(&args.batch_id)?; + + // Resolve existence up front so the dry run still emits the same + // structured `existed` field operators check against. + let existed = db + .has_acct_proof(batch_id) + .internal_error("Failed to read acct proof")?; + + if !args.force { + let ack = DeletedEeReceiptInfo { + address: args.batch_id, + kind: "acct", + existed, + }; + output(&ack, args.output_format)?; + print_force_hint(); + return Ok(()); + } + + let actually_existed = db + .delete_acct_proof(batch_id) + .internal_error("Failed to delete acct proof")?; + + let ack = DeletedEeReceiptInfo { + address: args.batch_id, + kind: "acct", + existed: actually_existed, + }; + output(&ack, args.output_format) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_batch_id_accepts_two_32_byte_halves() { + let prev = "11".repeat(32); + let last = "22".repeat(32); + let bid = parse_batch_id(&format!("{prev}:{last}")).unwrap(); + assert_eq!(bid.prev_block().0, [0x11u8; 32]); + assert_eq!(bid.last_block().0, [0x22u8; 32]); + } + + #[test] + fn parse_batch_id_accepts_0x_prefix_on_each_half() { + let prev = format!("0x{}", "aa".repeat(32)); + let last = format!("0x{}", "bb".repeat(32)); + let bid = parse_batch_id(&format!("{prev}:{last}")).unwrap(); + assert_eq!(bid.prev_block().0, [0xaau8; 32]); + assert_eq!(bid.last_block().0, [0xbbu8; 32]); + } + + #[test] + fn parse_batch_id_rejects_missing_colon() { + assert!(parse_batch_id(&"11".repeat(64)).is_err()); + } + + #[test] + fn parse_batch_id_rejects_wrong_length_halves() { + // 31 bytes — short + let prev = "11".repeat(31); + let last = "22".repeat(32); + assert!(parse_batch_id(&format!("{prev}:{last}")).is_err()); + + // 33 bytes — long + let prev = "11".repeat(33); + let last = "22".repeat(32); + assert!(parse_batch_id(&format!("{prev}:{last}")).is_err()); + } + + #[test] + fn parse_batch_id_rejects_non_hex() { + let last = "22".repeat(32); + assert!(parse_batch_id(&format!("not-hex:{last}")).is_err()); + } +} diff --git a/bin/strata-dbtool/src/cmd/mod.rs b/bin/strata-dbtool/src/cmd/mod.rs index eee9462939..e561577b51 100644 --- a/bin/strata-dbtool/src/cmd/mod.rs +++ b/bin/strata-dbtool/src/cmd/mod.rs @@ -1,8 +1,13 @@ pub(crate) mod broadcaster; pub(crate) mod checkpoint; +pub(crate) mod checkpoint_proof; pub(crate) mod client_state; +pub(crate) mod ee_prover_task; +pub(crate) mod ee_receipts; pub(crate) mod l1; pub(crate) mod ol; pub(crate) mod ol_state; +pub(crate) mod prover_task; +pub(crate) mod prover_task_common; pub(crate) mod syncinfo; pub(crate) mod writer; diff --git a/bin/strata-dbtool/src/cmd/prover_task.rs b/bin/strata-dbtool/src/cmd/prover_task.rs new file mode 100644 index 0000000000..aa6b79251e --- /dev/null +++ b/bin/strata-dbtool/src/cmd/prover_task.rs @@ -0,0 +1,447 @@ +//! Admin commands operating on the OL prover task store. +//! +//! These talk directly to [`strata_db_types::traits::ProverTaskDatabase`] +//! so they can manipulate records without going through the running +//! prover service — by design, the node must be offline. +//! +//! Every mutating verb follows the `revert-ol-state` UX: without +//! `-f/--force` the command is a dry run (prints what *would* happen +//! and a `Use --force to execute these changes.` hint); with `--force` +//! the mutation actually lands. + +use argh::FromArgs; +use strata_checkpoint_types::CheckpointProofTask; +use strata_cli_common::errors::{DisplayableError, DisplayedError}; +use strata_db_types::traits::{DatabaseBackend, ProverTaskDatabase}; +use strata_identifiers::Epoch; +use strata_paas::{TaskRecordData, TaskStatus}; + +use crate::{ + cli::OutputFormat, + cmd::{ + checkpoint::get_canonical_epoch_commitment_at, + prover_task_common::{parse_task_key, print_force_hint, StatusFilter, ABANDONED_REASON}, + }, + output::{ + output, + prover_task::{ProverTaskInfo, ProverTasksSummaryInfo}, + }, +}; + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "get-prover-task")] +/// Fetch a single prover task record by its hex-encoded key. +pub(crate) struct GetProverTaskArgs { + /// hex-encoded task key (as stored by `ProverTaskDatabase`) + #[argh(positional)] + pub(crate) key_hex: String, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "get-prover-tasks-summary")] +/// Summarize prover tasks by status, with a bounded slice of entries. +pub(crate) struct GetProverTasksSummaryArgs { + /// status filter: all (default), pending, proving, completed, + /// transient-failure, permanent-failure, unfinished, terminal + #[argh(option, default = "StatusFilter::All")] + pub(crate) status: StatusFilter, + + /// max number of matching entries to include in the output (default 20) + #[argh(option, default = "20")] + pub(crate) limit: usize, + + /// output format: "porcelain" (default) or "json" + #[argh(option, short = 'o', default = "OutputFormat::Porcelain")] + pub(crate) output_format: OutputFormat, +} + +/// Fetch a single prover task record by hex-encoded key. +pub(crate) fn get_prover_task( + db: &impl DatabaseBackend, + args: GetProverTaskArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + let record = db + .prover_task_db() + .get_task(key.clone()) + .internal_error("Failed to read prover task record")? + .ok_or_else(|| { + DisplayedError::UserError( + "No prover task found for key".to_string(), + Box::new(args.key_hex.clone()), + ) + })?; + + let info = ProverTaskInfo::from_record(&key, &record); + output(&info, args.output_format) +} + +/// Summarize prover task store contents. +pub(crate) fn get_prover_tasks_summary( + db: &impl DatabaseBackend, + args: GetProverTasksSummaryArgs, +) -> Result<(), DisplayedError> { + let task_db = db.prover_task_db(); + + let all = task_db + .list_all_tasks() + .internal_error("Failed to list prover tasks")?; + + let mut pending = 0usize; + let mut proving = 0usize; + let mut completed = 0usize; + let mut transient_failure = 0usize; + let mut permanent_failure = 0usize; + let mut matched = 0usize; + let mut entries: Vec = Vec::new(); + + for (key, record) in &all { + match record.status() { + TaskStatus::Pending => pending += 1, + TaskStatus::Proving { .. } => proving += 1, + TaskStatus::Completed => completed += 1, + TaskStatus::TransientFailure { .. } => transient_failure += 1, + TaskStatus::PermanentFailure { .. } => permanent_failure += 1, + } + if args.status.matches(record.status()) { + matched += 1; + if entries.len() < args.limit { + entries.push(ProverTaskInfo::from_record(key, record)); + } + } + } + + let summary = ProverTasksSummaryInfo { + total: all.len(), + pending, + proving, + completed, + transient_failure, + permanent_failure, + matched, + returned: entries.len(), + entries, + }; + + output(&summary, args.output_format) +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "abandon-prover-task")] +/// Mark a single prover task as `PermanentFailure { error: "abandoned via dbtool" }`. +/// +/// Leaves the record in the DB for audit; recovery will not respawn it. +/// Dry-run unless `--force` is passed. +pub(crate) struct AbandonProverTaskArgs { + /// hex-encoded task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "abandon-prover-tasks")] +/// Bulk-abandon every Pending/Proving prover task. +/// +/// Use case: after a crash or operator-induced restart, prevent stuck +/// in-progress tasks from being respawned by the recovery scanner. +/// Dry-run unless `--force` is passed. +pub(crate) struct AbandonProverTasksArgs { + /// only consider Pending/Proving tasks (currently the only supported + /// selector — kept explicit so future selectors can be added) + #[argh(switch)] + pub(crate) all_unfinished: bool, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "reset-prover-task")] +/// Reset a prover task to `Pending` and clear its retry-after timestamp. +/// +/// Use case: force a fresh prove attempt (drops accumulated retry count). +/// Dry-run unless `--force` is passed. +pub(crate) struct ResetProverTaskArgs { + /// hex-encoded task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "delete-prover-task")] +/// Hard-delete a prover task record. +/// +/// Prefer `abandon-prover-task` unless you really want the row gone. +/// Dry-run unless `--force` is passed. +pub(crate) struct DeleteProverTaskArgs { + /// hex-encoded task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +/// Abandon a single task by flipping its status to `PermanentFailure`. +pub(crate) fn abandon_prover_task( + db: &impl DatabaseBackend, + args: AbandonProverTaskArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + let task_db = db.prover_task_db(); + + let mut record = task_db + .get_task(key.clone()) + .internal_error("Failed to read prover task record")? + .ok_or_else(|| { + DisplayedError::UserError( + "No prover task found for key".to_string(), + Box::new(args.key_hex.clone()), + ) + })?; + + if record.status().is_terminal() { + return Err(DisplayedError::UserError( + "Task is already in a terminal state".to_string(), + Box::new(args.key_hex), + )); + } + + if !args.force { + println!("would abandon: {}", args.key_hex); + print_force_hint(); + return Ok(()); + } + + record.set_status(TaskStatus::PermanentFailure { + error: ABANDONED_REASON.to_string(), + }); + task_db + .put_task(key, record) + .internal_error("Failed to persist abandoned task")?; + + println!("abandoned: {}", args.key_hex); + Ok(()) +} + +/// Bulk-abandon every Pending/Proving task. +pub(crate) fn abandon_prover_tasks( + db: &impl DatabaseBackend, + args: AbandonProverTasksArgs, +) -> Result<(), DisplayedError> { + if !args.all_unfinished { + return Err(DisplayedError::UserError( + "--all-unfinished is the only currently supported selector".to_string(), + Box::new(()), + )); + } + + let task_db = db.prover_task_db(); + let unfinished = task_db + .list_unfinished() + .internal_error("Failed to list unfinished prover tasks")?; + + let mut abandoned = 0usize; + for (key, mut record) in unfinished { + let key_hex = hex::encode(&key); + if args.force { + record.set_status(TaskStatus::PermanentFailure { + error: ABANDONED_REASON.to_string(), + }); + task_db + .put_task(key, record) + .internal_error("Failed to persist abandoned task")?; + println!("abandoned: {key_hex}"); + } else { + println!("would abandon: {key_hex}"); + } + abandoned += 1; + } + + let verb = if args.force { + "abandoned" + } else { + "would abandon" + }; + println!("{verb} {abandoned} task(s)"); + if !args.force { + print_force_hint(); + } + Ok(()) +} + +/// Reset a task to `Pending` and clear its retry-after timestamp. +pub(crate) fn reset_prover_task( + db: &impl DatabaseBackend, + args: ResetProverTaskArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + let task_db = db.prover_task_db(); + + let mut record = task_db + .get_task(key.clone()) + .internal_error("Failed to read prover task record")? + .ok_or_else(|| { + DisplayedError::UserError( + "No prover task found for key".to_string(), + Box::new(args.key_hex.clone()), + ) + })?; + + if !args.force { + println!("would reset: {}", args.key_hex); + print_force_hint(); + return Ok(()); + } + + record.set_status(TaskStatus::Pending); + record.set_retry_after_secs(None); + task_db + .put_task(key, record) + .internal_error("Failed to persist reset task")?; + + println!("reset: {}", args.key_hex); + Ok(()) +} + +/// Hard-delete a task row. +pub(crate) fn delete_prover_task( + db: &impl DatabaseBackend, + args: DeleteProverTaskArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + let task_db = db.prover_task_db(); + + // Resolve existence up front so the dry run can surface a clear + // error rather than silently "previewing" a no-op delete. + let exists = task_db + .get_task(key.clone()) + .internal_error("Failed to read prover task record")? + .is_some(); + if !exists { + return Err(DisplayedError::UserError( + "No prover task found for key".to_string(), + Box::new(args.key_hex), + )); + } + + if !args.force { + println!("would delete: {}", args.key_hex); + print_force_hint(); + return Ok(()); + } + + task_db + .delete_task(key) + .internal_error("Failed to delete prover task")?; + + println!("deleted: {}", args.key_hex); + Ok(()) +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "backfill-checkpoint-proof-task")] +/// Queue a fresh `Pending` checkpoint-proof task for an epoch. +/// +/// Resolves the canonical commitment at the epoch and constructs the +/// task key via [`CheckpointProofTask`] so the running node picks it up +/// on next startup recovery. Dry-run unless `--force` is passed. +pub(crate) struct BackfillCheckpointProofTaskArgs { + /// checkpoint epoch + #[argh(positional)] + pub(crate) epoch: Epoch, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +#[derive(FromArgs, PartialEq, Debug)] +#[argh(subcommand, name = "backfill-prover-task-raw")] +/// Insert a `Pending` task record under a raw hex-encoded key. +/// +/// Escape hatch for proof kinds without a typed helper. Caller is +/// responsible for matching the key format the host expects. +/// Dry-run unless `--force` is passed. +pub(crate) struct BackfillProverTaskRawArgs { + /// hex-encoded task key + #[argh(positional)] + pub(crate) key_hex: String, + + /// force execution (without this flag, only a dry run is performed) + #[argh(switch, short = 'f')] + pub(crate) force: bool, +} + +/// Insert a Pending checkpoint-proof task for the canonical commitment at +/// the given epoch. +pub(crate) fn backfill_checkpoint_proof_task( + db: &impl DatabaseBackend, + args: BackfillCheckpointProofTaskArgs, +) -> Result<(), DisplayedError> { + let commitment = get_canonical_epoch_commitment_at(db, args.epoch)?.ok_or_else(|| { + DisplayedError::UserError( + "No canonical checkpoint commitment at epoch".to_string(), + Box::new(args.epoch), + ) + })?; + let task = CheckpointProofTask(commitment); + let key = task.to_key_bytes(); + let key_hex = hex::encode(&key); + + if !args.force { + println!( + "would backfill checkpoint proof task: {key_hex} (epoch {})", + args.epoch + ); + print_force_hint(); + return Ok(()); + } + + let record = TaskRecordData::new(TaskStatus::Pending); + db.prover_task_db() + .insert_task(key, record) + .internal_error("Failed to insert prover task")?; + + println!( + "backfilled checkpoint proof task: {key_hex} (epoch {})", + args.epoch + ); + Ok(()) +} + +/// Insert a Pending task under a caller-provided raw key. +pub(crate) fn backfill_prover_task_raw( + db: &impl DatabaseBackend, + args: BackfillProverTaskRawArgs, +) -> Result<(), DisplayedError> { + let key = parse_task_key(&args.key_hex)?; + + if !args.force { + println!("would backfill prover task: {}", args.key_hex); + print_force_hint(); + return Ok(()); + } + + let record = TaskRecordData::new(TaskStatus::Pending); + db.prover_task_db() + .insert_task(key, record) + .internal_error("Failed to insert prover task")?; + + println!("backfilled prover task: {}", args.key_hex); + Ok(()) +} diff --git a/bin/strata-dbtool/src/cmd/prover_task_common.rs b/bin/strata-dbtool/src/cmd/prover_task_common.rs new file mode 100644 index 0000000000..262488f4fe --- /dev/null +++ b/bin/strata-dbtool/src/cmd/prover_task_common.rs @@ -0,0 +1,189 @@ +//! Shared helpers for prover-task admin commands. +//! +//! Both the OL ([`super::prover_task`]) and EE ([`super::ee_prover_task`]) +//! task-store surfaces accept the same status filters, hex-key encoding, +//! `--confirm` guard, and abandon-reason string. Keeping these in one +//! place ensures the two surfaces stay in lockstep — operator workflows +//! migrate between them with no surprises. + +use std::{fmt, str::FromStr}; + +use strata_cli_common::errors::DisplayedError; +use strata_paas::TaskStatus; + +/// Status filter accepted by the summary commands. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StatusFilter { + All, + Pending, + Proving, + Completed, + TransientFailure, + PermanentFailure, + /// Pending or Proving — what the prover's startup recovery would respawn. + Unfinished, + /// Completed or PermanentFailure — won't be retried again. + Terminal, +} + +impl StatusFilter { + pub(crate) fn matches(&self, status: &TaskStatus) -> bool { + match self { + Self::All => true, + Self::Pending => matches!(status, TaskStatus::Pending), + Self::Proving => matches!(status, TaskStatus::Proving { .. }), + Self::Completed => matches!(status, TaskStatus::Completed), + Self::TransientFailure => matches!(status, TaskStatus::TransientFailure { .. }), + Self::PermanentFailure => matches!(status, TaskStatus::PermanentFailure { .. }), + Self::Unfinished => status.is_unfinished(), + Self::Terminal => status.is_terminal(), + } + } +} + +#[derive(Debug)] +pub(crate) struct UnsupportedStatusFilter; + +impl fmt::Display for UnsupportedStatusFilter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "must be one of: all, pending, proving, completed, transient-failure, \ + permanent-failure, unfinished, terminal" + ) + } +} + +impl FromStr for StatusFilter { + type Err = UnsupportedStatusFilter; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "all" => Ok(Self::All), + "pending" => Ok(Self::Pending), + "proving" => Ok(Self::Proving), + "completed" => Ok(Self::Completed), + "transient-failure" | "transient_failure" => Ok(Self::TransientFailure), + "permanent-failure" | "permanent_failure" => Ok(Self::PermanentFailure), + "unfinished" => Ok(Self::Unfinished), + "terminal" => Ok(Self::Terminal), + _ => Err(UnsupportedStatusFilter), + } + } +} + +/// Error string written into `PermanentFailure` by the abandon commands. +/// +/// Operator workflows grep on this exact phrase, so it must stay stable +/// across the OL and EE surfaces. +pub(crate) const ABANDONED_REASON: &str = "abandoned via dbtool"; + +/// Parse a hex string into a task key, normalizing a `0x` prefix. +pub(crate) fn parse_task_key(hex_str: &str) -> Result, DisplayedError> { + let trimmed = hex_str.strip_prefix("0x").unwrap_or(hex_str); + hex::decode(trimmed).map_err(|e| { + DisplayedError::UserError("Invalid hex-encoded task key".to_string(), Box::new(e)) + }) +} + +/// Standard tail line printed at the end of a dry-run, matching the +/// phrasing used by `revert-ol-state`. +pub(crate) fn print_force_hint() { + println!(); + println!("Use --force to execute these changes."); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_filter_matches_each_variant() { + let pending = TaskStatus::Pending; + let proving = TaskStatus::Proving { retry_count: 2 }; + let completed = TaskStatus::Completed; + let transient = TaskStatus::TransientFailure { + retry_count: 1, + error: "x".into(), + }; + let permanent = TaskStatus::PermanentFailure { error: "y".into() }; + + assert!(StatusFilter::All.matches(&pending)); + assert!(StatusFilter::All.matches(&permanent)); + + assert!(StatusFilter::Pending.matches(&pending)); + assert!(!StatusFilter::Pending.matches(&proving)); + + assert!(StatusFilter::Proving.matches(&proving)); + assert!(!StatusFilter::Proving.matches(&pending)); + + assert!(StatusFilter::Completed.matches(&completed)); + assert!(!StatusFilter::Completed.matches(&pending)); + + assert!(StatusFilter::TransientFailure.matches(&transient)); + assert!(!StatusFilter::TransientFailure.matches(&permanent)); + + assert!(StatusFilter::PermanentFailure.matches(&permanent)); + assert!(!StatusFilter::PermanentFailure.matches(&transient)); + + assert!(StatusFilter::Unfinished.matches(&pending)); + assert!(StatusFilter::Unfinished.matches(&proving)); + assert!(!StatusFilter::Unfinished.matches(&completed)); + assert!(!StatusFilter::Unfinished.matches(&transient)); + + assert!(StatusFilter::Terminal.matches(&completed)); + assert!(StatusFilter::Terminal.matches(&permanent)); + assert!(!StatusFilter::Terminal.matches(&pending)); + assert!(!StatusFilter::Terminal.matches(&transient)); + } + + #[test] + fn status_filter_from_str_accepts_canonical_and_aliases() { + assert_eq!("all".parse::().unwrap(), StatusFilter::All); + assert_eq!( + "PENDING".parse::().unwrap(), + StatusFilter::Pending + ); + assert_eq!( + "transient-failure".parse::().unwrap(), + StatusFilter::TransientFailure + ); + assert_eq!( + "transient_failure".parse::().unwrap(), + StatusFilter::TransientFailure + ); + assert_eq!( + "permanent-failure".parse::().unwrap(), + StatusFilter::PermanentFailure + ); + assert_eq!( + "unfinished".parse::().unwrap(), + StatusFilter::Unfinished + ); + assert_eq!( + "terminal".parse::().unwrap(), + StatusFilter::Terminal + ); + + assert!("bogus".parse::().is_err()); + } + + #[test] + fn parse_task_key_accepts_hex_with_and_without_prefix() { + assert_eq!( + parse_task_key("deadbeef").unwrap(), + vec![0xde, 0xad, 0xbe, 0xef] + ); + assert_eq!( + parse_task_key("0xdeadbeef").unwrap(), + vec![0xde, 0xad, 0xbe, 0xef] + ); + assert_eq!(parse_task_key("").unwrap(), Vec::::new()); + } + + #[test] + fn parse_task_key_rejects_invalid_hex() { + assert!(parse_task_key("not-hex").is_err()); + assert!(parse_task_key("abc").is_err()); + } +} diff --git a/bin/strata-dbtool/src/db.rs b/bin/strata-dbtool/src/db.rs index c2954b5998..abee437d3b 100644 --- a/bin/strata-dbtool/src/db.rs +++ b/bin/strata-dbtool/src/db.rs @@ -1,7 +1,9 @@ use std::{path::Path, sync::Arc}; +use alpen_ee_database::EeProverDbSled; use strata_cli_common::errors::{DisplayableError, DisplayedError}; use strata_db_store_sled::{open_sled_database, SledBackend, SledDbConfig, SLED_NAME}; +use typed_sled::SledDb; /// Returns a boxed trait-object that satisfies all the low-level traits. pub(crate) fn open_database(path: &Path) -> Result, DisplayedError> { @@ -15,3 +17,30 @@ pub(crate) fn open_database(path: &Path) -> Result, DisplayedEr Ok(backend) } + +/// Opens the EE prover sled store at `/sled`. +/// +/// Mirrors the alpen-client's [`alpen_ee_database::init_db_storage`] +/// opener but only constructs the prover-task / chunk-receipt / acct-proof +/// trees — the dbtool has no use for the other EE DBs (witness, broadcast, +/// chunked-envelope, DA context), and skipping them keeps the cold-start +/// surface smaller. +pub(crate) fn open_ee_database(ee_datadir: &Path) -> Result, DisplayedError> { + let database_dir = ee_datadir.join("sled"); + let sled_db = sled::open(&database_dir).map_err(|e| { + DisplayedError::UserError( + format!("Failed to open EE sled database at {database_dir:?}"), + Box::new(e), + ) + })?; + + let typed_sled = + Arc::new(SledDb::new(sled_db).internal_error("Could not initialize typed-sled wrapper")?); + + let config = SledDbConfig::new_with_constant_backoff(5, 200); + let prover_db = EeProverDbSled::new(typed_sled, config) + .internal_error("Could not open EE prover db") + .map(Arc::new)?; + + Ok(prover_db) +} diff --git a/bin/strata-dbtool/src/main.rs b/bin/strata-dbtool/src/main.rs index c5aadb8c5f..ca5c8229a8 100644 --- a/bin/strata-dbtool/src/main.rs +++ b/bin/strata-dbtool/src/main.rs @@ -7,8 +7,10 @@ mod db; mod output; mod utils; -use std::process::exit; +use std::{path::Path, process::exit}; +use alpen_ee_database::EeProverDbSled; +use strata_cli_common::errors::DisplayedError; use strata_db_types::traits::DatabaseBackend; use tracing_subscriber::fmt::init; @@ -17,14 +19,28 @@ use crate::{ cmd::{ broadcaster::{get_broadcaster_summary, get_broadcaster_tx}, checkpoint::{get_checkpoint, get_checkpoints_summary, get_epoch_summary}, + checkpoint_proof::{delete_checkpoint_proof, get_checkpoint_proof}, client_state::get_client_state_update, + ee_prover_task::{ + ee_abandon_prover_task, ee_abandon_prover_tasks, ee_backfill_prover_task_raw, + ee_delete_prover_task, ee_get_prover_task, ee_get_prover_tasks_summary, + ee_reset_prover_task, + }, + ee_receipts::{ + ee_delete_acct_proof, ee_delete_chunk_receipt, ee_get_acct_proof, ee_get_chunk_receipt, + }, l1::{get_l1_block, get_l1_summary}, ol::{get_ol_block, get_ol_summary}, ol_state::{get_ol_state, revert_ol_state}, + prover_task::{ + abandon_prover_task, abandon_prover_tasks, backfill_checkpoint_proof_task, + backfill_prover_task_raw, delete_prover_task, get_prover_task, + get_prover_tasks_summary, reset_prover_task, + }, syncinfo::get_syncinfo, writer::{get_writer_payload, get_writer_summary}, }, - db::open_database, + db::{open_database, open_ee_database}, }; fn main() { @@ -38,6 +54,13 @@ fn main() { }); let db = db.as_ref(); + // The EE DB is only opened when an `ee-*` command actually runs — + // OL-only invocations must not require `--ee-datadir` to be set, and + // sled itself takes an exclusive lock on the directory, so opening + // eagerly would block parallel dbtool invocations on the same OL + // datadir. + let ee_datadir = cli.ee_datadir.as_deref(); + let result = match cli.cmd { Command::GetOLState(args) => get_ol_state(db, args), Command::RevertOLState(args) => revert_ol_state(db, args), @@ -54,6 +77,45 @@ fn main() { Command::GetEpochSummary(args) => get_epoch_summary(db, args), Command::GetSyncinfo(args) => get_syncinfo(db, args), Command::GetClientStateUpdate(args) => get_client_state_update(db, args), + Command::GetProverTask(args) => get_prover_task(db, args), + Command::GetProverTasksSummary(args) => get_prover_tasks_summary(db, args), + Command::AbandonProverTask(args) => abandon_prover_task(db, args), + Command::AbandonProverTasks(args) => abandon_prover_tasks(db, args), + Command::ResetProverTask(args) => reset_prover_task(db, args), + Command::DeleteProverTask(args) => delete_prover_task(db, args), + Command::GetCheckpointProof(args) => get_checkpoint_proof(db, args), + Command::DeleteCheckpointProof(args) => delete_checkpoint_proof(db, args), + Command::BackfillCheckpointProofTask(args) => backfill_checkpoint_proof_task(db, args), + Command::BackfillProverTaskRaw(args) => backfill_prover_task_raw(db, args), + Command::EeGetProverTask(args) => with_ee_db(ee_datadir, |db| ee_get_prover_task(db, args)), + Command::EeGetProverTasksSummary(args) => { + with_ee_db(ee_datadir, |db| ee_get_prover_tasks_summary(db, args)) + } + Command::EeAbandonProverTask(args) => { + with_ee_db(ee_datadir, |db| ee_abandon_prover_task(db, args)) + } + Command::EeAbandonProverTasks(args) => { + with_ee_db(ee_datadir, |db| ee_abandon_prover_tasks(db, args)) + } + Command::EeResetProverTask(args) => { + with_ee_db(ee_datadir, |db| ee_reset_prover_task(db, args)) + } + Command::EeDeleteProverTask(args) => { + with_ee_db(ee_datadir, |db| ee_delete_prover_task(db, args)) + } + Command::EeBackfillProverTaskRaw(args) => { + with_ee_db(ee_datadir, |db| ee_backfill_prover_task_raw(db, args)) + } + Command::EeGetChunkReceipt(args) => { + with_ee_db(ee_datadir, |db| ee_get_chunk_receipt(db, args)) + } + Command::EeDeleteChunkReceipt(args) => { + with_ee_db(ee_datadir, |db| ee_delete_chunk_receipt(db, args)) + } + Command::EeGetAcctProof(args) => with_ee_db(ee_datadir, |db| ee_get_acct_proof(db, args)), + Command::EeDeleteAcctProof(args) => { + with_ee_db(ee_datadir, |db| ee_delete_acct_proof(db, args)) + } }; if let Err(e) = result { @@ -61,3 +123,21 @@ fn main() { exit(1); } } + +/// Opens the EE prover db lazily and runs `f` against it. +/// +/// Returns a user-facing error if `--ee-datadir` was not supplied, so +/// the operator sees the missing flag rather than a sled open failure. +fn with_ee_db(ee_datadir: Option<&Path>, f: F) -> Result<(), DisplayedError> +where + F: FnOnce(&EeProverDbSled) -> Result<(), DisplayedError>, +{ + let ee_datadir = ee_datadir.ok_or_else(|| { + DisplayedError::UserError( + "--ee-datadir is required for ee-* subcommands".to_string(), + Box::new(()), + ) + })?; + let ee_db = open_ee_database(ee_datadir)?; + f(ee_db.as_ref()) +} diff --git a/bin/strata-dbtool/src/output/checkpoint_proof.rs b/bin/strata-dbtool/src/output/checkpoint_proof.rs new file mode 100644 index 0000000000..d08803aa6a --- /dev/null +++ b/bin/strata-dbtool/src/output/checkpoint_proof.rs @@ -0,0 +1,127 @@ +//! Output structs for the checkpoint-proof admin commands. + +use serde::Serialize; +use strata_identifiers::{Epoch, OLBlockId}; +use zkaleido::ProofReceiptWithMetadata; + +use super::{helpers::porcelain_field, traits::Formattable}; + +/// Per-receipt detail emitted by `get-checkpoint-proof`. +#[derive(Serialize)] +pub(crate) struct CheckpointProofInfo { + pub(crate) epoch: Epoch, + pub(crate) terminal_blkid: OLBlockId, + pub(crate) zkvm: String, + pub(crate) proof_type: String, + pub(crate) program_id_hex: String, + pub(crate) program_version: String, + pub(crate) proof_len: usize, + pub(crate) public_values_len: usize, +} + +impl CheckpointProofInfo { + pub(crate) fn from_receipt( + epoch: Epoch, + terminal_blkid: OLBlockId, + receipt: &ProofReceiptWithMetadata, + ) -> Self { + let metadata = receipt.metadata(); + Self { + epoch, + terminal_blkid, + zkvm: format!("{:?}", metadata.zkvm()), + proof_type: format!("{:?}", metadata.proof_type()), + program_id_hex: hex::encode(metadata.program_id().0), + program_version: metadata.version().to_string(), + proof_len: receipt.receipt().proof().as_bytes().len(), + public_values_len: receipt.receipt().public_values().as_bytes().len(), + } + } +} + +impl Formattable for CheckpointProofInfo { + fn format_porcelain(&self) -> String { + let mut out = Vec::new(); + out.push(porcelain_field("epoch", self.epoch)); + out.push(porcelain_field( + "terminal_blkid", + format!("{:?}", self.terminal_blkid), + )); + out.push(porcelain_field("zkvm", &self.zkvm)); + out.push(porcelain_field("proof_type", &self.proof_type)); + out.push(porcelain_field("program_id_hex", &self.program_id_hex)); + out.push(porcelain_field("program_version", &self.program_version)); + out.push(porcelain_field("proof_len", self.proof_len)); + out.push(porcelain_field("public_values_len", self.public_values_len)); + out.join("\n") + } +} + +/// Acknowledgement payload for `delete-checkpoint-proof`. +#[derive(Serialize)] +pub(crate) struct DeletedCheckpointProofInfo { + pub(crate) epoch: Epoch, + pub(crate) terminal_blkid: OLBlockId, + pub(crate) existed: bool, +} + +impl Formattable for DeletedCheckpointProofInfo { + fn format_porcelain(&self) -> String { + [ + porcelain_field("epoch", self.epoch), + porcelain_field("terminal_blkid", format!("{:?}", self.terminal_blkid)), + porcelain_field("existed", self.existed), + ] + .join("\n") + } +} + +#[cfg(test)] +mod tests { + use zkaleido::{ + ProgramId, Proof, ProofMetadata, ProofReceipt, ProofReceiptWithMetadata, ProofType, + PublicValues, ZkVm, + }; + + use super::*; + + fn fake_receipt() -> ProofReceiptWithMetadata { + let metadata = ProofMetadata::new( + ZkVm::Native, + ProgramId([7u8; 32]), + "0.1".to_string(), + ProofType::Groth16, + ); + let receipt = ProofReceipt::new(Proof::new(vec![1, 2, 3]), PublicValues::new(vec![9, 8])); + ProofReceiptWithMetadata::new(receipt, metadata) + } + + #[test] + fn checkpoint_proof_info_records_metadata_and_lengths() { + let receipt = fake_receipt(); + let info = CheckpointProofInfo::from_receipt(7u32, OLBlockId::default(), &receipt); + + assert_eq!(info.epoch, 7); + assert_eq!(info.zkvm, "Native"); + assert_eq!(info.proof_type, "Groth16"); + assert_eq!( + info.program_id_hex, + "0707070707070707070707070707070707070707070707070707070707070707" + ); + assert_eq!(info.program_version, "0.1"); + assert_eq!(info.proof_len, 3); + assert_eq!(info.public_values_len, 2); + } + + #[test] + fn deleted_info_porcelain_is_stable() { + let ack = DeletedCheckpointProofInfo { + epoch: 12u32, + terminal_blkid: OLBlockId::default(), + existed: true, + }; + let out = ack.format_porcelain(); + assert!(out.contains("epoch: 12")); + assert!(out.contains("existed: true")); + } +} diff --git a/bin/strata-dbtool/src/output/ee_receipts.rs b/bin/strata-dbtool/src/output/ee_receipts.rs new file mode 100644 index 0000000000..63ac406f04 --- /dev/null +++ b/bin/strata-dbtool/src/output/ee_receipts.rs @@ -0,0 +1,133 @@ +//! Output structs for the EE receipt admin commands. +//! +//! Chunk receipts and acct proofs share the same `ProofReceiptWithMetadata` +//! shape, so the inspection output is identical to OL's +//! `get-checkpoint-proof`. Only the addressing differs: chunk receipts +//! are keyed by an opaque task key (the chunk prover's task encoding), +//! and acct proofs are keyed by [`alpen_ee_common::BatchId`]. + +use serde::Serialize; +use zkaleido::ProofReceiptWithMetadata; + +use super::{helpers::porcelain_field, traits::Formattable}; + +/// Per-receipt detail for the EE inspection commands. Mirrors the OL +/// `CheckpointProofInfo` shape but carries the EE-side identifier +/// (`address`) rather than an epoch + terminal blkid. +#[derive(Serialize)] +pub(crate) struct EeReceiptInfo { + /// Human-readable identifier — chunk task key (hex) or batch id + /// (`prev_block_hex:last_block_hex`), depending on the kind. + pub(crate) address: String, + pub(crate) kind: &'static str, + pub(crate) zkvm: String, + pub(crate) proof_type: String, + pub(crate) program_id_hex: String, + pub(crate) program_version: String, + pub(crate) proof_len: usize, + pub(crate) public_values_len: usize, +} + +impl EeReceiptInfo { + pub(crate) fn from_receipt( + address: String, + kind: &'static str, + receipt: &ProofReceiptWithMetadata, + ) -> Self { + let metadata = receipt.metadata(); + Self { + address, + kind, + zkvm: format!("{:?}", metadata.zkvm()), + proof_type: format!("{:?}", metadata.proof_type()), + program_id_hex: hex::encode(metadata.program_id().0), + program_version: metadata.version().to_string(), + proof_len: receipt.receipt().proof().as_bytes().len(), + public_values_len: receipt.receipt().public_values().as_bytes().len(), + } + } +} + +impl Formattable for EeReceiptInfo { + fn format_porcelain(&self) -> String { + [ + porcelain_field("address", &self.address), + porcelain_field("kind", self.kind), + porcelain_field("zkvm", &self.zkvm), + porcelain_field("proof_type", &self.proof_type), + porcelain_field("program_id_hex", &self.program_id_hex), + porcelain_field("program_version", &self.program_version), + porcelain_field("proof_len", self.proof_len), + porcelain_field("public_values_len", self.public_values_len), + ] + .join("\n") + } +} + +/// Acknowledgement payload for the `ee-delete-*-receipt` / `ee-delete-acct-proof` +/// commands. +#[derive(Serialize)] +pub(crate) struct DeletedEeReceiptInfo { + pub(crate) address: String, + pub(crate) kind: &'static str, + pub(crate) existed: bool, +} + +impl Formattable for DeletedEeReceiptInfo { + fn format_porcelain(&self) -> String { + [ + porcelain_field("address", &self.address), + porcelain_field("kind", self.kind), + porcelain_field("existed", self.existed), + ] + .join("\n") + } +} + +#[cfg(test)] +mod tests { + use zkaleido::{ProgramId, Proof, ProofMetadata, ProofReceipt, ProofType, PublicValues, ZkVm}; + + use super::*; + + fn fake_receipt() -> ProofReceiptWithMetadata { + let metadata = ProofMetadata::new( + ZkVm::Native, + ProgramId([3u8; 32]), + "0.2".to_string(), + ProofType::Groth16, + ); + let receipt = ProofReceipt::new(Proof::new(vec![1, 2]), PublicValues::new(vec![7])); + ProofReceiptWithMetadata::new(receipt, metadata) + } + + #[test] + fn ee_receipt_info_records_kind_and_metadata() { + let receipt = fake_receipt(); + let info = EeReceiptInfo::from_receipt("deadbeef".into(), "chunk", &receipt); + + assert_eq!(info.address, "deadbeef"); + assert_eq!(info.kind, "chunk"); + assert_eq!(info.zkvm, "Native"); + assert_eq!(info.proof_type, "Groth16"); + assert_eq!(info.proof_len, 2); + assert_eq!(info.public_values_len, 1); + + let out = info.format_porcelain(); + assert!(out.contains("address: deadbeef")); + assert!(out.contains("kind: chunk")); + } + + #[test] + fn deleted_info_porcelain_is_stable() { + let ack = DeletedEeReceiptInfo { + address: "abcd".into(), + kind: "acct", + existed: false, + }; + let out = ack.format_porcelain(); + assert!(out.contains("address: abcd")); + assert!(out.contains("kind: acct")); + assert!(out.contains("existed: false")); + } +} diff --git a/bin/strata-dbtool/src/output/mod.rs b/bin/strata-dbtool/src/output/mod.rs index 7754ac211a..b0f7ace4a1 100644 --- a/bin/strata-dbtool/src/output/mod.rs +++ b/bin/strata-dbtool/src/output/mod.rs @@ -1,10 +1,13 @@ pub(crate) mod broadcaster; pub(crate) mod checkpoint; +pub(crate) mod checkpoint_proof; pub(crate) mod client_state; +pub(crate) mod ee_receipts; pub(crate) mod helpers; pub(crate) mod l1; pub(crate) mod ol; pub(crate) mod ol_state; +pub(crate) mod prover_task; pub(crate) mod syncinfo; pub(crate) mod traits; pub(crate) mod writer; diff --git a/bin/strata-dbtool/src/output/prover_task.rs b/bin/strata-dbtool/src/output/prover_task.rs new file mode 100644 index 0000000000..b468fbf568 --- /dev/null +++ b/bin/strata-dbtool/src/output/prover_task.rs @@ -0,0 +1,288 @@ +//! Prover task formatting implementations. + +use serde::Serialize; +use strata_paas::{TaskRecordData, TaskStatus}; + +use super::{helpers::porcelain_field, traits::Formattable}; + +/// Compact status descriptor used for both JSON and porcelain output. +/// +/// Borrows the `TaskStatus` variant name plus the human-relevant fields +/// (retry count, error string) into a single shape so the consumer +/// doesn't need to switch on the enum. +#[derive(Serialize)] +pub(crate) struct StatusInfo { + pub(crate) name: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) retry_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) error: Option, +} + +impl From<&TaskStatus> for StatusInfo { + fn from(status: &TaskStatus) -> Self { + match status { + TaskStatus::Pending => Self { + name: "pending", + retry_count: None, + error: None, + }, + TaskStatus::Proving { retry_count } => Self { + name: "proving", + retry_count: Some(*retry_count), + error: None, + }, + TaskStatus::Completed => Self { + name: "completed", + retry_count: None, + error: None, + }, + TaskStatus::TransientFailure { retry_count, error } => Self { + name: "transient_failure", + retry_count: Some(*retry_count), + error: Some(error.clone()), + }, + TaskStatus::PermanentFailure { error } => Self { + name: "permanent_failure", + retry_count: None, + error: Some(error.clone()), + }, + } + } +} + +/// EE task-key prefix carried by the alpen-client's shared chunk+acct +/// task tree. Documented in `bin/alpen-client/src/prover/storage.rs`. +const CHUNK_TASK_TAG: u8 = b'c'; +const ACCT_TASK_TAG: u8 = b'a'; + +/// Classifies an EE task key by its kind tag. +/// +/// Returns `None` for OL keys (which don't carry a tag — they're borsh +/// commitments) and for EE raw-backfilled keys that don't follow the +/// chunk/acct convention. +pub(crate) fn ee_kind_for_key(key: &[u8]) -> Option<&'static str> { + match key.first().copied() { + Some(CHUNK_TASK_TAG) => Some("chunk"), + Some(ACCT_TASK_TAG) => Some("acct"), + _ => Some("unknown"), + } +} + +/// Per-task detail emitted by `get-prover-task` and as an element of the +/// summary command's `entries` list. +#[derive(Serialize)] +pub(crate) struct ProverTaskInfo { + pub(crate) key_hex: String, + /// Set only for EE entries — distinguishes chunk vs acct vs unknown. + /// Omitted from OL output to keep the existing JSON shape unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) kind: Option<&'static str>, + pub(crate) status: StatusInfo, + pub(crate) updated_at_secs: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) retry_after_secs: Option, + pub(crate) metadata_len: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) metadata_hex: Option, +} + +impl ProverTaskInfo { + pub(crate) fn from_record(key: &[u8], data: &TaskRecordData) -> Self { + let metadata = data.metadata(); + Self { + key_hex: hex::encode(key), + kind: None, + status: StatusInfo::from(data.status()), + updated_at_secs: data.updated_at_secs(), + retry_after_secs: data.retry_after_secs(), + metadata_len: metadata.map(|m| m.len()).unwrap_or(0), + metadata_hex: metadata.map(hex::encode), + } + } + + /// Same as [`Self::from_record`] but tags the entry with its EE kind + /// derived from the key's first byte. + pub(crate) fn from_ee_record(key: &[u8], data: &TaskRecordData) -> Self { + Self { + kind: ee_kind_for_key(key), + ..Self::from_record(key, data) + } + } +} + +impl Formattable for ProverTaskInfo { + fn format_porcelain(&self) -> String { + let mut out = Vec::new(); + out.push(porcelain_field("key_hex", &self.key_hex)); + if let Some(kind) = self.kind { + out.push(porcelain_field("kind", kind)); + } + out.push(porcelain_field("status", self.status.name)); + if let Some(rc) = self.status.retry_count { + out.push(porcelain_field("status.retry_count", rc)); + } + if let Some(err) = &self.status.error { + out.push(porcelain_field("status.error", err)); + } + out.push(porcelain_field("updated_at_secs", self.updated_at_secs)); + if let Some(when) = self.retry_after_secs { + out.push(porcelain_field("retry_after_secs", when)); + } + out.push(porcelain_field("metadata_len", self.metadata_len)); + if let Some(meta) = &self.metadata_hex { + out.push(porcelain_field("metadata_hex", meta)); + } + out.join("\n") + } +} + +/// Aggregate counts emitted by `get-prover-tasks-summary`, plus a bounded +/// slice of matching entries for inspection. +#[derive(Serialize)] +pub(crate) struct ProverTasksSummaryInfo { + pub(crate) total: usize, + pub(crate) pending: usize, + pub(crate) proving: usize, + pub(crate) completed: usize, + pub(crate) transient_failure: usize, + pub(crate) permanent_failure: usize, + pub(crate) matched: usize, + pub(crate) returned: usize, + pub(crate) entries: Vec, +} + +impl Formattable for ProverTasksSummaryInfo { + fn format_porcelain(&self) -> String { + let mut out = vec![ + porcelain_field("total", self.total), + porcelain_field("pending", self.pending), + porcelain_field("proving", self.proving), + porcelain_field("completed", self.completed), + porcelain_field("transient_failure", self.transient_failure), + porcelain_field("permanent_failure", self.permanent_failure), + porcelain_field("matched", self.matched), + porcelain_field("returned", self.returned), + ]; + for (i, entry) in self.entries.iter().enumerate() { + out.push(porcelain_field( + &format!("entries[{i}].key_hex"), + &entry.key_hex, + )); + if let Some(kind) = entry.kind { + out.push(porcelain_field(&format!("entries[{i}].kind"), kind)); + } + out.push(porcelain_field( + &format!("entries[{i}].status"), + entry.status.name, + )); + if let Some(rc) = entry.status.retry_count { + out.push(porcelain_field( + &format!("entries[{i}].status.retry_count"), + rc, + )); + } + out.push(porcelain_field( + &format!("entries[{i}].updated_at_secs"), + entry.updated_at_secs, + )); + } + out.join("\n") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_info_carries_retry_count_and_error_where_relevant() { + let pending = StatusInfo::from(&TaskStatus::Pending); + assert_eq!(pending.name, "pending"); + assert_eq!(pending.retry_count, None); + assert_eq!(pending.error, None); + + let proving = StatusInfo::from(&TaskStatus::Proving { retry_count: 3 }); + assert_eq!(proving.name, "proving"); + assert_eq!(proving.retry_count, Some(3)); + assert_eq!(proving.error, None); + + let completed = StatusInfo::from(&TaskStatus::Completed); + assert_eq!(completed.name, "completed"); + assert_eq!(completed.retry_count, None); + assert_eq!(completed.error, None); + + let transient = StatusInfo::from(&TaskStatus::TransientFailure { + retry_count: 2, + error: "rpc down".into(), + }); + assert_eq!(transient.name, "transient_failure"); + assert_eq!(transient.retry_count, Some(2)); + assert_eq!(transient.error.as_deref(), Some("rpc down")); + + let permanent = StatusInfo::from(&TaskStatus::PermanentFailure { + error: "abandoned via dbtool".into(), + }); + assert_eq!(permanent.name, "permanent_failure"); + assert_eq!(permanent.retry_count, None); + assert_eq!(permanent.error.as_deref(), Some("abandoned via dbtool")); + } + + #[test] + fn prover_task_info_encodes_key_and_metadata_preview() { + let key = vec![0xde, 0xad, 0xbe, 0xef]; + let mut record = TaskRecordData::new(TaskStatus::Pending); + record.set_metadata(Some(vec![1, 2, 3])); + + let info = ProverTaskInfo::from_record(&key, &record); + assert_eq!(info.key_hex, "deadbeef"); + assert_eq!(info.status.name, "pending"); + assert_eq!(info.metadata_len, 3); + assert_eq!(info.metadata_hex.as_deref(), Some("010203")); + assert_eq!(info.retry_after_secs, None); + } + + #[test] + fn prover_task_info_omits_metadata_when_absent() { + let key = vec![0xaa]; + let record = TaskRecordData::new(TaskStatus::Completed); + + let info = ProverTaskInfo::from_record(&key, &record); + assert_eq!(info.metadata_len, 0); + assert_eq!(info.metadata_hex, None); + } + + #[test] + fn porcelain_output_includes_known_keys() { + let key = vec![0xaa, 0xbb]; + let record = TaskRecordData::new(TaskStatus::Proving { retry_count: 1 }); + let info = ProverTaskInfo::from_record(&key, &record); + + let out = info.format_porcelain(); + assert!(out.contains("key_hex: aabb")); + assert!(out.contains("status: proving")); + assert!(out.contains("status.retry_count: 1")); + // OL entries leave `kind` unset; porcelain must not surface it. + assert!(!out.contains("kind:")); + } + + #[test] + fn ee_kind_for_key_recognizes_chunk_and_acct_tags() { + assert_eq!(ee_kind_for_key(b"c-something"), Some("chunk")); + assert_eq!(ee_kind_for_key(b"a-something"), Some("acct")); + assert_eq!(ee_kind_for_key(b"zzz"), Some("unknown")); + assert_eq!(ee_kind_for_key(&[]), Some("unknown")); + } + + #[test] + fn from_ee_record_tags_kind_and_surfaces_it_in_porcelain() { + let record = TaskRecordData::new(TaskStatus::Pending); + let chunk = ProverTaskInfo::from_ee_record(b"c-xyz", &record); + assert_eq!(chunk.kind, Some("chunk")); + assert!(chunk.format_porcelain().contains("kind: chunk")); + + let acct = ProverTaskInfo::from_ee_record(b"a-xyz", &record); + assert_eq!(acct.kind, Some("acct")); + assert!(acct.format_porcelain().contains("kind: acct")); + } +} diff --git a/bin/strata/Cargo.toml b/bin/strata/Cargo.toml index 8b95cccdfc..e696e7cf59 100644 --- a/bin/strata/Cargo.toml +++ b/bin/strata/Cargo.toml @@ -23,7 +23,6 @@ debug-utils = [ "strata-ol-sequencer?/debug-utils", ] prover = [ - "dep:borsh", "dep:strata-ol-state-support-types", "dep:strata-ol-stf", "dep:strata-paas", @@ -94,7 +93,6 @@ argh.workspace = true async-trait.workspace = true bitcoin.workspace = true bitcoind-async-client.workspace = true -borsh = { workspace = true, optional = true } format_serde_error.workspace = true http.workspace = true jsonrpsee = { workspace = true, features = ["server", "macros"] } diff --git a/bin/strata/src/prover/spec.rs b/bin/strata/src/prover/spec.rs index c9ebfc2e1e..449b4b5a5a 100644 --- a/bin/strata/src/prover/spec.rs +++ b/bin/strata/src/prover/spec.rs @@ -4,10 +4,10 @@ //! as an [`EpochCommitment`] and fetches the proof input from local //! [`NodeStorage`] without any RPC round-trip. -use std::{fmt, sync::Arc}; +use std::sync::Arc; use async_trait::async_trait; -use borsh::{BorshDeserialize, BorshSerialize, io::Error as BorshIoError}; +pub(crate) use strata_checkpoint_types::CheckpointProofTask as CheckpointTask; use strata_identifiers::{Epoch, EpochCommitment}; use strata_ol_state_support_types::{DaAccumulatingState, MemoryStateBaseLayer}; use strata_ol_stf::execute_block_batch_preseal; @@ -19,34 +19,6 @@ use tracing::debug; use super::errors::ProverError; -/// Task identifier for checkpoint proofs. -/// -/// Newtype over [`EpochCommitment`] so we can attach the byte-encoding and -/// display bounds that [`strata_paas::TaskKey`] requires without polluting -/// the domain type. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, BorshSerialize, BorshDeserialize)] -pub(crate) struct CheckpointTask(pub EpochCommitment); - -impl fmt::Display for CheckpointTask { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl From for Vec { - fn from(task: CheckpointTask) -> Self { - borsh::to_vec(&task).expect("CheckpointTask borsh-serializable") - } -} - -impl TryFrom> for CheckpointTask { - type Error = BorshIoError; - - fn try_from(bytes: Vec) -> Result { - borsh::from_slice(&bytes) - } -} - /// Proof specification for integrated checkpoint proving. #[derive(Clone)] pub(crate) struct CheckpointSpec { diff --git a/crates/alpen-ee/database/src/sleddb/prover_db.rs b/crates/alpen-ee/database/src/sleddb/prover_db.rs index 1392d57e89..2b72733121 100644 --- a/crates/alpen-ee/database/src/sleddb/prover_db.rs +++ b/crates/alpen-ee/database/src/sleddb/prover_db.rs @@ -72,6 +72,14 @@ impl EeProverDbSled { Ok(self.chunk_receipt_tree.get(&key.to_vec())?) } + /// Removes a chunk receipt, returning `true` if a row existed. + /// + /// Admin-only path (offline dbtool). Callers must keep the node down + /// to avoid racing with chunk-prover writes. + pub fn delete_chunk_receipt(&self, key: &[u8]) -> DbResult { + Ok(self.chunk_receipt_tree.take(&key.to_vec())?.is_some()) + } + // ---- Acct proof store (typed BatchId API) ---- pub fn put_acct_proof( @@ -108,6 +116,21 @@ impl EeProverDbSled { let batch_id: BatchId = db_id.into(); self.get_acct_proof(batch_id) } + + /// Removes an acct proof along with its secondary index entry, + /// returning `true` if the proof row existed. + /// + /// Admin-only path (offline dbtool). The two trees are not deleted + /// in a single transaction — acceptable because callers stop the + /// node before invoking this, so no concurrent writer can observe + /// the intermediate state. + pub fn delete_acct_proof(&self, batch_id: BatchId) -> DbResult { + let db_id: DBBatchId = batch_id.into(); + let proof_id = proof_id_for(batch_id); + let existed = self.acct_proof_tree.take(&db_id)?.is_some(); + self.acct_proof_id_index_tree.remove(&proof_id)?; + Ok(existed) + } } impl ProverTaskDatabase for EeProverDbSled { @@ -131,6 +154,13 @@ impl ProverTaskDatabase for EeProverDbSled { Ok(()) } + fn delete_task(&self, key: Vec) -> DbResult { + let old = self.prover_task_tree.get(&key)?; + let existed = old.is_some(); + self.prover_task_tree.compare_and_swap(key, old, None)?; + Ok(existed) + } + fn list_retriable(&self, now_secs: u64) -> DbResult, TaskRecordData)>> { let mut out = Vec::new(); for item in self.prover_task_tree.iter() { @@ -155,6 +185,14 @@ impl ProverTaskDatabase for EeProverDbSled { Ok(out) } + fn list_all_tasks(&self) -> DbResult, TaskRecordData)>> { + let mut out = Vec::new(); + for item in self.prover_task_tree.iter() { + out.push(item?); + } + Ok(out) + } + fn count_tasks(&self) -> DbResult { let mut n = 0; for item in self.prover_task_tree.iter() { @@ -173,3 +211,76 @@ impl ProverTaskDatabase for EeProverDbSled { fn proof_id_for(batch_id: BatchId) -> ProofId { batch_id.last_block() } + +#[cfg(test)] +mod tests { + use strata_acct_types::Hash; + use zkaleido::{ProgramId, Proof, ProofMetadata, ProofReceipt, ProofType, PublicValues, ZkVm}; + + use super::*; + + fn setup_db() -> EeProverDbSled { + let sled_db = sled::Config::new().temporary(true).open().unwrap(); + let typed_sled = Arc::new(SledDb::new(sled_db).unwrap()); + let config = SledDbConfig::new_with_constant_backoff(2, 0); + EeProverDbSled::new(typed_sled, config).unwrap() + } + + fn dummy_receipt() -> ProofReceiptWithMetadata { + let receipt = ProofReceipt::new(Proof::default(), PublicValues::default()); + let metadata = ProofMetadata::new( + ZkVm::Native, + ProgramId([0u8; 32]), + "0.1".to_string(), + ProofType::Groth16, + ); + ProofReceiptWithMetadata::new(receipt, metadata) + } + + fn hash_from_u8(seed: u8) -> Hash { + let mut bytes = [0u8; 32]; + bytes[0] = 1; + bytes[31] = seed; + Hash::from(bytes) + } + + #[test] + fn delete_chunk_receipt_roundtrip() { + let db = setup_db(); + let key = b"chunk-key".to_vec(); + let receipt = dummy_receipt(); + + assert!(matches!(db.delete_chunk_receipt(&key), Ok(false))); + + db.put_chunk_receipt(key.clone(), receipt).unwrap(); + assert!(db.get_chunk_receipt(&key).unwrap().is_some()); + + assert!(matches!(db.delete_chunk_receipt(&key), Ok(true))); + assert!(matches!(db.delete_chunk_receipt(&key), Ok(false))); + assert!(db.get_chunk_receipt(&key).unwrap().is_none()); + } + + #[test] + fn delete_acct_proof_clears_primary_and_secondary_rows() { + let db = setup_db(); + let batch_id = BatchId::from_parts(hash_from_u8(1), hash_from_u8(2)); + let proof_id: ProofId = batch_id.last_block(); + + // Missing primary row reports false; secondary index already absent + // so the call is a clean no-op. + assert!(matches!(db.delete_acct_proof(batch_id), Ok(false))); + + db.put_acct_proof(batch_id, dummy_receipt()).unwrap(); + assert!(db.has_acct_proof(batch_id).unwrap()); + assert!(db.get_acct_proof_by_id(proof_id).unwrap().is_some()); + + assert!(matches!(db.delete_acct_proof(batch_id), Ok(true))); + assert!(!db.has_acct_proof(batch_id).unwrap()); + // Secondary index entry must be cleared so the by-id lookup also + // misses — otherwise the index would dangle. + assert!(db.get_acct_proof_by_id(proof_id).unwrap().is_none()); + + // Second delete is idempotent. + assert!(matches!(db.delete_acct_proof(batch_id), Ok(false))); + } +} diff --git a/crates/checkpoint-types/src/lib.rs b/crates/checkpoint-types/src/lib.rs index 06f647f01a..f5f3288cc7 100644 --- a/crates/checkpoint-types/src/lib.rs +++ b/crates/checkpoint-types/src/lib.rs @@ -2,6 +2,8 @@ mod batch; mod checkpoint; +mod prover_task; pub use batch::*; pub use checkpoint::*; +pub use prover_task::CheckpointProofTask; diff --git a/crates/checkpoint-types/src/prover_task.rs b/crates/checkpoint-types/src/prover_task.rs new file mode 100644 index 0000000000..f664fd084e --- /dev/null +++ b/crates/checkpoint-types/src/prover_task.rs @@ -0,0 +1,83 @@ +//! Task-key wrapper used by the integrated checkpoint prover. +//! +//! Lives in a shared crate so the running node (`bin/strata`) and offline +//! admin tooling (`bin/strata-dbtool`) agree on the on-disk byte format +//! for entries in the [`strata_db_types::traits::ProverTaskDatabase`]. +//! +//! Wire format is `borsh::to_vec(&CheckpointProofTask(commitment))`. +//! Because borsh serializes a tuple newtype as its inner field, the +//! resulting bytes are identical to `borsh::to_vec(&commitment)` — but +//! the explicit wrapper documents the contract and gives both sides a +//! single import point. + +use std::fmt; + +use borsh::{io::Error as BorshIoError, BorshDeserialize, BorshSerialize}; +use strata_identifiers::EpochCommitment; + +/// Task identifier for an integrated checkpoint proof. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, BorshSerialize, BorshDeserialize)] +pub struct CheckpointProofTask(pub EpochCommitment); + +impl CheckpointProofTask { + /// Returns the underlying epoch commitment. + pub fn commitment(&self) -> EpochCommitment { + self.0 + } + + /// Encodes the task into its stored byte form. + pub fn to_key_bytes(&self) -> Vec { + borsh::to_vec(self).expect("CheckpointProofTask borsh-serializable") + } +} + +impl fmt::Display for CheckpointProofTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for Vec { + fn from(task: CheckpointProofTask) -> Self { + task.to_key_bytes() + } +} + +impl TryFrom> for CheckpointProofTask { + type Error = BorshIoError; + + fn try_from(bytes: Vec) -> Result { + borsh::from_slice(&bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn key_bytes_roundtrip() { + let task = CheckpointProofTask(EpochCommitment::null()); + let bytes = task.to_key_bytes(); + let decoded = CheckpointProofTask::try_from(bytes).unwrap(); + assert_eq!(decoded, task); + } + + #[test] + fn key_bytes_match_borsh_of_inner_commitment() { + // The on-disk format is documented as `borsh::to_vec(&task)` — which, + // because borsh serializes a tuple newtype as its inner field, must + // equal `borsh::to_vec(&commitment)`. This invariant lets external + // tooling reconstruct the key without depending on this wrapper. + let commit = EpochCommitment::null(); + let task = CheckpointProofTask(commit); + assert_eq!(task.to_key_bytes(), borsh::to_vec(&commit).unwrap()); + } + + #[test] + fn commitment_accessor_returns_inner() { + let commit = EpochCommitment::null(); + let task = CheckpointProofTask(commit); + assert_eq!(task.commitment(), commit); + } +} diff --git a/crates/db/store-sled/src/prover/db.rs b/crates/db/store-sled/src/prover/db.rs index 33958c7c11..7e2c0a4c61 100644 --- a/crates/db/store-sled/src/prover/db.rs +++ b/crates/db/store-sled/src/prover/db.rs @@ -61,6 +61,13 @@ impl ProverTaskDatabase for ProofDBSled { Ok(()) } + fn delete_task(&self, key: Vec) -> DbResult { + let old = self.prover_task_tree.get(&key)?; + let existed = old.is_some(); + self.prover_task_tree.compare_and_swap(key, old, None)?; + Ok(existed) + } + fn list_retriable(&self, now_secs: u64) -> DbResult, TaskRecordData)>> { let mut out = Vec::new(); for item in self.prover_task_tree.iter() { @@ -85,6 +92,14 @@ impl ProverTaskDatabase for ProofDBSled { Ok(out) } + fn list_all_tasks(&self) -> DbResult, TaskRecordData)>> { + let mut out = Vec::new(); + for item in self.prover_task_tree.iter() { + out.push(item?); + } + Ok(out) + } + fn count_tasks(&self) -> DbResult { let mut n = 0; for item in self.prover_task_tree.iter() { diff --git a/crates/db/tests/Cargo.toml b/crates/db/tests/Cargo.toml index 72e0e2133d..675d4e3efe 100644 --- a/crates/db/tests/Cargo.toml +++ b/crates/db/tests/Cargo.toml @@ -21,6 +21,7 @@ strata-ol-chain-types.workspace = true strata-ol-chain-types-new = { workspace = true, features = ["test-utils"] } strata-ol-chainstate-types.workspace = true strata-ol-state-types = { workspace = true, features = ["test-utils"] } +strata-paas.workspace = true strata-primitives.workspace = true strata-state.workspace = true strata-test-utils.workspace = true diff --git a/crates/db/tests/src/proof_tests.rs b/crates/db/tests/src/proof_tests.rs index 6f0b78552f..b29d56483b 100644 --- a/crates/db/tests/src/proof_tests.rs +++ b/crates/db/tests/src/proof_tests.rs @@ -1,5 +1,6 @@ -use strata_db_types::traits::CheckpointProofDatabase; +use strata_db_types::traits::{CheckpointProofDatabase, ProverTaskDatabase}; use strata_identifiers::EpochCommitment; +use strata_paas::{TaskRecordData, TaskStatus}; use zkaleido::{ ProgramId, Proof, ProofMetadata, ProofReceipt, ProofReceiptWithMetadata, ProofType, PublicValues, ZkVm, @@ -41,6 +42,22 @@ pub fn test_get_nonexistent_proof(db: &impl CheckpointProofDatabase) { assert_eq!(stored_proof, None, "Nonexistent proof should return None"); } +pub fn test_delete_task_roundtrip(db: &impl ProverTaskDatabase) { + let key = b"task-key-1".to_vec(); + let record = TaskRecordData::new(TaskStatus::Pending); + + // Deleting a missing key reports false. + assert!(matches!(db.delete_task(key.clone()), Ok(false))); + + db.insert_task(key.clone(), record).unwrap(); + assert!(db.get_task(key.clone()).unwrap().is_some()); + + // First delete reports true; second reports false. + assert!(matches!(db.delete_task(key.clone()), Ok(true))); + assert!(matches!(db.delete_task(key.clone()), Ok(false))); + assert!(db.get_task(key).unwrap().is_none()); +} + // Helper functions fn generate_proof() -> (EpochCommitment, ProofReceiptWithMetadata) { let epoch = EpochCommitment::null(); @@ -77,5 +94,11 @@ macro_rules! proof_db_tests { let db = $setup_expr; $crate::proof_tests::test_get_nonexistent_proof(&db); } + + #[test] + fn test_delete_task_roundtrip() { + let db = $setup_expr; + $crate::proof_tests::test_delete_task_roundtrip(&db); + } }; } diff --git a/crates/db/types/src/traits.rs b/crates/db/types/src/traits.rs index 94f849d780..2cacd53565 100644 --- a/crates/db/types/src/traits.rs +++ b/crates/db/types/src/traits.rs @@ -489,12 +489,25 @@ pub trait ProverTaskDatabase: Send + Sync + 'static { /// Upsert a record — overwrites any existing entry under the key. fn put_task(&self, key: Vec, record: TaskRecordData) -> DbResult<()>; + /// Removes a task record. Returns `true` if the key existed prior to the + /// call, `false` otherwise. + /// + /// Intended for offline admin tooling (e.g. `strata-dbtool`) — the + /// runtime task lifecycle is driven by status transitions, not deletion. + fn delete_task(&self, key: Vec) -> DbResult; + /// All records where `status` is retriable and `retry_after_secs <= now_secs`. fn list_retriable(&self, now_secs: u64) -> DbResult, TaskRecordData)>>; /// All records whose status is not yet terminal (Pending / Proving). fn list_unfinished(&self) -> DbResult, TaskRecordData)>>; + /// Every record in the store, in implementation-defined order. + /// + /// Intended for offline admin tooling — the runtime path uses the + /// filtered iterators above to avoid scanning terminal entries. + fn list_all_tasks(&self) -> DbResult, TaskRecordData)>>; + /// Number of records in the store. fn count_tasks(&self) -> DbResult; } diff --git a/functional-tests/tests/dbtool/ee_prover_task_admin.py b/functional-tests/tests/dbtool/ee_prover_task_admin.py new file mode 100644 index 0000000000..f18bf08e3a --- /dev/null +++ b/functional-tests/tests/dbtool/ee_prover_task_admin.py @@ -0,0 +1,167 @@ +"""End-to-end exercise of the EE prover-task admin commands. + +Mirrors `prover_task_admin.py` but drives the alpen-client's prover +store (`/sled`) via `--ee-datadir`. We deliberately don't +drive the chunk/acct provers — the value of this test is to lock in +the DB-level admin contract, kind-tag filtering, and the dry-run-by- +default `--force` semantics that the dbtool delivers. +""" + +import logging + +import flexitest + +from common.base_test import AlpenClientTest +from common.config import ServiceType +from envconfigs.alpen_client import AlpenClientEnv +from tests.dbtool.helpers import run_dbtool_ee, run_dbtool_ee_json + +logger = logging.getLogger(__name__) + +# Kind-tagged raw keys. The alpen-client's task encoders prefix every +# key with `b'c'` (chunk) or `b'a'` (acct); we mirror that here so +# `--kind` filters land correctly. +_CHUNK_KEY = "63" + "11" * 8 # b'c' + 8 arbitrary bytes +_ACCT_KEY = "61" + "22" * 8 # b'a' + 8 arbitrary bytes + + +def _summary(ee_datadir: str, *extra: str) -> dict: + return run_dbtool_ee_json(ee_datadir, "ee-get-prover-tasks-summary", "--limit", "100", *extra) + + +def _task(ee_datadir: str, key_hex: str) -> dict: + return run_dbtool_ee_json(ee_datadir, "ee-get-prover-task", key_hex) + + +@flexitest.register +class DbtoolEeProverTaskAdminTest(AlpenClientTest): + def __init__(self, ctx: flexitest.InitContext): + # Use a private env: the test stops the sequencer to poke at the + # sled DB directly, so it must not share the `alpen_ee` env with + # other tests that expect the sequencer to be live. + ctx.set_env(AlpenClientEnv(fullnode_count=0, enable_l1_da=True)) + + def main(self, ctx): + seq_service = self.get_service(ServiceType.AlpenSequencer) + seq_service.wait_for_ready(timeout=60) + seq_service.stop() + ee_datadir = seq_service.props["datadir"] + + # The alpen-client may already have written real chunk/acct + # tasks before we stopped it; record the baseline and reason + # in deltas so the test stays correct under any starting state. + baseline = _summary(ee_datadir) + baseline_total = baseline["total"] + baseline_pending = baseline["pending"] + baseline_permanent = baseline["permanent_failure"] + + # Dry-run-by-default: without --force, the backfill must print + # "would backfill" + force hint and leave the DB unchanged. + code, stdout, stderr = run_dbtool_ee(ee_datadir, "ee-backfill-prover-task-raw", _CHUNK_KEY) + assert code == 0, stderr + assert "would backfill" in stdout, stdout + assert "Use --force to execute these changes." in stdout, stdout + assert _summary(ee_datadir)["total"] == baseline_total, "dry-run backfill must not write" + + # Backfill one chunk-tagged and one acct-tagged task for real; + # both should land in Pending. + for key in (_CHUNK_KEY, _ACCT_KEY): + code, _, stderr = run_dbtool_ee( + ee_datadir, "ee-backfill-prover-task-raw", key, "--force" + ) + assert code == 0, stderr + + after_backfill = _summary(ee_datadir) + assert after_backfill["total"] == baseline_total + 2, after_backfill + assert after_backfill["pending"] == baseline_pending + 2, after_backfill + + # Kind filters: the chunk-tagged key shows up under --kind chunk, + # the acct-tagged key under --kind acct, and neither leaks. + chunk_summary = _summary(ee_datadir, "--kind", "chunk") + acct_summary = _summary(ee_datadir, "--kind", "acct") + chunk_keys = {entry["key_hex"] for entry in chunk_summary["entries"]} + acct_keys = {entry["key_hex"] for entry in acct_summary["entries"]} + assert _CHUNK_KEY in chunk_keys, chunk_summary + assert _ACCT_KEY not in chunk_keys, chunk_summary + assert _ACCT_KEY in acct_keys, acct_summary + assert _CHUNK_KEY not in acct_keys, acct_summary + + # Each surfaced entry should carry its derived kind label, which + # is the field other admin tooling can grep on. + for entry in chunk_summary["entries"]: + if entry["key_hex"] == _CHUNK_KEY: + assert entry["kind"] == "chunk", entry + for entry in acct_summary["entries"]: + if entry["key_hex"] == _ACCT_KEY: + assert entry["kind"] == "acct", entry + + # Single-row dry runs against existing keys preview the action + # and emit the same force hint, without writing. + for verb in ( + "ee-abandon-prover-task", + "ee-reset-prover-task", + "ee-delete-prover-task", + ): + code, stdout, stderr = run_dbtool_ee(ee_datadir, verb, _CHUNK_KEY) + assert code == 0, stderr + assert "Use --force to execute these changes." in stdout, (verb, stdout) + # State unchanged after dry runs. + assert _task(ee_datadir, _CHUNK_KEY)["status"]["name"] == "pending" + + # Abandon the chunk-tagged task → PermanentFailure with the + # documented reason. The single-key path operates on the opaque + # key, so no kind flag is needed. + code, _, stderr = run_dbtool_ee(ee_datadir, "ee-abandon-prover-task", _CHUNK_KEY, "--force") + assert code == 0, stderr + record = _task(ee_datadir, _CHUNK_KEY) + assert record["status"]["name"] == "permanent_failure", record + assert record["status"]["error"] == "abandoned via dbtool", record + assert record["kind"] == "chunk", record + + # Reset moves the acct-tagged task back to Pending and clears + # any retry_after. + code, _, stderr = run_dbtool_ee(ee_datadir, "ee-reset-prover-task", _ACCT_KEY, "--force") + assert code == 0, stderr + record = _task(ee_datadir, _ACCT_KEY) + assert record["status"]["name"] == "pending", record + assert "retry_after_secs" not in record, record + assert record["kind"] == "acct", record + + # Delete the abandoned one; it should drop from the summary. + code, _, stderr = run_dbtool_ee(ee_datadir, "ee-delete-prover-task", _CHUNK_KEY, "--force") + assert code == 0, stderr + after_delete = _summary(ee_datadir) + assert after_delete["total"] == baseline_total + 1, after_delete + assert after_delete["permanent_failure"] == baseline_permanent, after_delete + + # Bulk dry run (no --force) prints intent without writing. + code, stdout, stderr = run_dbtool_ee( + ee_datadir, + "ee-abandon-prover-tasks", + "--all-unfinished", + "--kind", + "acct", + ) + assert code == 0, stderr + assert "would abandon" in stdout, stdout + assert "Use --force to execute these changes." in stdout, stdout + # The acct-tagged task we reset is still Pending after dry-run. + record = _task(ee_datadir, _ACCT_KEY) + assert record["status"]["name"] == "pending", record + + # Bulk for-real (kind=acct) flips just our acct task — it must + # not touch any pre-existing chunk tasks the alpen-client may + # have left behind. + code, _, stderr = run_dbtool_ee( + ee_datadir, + "ee-abandon-prover-tasks", + "--all-unfinished", + "--kind", + "acct", + "--force", + ) + assert code == 0, stderr + record = _task(ee_datadir, _ACCT_KEY) + assert record["status"]["name"] == "permanent_failure", record + + return True diff --git a/functional-tests/tests/dbtool/helpers.py b/functional-tests/tests/dbtool/helpers.py index fa1018d28b..ba66a58f74 100644 --- a/functional-tests/tests/dbtool/helpers.py +++ b/functional-tests/tests/dbtool/helpers.py @@ -91,6 +91,40 @@ def run_dbtool_json(datadir: str, *args: str, timeout: int = 60) -> dict[str, An return extract_json_from_output(stdout) +def run_dbtool_ee(ee_datadir: str, *args: str, timeout: int = 60) -> tuple[int, str, str]: + """Run strata-dbtool against an alpen-client datadir using --ee-datadir. + + `ee-*` subcommands run against a separate sled instance, so they need + --ee-datadir rather than -d. + """ + cmd = ["strata-dbtool", "--ee-datadir", ee_datadir, *args] + logger.info("Running command: %s", " ".join(cmd)) + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=str(Path(ee_datadir).parent), + timeout=timeout, + ) + if result.returncode == 0: + if result.stdout: + logger.info("Stdout: %s", result.stdout.strip()) + else: + if result.stderr: + logger.info("Stderr: %s", result.stderr.strip()) + return result.returncode, result.stdout, result.stderr + + +def run_dbtool_ee_json(ee_datadir: str, *args: str, timeout: int = 60) -> dict[str, Any]: + """Run strata-dbtool ee-* command with JSON output and parse response.""" + code, stdout, stderr = run_dbtool_ee(ee_datadir, *args, "-o", "json", timeout=timeout) + if code != 0: + raise RuntimeError( + f"strata-dbtool ee command failed ({' '.join(args)}): {stderr or stdout}" + ) + return extract_json_from_output(stdout) + + def _load_rollup_params(datadir: str) -> dict[str, Any]: """Load rollup params from rollup-params.json in the node datadir.""" rollup_path = Path(datadir) / "rollup-params.json" diff --git a/functional-tests/tests/dbtool/prover_task_admin.py b/functional-tests/tests/dbtool/prover_task_admin.py new file mode 100644 index 0000000000..b522589098 --- /dev/null +++ b/functional-tests/tests/dbtool/prover_task_admin.py @@ -0,0 +1,133 @@ +"""End-to-end exercise of the prover-task admin commands. + +Operates entirely on the offline datadir: backfills a synthetic task via +the raw-key escape hatch, then walks it through the abandon / reset / +delete verbs and asserts each transition lands in the DB as documented. + +We deliberately don't drive the prover here — the value of this test is +to lock in the DB-level admin contract, which is the surface STR-3414 +delivers. +""" + +import logging + +import flexitest + +from common.base_test import StrataNodeTest +from common.config import EpochSealingConfig, ServiceType +from envconfigs.strata import StrataEnvConfig +from tests.dbtool.helpers import run_dbtool, run_dbtool_json + +logger = logging.getLogger(__name__) + +# Arbitrary hex key — the dbtool's raw backfill accepts any byte string, +# so we don't need to construct a real `CheckpointProofTask` here. The +# typed-backfill path is exercised separately by callers that have a +# canonical epoch commitment to resolve. +_RAW_KEY_A = "deadbeef" +_RAW_KEY_B = "cafebabe" + + +def _summary(datadir: str) -> dict: + return run_dbtool_json(datadir, "get-prover-tasks-summary", "--limit", "100") + + +def _task(datadir: str, key_hex: str) -> dict: + return run_dbtool_json(datadir, "get-prover-task", key_hex) + + +@flexitest.register +class DbtoolProverTaskAdminTest(StrataNodeTest): + def __init__(self, ctx: flexitest.InitContext): + ctx.set_env( + StrataEnvConfig( + pre_generate_blocks=10, + epoch_sealing=EpochSealingConfig.new_fixed_slot(4), + ) + ) + + def main(self, ctx): + seq_service = self.get_service(ServiceType.Strata) + seq_service.wait_for_rpc_ready(timeout=20) + seq_service.stop() + datadir = seq_service.props["datadir"] + + # Baseline: no prover tasks in a fresh node datadir. + summary = _summary(datadir) + assert summary["total"] == 0, summary + + # Dry-run-by-default: without --force the mutates must print + # "would X" + the force hint, exit 0, and leave the DB alone. + # Backfill needs to come first since the other verbs need a row + # to operate on; we run its dry-run on a key that doesn't exist + # yet so we can confirm it didn't create one. + code, stdout, stderr = run_dbtool(datadir, "backfill-prover-task-raw", _RAW_KEY_A) + assert code == 0, stderr + assert "would backfill" in stdout, stdout + assert "Use --force to execute these changes." in stdout, stdout + assert _summary(datadir)["total"] == 0, "dry-run backfill must not write" + + # Backfill two raw tasks for real; both should land in Pending. + for key in (_RAW_KEY_A, _RAW_KEY_B): + code, _, stderr = run_dbtool(datadir, "backfill-prover-task-raw", key, "--force") + assert code == 0, stderr + + summary = _summary(datadir) + assert summary["total"] == 2, summary + assert summary["pending"] == 2, summary + + # Single-row dry runs against existing keys preview the action + # and emit the same force hint, without writing. + for verb in ("abandon-prover-task", "reset-prover-task", "delete-prover-task"): + code, stdout, stderr = run_dbtool(datadir, verb, _RAW_KEY_A) + assert code == 0, stderr + assert "Use --force to execute these changes." in stdout, (verb, stdout) + # State after the dry runs is unchanged. + assert _task(datadir, _RAW_KEY_A)["status"]["name"] == "pending" + + # Abandon the first → PermanentFailure with the documented reason. + code, _, stderr = run_dbtool(datadir, "abandon-prover-task", _RAW_KEY_A, "--force") + assert code == 0, stderr + record = _task(datadir, _RAW_KEY_A) + assert record["status"]["name"] == "permanent_failure", record + assert record["status"]["error"] == "abandoned via dbtool", record + + # Reset moves the second back to Pending and clears any retry_after. + code, _, stderr = run_dbtool(datadir, "reset-prover-task", _RAW_KEY_B, "--force") + assert code == 0, stderr + record = _task(datadir, _RAW_KEY_B) + assert record["status"]["name"] == "pending", record + assert "retry_after_secs" not in record, record + + # Delete the abandoned one; it should drop from the summary. + code, _, stderr = run_dbtool(datadir, "delete-prover-task", _RAW_KEY_A, "--force") + assert code == 0, stderr + summary = _summary(datadir) + assert summary["total"] == 1, summary + assert summary["pending"] == 1, summary + + # Bulk dry run (no --force) prints intent without writing. + code, stdout, stderr = run_dbtool( + datadir, + "abandon-prover-tasks", + "--all-unfinished", + ) + assert code == 0, stderr + assert "would abandon" in stdout, stdout + assert "Use --force to execute these changes." in stdout, stdout + summary = _summary(datadir) + assert summary["pending"] == 1, summary + + # Bulk for-real flips the remaining task to permanent_failure. + code, _, stderr = run_dbtool( + datadir, + "abandon-prover-tasks", + "--all-unfinished", + "--force", + ) + assert code == 0, stderr + summary = _summary(datadir) + assert summary["pending"] == 0, summary + assert summary["permanent_failure"] == 1, summary + + return True