From cdc7e7ab5f7f8f96dd08ecc82a969d88729a37de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Tue, 6 May 2025 17:39:50 +0200 Subject: [PATCH 1/3] chore(web): refactor audit dashboard filters --- entity/src/audit_result_latest.rs | 35 +----------- entity/src/content.rs | 52 +++++++++++++++++- entity/src/content_audit.rs | 38 ++++++++----- glados-audit/src/lib.rs | 2 +- glados-audit/src/selection.rs | 8 +-- glados-core/src/stats.rs | 2 +- glados-web/src/routes.rs | 13 +++-- glados-web/src/templates.rs | 5 +- glados-web/templates/audit_dashboard.html | 65 ++++++----------------- 9 files changed, 111 insertions(+), 109 deletions(-) diff --git a/entity/src/audit_result_latest.rs b/entity/src/audit_result_latest.rs index e1b18c31..ed8f6429 100644 --- a/entity/src/audit_result_latest.rs +++ b/entity/src/audit_result_latest.rs @@ -1,40 +1,9 @@ //! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.2 use chrono::{DateTime, Utc}; -use sea_orm::{ - entity::prelude::*, - strum::{EnumMessage, EnumString}, -}; -use serde::Deserialize; +use sea_orm::entity::prelude::*; -use crate::content_audit::AuditResult; - -// Not using the constants in ethportal-api because seaorm does not support DeriveActiveEnum from a -// variable -#[derive( - Debug, - Clone, - Copy, - Eq, - PartialEq, - EnumIter, - DeriveActiveEnum, - Deserialize, - EnumMessage, - EnumString, -)] -#[sea_orm(rs_type = "u8", db_type = "Integer")] -#[strum(serialize_all = "snake_case")] -pub enum ContentType { - #[strum(message = "Block headers by hash")] - BlockHeadersByHash = 0, - #[strum(message = "Block bodies")] - BlockBodies = 1, - #[strum(message = "Block receipts")] - BlockReceipts = 2, - #[strum(message = "Block headers by number")] - BlockHeadersByNumber = 3, -} +use crate::{content::ContentType, content_audit::AuditResult}; #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "audit_result_latest")] diff --git a/entity/src/content.rs b/entity/src/content.rs index 56960ecf..bcea5b0b 100644 --- a/entity/src/content.rs +++ b/entity/src/content.rs @@ -1,13 +1,19 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7 -use crate::utils; use alloy_primitives::B256; use anyhow::Result; use chrono::{DateTime, Utc}; use ethportal_api::utils::bytes::{hex_encode, hex_encode_compact}; use ethportal_api::OverlayContentKey; -use sea_orm::{entity::prelude::*, ActiveValue::NotSet, Set}; +use sea_orm::{ + entity::prelude::*, + strum::{EnumMessage, EnumProperty, EnumString}, + ActiveValue::NotSet, + Iterable, Set, +}; use serde::Deserialize; +use crate::utils; + /// Portal network sub-protocol. History, state, transactions etc. #[derive(Debug, Clone, Copy, Eq, PartialEq, EnumIter, DeriveActiveEnum, Deserialize)] #[sea_orm(rs_type = "i32", db_type = "Integer")] @@ -27,6 +33,48 @@ impl SubProtocol { } } +// Not using the constants in ethportal-api because seaorm does not support DeriveActiveEnum from a variable +// Newer versions of strum support int type properties +#[derive( + Debug, + Clone, + Eq, + Hash, + PartialEq, + EnumIter, + DeriveActiveEnum, + EnumMessage, + EnumString, + EnumProperty, +)] +#[sea_orm(rs_type = "u8", db_type = "Integer")] +#[strum(serialize_all = "snake_case")] +pub enum ContentType { + #[strum(message = "Headers by hash", props(subprotocol = "0"))] + BlockHeadersByHash = 0, + #[strum(message = "Bodies", props(subprotocol = "0"))] + BlockBodies = 1, + #[strum(message = "Receipts", props(subprotocol = "0"))] + BlockReceipts = 2, + #[strum(message = "Headers by number", props(subprotocol = "0"))] + BlockHeadersByNumber = 3, + #[strum[message = "Block Roots", props(subprotocol="2")]] + BlockRoots = 16, // 0x10 + #[strum[message = "Account Trie Nodes", props(subprotocol="1")]] + AccountTrieNodes = 32, // 0x20 +} + +impl ContentType { + pub fn vec_subprotocol(subprotocol: SubProtocol) -> Vec { + ContentType::iter() + .filter(|content_type| { + content_type.get_str("subprotocol").unwrap_or("-1") + == (subprotocol as usize).to_string() + }) + .collect() + } +} + #[derive(Debug, PartialEq)] pub struct InvalidSubProtocolError; diff --git a/entity/src/content_audit.rs b/entity/src/content_audit.rs index 45c9a799..6e5db0cc 100644 --- a/entity/src/content_audit.rs +++ b/entity/src/content_audit.rs @@ -38,7 +38,7 @@ pub enum HistorySelectionStrategy { /// Content that is: /// 1. Not yet audited. /// 2. Sorted by date entered into glados database (oldest first). - SelectOldestUnaudited = 3, + OldestUnaudited = 3, /// Perform a single audit for a previously audited content key. SpecificContentKey = 4, /// Perform audits of random fourfours data. @@ -51,7 +51,7 @@ impl From for HistorySelectionStrategy { 0 => HistorySelectionStrategy::Latest, 1 => HistorySelectionStrategy::Random, 2 => HistorySelectionStrategy::Failed, - 3 => HistorySelectionStrategy::SelectOldestUnaudited, + 3 => HistorySelectionStrategy::OldestUnaudited, 4 => HistorySelectionStrategy::SpecificContentKey, 5 => HistorySelectionStrategy::FourFours, _ => panic!("Invalid value for HistorySelectionStrategy"), @@ -66,7 +66,7 @@ impl TryFrom for HistorySelectionStrategy { "Latest" => Ok(HistorySelectionStrategy::Latest), "Random" => Ok(HistorySelectionStrategy::Random), "Failed" => Ok(HistorySelectionStrategy::Failed), - "SelectOldestUnaudited" => Ok(HistorySelectionStrategy::SelectOldestUnaudited), + "OldestUnaudited" => Ok(HistorySelectionStrategy::OldestUnaudited), "SpecificContentKey" => Ok(HistorySelectionStrategy::SpecificContentKey), "FourFours" => Ok(HistorySelectionStrategy::FourFours), _ => bail!("Invalid value for HistorySelectionStrategy {}", value), @@ -454,8 +454,8 @@ impl SelectionStrategy { SelectionStrategy::History(HistorySelectionStrategy::FourFours) => { "FourFours".to_string() } - SelectionStrategy::History(HistorySelectionStrategy::SelectOldestUnaudited) => { - "Select Oldest Unaudited".to_string() + SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited) => { + "Oldest Unaudited".to_string() } SelectionStrategy::History(HistorySelectionStrategy::SpecificContentKey) => { "Specific Content Key".to_string() @@ -467,6 +467,20 @@ impl SelectionStrategy { SelectionStrategy::State(StateSelectionStrategy::Latest) => "Latest".to_string(), } } + + pub fn vec_subprotocol(subprotocol: SubProtocol) -> Vec { + match subprotocol { + SubProtocol::History => HistorySelectionStrategy::iter() + .map(SelectionStrategy::History) + .collect(), + SubProtocol::State => StateSelectionStrategy::iter() + .map(SelectionStrategy::State) + .collect(), + SubProtocol::Beacon => BeaconSelectionStrategy::iter() + .map(SelectionStrategy::Beacon) + .collect(), + } + } } impl Model { @@ -513,7 +527,7 @@ mod tests { 2 ); assert_eq!( - SelectionStrategy::History(HistorySelectionStrategy::SelectOldestUnaudited).to_value(), + SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited).to_value(), 3 ); assert_eq!( @@ -550,7 +564,7 @@ mod tests { ); assert_eq!( SelectionStrategy::try_from_value(&3).unwrap(), - SelectionStrategy::History(HistorySelectionStrategy::SelectOldestUnaudited) + SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited) ); assert_eq!( SelectionStrategy::try_from_value(&4).unwrap(), @@ -585,8 +599,8 @@ mod tests { "Failed" ); assert_eq!( - SelectionStrategy::History(HistorySelectionStrategy::SelectOldestUnaudited).as_text(), - "Select Oldest Unaudited" + SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited).as_text(), + "Oldest Unaudited" ); assert_eq!( SelectionStrategy::History(HistorySelectionStrategy::SpecificContentKey).as_text(), @@ -621,8 +635,8 @@ mod tests { HistorySelectionStrategy::Failed ); assert_eq!( - HistorySelectionStrategy::try_from("SelectOldestUnaudited".to_string()).unwrap(), - HistorySelectionStrategy::SelectOldestUnaudited + HistorySelectionStrategy::try_from("OldestUnaudited".to_string()).unwrap(), + HistorySelectionStrategy::OldestUnaudited ); assert_eq!( HistorySelectionStrategy::try_from("SpecificContentKey".to_string()).unwrap(), @@ -658,7 +672,7 @@ mod tests { ); assert_eq!( Value::from(SelectionStrategy::History( - HistorySelectionStrategy::SelectOldestUnaudited + HistorySelectionStrategy::OldestUnaudited )), Value::Int(Some(3)) ); diff --git a/glados-audit/src/lib.rs b/glados-audit/src/lib.rs index 26e486ed..bc6d445a 100644 --- a/glados-audit/src/lib.rs +++ b/glados-audit/src/lib.rs @@ -103,7 +103,7 @@ impl AuditConfig { HistorySelectionStrategy::Latest => args.latest_strategy_weight, HistorySelectionStrategy::Random => args.random_strategy_weight, HistorySelectionStrategy::Failed => args.failed_strategy_weight, - HistorySelectionStrategy::SelectOldestUnaudited => args.oldest_strategy_weight, + HistorySelectionStrategy::OldestUnaudited => args.oldest_strategy_weight, HistorySelectionStrategy::FourFours => args.four_fours_strategy_weight, HistorySelectionStrategy::SpecificContentKey => 0, }; diff --git a/glados-audit/src/selection.rs b/glados-audit/src/selection.rs index 65e05312..02d70d3f 100644 --- a/glados-audit/src/selection.rs +++ b/glados-audit/src/selection.rs @@ -43,7 +43,7 @@ pub async fn start_audit_selection_task( SelectionStrategy::History(HistorySelectionStrategy::Failed) => { warn!("Need to implement SelectionStrategy::Failed") } - SelectionStrategy::History(HistorySelectionStrategy::SelectOldestUnaudited) => { + SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited) => { select_oldest_unaudited_content_for_audit(tx, conn).await } SelectionStrategy::History(HistorySelectionStrategy::SpecificContentKey) => { @@ -302,7 +302,7 @@ pub struct MaxContentId { pub id: i32, } -/// Finds and sends audit tasks for [SelectionStrategy::SelectOldestUnaudited]. +/// Finds and sends audit tasks for [SelectionStrategy::OldestUnaudited]. /// /// Strategy achieved by: /// 1. Find oldest content @@ -370,7 +370,7 @@ async fn select_oldest_unaudited_content_for_audit( ); add_to_queue( tx.clone(), - SelectionStrategy::History(HistorySelectionStrategy::SelectOldestUnaudited), + SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited), content_key_db_entries, ) .await; @@ -551,7 +551,7 @@ mod tests { assert_eq!(checked_ids.len(), expected_key_ids.len()); } - /// Tests that the `SelectionStrategy::SelectOldestUnaudited` selects the correct values + /// Tests that the `SelectionStrategy::OldestUnaudited` selects the correct values /// from the test database. #[tokio::test] async fn test_select_oldest_unaudited_strategy() { diff --git a/glados-core/src/stats.rs b/glados-core/src/stats.rs index 9725869e..089cc262 100644 --- a/glados-core/src/stats.rs +++ b/glados-core/src/stats.rs @@ -49,7 +49,7 @@ pub fn filter_audits(filters: AuditFilters) -> Select { }, )), StrategyFilter::Oldest => audits.filter(content_audit::Column::StrategyUsed.eq( - SelectionStrategy::History(HistorySelectionStrategy::SelectOldestUnaudited), + SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited), )), StrategyFilter::FourFours => audits.filter(content_audit::Column::StrategyUsed.eq( SelectionStrategy::History(HistorySelectionStrategy::FourFours), diff --git a/glados-web/src/routes.rs b/glados-web/src/routes.rs index 0e2b1665..a8e8c238 100644 --- a/glados-web/src/routes.rs +++ b/glados-web/src/routes.rs @@ -40,12 +40,11 @@ use crate::templates::{ }; use crate::{state::State, templates::AuditTuple}; use entity::{ - audit_result_latest::ContentType, audit_stats, census, census_node, census_node::{Client, OperatingSystem, Version}, client_info, content, - content::SubProtocol, - content_audit::{self, AuditResult}, + content::{ContentType, SubProtocol}, + content_audit::{self, AuditResult, SelectionStrategy}, execution_metadata, key_value, node, record, }; use glados_core::stats::{ @@ -144,7 +143,7 @@ pub async fn network_overview( average_radius_chart: radius_percentages, stats: [hour_stats, day_stats, week_stats], new_content: [hour_new, day_new, week_new], - content_types: ContentType::iter().collect(), + content_types: ContentType::vec_subprotocol(subprotocol), }; HtmlTemplate(template) } @@ -464,7 +463,11 @@ pub async fn contentaudit_dashboard( params: HttpQuery>, ) -> Result, StatusCode> { let subprotocol = get_subprotocol_from_params(¶ms); - let template = AuditDashboardTemplate { subprotocol }; + let template = AuditDashboardTemplate { + subprotocol, + content_types: ContentType::vec_subprotocol(subprotocol), + strategies: SelectionStrategy::vec_subprotocol(subprotocol), + }; Ok(HtmlTemplate(template)) } diff --git a/glados-web/src/templates.rs b/glados-web/src/templates.rs index 10c86e6f..1c333176 100644 --- a/glados-web/src/templates.rs +++ b/glados-web/src/templates.rs @@ -9,10 +9,9 @@ use crate::routes::{ CalculatedRadiusChartData, ClientDiversityResult, PaginatedCensusListResult, RawEnr, }; use entity::{ - audit_result_latest::ContentType, census_node::{Client, OperatingSystem}, client_info, - content::{self, SubProtocol}, + content::{self, ContentType, SubProtocol}, content_audit, execution_metadata, key_value, node, record, }; use glados_core::stats::{AuditStats, StrategyFilter}; @@ -103,6 +102,8 @@ pub struct ContentKeyListTemplate { #[template(path = "audit_dashboard.html")] pub struct AuditDashboardTemplate { pub subprotocol: SubProtocol, + pub content_types: Vec, + pub strategies: Vec, } #[derive(Template)] diff --git a/glados-web/templates/audit_dashboard.html b/glados-web/templates/audit_dashboard.html index 0b7a3a97..576999ab 100644 --- a/glados-web/templates/audit_dashboard.html +++ b/glados-web/templates/audit_dashboard.html @@ -10,64 +10,31 @@

Audit Dashboard

- {% if subprotocol == SubProtocol::History %} - -
- - - - -
-
- - - - - -
- - {% elseif subprotocol == SubProtocol::State %} -
- + {% for content_type in content_types %} + + {% endfor %}
- -
- - -
- - {% elseif subprotocol == SubProtocol::Beacon %} - -
- - -
-
- + {% for strategy in strategies %} + + {% endfor %}
- {% endif %}
From b178e9c531d3080c530d2ad256e2b9030d9a6c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Mon, 19 May 2025 18:18:29 +0200 Subject: [PATCH 2/3] chore: refactor audit dashboard --- entity/src/content.rs | 56 +-- entity/src/content_audit.rs | 293 ++++++++++----- glados-audit/src/lib.rs | 17 +- glados-audit/src/selection.rs | 4 +- glados-audit/src/stats.rs | 159 ++++---- glados-core/src/stats.rs | 231 +++++------- glados-web/assets/js/audit_dashboard.js | 351 ++++++++++++++++++ glados-web/src/lib.rs | 3 +- glados-web/src/routes.rs | 160 +++++--- glados-web/src/templates.rs | 21 +- glados-web/templates/audit_dashboard.html | 211 +++++------ glados-web/templates/audit_table.html | 88 ----- glados-web/templates/contentaudit_detail.html | 2 +- glados-web/templates/index.html | 12 +- 14 files changed, 973 insertions(+), 635 deletions(-) create mode 100644 glados-web/assets/js/audit_dashboard.js delete mode 100644 glados-web/templates/audit_table.html diff --git a/entity/src/content.rs b/entity/src/content.rs index bcea5b0b..38252c91 100644 --- a/entity/src/content.rs +++ b/entity/src/content.rs @@ -15,20 +15,41 @@ use serde::Deserialize; use crate::utils; /// Portal network sub-protocol. History, state, transactions etc. -#[derive(Debug, Clone, Copy, Eq, PartialEq, EnumIter, DeriveActiveEnum, Deserialize)] +#[derive( + Debug, + Clone, + Copy, + Eq, + PartialEq, + EnumIter, + EnumString, + EnumMessage, + DeriveActiveEnum, + Deserialize, +)] #[sea_orm(rs_type = "i32", db_type = "Integer")] +#[strum(ascii_case_insensitive)] +#[serde(rename_all = "snake_case")] pub enum SubProtocol { + #[strum(message = "History")] History = 0, + #[strum(message = "State")] State = 1, + #[strum(message = "Beacon")] Beacon = 2, } -impl SubProtocol { - pub fn as_text(&self) -> String { - match self { - SubProtocol::History => "History".to_string(), - SubProtocol::State => "State".to_string(), - SubProtocol::Beacon => "Beacon".to_string(), +#[derive(Debug, PartialEq)] +pub struct InvalidSubProtocolError; + +impl TryFrom for SubProtocol { + type Error = InvalidSubProtocolError; + fn try_from(value: u8) -> std::result::Result { + match value { + 0 => Ok(SubProtocol::History), + 1 => Ok(SubProtocol::State), + 2 => Ok(SubProtocol::Beacon), + _ => Err(InvalidSubProtocolError), } } } @@ -46,9 +67,12 @@ impl SubProtocol { EnumMessage, EnumString, EnumProperty, + Copy, + Deserialize, )] -#[sea_orm(rs_type = "u8", db_type = "Integer")] +#[sea_orm(rs_type = "i32", db_type = "Integer")] #[strum(serialize_all = "snake_case")] +#[serde(rename_all = "snake_case")] pub enum ContentType { #[strum(message = "Headers by hash", props(subprotocol = "0"))] BlockHeadersByHash = 0, @@ -75,22 +99,6 @@ impl ContentType { } } -#[derive(Debug, PartialEq)] -pub struct InvalidSubProtocolError; - -impl TryFrom<&String> for SubProtocol { - type Error = InvalidSubProtocolError; - - fn try_from(value: &String) -> Result { - match value.to_lowercase().as_str() { - "history" => Ok(SubProtocol::History), - "state" => Ok(SubProtocol::State), - "beacon" => Ok(SubProtocol::Beacon), - _ => Err(InvalidSubProtocolError), - } - } -} - #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] #[sea_orm(table_name = "content")] pub struct Model { diff --git a/entity/src/content_audit.rs b/entity/src/content_audit.rs index 6e5db0cc..a1380d78 100644 --- a/entity/src/content_audit.rs +++ b/entity/src/content_audit.rs @@ -1,47 +1,91 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.10.7 -use crate::content; -use crate::utils; +use std::fmt; + use anyhow::{bail, Result}; use chrono::{DateTime, Utc}; use clap::ValueEnum; use content::SubProtocol; use ethportal_api::{utils::bytes::hex_encode, OverlayContentKey}; use sea_orm::{ - entity::prelude::*, strum::IntoEnumIterator, ActiveValue::NotSet, DbBackend, DeriveActiveEnum, - FromQueryResult, Set, Statement, TryGetable, + entity::prelude::*, + strum::{EnumMessage, EnumString, IntoEnumIterator}, + ActiveValue::NotSet, + DbBackend, DeriveActiveEnum, FromQueryResult, Set, Statement, TryGetable, }; use sea_query::{ArrayType, Nullable, SeaRc, ValueType, ValueTypeErr}; +use serde::{Deserialize, Serialize}; + +use crate::content; +use crate::utils; -#[derive(Debug, Clone, Eq, PartialEq, EnumIter, DeriveActiveEnum)] +#[derive( + Debug, + Clone, + Eq, + PartialEq, + EnumIter, + EnumMessage, + EnumString, + DeriveActiveEnum, + Copy, + Deserialize, + Serialize, +)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] #[sea_orm(rs_type = "i32", db_type = "Integer")] pub enum AuditResult { + #[strum(message = "Failure")] Failure = 0, + #[strum(message = "Success")] Success = 1, } -#[derive(Debug, Clone, Eq, Hash, PartialEq, EnumIter, DeriveActiveEnum, ValueEnum)] +#[derive( + Debug, + Clone, + Eq, + Hash, + PartialEq, + EnumIter, + EnumMessage, + EnumString, + DeriveActiveEnum, + ValueEnum, + Serialize, + Deserialize, + Copy, +)] #[clap(rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] #[sea_orm(rs_type = "i32", db_type = "Integer")] /// Each strategy is responsible for selecting which content key(s) to begin audits for. pub enum HistorySelectionStrategy { /// Content that is: /// 1. Not yet audited /// 2. Sorted by date entered into glados database (newest first). + #[strum(message = "Latest", props(subprotocol = "0"))] Latest = 0, /// Randomly selected content. + #[strum(message = "Random", props(subprotocol = "0"))] Random = 1, /// Content that looks for failed audits and checks whether the data is still missing. /// 1. Key was audited previously /// 2. Latest audit for the key failed (data absent) /// 3. Keys sorted by date audited (keys with oldest failed audit first) + #[strum(message = "Failed", props(subprotocol = "0"))] Failed = 2, /// Content that is: /// 1. Not yet audited. /// 2. Sorted by date entered into glados database (oldest first). + #[strum(message = "Oldest Unaudited", props(subprotocol = "0"))] OldestUnaudited = 3, /// Perform a single audit for a previously audited content key. + #[strum(message = "Specific Content Key", props(subprotocol = "0"))] SpecificContentKey = 4, /// Perform audits of random fourfours data. + #[strum(message = "Four Fours", props(subprotocol = "0"))] FourFours = 5, } @@ -59,29 +103,31 @@ impl From for HistorySelectionStrategy { } } -impl TryFrom for HistorySelectionStrategy { - type Error = anyhow::Error; - fn try_from(value: String) -> Result { - match value.as_str() { - "Latest" => Ok(HistorySelectionStrategy::Latest), - "Random" => Ok(HistorySelectionStrategy::Random), - "Failed" => Ok(HistorySelectionStrategy::Failed), - "OldestUnaudited" => Ok(HistorySelectionStrategy::OldestUnaudited), - "SpecificContentKey" => Ok(HistorySelectionStrategy::SpecificContentKey), - "FourFours" => Ok(HistorySelectionStrategy::FourFours), - _ => bail!("Invalid value for HistorySelectionStrategy {}", value), - } - } -} - -#[derive(Debug, Clone, Eq, Hash, PartialEq, EnumIter, DeriveActiveEnum, ValueEnum)] +#[derive( + Debug, + Clone, + Eq, + Hash, + PartialEq, + EnumIter, + EnumMessage, + EnumString, + DeriveActiveEnum, + ValueEnum, + Serialize, + Deserialize, + Copy, +)] #[clap(rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] #[sea_orm(rs_type = "i32", db_type = "Integer")] /// Each strategy is responsible for selecting which content key(s) to begin audits for. pub enum BeaconSelectionStrategy { /// Content that is: /// 1. Not yet audited /// 2. Sorted by date entered into glados database (newest first). + #[strum(message = "Latest", props(subprotocol = "1"))] Latest = 0, } @@ -94,23 +140,31 @@ impl From for BeaconSelectionStrategy { } } -impl TryFrom for BeaconSelectionStrategy { - type Error = anyhow::Error; - fn try_from(value: String) -> Result { - match value.as_str() { - "Latest" => Ok(BeaconSelectionStrategy::Latest), - _ => bail!("Invalid value for BeaconSelectionStrategy {}", value), - } - } -} - -#[derive(Debug, Clone, Eq, Hash, PartialEq, EnumIter, DeriveActiveEnum, ValueEnum)] +#[derive( + Debug, + Clone, + Eq, + Hash, + PartialEq, + EnumIter, + EnumMessage, + EnumString, + DeriveActiveEnum, + ValueEnum, + Serialize, + Deserialize, + Copy, +)] #[clap(rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case", ascii_case_insensitive)] #[sea_orm(rs_type = "i32", db_type = "Integer")] /// Each strategy is responsible for selecting which content key(s) to begin audits for. pub enum StateSelectionStrategy { /// Does a random walk of the state at a random walk. + #[strum(message = "State Roots", props(subprotocol = "2"))] StateRoots = 0, + #[strum(message = "Latest", props(subprotocol = "2"))] Latest = 1, } @@ -124,18 +178,8 @@ impl From for StateSelectionStrategy { } } -impl TryFrom for StateSelectionStrategy { - type Error = anyhow::Error; - fn try_from(value: String) -> Result { - match value.as_str() { - "StateRoots" => Ok(StateSelectionStrategy::StateRoots), - "Latest" => Ok(StateSelectionStrategy::Latest), - _ => bail!("Invalid value for StateSelectionStrategy {}", value), - } - } -} - -#[derive(Debug, Clone, Eq, Hash, PartialEq)] +#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum SelectionStrategy { History(HistorySelectionStrategy), Beacon(BeaconSelectionStrategy), @@ -267,11 +311,37 @@ impl TryGetable for SelectionStrategy { } } -impl AuditResult { - pub fn as_text(&self) -> String { +impl From for SelectionStrategy { + fn from(value: HistorySelectionStrategy) -> Self { + Self::History(value) + } +} + +impl From for SelectionStrategy { + fn from(value: StateSelectionStrategy) -> Self { + Self::State(value) + } +} + +impl From for SelectionStrategy { + fn from(value: BeaconSelectionStrategy) -> Self { + Self::Beacon(value) + } +} + +impl SelectionStrategy { + pub fn get_message(&self) -> Option<&str> { match self { - AuditResult::Failure => "fail".to_string(), - AuditResult::Success => "success".to_string(), + SelectionStrategy::History(h) => h.get_message(), + SelectionStrategy::Beacon(b) => b.get_message(), + SelectionStrategy::State(s) => s.get_message(), + } + } + pub fn get_serializations(&self) -> &[&str] { + match self { + SelectionStrategy::History(h) => h.get_serializations(), + SelectionStrategy::Beacon(b) => b.get_serializations(), + SelectionStrategy::State(s) => s.get_serializations(), } } } @@ -379,6 +449,24 @@ pub async fn get_audits( .await?) } +pub fn serialize_selection_strategy( + network: SubProtocol, + selection_strategy_str: &str, +) -> std::prelude::v1::Result { + match network { + SubProtocol::History => Ok(SelectionStrategy::History( + HistorySelectionStrategy::from_str(selection_strategy_str, true)?, + )), + SubProtocol::State => Ok(SelectionStrategy::State(StateSelectionStrategy::from_str( + selection_strategy_str, + true, + )?)), + SubProtocol::Beacon => Ok(SelectionStrategy::Beacon( + BeaconSelectionStrategy::from_str(selection_strategy_str, true)?, + )), + } +} + pub async fn get_failed_keys( subprotocol: SubProtocol, strategy_used: String, @@ -387,11 +475,13 @@ pub async fn get_failed_keys( ) -> Result> { const PAGE_SIZE: u32 = 1000; - let subprotocol_strategy: SelectionStrategy = match subprotocol { - SubProtocol::History => SelectionStrategy::History(strategy_used.try_into()?), - SubProtocol::State => SelectionStrategy::State(strategy_used.try_into()?), - SubProtocol::Beacon => SelectionStrategy::Beacon(strategy_used.try_into()?), - }; + let subprotocol_strategy = + serialize_selection_strategy(subprotocol, &strategy_used).map_err(|_| { + anyhow::Error::msg(format!( + "unknown variant for {}: {}", + subprotocol, strategy_used + )) + })?; let keys_result = FailedKeysResult::find_by_statement(Statement::from_sql_and_values( DbBackend::Postgres, @@ -440,34 +530,32 @@ pub async fn get_failed_keys( .collect::>()) } -impl SelectionStrategy { - /// This performs the function of Display, which is not able to be implemented - /// for this enum. - /// - /// SelectionStrategy derive macro DeriveActiveEnum introduces a conflicting - /// Display implementation. - pub fn as_text(&self) -> String { +impl fmt::Display for SelectionStrategy { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - SelectionStrategy::History(HistorySelectionStrategy::Latest) => "Latest".to_string(), - SelectionStrategy::History(HistorySelectionStrategy::Random) => "Random".to_string(), - SelectionStrategy::History(HistorySelectionStrategy::Failed) => "Failed".to_string(), - SelectionStrategy::History(HistorySelectionStrategy::FourFours) => { - "FourFours".to_string() - } - SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited) => { - "Oldest Unaudited".to_string() - } - SelectionStrategy::History(HistorySelectionStrategy::SpecificContentKey) => { - "Specific Content Key".to_string() - } - SelectionStrategy::Beacon(BeaconSelectionStrategy::Latest) => "Latest".to_string(), - SelectionStrategy::State(StateSelectionStrategy::StateRoots) => { - "State Roots".to_string() - } - SelectionStrategy::State(StateSelectionStrategy::Latest) => "Latest".to_string(), + SelectionStrategy::History(h) => write!( + f, + "{}", + h.get_message() + .expect("HistorySelectionStrategy missing message") + ), + SelectionStrategy::Beacon(b) => write!( + f, + "{}", + b.get_message() + .expect("BeaconSelectionStrategy missing message") + ), + SelectionStrategy::State(s) => write!( + f, + "{}", + s.get_message() + .expect("StateSelectionStrategy missing message") + ), } } +} +impl SelectionStrategy { pub fn vec_subprotocol(subprotocol: SubProtocol) -> Vec { match subprotocol { SubProtocol::History => HistorySelectionStrategy::iter() @@ -498,7 +586,7 @@ impl Model { /// A few early database entries do not have a recorded strategy. pub fn strategy_as_text(&self) -> String { match &self.strategy_used { - Some(s) => s.as_text(), + Some(s) => s.to_string(), None => "No strategy recorded".to_string(), } } @@ -508,9 +596,10 @@ impl Model { mod tests { use sea_orm::{ActiveEnum, Value}; - use crate::content_audit::StateSelectionStrategy; - - use super::{BeaconSelectionStrategy, HistorySelectionStrategy, SelectionStrategy}; + use super::{ + BeaconSelectionStrategy, HistorySelectionStrategy, SelectionStrategy, + StateSelectionStrategy, + }; #[test] fn test_selection_strategy_to_value() { @@ -585,37 +674,37 @@ mod tests { } #[test] - fn test_selection_strategy_as_text() { + fn test_selection_strategy_to_string() { assert_eq!( - SelectionStrategy::History(HistorySelectionStrategy::Latest).as_text(), + SelectionStrategy::History(HistorySelectionStrategy::Latest).to_string(), "Latest" ); assert_eq!( - SelectionStrategy::History(HistorySelectionStrategy::Random).as_text(), + SelectionStrategy::History(HistorySelectionStrategy::Random).to_string(), "Random" ); assert_eq!( - SelectionStrategy::History(HistorySelectionStrategy::Failed).as_text(), + SelectionStrategy::History(HistorySelectionStrategy::Failed).to_string(), "Failed" ); assert_eq!( - SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited).as_text(), + SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited).to_string(), "Oldest Unaudited" ); assert_eq!( - SelectionStrategy::History(HistorySelectionStrategy::SpecificContentKey).as_text(), + SelectionStrategy::History(HistorySelectionStrategy::SpecificContentKey).to_string(), "Specific Content Key" ); assert_eq!( - SelectionStrategy::History(HistorySelectionStrategy::FourFours).as_text(), - "FourFours" + SelectionStrategy::History(HistorySelectionStrategy::FourFours).to_string(), + "Four Fours" ); assert_eq!( - SelectionStrategy::Beacon(BeaconSelectionStrategy::Latest).as_text(), + SelectionStrategy::Beacon(BeaconSelectionStrategy::Latest).to_string(), "Latest" ); assert_eq!( - SelectionStrategy::State(StateSelectionStrategy::StateRoots).as_text(), + SelectionStrategy::State(StateSelectionStrategy::StateRoots).to_string(), "State Roots" ); } @@ -623,35 +712,39 @@ mod tests { #[test] fn test_selection_strategy_from_text() { assert_eq!( - HistorySelectionStrategy::try_from("Latest".to_string()).unwrap(), + HistorySelectionStrategy::try_from("Latest").unwrap(), + HistorySelectionStrategy::Latest, + ); + assert_eq!( + HistorySelectionStrategy::try_from("latest").unwrap(), HistorySelectionStrategy::Latest, ); assert_eq!( - HistorySelectionStrategy::try_from("Random".to_string()).unwrap(), + HistorySelectionStrategy::try_from("random").unwrap(), HistorySelectionStrategy::Random, ); assert_eq!( - HistorySelectionStrategy::try_from("Failed".to_string()).unwrap(), + HistorySelectionStrategy::try_from("failed").unwrap(), HistorySelectionStrategy::Failed ); assert_eq!( - HistorySelectionStrategy::try_from("OldestUnaudited".to_string()).unwrap(), + HistorySelectionStrategy::try_from("oldest_unaudited").unwrap(), HistorySelectionStrategy::OldestUnaudited ); assert_eq!( - HistorySelectionStrategy::try_from("SpecificContentKey".to_string()).unwrap(), + HistorySelectionStrategy::try_from("specific_content_key").unwrap(), HistorySelectionStrategy::SpecificContentKey ); assert_eq!( - HistorySelectionStrategy::try_from("FourFours".to_string()).unwrap(), + HistorySelectionStrategy::try_from("four_fours").unwrap(), HistorySelectionStrategy::FourFours ); assert_eq!( - BeaconSelectionStrategy::try_from("Latest".to_string()).unwrap(), + BeaconSelectionStrategy::try_from("latest").unwrap(), BeaconSelectionStrategy::Latest ); assert_eq!( - StateSelectionStrategy::try_from("StateRoots".to_string()).unwrap(), + StateSelectionStrategy::try_from("state_roots").unwrap(), StateSelectionStrategy::StateRoots ); } diff --git a/glados-audit/src/lib.rs b/glados-audit/src/lib.rs index bc6d445a..ff9a495d 100644 --- a/glados-audit/src/lib.rs +++ b/glados-audit/src/lib.rs @@ -107,7 +107,7 @@ impl AuditConfig { HistorySelectionStrategy::FourFours => args.four_fours_strategy_weight, HistorySelectionStrategy::SpecificContentKey => 0, }; - weights.insert(strat.clone(), weight); + weights.insert(*strat, weight); } let mut portal_clients: Vec = vec![]; for client_url in args.portal_client { @@ -178,10 +178,7 @@ pub async fn run_glados_audit(conn: DatabaseConnection, config: AuditConfig) { .beacon_strategies .iter() .map(|strats| { - ( - SelectionStrategy::Beacon(strats.clone()), - /* weight= */ 1, - ) + (SelectionStrategy::Beacon(*strats), /* weight= */ 1) }) .collect(); start_audit(conn.clone(), config.clone(), strategies).await; @@ -192,7 +189,7 @@ pub async fn run_glados_audit(conn: DatabaseConnection, config: AuditConfig) { .history_strategies .iter() .filter_map(|strats| { - let strategy = SelectionStrategy::History(strats.clone()); + let strategy = SelectionStrategy::History(*strats); match config.weights.get(strats) { Some(weight) => Some((strategy, *weight)), None => { @@ -224,17 +221,13 @@ async fn start_audit( // Each strategy sends tasks to a separate channel. let (tx, rx) = mpsc::channel::(100); let task_channel = TaskChannel { - strategy: strategy.clone(), + strategy, weight, rx, }; task_channels.push(task_channel); // Strategies generate tasks in their own thread for their own channel. - tokio::spawn(start_audit_selection_task( - strategy.clone(), - tx, - conn.clone(), - )); + tokio::spawn(start_audit_selection_task(strategy, tx, conn.clone())); } // Collation of generated tasks, taken proportional to weights. let (collation_tx, collation_rx) = mpsc::channel::(100); diff --git a/glados-audit/src/selection.rs b/glados-audit/src/selection.rs index 02d70d3f..478fb5ab 100644 --- a/glados-audit/src/selection.rs +++ b/glados-audit/src/selection.rs @@ -135,7 +135,7 @@ async fn select_latest_content_for_audit( strategy = "latest", item_count, "Adding content keys to the audit queue." ); - add_to_queue(tx.clone(), strategy.clone(), content_key_db_entries).await; + add_to_queue(tx.clone(), strategy, content_key_db_entries).await; } } @@ -211,7 +211,7 @@ async fn add_to_queue( ); for content_key_model in items { let task = AuditTask { - strategy: strategy.clone(), + strategy, content: content_key_model, }; if let Err(e) = tx.send(task).await { diff --git a/glados-audit/src/stats.rs b/glados-audit/src/stats.rs index 36a2aaed..d43574be 100644 --- a/glados-audit/src/stats.rs +++ b/glados-audit/src/stats.rs @@ -1,9 +1,10 @@ use chrono::Utc; -use entity::{audit_stats, content::SubProtocol}; -use glados_core::stats::{ - filter_audits, get_audit_stats, AuditFilters, ContentTypeFilter, Period, StrategyFilter, - SuccessFilter, +use entity::{ + audit_stats, + content::{ContentType, SubProtocol}, + content_audit::{BeaconSelectionStrategy, HistorySelectionStrategy, StateSelectionStrategy}, }; +use glados_core::stats::{filter_audits, get_audit_stats, AuditFilters, Period}; use sea_orm::{DatabaseConnection, DbErr}; use tokio::time::{interval, Duration}; use tracing::{debug, error}; @@ -54,9 +55,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ) = tokio::join!( get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::All, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: None, + content_type: None, + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -64,9 +65,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Latest, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Latest.into()), + content_type: None, + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -74,9 +75,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Random, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Random.into()), + content_type: None, + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -84,9 +85,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::FourFours, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::FourFours.into()), + content_type: None, + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -94,9 +95,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::All, - content_type: ContentTypeFilter::Headers, - success: SuccessFilter::All, + strategy: None, + content_type: Some(ContentType::BlockHeadersByHash), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -104,9 +105,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::All, - content_type: ContentTypeFilter::HeadersByNumber, - success: SuccessFilter::All, + strategy: None, + content_type: Some(ContentType::BlockHeadersByNumber), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -114,9 +115,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::All, - content_type: ContentTypeFilter::Bodies, - success: SuccessFilter::All, + strategy: None, + content_type: Some(ContentType::BlockBodies), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -124,9 +125,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::All, - content_type: ContentTypeFilter::Receipts, - success: SuccessFilter::All, + strategy: None, + content_type: Some(ContentType::BlockReceipts), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -134,9 +135,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Latest, - content_type: ContentTypeFilter::Headers, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Latest.into()), + content_type: Some(ContentType::BlockHeadersByHash), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -144,9 +145,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Latest, - content_type: ContentTypeFilter::HeadersByNumber, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Latest.into()), + content_type: Some(ContentType::BlockHeadersByNumber), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -154,9 +155,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Latest, - content_type: ContentTypeFilter::Bodies, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Latest.into()), + content_type: Some(ContentType::BlockBodies), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -164,9 +165,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Latest, - content_type: ContentTypeFilter::Receipts, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Latest.into()), + content_type: Some(ContentType::BlockReceipts), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -174,9 +175,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Random, - content_type: ContentTypeFilter::Headers, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Random.into()), + content_type: Some(ContentType::BlockHeadersByHash), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -184,9 +185,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Random, - content_type: ContentTypeFilter::HeadersByNumber, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Random.into()), + content_type: Some(ContentType::BlockHeadersByNumber), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -194,9 +195,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Random, - content_type: ContentTypeFilter::Bodies, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Random.into()), + content_type: Some(ContentType::BlockBodies), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -204,9 +205,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Random, - content_type: ContentTypeFilter::Receipts, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::Random.into()), + content_type: Some(ContentType::BlockReceipts), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -214,9 +215,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::FourFours, - content_type: ContentTypeFilter::Headers, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::FourFours.into()), + content_type: Some(ContentType::BlockHeadersByHash), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -224,9 +225,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::FourFours, - content_type: ContentTypeFilter::HeadersByNumber, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::FourFours.into()), + content_type: Some(ContentType::BlockHeadersByNumber), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -234,9 +235,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::FourFours, - content_type: ContentTypeFilter::Bodies, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::FourFours.into()), + content_type: Some(ContentType::BlockBodies), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -244,9 +245,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::FourFours, - content_type: ContentTypeFilter::Receipts, - success: SuccessFilter::All, + strategy: Some(HistorySelectionStrategy::FourFours.into()), + content_type: Some(ContentType::BlockReceipts), + audit_result: None, network: SubProtocol::History },), Period::Hour, @@ -254,9 +255,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::All, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: None, + content_type: None, + audit_result: None, network: SubProtocol::State },), Period::Hour, @@ -264,9 +265,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Latest, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(StateSelectionStrategy::Latest.into()), + content_type: None, + audit_result: None, network: SubProtocol::State },), Period::Hour, @@ -274,9 +275,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::StateRoots, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(StateSelectionStrategy::StateRoots.into()), + content_type: None, + audit_result: None, network: SubProtocol::State },), Period::Hour, @@ -284,9 +285,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::All, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: None, + content_type: None, + audit_result: None, network: SubProtocol::Beacon },), Period::Hour, @@ -294,9 +295,9 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { ), get_audit_stats( filter_audits(AuditFilters { - strategy: StrategyFilter::Latest, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(BeaconSelectionStrategy::Latest.into()), + content_type: None, + audit_result: None, network: SubProtocol::Beacon },), Period::Hour, diff --git a/glados-core/src/stats.rs b/glados-core/src/stats.rs index 089cc262..200f0a0d 100644 --- a/glados-core/src/stats.rs +++ b/glados-core/src/stats.rs @@ -1,19 +1,17 @@ -use std::fmt::Display; +use std::fmt; use chrono::{DateTime, Utc}; use sea_orm::{ sea_query::{Expr, IntoCondition}, + strum::EnumMessage, ColumnTrait, DatabaseConnection, DbErr, EntityTrait, JoinType, PaginatorTrait, QueryFilter, - QuerySelect, RelationTrait, Select, + QuerySelect, QueryTrait, RelationTrait, Select, }; -use serde::Deserialize; +use serde::{de, Deserialize, Serialize}; use entity::{ - content::{self, SubProtocol}, - content_audit::{ - self, AuditResult, BeaconSelectionStrategy, HistorySelectionStrategy, SelectionStrategy, - StateSelectionStrategy, - }, + content::{self, ContentType, SubProtocol}, + content_audit::{self, serialize_selection_strategy, AuditResult, SelectionStrategy}, }; /// Generates a SeaORM select query for audits based on the provided filters. @@ -21,76 +19,35 @@ use entity::{ /// TODO: add support for filtering by portal client pub fn filter_audits(filters: AuditFilters) -> Select { // This base query will have filters added to it - let audits = content_audit::Entity::find(); - let audits = audits.join( - JoinType::Join, - content_audit::Relation::Content - .def() - .on_condition(move |_left, _right| { - content::Column::ProtocolId - .eq(filters.network) - .into_condition() - }), - ); - // Strategy filters - let audits = match filters.strategy { - StrategyFilter::All => audits, - StrategyFilter::Random => audits.filter( - content_audit::Column::StrategyUsed - .eq(SelectionStrategy::History(HistorySelectionStrategy::Random)), - ), - StrategyFilter::Latest => audits.filter(content_audit::Column::StrategyUsed.eq( - match filters.network { - SubProtocol::History => { - SelectionStrategy::History(HistorySelectionStrategy::Latest) - } - SubProtocol::State => SelectionStrategy::State(StateSelectionStrategy::Latest), - SubProtocol::Beacon => SelectionStrategy::Beacon(BeaconSelectionStrategy::Latest), - }, - )), - StrategyFilter::Oldest => audits.filter(content_audit::Column::StrategyUsed.eq( - SelectionStrategy::History(HistorySelectionStrategy::OldestUnaudited), - )), - StrategyFilter::FourFours => audits.filter(content_audit::Column::StrategyUsed.eq( - SelectionStrategy::History(HistorySelectionStrategy::FourFours), - )), - StrategyFilter::StateRoots => audits.filter( - content_audit::Column::StrategyUsed - .eq(SelectionStrategy::State(StateSelectionStrategy::StateRoots)), - ), - }; - // Success filters - let audits = match filters.success { - SuccessFilter::All => audits, - SuccessFilter::Success => { - audits.filter(content_audit::Column::Result.eq(AuditResult::Success)) - } - SuccessFilter::Failure => { - audits.filter(content_audit::Column::Result.eq(AuditResult::Failure)) - } - }; - // Content type filters - match filters.content_type { - ContentTypeFilter::All => audits, - ContentTypeFilter::Headers => { - audits.filter(Expr::cust("get_byte(content.content_key, 0) = 0x00").into_condition()) - } - ContentTypeFilter::Bodies => { - audits.filter(Expr::cust("get_byte(content.content_key, 0) = 0x01").into_condition()) - } - ContentTypeFilter::Receipts => { - audits.filter(Expr::cust("get_byte(content.content_key, 0) = 0x02").into_condition()) - } - ContentTypeFilter::HeadersByNumber => { - audits.filter(Expr::cust("get_byte(content.content_key, 0) = 0x03").into_condition()) - } - ContentTypeFilter::AccountTrieNodes => { - audits.filter(Expr::cust("get_byte(content.content_key, 0) = 0x20").into_condition()) - } - ContentTypeFilter::BlockRoots => { - audits.filter(Expr::cust("get_byte(content.content_key, 0) = 0x10").into_condition()) - } - } + content_audit::Entity::find() + .join( + JoinType::Join, + content_audit::Relation::Content + .def() + .on_condition(move |_left, _right| { + content::Column::ProtocolId + .eq(filters.network) + .into_condition() + }), + ) + // Strategy filters + .apply_if(filters.strategy, |query, audit_strategy| { + query.filter(content_audit::Column::StrategyUsed.eq(audit_strategy)) + }) + // Success filters + .apply_if(filters.audit_result, |query, audit_result| { + query.filter(content_audit::Column::Result.eq(audit_result)) + }) + // Content type filters + .apply_if(filters.content_type, |query, content_type| { + query.filter( + Expr::cust( + &("get_byte(content.content_key, 0) = ".to_string() + + &content_type.to_string()), + ) + .into_condition(), + ) + }) } /// Counts new content items for the given subprotocol and period @@ -117,24 +74,22 @@ pub async fn get_audit_stats( ) -> Result { let cutoff = period.cutoff_time(); - let total_audits = filtered + let (total_audits, total_passes): (i64, i64) = (filtered .clone() .filter(content_audit::Column::CreatedAt.gt(cutoff)) - .count(conn) - .await? as u32; - - let total_passes = filtered - .filter(content_audit::Column::CreatedAt.gt(cutoff)) - .filter(content_audit::Column::Result.eq(AuditResult::Success)) - .count(conn) - .await? as u32; - - // In case the numbers change in between queries, make sure passes don't exceed total audits - let total_passes = std::cmp::min(total_passes, total_audits); + .select_only() + .column_as(Expr::cust("COUNT(1)"), "total_audits") + .column_as( + Expr::cust("COALESCE(SUM(CASE WHEN result = 1 THEN 1 ELSE 0 END),0)"), + "total_passes", + ) + .into_tuple() + .all(conn) + .await?)[0]; let total_failures = total_audits - total_passes; - let audits_per_minute = (60 * total_audits) + let audits_per_minute = (60 * total_audits as u32) .checked_div(period.total_seconds()) .unwrap_or(0); @@ -150,15 +105,16 @@ pub async fn get_audit_stats( Ok(AuditStats { period, - total_audits, - total_passes, + total_audits: total_audits as u32, + total_passes: total_passes as u32, pass_percent, - total_failures, + total_failures: total_failures as u32, fail_percent, audits_per_minute, }) } +#[derive(Serialize)] pub struct AuditStats { pub period: Period, pub total_audits: u32, @@ -169,14 +125,15 @@ pub struct AuditStats { pub audits_per_minute: u32, } +#[derive(Serialize)] pub enum Period { Hour, Day, Week, } -impl Display for Period { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for Period { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let time_period = match self { Period::Hour => "hour", Period::Day => "day", @@ -205,52 +162,52 @@ impl Period { } } -#[derive(Deserialize, Copy, Clone)] +#[derive(Clone, Copy)] pub struct AuditFilters { - pub strategy: StrategyFilter, - pub content_type: ContentTypeFilter, - pub success: SuccessFilter, pub network: SubProtocol, + pub strategy: Option, + pub content_type: Option, + pub audit_result: Option, } -#[derive(Deserialize, Copy, Clone)] -pub enum StrategyFilter { - All, - Random, - Latest, - Oldest, - FourFours, - StateRoots, -} +impl<'de> de::Deserialize<'de> for AuditFilters { + fn deserialize(deserializer: D) -> Result + where + D: de::Deserializer<'de>, + { + #[derive(Deserialize)] + struct IntermediateAuditFilters { + network: SubProtocol, + content_type: Option, + audit_result: Option, + strategy: Option, + } -impl Display for StrategyFilter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let name = match &self { - StrategyFilter::All => "All", - StrategyFilter::Random => "Random", - StrategyFilter::Latest => "Latest", - StrategyFilter::Oldest => "Oldest", - StrategyFilter::FourFours => "4444s", - StrategyFilter::StateRoots => "State Roots", + let intermediate_filter = IntermediateAuditFilters::deserialize(deserializer)?; + + let strategy = match intermediate_filter.strategy { + Some(strat) => Some( + serialize_selection_strategy(intermediate_filter.network, &strat).map_err( + |_| { + de::Error::custom(format!( + "unknown variant for {}: {}", + intermediate_filter + .network + .get_message() + .expect("Subprotocol missing message"), + strat + )) + }, + )?, + ), + None => None, }; - write!(f, "{}", name) - } -} -#[derive(Deserialize, Copy, Clone)] -pub enum SuccessFilter { - All, - Success, - Failure, -} - -#[derive(Deserialize, Copy, Clone)] -pub enum ContentTypeFilter { - All, - Headers, - Bodies, - Receipts, - AccountTrieNodes, - BlockRoots, - HeadersByNumber, + Ok(AuditFilters { + network: intermediate_filter.network, + content_type: intermediate_filter.content_type, + audit_result: intermediate_filter.audit_result, + strategy, + }) + } } diff --git a/glados-web/assets/js/audit_dashboard.js b/glados-web/assets/js/audit_dashboard.js new file mode 100644 index 00000000..e8fc0fd5 --- /dev/null +++ b/glados-web/assets/js/audit_dashboard.js @@ -0,0 +1,351 @@ +import { Spinner, spinnerOpts } from "./spin_conf.js"; + +let customSpinnerOpts = { + ...spinnerOpts, + width: 30, + radius: 25, +}; + +let summaryController = null; +let listController = null; + +function shorten(hex) { + return `${hex.substring(0, 6)}...${hex.substring(hex.length - 4)}`; +} + +function formatTimeAgo(dateStr) { + const YEAR_MS = 365 * 24 * 60 * 60 * 1000; + const DAY_MS = 24 * 60 * 60 * 1000; + const HOUR_MS = 60 * 60 * 1000; + const MINUTE_MS = 60 * 1000; + const SECOND_MS = 1000; + + const date = Date.parse(dateStr); + const delta = Date.now() - date; + + const years = Math.floor(delta / YEAR_MS); + const days = Math.floor((delta - years * YEAR_MS) / DAY_MS); + const hours = Math.floor((delta - years * YEAR_MS - days * DAY_MS) / HOUR_MS); + const minutes = Math.floor( + (delta - years * YEAR_MS - days * DAY_MS - hours * HOUR_MS) / MINUTE_MS, + ); + const seconds = Math.floor( + (delta - + years * YEAR_MS - + days * DAY_MS - + hours * HOUR_MS - + minutes * MINUTE_MS) / + SECOND_MS, + ); + + let formated = ""; + if (years > 0) { + formated += `${years}y`; + } + if (days > 0) { + formated += `${days}d`; + } + if (hours > 0) { + formated += `${hours}h`; + } + if (minutes > 0) { + formated += `${minutes}m`; + } + // Don't show seconds after a day + if ((seconds > 0 || formated == "") && days == 0 && years == 0) { + formated += `${seconds}s`; + } + + return formated + " ago"; +} + +function updateSummary(queryString) { + const auditListSummary = document.getElementById("audit-summary"); + + if (summaryController) { + summaryController.abort("Changed filters"); + } + + summaryController = new AbortController(); + let signal = summaryController.signal; + + auditListSummary.tBodies[0].innerHTML = ""; + const spinner = new Spinner(customSpinnerOpts).spin(auditListSummary); + + fetch(`/api/audits-stats/?${queryString}`, { signal }) + .then((response) => { + if (!response.ok) { + throw new Error("Network response was not ok"); + } + return response.json(); + }) + .then((data) => { + for (const periodStats of data) { + const row = document.createElement("tr"); + + const period = document.createElement("td"); + period.innerHTML = periodStats.period; + row.appendChild(period); + + const totalAudits = document.createElement("td"); + totalAudits.innerHTML = periodStats.total_audits.toLocaleString(); + row.appendChild(totalAudits); + + const totalPasses = document.createElement("td"); + totalPasses.innerHTML = periodStats.total_passes.toLocaleString(); + row.appendChild(totalPasses); + + const totalFailures = document.createElement("td"); + totalFailures.innerHTML = periodStats.total_failures.toLocaleString(); + row.appendChild(totalFailures); + + const passRate = document.createElement("td"); + passRate.innerHTML = periodStats.pass_percent.toFixed(1) + "%"; + row.appendChild(passRate); + + const failRate = document.createElement("td"); + failRate.innerHTML = periodStats.fail_percent.toFixed(1) + "%"; + row.appendChild(failRate); + + const auditsPerMinute = document.createElement("td"); + auditsPerMinute.innerHTML = + periodStats.audits_per_minute.toLocaleString(); + row.appendChild(auditsPerMinute); + + auditListSummary.tBodies[0].appendChild(row); + } + }) + .catch((error) => { + console.error("There was a problem with the fetch operation:", error); + }) + .finally(() => { + listController = null; + spinner.stop(); + }); +} +function updateList(queryString) { + const auditListTable = document.getElementById("audit-list"); + + if (listController) { + listController.abort("Changed filters"); + } + + listController = new AbortController(); + let signal = listController.signal; + + auditListTable.tBodies[0].innerHTML = ""; + const spinner = new Spinner(customSpinnerOpts).spin(auditListTable); + + fetch(`/api/audits/?${queryString}`, { signal }) + .then((response) => { + if (!response.ok) { + throw new Error("Network response was not ok"); + } + return response.json(); + }) + .then((data) => { + for (const audit of data) { + const row = document.createElement("tr"); + + const id = document.createElement("td"); + if (audit.has_trace) { + const idAnchor = document.createElement("a"); + idAnchor.href = `/audit/id/${audit.id}`; + idAnchor.innerHTML = audit.id; + id.appendChild(idAnchor); + } else { + id.innerHTML = audit.id; + } + row.appendChild(id); + + const result = document.createElement("td"); + const resultSpan = document.createElement("span"); + resultSpan.classList = ["badge"]; + if (audit.is_success) { + resultSpan.classList.add("text-bg-success"); + resultSpan.innerHTML = "Success"; + } else { + resultSpan.classList.add("text-bg-danger"); + resultSpan.innerHTML = "Fail"; + } + resultSpan.innerHTML = audit.is_success ? "Success" : "Fail"; + result.appendChild(resultSpan); + row.appendChild(result); + + const contentType = document.createElement("td"); + contentType.innerHTML = audit.content_type; + row.appendChild(contentType); + + const strategy = document.createElement("td"); + strategy.innerHTML = audit.strategy; + row.appendChild(strategy); + + const contentKey = document.createElement("td"); + const contentKeyAnchor = document.createElement("a"); + contentKeyAnchor.href = `/content/key/${audit.content_key}/`; + contentKeyAnchor.innerHTML = shorten(audit.content_key); + contentKey.appendChild(contentKeyAnchor); + row.appendChild(contentKey); + + const contentId = document.createElement("td"); + const contentIdAnchor = document.createElement("a"); + contentIdAnchor.href = `/content/id/${audit.content_id}/`; + contentIdAnchor.innerHTML = shorten(audit.content_id); + contentId.appendChild(contentIdAnchor); + row.appendChild(contentId); + + const firstAvailable = document.createElement("td"); + firstAvailable.title = audit.content_available_at; + firstAvailable.innerHTML = formatTimeAgo(audit.content_available_at); + row.appendChild(firstAvailable); + + const auditedAt = document.createElement("td"); + auditedAt.title = audit.audited_at; + auditedAt.innerHTML = formatTimeAgo(audit.audited_at); + row.appendChild(auditedAt); + + const client = document.createElement("td"); + client.innerHTML = audit.client_version_info; + row.appendChild(client); + + auditListTable.tBodies[0].appendChild(row); + } + }) + .catch((error) => { + console.error( + "There was a problem with the fetch operation:", + error.message, + ); + }) + .finally(() => { + listController = null; + spinner.stop(); + }); +} + +function updateDashboard(network, strategy, contentType, auditResult) { + const params = { + network: network, + }; + if (strategy) { + params.strategy = strategy; + } + if (contentType) { + params.content_type = contentType; + } + if (auditResult) { + params.audit_result = auditResult; + } + + const queryString = new URLSearchParams(params).toString(); + + updateList(queryString); + updateSummary(queryString); +} + +export var initAuditDashboard = async function () { + const network = new URL(window.location).searchParams + .get("network") + .toLowerCase(); + const contentGroup = document.querySelector("#content-buttons"); + const strategyGroup = document.querySelector("#strategy-buttons"); + const auditResultGroup = document.querySelector("#audit-result-buttons"); + + const activateButton = (btn, group) => { + // Deactivate all buttons in the group + group.querySelectorAll(".btn").forEach((button) => { + button.classList.remove("active"); + }); + // Activate the clicked button + btn.classList.add("active"); + }; + + const handleButtonClick = (event, group) => { + if (event.target.classList.contains("btn")) { + activateButton(event.target, group); + } + + // Store the active button in each group in session storage + sessionStorage.setItem( + `${network}-content-filter`, + `#${contentGroup.querySelector(".active").id}`, + ); + sessionStorage.setItem( + `${network}-strategy-filter`, + `#${strategyGroup.querySelector(".active").id}`, + ); + sessionStorage.setItem( + `${network}-audit-result-filter`, + `#${auditResultGroup.querySelector(".active").id}`, + ); + + // Get the active button's filter string in each group + const selectedContent = contentGroup + .querySelector(".active") + .getAttribute("filter"); + const selectedStrategy = strategyGroup + .querySelector(".active") + .getAttribute("filter"); + const selectedSuccess = auditResultGroup + .querySelector(".active") + .getAttribute("filter"); + + updateDashboard( + network, + selectedStrategy, + selectedContent, + selectedSuccess, + ); + }; + + // Check whether the browser's session storage contains a filter for the given group, otherwise use default + const setInitialButton = (filter, defaultButton, group) => { + if (sessionStorage.getItem(filter) !== null) { + activateButton( + document.querySelector(`${sessionStorage.getItem(filter)}`), + group, + ); + } else { + activateButton(document.querySelector(defaultButton), group); + } + }; + + // Attach event listeners to each button group + contentGroup.addEventListener("click", (event) => + handleButtonClick(event, contentGroup), + ); + strategyGroup.addEventListener("click", (event) => + handleButtonClick(event, strategyGroup), + ); + auditResultGroup.addEventListener("click", (event) => + handleButtonClick(event, auditResultGroup), + ); + + setInitialButton( + `${network}-content-filter`, + "#all-content-button", + contentGroup, + ); + setInitialButton( + `${network}-strategy-filter`, + "#all-strategy-button", + strategyGroup, + ); + setInitialButton( + `${network}-audit-result-filter`, + "#all-audit-result-button", + auditResultGroup, + ); + + const selectedContent = contentGroup + .querySelector(".active") + .getAttribute("filter"); + const selectedStrategy = strategyGroup + .querySelector(".active") + .getAttribute("filter"); + const selectedSuccess = auditResultGroup + .querySelector(".active") + .getAttribute("filter"); + + updateDashboard(network, selectedStrategy, selectedContent, selectedSuccess); +}; diff --git a/glados-web/src/lib.rs b/glados-web/src/lib.rs index 4db1741b..4cfdf911 100644 --- a/glados-web/src/lib.rs +++ b/glados-web/src/lib.rs @@ -83,7 +83,6 @@ pub async fn run_glados_web(config: Arc) -> Result<()> { ) .route("/audit/id/:audit_id", get(routes::contentaudit_detail)) .route("/audits/", get(routes::contentaudit_dashboard)) - .route("/audits/filter/", get(routes::contentaudit_filter)) .route( "/api/hourly-success-rate/", get(routes::hourly_success_rate), @@ -135,6 +134,8 @@ pub async fn run_glados_web(config: Arc) -> Result<()> { get(routes::weekly_transfer_failures), ) .route("/api/audit-block-status/", get(routes::audit_block_status)) + .route("/api/audits/", get(routes::audits_filter_api)) + .route("/api/audits-stats/", get(routes::audits_filter_stats_api)) .nest_service("/static/", serve_dir.clone()) .fallback_service(serve_dir) .layer(Extension(config)); diff --git a/glados-web/src/routes.rs b/glados-web/src/routes.rs index a8e8c238..148b86be 100644 --- a/glados-web/src/routes.rs +++ b/glados-web/src/routes.rs @@ -25,18 +25,20 @@ use ethportal_api::{ BeaconContentKey, HistoryContentKey, OverlayContentKey, StateContentKey, }; use sea_orm::{ - sea_query::{Expr, Query, SimpleExpr}, + sea_query::{Expr, JoinType, Query, SimpleExpr}, + strum::EnumMessage, ColumnTrait, ConnectionTrait, DatabaseConnection, DbBackend, EntityTrait, FromQueryResult, - Iterable, LoaderTrait, ModelTrait, QueryFilter, QueryOrder, QuerySelect, Statement, + Iterable, LoaderTrait, ModelTrait, QueryFilter, QueryOrder, QuerySelect, RelationTrait, + Statement, }; -use serde::Serialize; +use serde::{ser, ser::SerializeStruct, Serialize}; use tracing::{debug, error, info, warn}; use crate::templates::{ - AuditDashboardTemplate, AuditTableTemplate, CensusExplorerTemplate, ClientsTemplate, - ContentAuditDetailTemplate, ContentIdDetailTemplate, ContentIdListTemplate, - ContentKeyDetailTemplate, ContentKeyListTemplate, EnrDetailTemplate, HtmlTemplate, - IndexTemplate, NodeDetailTemplate, PaginatedCensusListTemplate, SingleCensusViewTemplate, + AuditDashboardTemplate, CensusExplorerTemplate, ClientsTemplate, ContentAuditDetailTemplate, + ContentIdDetailTemplate, ContentIdListTemplate, ContentKeyDetailTemplate, + ContentKeyListTemplate, EnrDetailTemplate, HtmlTemplate, IndexTemplate, NodeDetailTemplate, + PaginatedCensusListTemplate, SingleCensusViewTemplate, }; use crate::{state::State, templates::AuditTuple}; use entity::{ @@ -44,12 +46,14 @@ use entity::{ census_node::{Client, OperatingSystem, Version}, client_info, content, content::{ContentType, SubProtocol}, - content_audit::{self, AuditResult, SelectionStrategy}, + content_audit::{ + self, AuditResult, BeaconSelectionStrategy, HistorySelectionStrategy, SelectionStrategy, + StateSelectionStrategy, + }, execution_metadata, key_value, node, record, }; use glados_core::stats::{ - filter_audits, get_audit_stats, get_new_content_count, AuditFilters, ContentTypeFilter, Period, - StrategyFilter, SuccessFilter, + filter_audits, get_audit_stats, get_new_content_count, AuditFilters, Period, }; use migration::{Alias, Order}; @@ -62,13 +66,8 @@ pub async fn handle_error(_err: io::Error) -> impl IntoResponse { // Get the subprotocol from the query parameters, defaulting to History pub fn get_subprotocol_from_params(params: &HashMap) -> SubProtocol { - match params.get("network") { - None => SubProtocol::History, - Some(subprotocol) => match subprotocol.try_into().ok() { - Some(subprotocol) => subprotocol, - None => SubProtocol::History, - }, - } + SubProtocol::from_str(params.get("network").unwrap_or(&"".to_string())) + .unwrap_or(SubProtocol::History) } pub async fn network_overview( @@ -86,10 +85,10 @@ pub async fn network_overview( let radius_percentages = generate_radius_graph_data(&state, subprotocol).await; - let strategy: StrategyFilter = match subprotocol { - SubProtocol::History => StrategyFilter::FourFours, - SubProtocol::State => StrategyFilter::StateRoots, - SubProtocol::Beacon => StrategyFilter::Latest, + let strategy: SelectionStrategy = match subprotocol { + SubProtocol::History => HistorySelectionStrategy::Latest.into(), + SubProtocol::State => StateSelectionStrategy::StateRoots.into(), + SubProtocol::Beacon => BeaconSelectionStrategy::Latest.into(), }; // Run queries for content dashboard data concurrently @@ -97,9 +96,9 @@ pub async fn network_overview( get_new_content_count(subprotocol, Period::Hour, &state.database_connection,), get_audit_stats( filter_audits(AuditFilters { - strategy, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(strategy), + content_type: None, + audit_result: None, network: subprotocol, },), Period::Hour, @@ -108,9 +107,9 @@ pub async fn network_overview( get_new_content_count(subprotocol, Period::Day, &state.database_connection,), get_audit_stats( filter_audits(AuditFilters { - strategy, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(strategy), + content_type: None, + audit_result: None, network: subprotocol, },), Period::Day, @@ -119,9 +118,9 @@ pub async fn network_overview( get_new_content_count(subprotocol, Period::Week, &state.database_connection,), get_audit_stats( filter_audits(AuditFilters { - strategy, - content_type: ContentTypeFilter::All, - success: SuccessFilter::All, + strategy: Some(strategy), + content_type: None, + audit_result: None, network: subprotocol, },), Period::Week, @@ -467,6 +466,7 @@ pub async fn contentaudit_dashboard( subprotocol, content_types: ContentType::vec_subprotocol(subprotocol), strategies: SelectionStrategy::vec_subprotocol(subprotocol), + audit_results: AuditResult::iter().collect(), }; Ok(HtmlTemplate(template)) } @@ -694,27 +694,94 @@ pub struct NodeWithRadius { pub data_radius: Vec, } +#[derive(FromQueryResult, Debug, Clone)] +pub struct AuditWide { + id: i32, + has_trace: bool, + is_success: bool, + content_type: ContentType, + strategy: SelectionStrategy, + protocol: SubProtocol, + content_key: Vec, + content_id: Vec, + content_available_at: DateTime, + audited_at: DateTime, + client_version_info: String, +} + +impl ser::Serialize for AuditWide { + fn serialize(&self, serializer: S) -> Result + where + S: ser::Serializer, + { + let mut s = serializer.serialize_struct("AuditWide", 9)?; + s.serialize_field("id", &self.id)?; + s.serialize_field("has_trace", &self.has_trace)?; + s.serialize_field("is_success", &self.is_success)?; + s.serialize_field("content_type", &self.content_type.get_message())?; + s.serialize_field("strategy", &self.strategy.get_message())?; + s.serialize_field("protocol", &self.protocol.get_message())?; + s.serialize_field("content_key", &hex_encode(&self.content_key))?; + s.serialize_field("content_id", &hex_encode(&self.content_id))?; + s.serialize_field("content_available_at", &self.content_available_at)?; + s.serialize_field("audited_at", &self.audited_at)?; + s.serialize_field("client_version_info", &self.client_version_info)?; + s.end() + } +} + +/// Takes an AuditFilter object generated from http query params +/// Conditionally creates a query based on the filters +pub async fn audits_filter_api( + Extension(state): Extension>, + filters: HttpQuery, +) -> Result>, StatusCode> { + let audits = filter_audits(filters.0); + let filtered_audits = audits + .join(JoinType::Join, content_audit::Relation::ClientInfo.def()) + .order_by_desc(content_audit::Column::CreatedAt) + .select_only() + .column_as(content_audit::Column::Id, "id") + .column_as(Expr::col(content_audit::Column::Trace).ne(""), "has_trace") + .column_as( + Expr::col(content_audit::Column::Result).eq(AuditResult::Success), + "is_success", + ) + .column_as( + Expr::cust("GET_BYTE(content.content_key, 0)"), + "content_type", + ) + .column_as(content_audit::Column::StrategyUsed, "strategy") + .column_as(content::Column::ProtocolId, "protocol") + .column(content::Column::ContentKey) + .column(content::Column::ContentId) + .column_as(content::Column::FirstAvailableAt, "content_available_at") + .column_as(content_audit::Column::CreatedAt, "audited_at") + .column_as(client_info::Column::VersionInfo, "client_version_info") + .limit(30) + .into_model::() + .all(&state.database_connection) + .await + .map_err(|e| { + error!(err=?e, "Could not look up audits"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + Ok(Json(filtered_audits)) +} + /// Takes an AuditFilter object generated from http query params /// Conditionally creates a query based on the filters -pub async fn contentaudit_filter( +pub async fn audits_filter_stats_api( Extension(state): Extension>, filters: HttpQuery, -) -> Result, StatusCode> { +) -> Result, StatusCode> { let audits = filter_audits(filters.0); - let (hour_stats, day_stats, week_stats, filtered_audits) = tokio::join!( + let (hour_stats, day_stats, week_stats) = tokio::join!( get_audit_stats(audits.clone(), Period::Hour, &state.database_connection), get_audit_stats(audits.clone(), Period::Day, &state.database_connection), get_audit_stats(audits.clone(), Period::Week, &state.database_connection), - audits - .order_by_desc(content_audit::Column::CreatedAt) - .limit(30) - .all(&state.database_connection), ); - let filtered_audits = filtered_audits.map_err(|e| { - error!(err=?e, "Could not look up audits"); - StatusCode::INTERNAL_SERVER_ERROR - })?; let hour_stats = hour_stats.map_err(|e| { error!(err=?e, "Could not look up audit hourly stats"); StatusCode::INTERNAL_SERVER_ERROR @@ -728,17 +795,8 @@ pub async fn contentaudit_filter( StatusCode::INTERNAL_SERVER_ERROR })?; - let filtered_audits: Vec = - get_audit_tuples_from_audit_models(filtered_audits, &state.database_connection).await?; - - let template = AuditTableTemplate { - stats: [hour_stats, day_stats, week_stats], - audits: filtered_audits, - }; - - Ok(HtmlTemplate(template)) + Ok(Json([hour_stats, day_stats, week_stats])) } - #[derive(FromQueryResult, Serialize, Debug)] pub struct DeadZoneData { pub data_radius: Vec, diff --git a/glados-web/src/templates.rs b/glados-web/src/templates.rs index 1c333176..1e255ce7 100644 --- a/glados-web/src/templates.rs +++ b/glados-web/src/templates.rs @@ -12,15 +12,20 @@ use entity::{ census_node::{Client, OperatingSystem}, client_info, content::{self, ContentType, SubProtocol}, - content_audit, execution_metadata, key_value, node, record, + content_audit, + content_audit::{ + AuditResult, BeaconSelectionStrategy, HistorySelectionStrategy, SelectionStrategy, + StateSelectionStrategy, + }, + execution_metadata, key_value, node, record, }; -use glados_core::stats::{AuditStats, StrategyFilter}; +use glados_core::stats::AuditStats; #[derive(Template)] #[template(path = "index.html")] pub struct IndexTemplate { pub subprotocol: SubProtocol, - pub strategy: StrategyFilter, + pub strategy: SelectionStrategy, pub client_diversity_data: Vec, pub average_radius_chart: Vec, pub stats: [AuditStats; 3], @@ -103,14 +108,8 @@ pub struct ContentKeyListTemplate { pub struct AuditDashboardTemplate { pub subprotocol: SubProtocol, pub content_types: Vec, - pub strategies: Vec, -} - -#[derive(Template)] -#[template(path = "audit_table.html")] -pub struct AuditTableTemplate { - pub stats: [AuditStats; 3], - pub audits: Vec, + pub strategies: Vec, + pub audit_results: Vec, } #[derive(Template)] diff --git a/glados-web/templates/audit_dashboard.html b/glados-web/templates/audit_dashboard.html index 576999ab..7f38c4ca 100644 --- a/glados-web/templates/audit_dashboard.html +++ b/glados-web/templates/audit_dashboard.html @@ -3,6 +3,7 @@ {% block title %}Audit Dashboard{% endblock %} {% block head %} + {% endblock %} {% block content %}
@@ -10,44 +11,93 @@

Audit Dashboard

-
- - {% for content_type in content_types %} - - {% endfor %} -
-
- - {% for strategy in strategies %} - - {% endfor %} -
- -
- + {% for content_type in content_types %} + + {% endfor %} +
+
+ + {% for strategy in strategies %} + + {% endfor %} +
+
+ - - + {% for audit_result in audit_results %} + + {% endfor %}

-
+ +
+
+
+ + + + + + + + + + + + + + +
PeriodTotal auditsTotal audit passesTotal audit failuresPass rate (%)Failure rate (%)Audits per minute
+
+
+
+
+ + + + + + + + + + + + + + + + +
AuditResultContent TypeStrategyContent KeyContent IDContent first availableAuditedClient
+
+
+
+ - diff --git a/glados-web/templates/audit_table.html b/glados-web/templates/audit_table.html deleted file mode 100644 index 369d3ca4..00000000 --- a/glados-web/templates/audit_table.html +++ /dev/null @@ -1,88 +0,0 @@ -
-
-
- - - - - - - - - - - - - - {% for stat in stats %} - - - - - - - - - - {% endfor %} - -
PeriodTotal auditsTotal audit passesTotal audit failuresPass rate (%)Failure rate (%)Audits per minute
{{ stat.period.to_string() }}{{ stat.total_audits }}{{ stat.total_passes }}{{ stat.total_failures }}{{ "{:.1}"|format(stat.pass_percent) }}%{{ "{:.1}"|format(stat.fail_percent) }}%{{ stat.audits_per_minute }}
-
-
-
-
-
    -
    -
    -
    - - - - - - - - - - - - - - - - {% for (audit, content, client_info) in audits %} - - - - - - - - - - - - {% endfor %} - -
    AuditResultSub-protocolStrategyContent KeyContent IDContent first availableAudited atClient
    {% if audit.trace != "" %}{{ audit.id - }}{% - else - %} - {{ audit.id }}{% endif %} - {% - if audit.is_success() %}Success{% else %}Fail{% endif %}{{ content.protocol_id.as_text() }}{{ audit.strategy_as_text() }}{{ content.key_as_hex_short() - }} - {{ content.id_as_hex_short() - }} - {{ - content.available_at_humanized() - }}{{ audit.created_at_humanized() }} - {{ client_info.version_info }}
    -
    -
    -
    -
-
-
-
diff --git a/glados-web/templates/contentaudit_detail.html b/glados-web/templates/contentaudit_detail.html index f698845e..2ef2ee58 100644 --- a/glados-web/templates/contentaudit_detail.html +++ b/glados-web/templates/contentaudit_detail.html @@ -23,7 +23,7 @@ {% endif %}
  • Started: {{ audit.created_at }}
    • -
  • Result: {{ audit.result.as_text() }}
    • +
  • Result: {{ audit.result.get_message().unwrap() }}
  • Strategy: {{ audit.strategy_as_text() }}
    • diff --git a/glados-web/templates/index.html b/glados-web/templates/index.html index 5ef4621e..6f2ae124 100644 --- a/glados-web/templates/index.html +++ b/glados-web/templates/index.html @@ -24,21 +24,21 @@
    {% match strategy %} - {% when StrategyFilter::FourFours %} + {% when SelectionStrategy::History(HistorySelectionStrategy::FourFours) %} Glados continuously queries the Portal network for random 4444s headers, blocks, and receipts. Statistical results of these audits are displayed here and can be viewed in more granular detail on the Audit Dashboard. - {% when StrategyFilter::StateRoots %} + {% when SelectionStrategy::State(StateSelectionStrategy::StateRoots) %} Glados continuously queries the Portal network for random state roots. Statistical results of these audits are displayed here and can be viewed in more granular detail on the Audit Dashboard. - {% when StrategyFilter::Latest %} + {% when SelectionStrategy::Beacon(BeaconSelectionStrategy::Latest) %} Glados continuously queries the Portal network recent beacon block roots. Statistical results of these audits are displayed here and can be viewed in more granular detail on the Audit Dashboard. {% else %} - No explanation available for {{ strategy }} + No explanation available for {{ strategy.get_message().expect("Strategy missing message") }} {% endmatch %}
    -

    {{ strategy }} stats

    +

    {{ strategy.get_message().expect("Strategy missing message") }} stats

    @@ -220,7 +220,7 @@

    Client Count

    style="width: auto" > {% for content_type in content_types %} - + {% endfor %}
    From ed6835263b5e8106c1f3ee6ddd23e1d674ff658b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20P=C3=A9rez?= Date: Wed, 21 May 2025 22:16:31 +0200 Subject: [PATCH 3/3] chore: remove latest state audit strategy --- entity/src/audit_stats.rs | 5 --- entity/src/content_audit.rs | 3 -- glados-audit/src/stats.rs | 13 ------ glados-web/assets/js/stats_history.js | 10 ++--- migration/src/lib.rs | 2 + ...250521_164847_remove_state_latest_stats.rs | 40 +++++++++++++++++++ 6 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 migration/src/m20250521_164847_remove_state_latest_stats.rs diff --git a/entity/src/audit_stats.rs b/entity/src/audit_stats.rs index 371e6713..f78de621 100644 --- a/entity/src/audit_stats.rs +++ b/entity/src/audit_stats.rs @@ -34,7 +34,6 @@ pub struct Model { pub success_rate_history_four_fours_bodies: f32, pub success_rate_history_four_fours_receipts: f32, pub success_rate_state_all: f32, - pub success_rate_state_latest: f32, pub success_rate_state_state_roots: f32, pub success_rate_beacon_all: f32, pub success_rate_beacon_latest: f32, @@ -71,7 +70,6 @@ pub async fn create( success_rate_history_four_fours_bodies: f32, success_rate_history_four_fours_receipts: f32, success_rate_state_all: f32, - success_rate_state_latest: f32, success_rate_state_state_roots: f32, success_rate_beacon_all: f32, success_rate_beacon_latest: f32, @@ -109,7 +107,6 @@ pub async fn create( success_rate_history_four_fours_bodies: Set(success_rate_history_four_fours_bodies), success_rate_history_four_fours_receipts: Set(success_rate_history_four_fours_receipts), success_rate_state_all: Set(success_rate_state_all), - success_rate_state_latest: Set(success_rate_state_latest), success_rate_state_state_roots: Set(success_rate_state_state_roots), success_rate_beacon_all: Set(success_rate_beacon_all), success_rate_beacon_latest: Set(success_rate_beacon_latest), @@ -150,7 +147,6 @@ pub struct StateStats { id: i32, timestamp: DateTime, success_rate_state_all: f32, - success_rate_state_latest: f32, success_rate_state_state_roots: f32, } @@ -228,7 +224,6 @@ pub async fn get_weekly_state_stats( Column::Id, Column::Timestamp, Column::SuccessRateStateAll, - Column::SuccessRateStateLatest, Column::SuccessRateStateStateRoots, ]) .filter(Column::Timestamp.gt(beginning)) diff --git a/entity/src/content_audit.rs b/entity/src/content_audit.rs index a1380d78..63026aec 100644 --- a/entity/src/content_audit.rs +++ b/entity/src/content_audit.rs @@ -164,15 +164,12 @@ pub enum StateSelectionStrategy { /// Does a random walk of the state at a random walk. #[strum(message = "State Roots", props(subprotocol = "2"))] StateRoots = 0, - #[strum(message = "Latest", props(subprotocol = "2"))] - Latest = 1, } impl From for StateSelectionStrategy { fn from(value: i32) -> Self { match value { 0 => StateSelectionStrategy::StateRoots, - 1 => StateSelectionStrategy::Latest, _ => panic!("Invalid value for StateSelectionStrategy"), } } diff --git a/glados-audit/src/stats.rs b/glados-audit/src/stats.rs index d43574be..2b5ae84b 100644 --- a/glados-audit/src/stats.rs +++ b/glados-audit/src/stats.rs @@ -48,7 +48,6 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { history_fourfours_bodies, history_fourfours_receipts, state_all, - state_latest, state_state_roots, beacon_all, beacon_latest, @@ -263,16 +262,6 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { Period::Hour, conn ), - get_audit_stats( - filter_audits(AuditFilters { - strategy: Some(StateSelectionStrategy::Latest.into()), - content_type: None, - audit_result: None, - network: SubProtocol::State - },), - Period::Hour, - conn - ), get_audit_stats( filter_audits(AuditFilters { strategy: Some(StateSelectionStrategy::StateRoots.into()), @@ -331,7 +320,6 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { let success_rate_history_fourfours_bodies = history_fourfours_bodies?.pass_percent; let success_rate_history_fourfours_receipts = history_fourfours_receipts?.pass_percent; let success_rate_state_all = state_all?.pass_percent; - let success_rate_state_latest = state_latest?.pass_percent; let success_rate_state_state_roots = state_state_roots?.pass_percent; let success_rate_beacon_all = beacon_all?.pass_percent; let success_rate_beacon_latest = beacon_latest?.pass_percent; @@ -361,7 +349,6 @@ async fn record_current_stats(conn: &DatabaseConnection) -> Result<(), DbErr> { success_rate_history_fourfours_bodies, success_rate_history_fourfours_receipts, success_rate_state_all, - success_rate_state_latest, success_rate_state_state_roots, success_rate_beacon_all, success_rate_beacon_latest, diff --git a/glados-web/assets/js/stats_history.js b/glados-web/assets/js/stats_history.js index 36f91acd..45ab0d7b 100644 --- a/glados-web/assets/js/stats_history.js +++ b/glados-web/assets/js/stats_history.js @@ -352,13 +352,9 @@ function getCurrentSubprotocol() { }, state: { baseUrl: "api/stats-state/?weeks-ago=", - keys: [ - "success_rate_state_all", - "success_rate_state_latest", - "success_rate_state_state_roots", - ], - selectedIndexes: [2], - labels: ["All", "Latest", "State Roots"], + keys: ["success_rate_state_all", "success_rate_state_state_roots"], + selectedIndexes: [1], + labels: ["All", "State Roots"], }, beacon: { baseUrl: "api/stats-beacon/?weeks-ago=", diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 128a68b4..2c301469 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -28,6 +28,7 @@ mod m20250314_144135_add_history_headers_by_number_audit_stats; mod m20250317_183352_refactor_history_audit_stats; mod m20250404_202958_internal_failures_replace_node_with_record; mod m20250404_220628_add_client_info_to_census_node; +mod m20250521_164847_remove_state_latest_stats; pub struct Migrator; @@ -59,6 +60,7 @@ impl MigratorTrait for Migrator { Box::new(m20250317_183352_refactor_history_audit_stats::Migration), Box::new(m20250404_202958_internal_failures_replace_node_with_record::Migration), Box::new(m20250404_220628_add_client_info_to_census_node::Migration), + Box::new(m20250521_164847_remove_state_latest_stats::Migration), ] } } diff --git a/migration/src/m20250521_164847_remove_state_latest_stats.rs b/migration/src/m20250521_164847_remove_state_latest_stats.rs new file mode 100644 index 00000000..4d6a897d --- /dev/null +++ b/migration/src/m20250521_164847_remove_state_latest_stats.rs @@ -0,0 +1,40 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(AuditStats::Table) + .drop_column(AuditStats::SuccessRateStateLatest) + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(AuditStats::Table) + .add_column_if_not_exists( + ColumnDef::new(AuditStats::SuccessRateStateLatest) + .float() + .default(0.0), + ) + .to_owned(), + ) + .await + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum AuditStats { + Table, + SuccessRateStateLatest, +}