diff --git a/cmd/loadeljson.go b/cmd/loadeljson.go new file mode 100644 index 00000000..9d5a4967 --- /dev/null +++ b/cmd/loadeljson.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "fmt" + "os" + "pbench/cmd/loadeljson" + "pbench/utils" + "runtime" + "time" + + "github.com/spf13/cobra" +) + +// loadElJsonCmd represents the loadeljson command +var loadElJsonCmd = &cobra.Command{ + Use: `loadeljson [flags] [list of files or directories to process]`, + Short: "Load event listener JSON files into database and run recorders", + Long: `Load event listener JSON files (QueryCompletedEvent) into database and run recorders`, + DisableFlagsInUseLine: true, + Args: func(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return fmt.Errorf("requires at least 1 arg, only received %d", len(args)) + } + if loadeljson.Parallelism < 1 || loadeljson.Parallelism > runtime.NumCPU() { + return fmt.Errorf("invalid parallelism: %d, it should be >= 1 and <= %d", loadeljson.Parallelism, runtime.NumCPU()) + } + utils.ExpandHomeDirectory(&loadeljson.MySQLCfgPath) + utils.ExpandHomeDirectory(&loadeljson.InfluxCfgPath) + utils.ExpandHomeDirectory(&loadeljson.OutputPath) + return nil + }, + Run: loadeljson.Run, +} + +func init() { + RootCmd.AddCommand(loadElJsonCmd) + wd, _ := os.Getwd() + loadElJsonCmd.Flags().StringVarP(&loadeljson.RunName, "name", "n", fmt.Sprintf("load_el_%s", time.Now().Format(utils.DirectoryNameTimeFormat)), `Assign a name to this run. (default: "load_el_")`) + loadElJsonCmd.Flags().StringVarP(&loadeljson.Comment, "comment", "c", "", `Add a comment to this run (optional)`) + loadElJsonCmd.Flags().BoolVarP(&loadeljson.RecordRun, "record-run", "r", false, "Record all the loaded JSON as a run") + loadElJsonCmd.Flags().StringVarP(&loadeljson.OutputPath, "output-path", "o", wd, "Output directory path") + loadElJsonCmd.Flags().IntVarP(&loadeljson.Parallelism, "parallel", "P", runtime.NumCPU(), "Number of parallel threads to load json files") + loadElJsonCmd.Flags().StringVar(&loadeljson.InfluxCfgPath, "influx", "", "InfluxDB connection config for run recorder (optional)") + loadElJsonCmd.Flags().StringVar(&loadeljson.MySQLCfgPath, "mysql", "", "MySQL connection config for event listener and run recorder (optional)") + loadElJsonCmd.Flags().BoolVar(&loadeljson.IsNDJSON, "ndjson", false, "Process files as NDJSON (newline-delimited JSON) format") +} diff --git a/cmd/loadeljson/202506160849_0_error.ndjson b/cmd/loadeljson/202506160849_0_error.ndjson new file mode 100644 index 00000000..ddd13436 --- /dev/null +++ b/cmd/loadeljson/202506160849_0_error.ndjson @@ -0,0 +1 @@ +{"instanceId":"978ecaae-0fe1-4997-8bfc-78b04e54662a","clusterName":"cluster1","queryCreatedEvent":null,"queryCompletedEvent":{"metadata":{"queryId":"20250616_084808_00027_ux3a6","transactionId":"55e64835-74b8-49c3-8316-e3bdbbee6e04","tracingId":"noop_dummy_id","query":"--TPC-DS Q3<<>>select dt.d_year<<>> , item.i_brand_id brand_id<<>> , item.i_brand brand<<>> , sum(ss_net_profit) sum_agg<<>>from date_dim dt<<>> , store_sales<<>> , item<<>>where dt.d_date_sk = store_sales.ss_sold_date_sk<<>> and store_sales.ss_item_sk = item.i_item_sk<<>> and item.i_manufact_id = 445<<>> and dt.d_moy = 12<<>>group by dt.d_year<<>> , item.i_brand<<>> , item.i_brand_id<<>>order by dt.d_year<<>> , sum_agg desc<<>> , brand_id<<>>limit 100","queryHash":"eb3623724f2f753c","preparedQuery":null,"queryState":"FINISHED","uri":"http://10.128.58.157:8585/v1/query/20250616_084808_00027_ux3a6","plan":"","jsonPlan":"{\n \"0\" : {\n \"plan\" : {\n \"id\" : \"16\",\n \"name\" : \"Output\",\n \"identifier\" : \"[d_year, brand_id, brand, sum_agg]\",\n \"details\" : \"brand_id := i_brand_id (3:8)\\nbrand := i_brand (4:8)\\nsum_agg := sum (5:8)\\n\",\n \"children\" : [ {\n \"id\" : \"523\",\n \"name\" : \"TopN\",\n \"identifier\" : \"[100 by (d_year ASC_NULLS_LAST, sum DESC_NULLS_LAST, i_brand_id ASC_NULLS_LAST)]\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"723\",\n \"name\" : \"LocalExchange\",\n \"identifier\" : \"[SINGLE] ()\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"637\",\n \"name\" : \"RemoteSource\",\n \"identifier\" : \"[1]\",\n \"details\" : \"\",\n \"children\" : [ ],\n \"remoteSources\" : [ \"1\" ],\n \"estimates\" : [ ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : { }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : { }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : { }\n } ]\n}\n },\n \"1\" : {\n \"plan\" : {\n \"id\" : \"522\",\n \"name\" : \"TopNPartial\",\n \"identifier\" : \"[100 by (d_year ASC_NULLS_LAST, sum DESC_NULLS_LAST, i_brand_id ASC_NULLS_LAST)]\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"830\",\n \"name\" : \"Project\",\n \"identifier\" : \"[projectLocality = LOCAL]\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"8\",\n \"name\" : \"Aggregate(FINAL)[d_year, i_brand, i_brand_id][$hashvalue]\",\n \"identifier\" : \"\",\n \"details\" : \"sum := \\\"presto.default.sum\\\"((sum_27)) (5:8)\\n\",\n \"children\" : [ {\n \"id\" : \"749\",\n \"name\" : \"LocalExchange\",\n \"identifier\" : \"[HASH][$hashvalue] (d_year, i_brand, i_brand_id)\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"755\",\n \"name\" : \"RemoteSource\",\n \"identifier\" : \"[2]\",\n \"details\" : \"\",\n \"children\" : [ ],\n \"remoteSources\" : [ \"2\" ],\n \"estimates\" : [ ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : { }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : { }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : { }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : { }\n } ]\n}\n },\n \"2\" : {\n \"plan\" : {\n \"id\" : \"753\",\n \"name\" : \"Aggregate(PARTIAL)[d_year, i_brand, i_brand_id][$hashvalue_37]\",\n \"identifier\" : \"\",\n \"details\" : \"sum_27 := \\\"presto.default.sum\\\"((ss_net_profit)) (5:8)\\n\",\n \"children\" : [ {\n \"id\" : \"829\",\n \"name\" : \"Project\",\n \"identifier\" : \"[projectLocality = LOCAL]\",\n \"details\" : \"$hashvalue_37 := combine_hash(combine_hash(combine_hash(BIGINT'0', COALESCE($operator$hash_code(d_year), BIGINT'0')), COALESCE($operator$hash_code(i_brand), BIGINT'0')), COALESCE($operator$hash_code(i_brand_id), BIGINT'0')) (13:10)\\n\",\n \"children\" : [ {\n \"id\" : \"539\",\n \"name\" : \"InnerJoin\",\n \"identifier\" : \"[(\\\"ss_sold_date_sk\\\" = \\\"cast\\\")][$hashvalue_33, $hashvalue_34]\",\n \"details\" : \"Distribution: REPLICATED\\n\",\n \"children\" : [ {\n \"id\" : \"827\",\n \"name\" : \"Project\",\n \"identifier\" : \"[projectLocality = LOCAL]\",\n \"details\" : \"$hashvalue_33 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(ss_sold_date_sk), BIGINT'0')) (7:7)\\n\",\n \"children\" : [ {\n \"id\" : \"538\",\n \"name\" : \"InnerJoin\",\n \"identifier\" : \"[(\\\"ss_item_sk\\\" = \\\"i_item_sk\\\")][$hashvalue_29, $hashvalue_30]\",\n \"details\" : \"Distribution: REPLICATED\\n\",\n \"children\" : [ {\n \"id\" : \"826\",\n \"name\" : \"ScanProject\",\n \"identifier\" : \"[table = TableHandle {connectorId='glue', connectorHandle='HiveTableHandle{schemaName=tpcds_sf1000_parquet_delta, tableName=store_sales, analyzePartitionValues=Optional.empty}', layout='Optional[tpcds_sf1000_parquet_delta.store_sales{}]'}, projectLocality = LOCAL]\",\n \"details\" : \"$hashvalue_29 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(ss_item_sk), BIGINT'0')) (7:7)\\nLAYOUT: tpcds_sf1000_parquet_delta.store_sales{}\\nss_item_sk := ss_item_sk:int:2:REGULAR (7:6)\\nss_sold_date_sk := ss_sold_date_sk:bigint:0:REGULAR (7:6)\\nss_net_profit := ss_net_profit:decimal(7,2):22:REGULAR (7:6)\\n\",\n \"children\" : [ ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 2.879987999E9,\n \"totalSize\" : \"NaN\",\n \"confident\" : true,\n \"variableStatistics\" : {\n \"ss_sold_date_sk\" : {\n \"lowValue\" : 2450816.0,\n \"highValue\" : 2452642.0,\n \"nullsFraction\" : 0.04500048022595944,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1820.0\n },\n \"ss_item_sk\" : {\n \"lowValue\" : 1.0,\n \"highValue\" : 300000.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 300000.0\n },\n \"ss_net_profit\" : {\n \"lowValue\" : -10000.0,\n \"highValue\" : 9986.0,\n \"nullsFraction\" : 0.044990789213354636,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1499246.0\n }\n }\n }, {\n \"outputRowCount\" : 2.879987999E9,\n \"totalSize\" : \"NaN\",\n \"confident\" : true,\n \"variableStatistics\" : {\n \"ss_sold_date_sk\" : {\n \"lowValue\" : 2450816.0,\n \"highValue\" : 2452642.0,\n \"nullsFraction\" : 0.04500048022595944,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1820.0\n },\n \"ss_item_sk\" : {\n \"lowValue\" : 1.0,\n \"highValue\" : 300000.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 300000.0\n },\n \"ss_net_profit\" : {\n \"lowValue\" : -10000.0,\n \"highValue\" : 9986.0,\n \"nullsFraction\" : 0.044990789213354636,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1499246.0\n }\n }\n } ]\n }, {\n \"id\" : \"720\",\n \"name\" : \"LocalExchange\",\n \"identifier\" : \"[HASH][$hashvalue_30] (i_item_sk)\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"633\",\n \"name\" : \"RemoteSource\",\n \"identifier\" : \"[3]\",\n \"details\" : \"\",\n \"children\" : [ ],\n \"remoteSources\" : [ \"3\" ],\n \"estimates\" : [ ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 306.2835209825998,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"i_item_sk\" : {\n \"lowValue\" : 1.0,\n \"highValue\" : 300000.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 306.2835209825998\n },\n \"i_brand_id\" : {\n \"lowValue\" : 1001001.0,\n \"highValue\" : 1.0016017E7,\n \"nullsFraction\" : 0.001271666666666671,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 305.8940304384169\n },\n \"i_brand\" : {\n \"lowValue\" : \"-Infinity\",\n \"highValue\" : \"Infinity\",\n \"nullsFraction\" : 0.0012766666666665927,\n \"averageRowSize\" : 20.157645855751685,\n \"distinctValuesCount\" : 305.892499020812\n }\n }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 2940309.5490711736,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"ss_sold_date_sk\" : {\n \"lowValue\" : 2450816.0,\n \"highValue\" : 2452642.0,\n \"nullsFraction\" : 0.04500048022595944,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1820.0\n },\n \"i_brand_id\" : {\n \"lowValue\" : 1001001.0,\n \"highValue\" : 1.0016017E7,\n \"nullsFraction\" : 0.001271666666666671,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 305.8940304384169\n },\n \"i_brand\" : {\n \"lowValue\" : \"-Infinity\",\n \"highValue\" : \"Infinity\",\n \"nullsFraction\" : 0.0012766666666665927,\n \"averageRowSize\" : 20.157645855751685,\n \"distinctValuesCount\" : 305.892499020812\n },\n \"ss_net_profit\" : {\n \"lowValue\" : -10000.0,\n \"highValue\" : 9986.0,\n \"nullsFraction\" : 0.044990789213354636,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1499246.0\n }\n }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 2940309.5490711736,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"ss_sold_date_sk\" : {\n \"lowValue\" : 2450816.0,\n \"highValue\" : 2452642.0,\n \"nullsFraction\" : 0.04500048022595944,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1820.0\n },\n \"i_brand_id\" : {\n \"lowValue\" : 1001001.0,\n \"highValue\" : 1.0016017E7,\n \"nullsFraction\" : 0.001271666666666671,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 305.8940304384169\n },\n \"i_brand\" : {\n \"lowValue\" : \"-Infinity\",\n \"highValue\" : \"Infinity\",\n \"nullsFraction\" : 0.0012766666666665927,\n \"averageRowSize\" : 20.157645855751685,\n \"distinctValuesCount\" : 305.892499020812\n },\n \"ss_net_profit\" : {\n \"lowValue\" : -10000.0,\n \"highValue\" : 9986.0,\n \"nullsFraction\" : 0.044990789213354636,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1499246.0\n }\n }\n } ]\n }, {\n \"id\" : \"721\",\n \"name\" : \"LocalExchange\",\n \"identifier\" : \"[HASH][$hashvalue_34] (cast)\",\n \"details\" : \"\",\n \"children\" : [ {\n \"id\" : \"635\",\n \"name\" : \"RemoteSource\",\n \"identifier\" : \"[4]\",\n \"details\" : \"\",\n \"children\" : [ ],\n \"remoteSources\" : [ \"4\" ],\n \"estimates\" : [ ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 6087.416666666666,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"d_year\" : {\n \"lowValue\" : 1900.0,\n \"highValue\" : 2100.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 201.0\n },\n \"cast\" : {\n \"lowValue\" : 2415022.0,\n \"highValue\" : 2488070.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 6087.416666666666\n }\n }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 2807994.207349996,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"d_year\" : {\n \"lowValue\" : 1900.0,\n \"highValue\" : 2100.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 201.0\n },\n \"i_brand_id\" : {\n \"lowValue\" : 1001001.0,\n \"highValue\" : 1.0016017E7,\n \"nullsFraction\" : 0.001271666666666671,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 305.8940304384169\n },\n \"i_brand\" : {\n \"lowValue\" : \"-Infinity\",\n \"highValue\" : \"Infinity\",\n \"nullsFraction\" : 0.0012766666666665927,\n \"averageRowSize\" : 20.157645855751685,\n \"distinctValuesCount\" : 305.892499020812\n },\n \"ss_net_profit\" : {\n \"lowValue\" : -10000.0,\n \"highValue\" : 9986.0,\n \"nullsFraction\" : 0.044990789213354636,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1499246.0\n }\n }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 2807994.207349996,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"d_year\" : {\n \"lowValue\" : 1900.0,\n \"highValue\" : 2100.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 201.0\n },\n \"i_brand_id\" : {\n \"lowValue\" : 1001001.0,\n \"highValue\" : 1.0016017E7,\n \"nullsFraction\" : 0.001271666666666671,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 305.8940304384169\n },\n \"i_brand\" : {\n \"lowValue\" : \"-Infinity\",\n \"highValue\" : \"Infinity\",\n \"nullsFraction\" : 0.0012766666666665927,\n \"averageRowSize\" : 20.157645855751685,\n \"distinctValuesCount\" : 305.892499020812\n },\n \"ss_net_profit\" : {\n \"lowValue\" : -10000.0,\n \"highValue\" : 9986.0,\n \"nullsFraction\" : 0.044990789213354636,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1499246.0\n }\n }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : \"NaN\",\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : { }\n } ]\n}\n },\n \"3\" : {\n \"plan\" : {\n \"id\" : \"249\",\n \"name\" : \"ScanFilterProject\",\n \"identifier\" : \"[table = TableHandle {connectorId='glue', connectorHandle='HiveTableHandle{schemaName=tpcds_sf1000_parquet_delta, tableName=item, analyzePartitionValues=Optional.empty}', layout='Optional[tpcds_sf1000_parquet_delta.item{domains={i_manufact_id=[ [[\\\"445\\\"]] ]}}]'}, filterPredicate = (i_manufact_id) = (INTEGER'445'), projectLocality = LOCAL]\",\n \"details\" : \"$hashvalue_32 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(i_item_sk), BIGINT'0')) (8:6)\\nLAYOUT: tpcds_sf1000_parquet_delta.item{domains={i_manufact_id=[ [[\\\"445\\\"]] ]}}\\ni_manufact_id := i_manufact_id:int:13:REGULAR (8:6)\\ni_brand_id := i_brand_id:int:7:REGULAR (8:6)\\ni_brand := i_brand:varchar(50):8:REGULAR (8:6)\\ni_item_sk := i_item_sk:int:0:REGULAR (8:6)\\n\",\n \"children\" : [ ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 300000.0,\n \"totalSize\" : \"NaN\",\n \"confident\" : true,\n \"variableStatistics\" : {\n \"i_item_sk\" : {\n \"lowValue\" : 1.0,\n \"highValue\" : 300000.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 300000.0\n },\n \"i_brand_id\" : {\n \"lowValue\" : 1001001.0,\n \"highValue\" : 1.0016017E7,\n \"nullsFraction\" : 0.002543333333333333,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 958.0\n },\n \"i_brand\" : {\n \"lowValue\" : \"-Infinity\",\n \"highValue\" : \"Infinity\",\n \"nullsFraction\" : 0.0025533333333333332,\n \"averageRowSize\" : 20.157645855751685,\n \"distinctValuesCount\" : 723.0\n },\n \"i_manufact_id\" : {\n \"lowValue\" : 1.0,\n \"highValue\" : 1000.0,\n \"nullsFraction\" : 0.0025366666666666667,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 977.0\n }\n }\n }, {\n \"outputRowCount\" : 306.2835209825998,\n \"totalSize\" : \"NaN\",\n \"confident\" : true,\n \"variableStatistics\" : {\n \"i_item_sk\" : {\n \"lowValue\" : 1.0,\n \"highValue\" : 300000.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 306.2835209825998\n },\n \"i_brand_id\" : {\n \"lowValue\" : 1001001.0,\n \"highValue\" : 1.0016017E7,\n \"nullsFraction\" : 0.001271666666666671,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 305.8940304384169\n },\n \"i_brand\" : {\n \"lowValue\" : \"-Infinity\",\n \"highValue\" : \"Infinity\",\n \"nullsFraction\" : 0.0012766666666665927,\n \"averageRowSize\" : 20.157645855751685,\n \"distinctValuesCount\" : 305.892499020812\n },\n \"i_manufact_id\" : {\n \"lowValue\" : 445.0,\n \"highValue\" : 445.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1.0\n }\n }\n }, {\n \"outputRowCount\" : 306.2835209825998,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"i_item_sk\" : {\n \"lowValue\" : 1.0,\n \"highValue\" : 300000.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 306.2835209825998\n },\n \"i_brand_id\" : {\n \"lowValue\" : 1001001.0,\n \"highValue\" : 1.0016017E7,\n \"nullsFraction\" : 0.001271666666666671,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 305.8940304384169\n },\n \"i_brand\" : {\n \"lowValue\" : \"-Infinity\",\n \"highValue\" : \"Infinity\",\n \"nullsFraction\" : 0.0012766666666665927,\n \"averageRowSize\" : 20.157645855751685,\n \"distinctValuesCount\" : 305.892499020812\n }\n }\n } ]\n}\n },\n \"4\" : {\n \"plan\" : {\n \"id\" : \"828\",\n \"name\" : \"Project\",\n \"identifier\" : \"[projectLocality = LOCAL]\",\n \"details\" : \"$hashvalue_36 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(cast), BIGINT'0')) (6:7)\\n\",\n \"children\" : [ {\n \"id\" : \"244\",\n \"name\" : \"ScanFilterProject\",\n \"identifier\" : \"[table = TableHandle {connectorId='glue', connectorHandle='HiveTableHandle{schemaName=tpcds_sf1000_parquet_delta, tableName=date_dim, analyzePartitionValues=Optional.empty}', layout='Optional[tpcds_sf1000_parquet_delta.date_dim{domains={d_moy=[ [[\\\"12\\\"]] ]}}]'}, filterPredicate = (d_moy) = (INTEGER'12'), projectLocality = LOCAL]\",\n \"details\" : \"cast := CAST(d_date_sk AS bigint) (6:7)\\nLAYOUT: tpcds_sf1000_parquet_delta.date_dim{domains={d_moy=[ [[\\\"12\\\"]] ]}}\\nd_moy := d_moy:int:8:REGULAR (6:6)\\nd_date_sk := d_date_sk:int:0:REGULAR (6:6)\\nd_year := d_year:int:6:REGULAR (6:6)\\n\",\n \"children\" : [ ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 73049.0,\n \"totalSize\" : \"NaN\",\n \"confident\" : true,\n \"variableStatistics\" : {\n \"d_year\" : {\n \"lowValue\" : 1900.0,\n \"highValue\" : 2100.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 201.0\n },\n \"d_moy\" : {\n \"lowValue\" : 1.0,\n \"highValue\" : 12.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 12.0\n },\n \"d_date_sk\" : {\n \"lowValue\" : 2415022.0,\n \"highValue\" : 2488070.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 69971.0\n }\n }\n }, {\n \"outputRowCount\" : 6087.416666666666,\n \"totalSize\" : \"NaN\",\n \"confident\" : true,\n \"variableStatistics\" : {\n \"d_year\" : {\n \"lowValue\" : 1900.0,\n \"highValue\" : 2100.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 201.0\n },\n \"d_moy\" : {\n \"lowValue\" : 12.0,\n \"highValue\" : 12.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 1.0\n },\n \"d_date_sk\" : {\n \"lowValue\" : 2415022.0,\n \"highValue\" : 2488070.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 6087.416666666666\n }\n }\n }, {\n \"outputRowCount\" : 6087.416666666666,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"d_year\" : {\n \"lowValue\" : 1900.0,\n \"highValue\" : 2100.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 201.0\n },\n \"cast\" : {\n \"lowValue\" : 2415022.0,\n \"highValue\" : 2488070.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 6087.416666666666\n }\n }\n } ]\n } ],\n \"remoteSources\" : [ ],\n \"estimates\" : [ {\n \"outputRowCount\" : 6087.416666666666,\n \"totalSize\" : \"NaN\",\n \"confident\" : false,\n \"variableStatistics\" : {\n \"d_year\" : {\n \"lowValue\" : 1900.0,\n \"highValue\" : 2100.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 201.0\n },\n \"cast\" : {\n \"lowValue\" : 2415022.0,\n \"highValue\" : 2488070.0,\n \"nullsFraction\" : 0.0,\n \"averageRowSize\" : \"NaN\",\n \"distinctValuesCount\" : 6087.416666666666\n }\n }\n } ]\n}\n }\n}","graphvizPlan":"digraph distributed_plan {\nsubgraph cluster_0 {\nlabel = \"SINGLE\"\nplannode_1[label=\"{Output[d_year, brand_id, brand, sum_agg]|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=white];\nplannode_2[label=\"{TopN[100]|d_year ASC_NULLS_LAST, sum DESC_NULLS_LAST, i_brand_id ASC_NULLS_LAST|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=darksalmon];\nplannode_3[label=\"{ExchangeNode[GATHER]|d_year, i_brand, i_brand_id, sum|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=gold];\nplannode_4[label=\"{Exchange 1:N|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=gold];\n}\nplannode_1 -> plannode_2;\nplannode_2 -> plannode_3;\nplannode_3 -> plannode_4;\nplannode_4 -> plannode_5;\nsubgraph cluster_1 {\nlabel = \"HASH\"\nplannode_5[label=\"{TopN[100]|d_year ASC_NULLS_LAST, sum DESC_NULLS_LAST, i_brand_id ASC_NULLS_LAST|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=darksalmon];\nplannode_6[label=\"{Project|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=bisque];\nplannode_7[label=\"{Aggregate[FINAL]|sum := sum(sum_27)\\n|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=chartreuse3];\nplannode_8[label=\"{ExchangeNode[REPARTITION]|d_year, i_brand, i_brand_id|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=gold];\nplannode_9[label=\"{Exchange 1:N|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=gold];\n}\nplannode_5 -> plannode_6;\nplannode_6 -> plannode_7;\nplannode_7 -> plannode_8;\nplannode_8 -> plannode_9;\nplannode_9 -> plannode_10;\nsubgraph cluster_2 {\nlabel = \"SOURCE\"\nplannode_10[label=\"{Aggregate[PARTIAL]|sum_27 := sum(ss_net_profit)\\n|Estimates: \\{rows: ? (?), cpu: ?, memory: ?, network: ?\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=chartreuse3];\nplannode_11[label=\"{Project|$hashvalue_37 := combine_hash(combine_hash(combine_hash(BIGINT'0', COALESCE($operator$hash_code(d_year), BIGINT'0')), COALESCE($operator$hash_code(i_brand), BIGINT'0')), COALESCE($operator$hash_code(i_brand_id), BIGINT'0'))\\n|Estimates: \\{rows: 2807994 (141.31MB), cpu: 245109888183.85, memory: 1228207.22, network: 1228207.22\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=bisque];\nplannode_12[label=\"{InnerJoin[REPLICATED]|(\\\"ss_sold_date_sk\\\" = \\\"cast\\\")|Estimates: \\{rows: 2807994 (117.20MB), cpu: 244961719039.14, memory: 1228207.22, network: 1228207.22\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=orange];\nplannode_13[label=\"{Project|$hashvalue_33 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(ss_sold_date_sk), BIGINT'0'))\\n|Estimates: \\{rows: 2940310 (158.17MB), cpu: 244668311238.19, memory: 108122.55, network: 108122.55\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=bisque];\nplannode_14[label=\"{InnerJoin[REPLICATED]|(\\\"ss_item_sk\\\" = \\\"i_item_sk\\\")|Estimates: \\{rows: 2940310 (132.93MB), cpu: 244502457507.84, memory: 108122.55, network: 108122.55\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=orange];\nplannode_15[label=\"{Project|$hashvalue_29 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(ss_item_sk), BIGINT'0'))\\n|Estimates: \\{rows: 2879987999 (83.90GB), cpu: 154252559529.00, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=bisque];\nplannode_16[label=\"{TableScan | [TableHandle \\{connectorId='glue', connectorHandle='HiveTableHandle\\{schemaName=tpcds_sf1000_parquet_delta, tableName=store_sales, analyzePartitionValues=Optional.empty\\}', layout='Optional[tpcds_sf1000_parquet_delta.store_sales\\{\\}]'\\}]|Estimates: \\{rows: 2879987999 (59.76GB), cpu: 64166333769.00, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=deepskyblue];\nplannode_17[label=\"{ExchangeNode[REPARTITION]|i_item_sk|Estimates: \\{rows: 306 (13.20kB), cpu: 24078544.64, memory: 0.00, network: 108122.55\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=gold];\nplannode_18[label=\"{Exchange 1:N|Estimates: \\{rows: 306 (13.20kB), cpu: 24065029.32, memory: 0.00, network: 108122.55\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=gold];\nplannode_19[label=\"{ExchangeNode[REPARTITION]|cast|Estimates: \\{rows: 6087 (136.73kB), cpu: 2556715.00, memory: 0.00, network: 1120084.67\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=gold];\nplannode_20[label=\"{Exchange 1:N|Estimates: \\{rows: 6087 (136.73kB), cpu: 2416704.42, memory: 0.00, network: 1120084.67\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=gold];\n}\nplannode_10 -> plannode_11;\nplannode_11 -> plannode_12;\nplannode_12 -> plannode_19 [label = \"Build\"];\nplannode_19 -> plannode_20;\nplannode_20 -> plannode_21;\nplannode_12 -> plannode_13 [label = \"Probe\"];\nplannode_13 -> plannode_14;\nplannode_14 -> plannode_17 [label = \"Build\"];\nplannode_17 -> plannode_18;\nplannode_18 -> plannode_22;\nplannode_14 -> plannode_15 [label = \"Probe\"];\nplannode_15 -> plannode_16;\nsubgraph cluster_3 {\nlabel = \"SOURCE\"\nplannode_22[label=\"{Project|$hashvalue_32 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(i_item_sk), BIGINT'0'))\\n|Estimates: \\{rows: 306 (13.20kB), cpu: 24065029.32, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=bisque];\nplannode_23[label=\"{Filter|(i_manufact_id) = (INTEGER'445')|Estimates: \\{rows: 306 (12.00kB), cpu: 24051514.00, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=yellow];\nplannode_24[label=\"{TableScan | [TableHandle \\{connectorId='glue', connectorHandle='HiveTableHandle\\{schemaName=tpcds_sf1000_parquet_delta, tableName=item, analyzePartitionValues=Optional.empty\\}', layout='Optional[tpcds_sf1000_parquet_delta.item\\{domains=\\{i_manufact_id=[ [[\\\"445\\\"]] ]\\}\\}]'\\}]|Estimates: \\{rows: 300000 (11.47MB), cpu: 12025757.00, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=deepskyblue];\n}\nplannode_22 -> plannode_23;\nplannode_23 -> plannode_24;\nsubgraph cluster_4 {\nlabel = \"SOURCE\"\nplannode_21[label=\"{Project|$hashvalue_36 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(cast), BIGINT'0'))\\n|Estimates: \\{rows: 6087 (136.73kB), cpu: 2416704.42, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=bisque];\nplannode_25[label=\"{Project|cast := CAST(d_date_sk AS bigint)\\n|Estimates: \\{rows: 6087 (83.23kB), cpu: 2276693.83, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=bisque];\nplannode_26[label=\"{Filter|(d_moy) = (INTEGER'12')|Estimates: \\{rows: 6087 (89.17kB), cpu: 2191470.00, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=yellow];\nplannode_27[label=\"{TableScan | [TableHandle \\{connectorId='glue', connectorHandle='HiveTableHandle\\{schemaName=tpcds_sf1000_parquet_delta, tableName=date_dim, analyzePartitionValues=Optional.empty\\}', layout='Optional[tpcds_sf1000_parquet_delta.date_dim\\{domains=\\{d_moy=[ [[\\\"12\\\"]] ]\\}\\}]'\\}]|Estimates: \\{rows: 73049 (1.04MB), cpu: 1095735.00, memory: 0.00, network: 0.00\\}\n}\", style=\"rounded, filled\", shape=record, fillcolor=deepskyblue];\n}\nplannode_21 -> plannode_25;\nplannode_25 -> plannode_26;\nplannode_26 -> plannode_27;\n}\n","payload":"","runtimeOptimizedStages":[],"planNodeRuntimeStats":"{\n \"16\" : {\n \"planNodeId\" : \"16\",\n \"planNodeScheduledTime\" : \"0.00ms\",\n \"planNodeCpuTime\" : \"0.00ms\",\n \"planNodeInputPositions\" : 100,\n \"planNodeInputDataSize\" : \"4.69kB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 100,\n \"planNodeOutputDataSize\" : \"4.69kB\",\n \"operatorTypes\" : [ \"TaskOutputOperator\" ]\n },\n \"244\" : {\n \"planNodeId\" : \"244\",\n \"planNodeScheduledTime\" : \"8.00ms\",\n \"planNodeCpuTime\" : \"7.00ms\",\n \"planNodeInputPositions\" : 73049,\n \"planNodeInputDataSize\" : \"297.02kB\",\n \"planNodeRawInputPositions\" : 73049,\n \"planNodeRawInputDataSize\" : \"297.02kB\",\n \"planNodeOutputPositions\" : 6200,\n \"planNodeOutputDataSize\" : \"84.77kB\",\n \"operatorTypes\" : [ \"ScanFilterAndProjectOperator\" ]\n },\n \"753\" : {\n \"planNodeId\" : \"753\",\n \"planNodeScheduledTime\" : \"3017.00ms\",\n \"planNodeCpuTime\" : \"2266.00ms\",\n \"planNodeInputPositions\" : 438836,\n \"planNodeInputDataSize\" : \"20.56MB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 310747,\n \"planNodeOutputDataSize\" : \"20.16MB\",\n \"operatorTypes\" : [ \"HashAggregationOperator\" ]\n },\n \"720\" : {\n \"planNodeId\" : \"720\",\n \"planNodeScheduledTime\" : \"0.00ms\",\n \"planNodeCpuTime\" : \"0.00ms\",\n \"planNodeInputPositions\" : 2288,\n \"planNodeInputDataSize\" : \"90.16kB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 2288,\n \"planNodeOutputDataSize\" : \"90.16kB\",\n \"operatorTypes\" : [ \"LocalExchangeSinkOperator\" ]\n },\n \"522\" : {\n \"planNodeId\" : \"522\",\n \"planNodeScheduledTime\" : \"319.00ms\",\n \"planNodeCpuTime\" : \"283.00ms\",\n \"planNodeInputPositions\" : 508,\n \"planNodeInputDataSize\" : \"23.84kB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 508,\n \"planNodeOutputDataSize\" : \"23.84kB\",\n \"operatorTypes\" : [ \"TopNOperator\" ]\n },\n \"830\" : {\n \"planNodeId\" : \"830\",\n \"planNodeScheduledTime\" : \"238.00ms\",\n \"planNodeCpuTime\" : \"234.00ms\",\n \"planNodeInputPositions\" : 508,\n \"planNodeInputDataSize\" : \"28.32kB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 508,\n \"planNodeOutputDataSize\" : \"23.84kB\",\n \"operatorTypes\" : [ \"FilterAndProjectOperator\" ]\n },\n \"721\" : {\n \"planNodeId\" : \"721\",\n \"planNodeScheduledTime\" : \"9.00ms\",\n \"planNodeCpuTime\" : \"4.00ms\",\n \"planNodeInputPositions\" : 49600,\n \"planNodeInputDataSize\" : \"1.09MB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 49600,\n \"planNodeOutputDataSize\" : \"1.09MB\",\n \"operatorTypes\" : [ \"LocalExchangeSinkOperator\" ]\n },\n \"633\" : {\n \"planNodeId\" : \"633\",\n \"planNodeScheduledTime\" : \"0.00ms\",\n \"planNodeCpuTime\" : \"0.00ms\",\n \"planNodeInputPositions\" : 2288,\n \"planNodeInputDataSize\" : \"90.16kB\",\n \"planNodeRawInputPositions\" : 2288,\n \"planNodeRawInputDataSize\" : \"81.84kB\",\n \"planNodeOutputPositions\" : 2288,\n \"planNodeOutputDataSize\" : \"90.16kB\",\n \"operatorTypes\" : [ \"ExchangeOperator\" ]\n },\n \"523\" : {\n \"planNodeId\" : \"523\",\n \"planNodeScheduledTime\" : \"3.00ms\",\n \"planNodeCpuTime\" : \"3.00ms\",\n \"planNodeInputPositions\" : 508,\n \"planNodeInputDataSize\" : \"23.84kB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 100,\n \"planNodeOutputDataSize\" : \"4.69kB\",\n \"operatorTypes\" : [ \"TopNOperator\" ]\n },\n \"249\" : {\n \"planNodeId\" : \"249\",\n \"planNodeScheduledTime\" : \"48.00ms\",\n \"planNodeCpuTime\" : \"19.00ms\",\n \"planNodeInputPositions\" : 300000,\n \"planNodeInputDataSize\" : \"2.62MB\",\n \"planNodeRawInputPositions\" : 300000,\n \"planNodeRawInputDataSize\" : \"2.62MB\",\n \"planNodeOutputPositions\" : 286,\n \"planNodeOutputDataSize\" : \"11.27kB\",\n \"operatorTypes\" : [ \"ScanFilterAndProjectOperator\" ]\n },\n \"755\" : {\n \"planNodeId\" : \"755\",\n \"planNodeScheduledTime\" : \"306.00ms\",\n \"planNodeCpuTime\" : \"166.00ms\",\n \"planNodeInputPositions\" : 310747,\n \"planNodeInputDataSize\" : \"20.36MB\",\n \"planNodeRawInputPositions\" : 310747,\n \"planNodeRawInputDataSize\" : \"20.21MB\",\n \"planNodeOutputPositions\" : 310747,\n \"planNodeOutputDataSize\" : \"20.36MB\",\n \"operatorTypes\" : [ \"ExchangeOperator\" ]\n },\n \"635\" : {\n \"planNodeId\" : \"635\",\n \"planNodeScheduledTime\" : \"0.00ms\",\n \"planNodeCpuTime\" : \"0.00ms\",\n \"planNodeInputPositions\" : 49600,\n \"planNodeInputDataSize\" : \"1.09MB\",\n \"planNodeRawInputPositions\" : 49600,\n \"planNodeRawInputDataSize\" : \"971.59kB\",\n \"planNodeOutputPositions\" : 49600,\n \"planNodeOutputDataSize\" : \"1.09MB\",\n \"operatorTypes\" : [ \"ExchangeOperator\" ]\n },\n \"8\" : {\n \"planNodeId\" : \"8\",\n \"planNodeScheduledTime\" : \"1107.00ms\",\n \"planNodeCpuTime\" : \"1072.00ms\",\n \"planNodeInputPositions\" : 310747,\n \"planNodeInputDataSize\" : \"20.36MB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 508,\n \"planNodeOutputDataSize\" : \"28.32kB\",\n \"operatorTypes\" : [ \"HashAggregationOperator\" ]\n },\n \"723\" : {\n \"planNodeId\" : \"723\",\n \"planNodeScheduledTime\" : \"3.00ms\",\n \"planNodeCpuTime\" : \"3.00ms\",\n \"planNodeInputPositions\" : 508,\n \"planNodeInputDataSize\" : \"23.84kB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 508,\n \"planNodeOutputDataSize\" : \"23.84kB\",\n \"operatorTypes\" : [ \"LocalExchangeSinkOperator\" ]\n },\n \"538\" : {\n \"planNodeId\" : \"538\",\n \"planNodeScheduledTime\" : \"180036.00ms\",\n \"planNodeCpuTime\" : \"150651.00ms\",\n \"planNodeInputPositions\" : 2879990287,\n \"planNodeInputDataSize\" : \"85.82GB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 2713615,\n \"planNodeOutputDataSize\" : \"127.03MB\",\n \"operatorTypes\" : [ \"LookupJoinOperator\", \"HashBuilderOperator\" ]\n },\n \"637\" : {\n \"planNodeId\" : \"637\",\n \"planNodeScheduledTime\" : \"41.00ms\",\n \"planNodeCpuTime\" : \"3.00ms\",\n \"planNodeInputPositions\" : 508,\n \"planNodeInputDataSize\" : \"23.84kB\",\n \"planNodeRawInputPositions\" : 508,\n \"planNodeRawInputDataSize\" : \"40.50kB\",\n \"planNodeOutputPositions\" : 508,\n \"planNodeOutputDataSize\" : \"23.84kB\",\n \"operatorTypes\" : [ \"ExchangeOperator\" ]\n },\n \"539\" : {\n \"planNodeId\" : \"539\",\n \"planNodeScheduledTime\" : \"2410.00ms\",\n \"planNodeCpuTime\" : \"2137.00ms\",\n \"planNodeInputPositions\" : 2763215,\n \"planNodeInputDataSize\" : \"138.63MB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 438836,\n \"planNodeOutputDataSize\" : \"21.82MB\",\n \"operatorTypes\" : [ \"LookupJoinOperator\", \"HashBuilderOperator\" ]\n },\n \"826\" : {\n \"planNodeId\" : \"826\",\n \"planNodeScheduledTime\" : \"1878000.00ms\",\n \"planNodeCpuTime\" : \"199210.00ms\",\n \"planNodeInputPositions\" : 2879987999,\n \"planNodeInputDataSize\" : \"25.12GB\",\n \"planNodeRawInputPositions\" : 2879987999,\n \"planNodeRawInputDataSize\" : \"25.12GB\",\n \"planNodeOutputPositions\" : 2879987999,\n \"planNodeOutputDataSize\" : \"85.82GB\",\n \"operatorTypes\" : [ \"ScanFilterAndProjectOperator\" ]\n },\n \"749\" : {\n \"planNodeId\" : \"749\",\n \"planNodeScheduledTime\" : \"2937.00ms\",\n \"planNodeCpuTime\" : \"2043.00ms\",\n \"planNodeInputPositions\" : 310747,\n \"planNodeInputDataSize\" : \"20.36MB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 310747,\n \"planNodeOutputDataSize\" : \"20.36MB\",\n \"operatorTypes\" : [ \"LocalExchangeSinkOperator\" ]\n },\n \"827\" : {\n \"planNodeId\" : \"827\",\n \"planNodeScheduledTime\" : \"11370.00ms\",\n \"planNodeCpuTime\" : \"10411.00ms\",\n \"planNodeInputPositions\" : 2713615,\n \"planNodeInputDataSize\" : \"127.03MB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 2713615,\n \"planNodeOutputDataSize\" : \"137.54MB\",\n \"operatorTypes\" : [ \"FilterAndProjectOperator\" ]\n },\n \"828\" : {\n \"planNodeId\" : \"828\",\n \"planNodeScheduledTime\" : \"1.00ms\",\n \"planNodeCpuTime\" : \"1.00ms\",\n \"planNodeInputPositions\" : 6200,\n \"planNodeInputDataSize\" : \"84.77kB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 6200,\n \"planNodeOutputDataSize\" : \"139.26kB\",\n \"operatorTypes\" : [ \"FilterAndProjectOperator\" ]\n },\n \"829\" : {\n \"planNodeId\" : \"829\",\n \"planNodeScheduledTime\" : \"1818.00ms\",\n \"planNodeCpuTime\" : \"1702.00ms\",\n \"planNodeInputPositions\" : 438836,\n \"planNodeInputDataSize\" : \"21.82MB\",\n \"planNodeRawInputPositions\" : 0,\n \"planNodeRawInputDataSize\" : \"0B\",\n \"planNodeOutputPositions\" : 438836,\n \"planNodeOutputDataSize\" : \"20.56MB\",\n \"operatorTypes\" : [ \"FilterAndProjectOperator\" ]\n }\n}"},"statistics":{"cpuTime":371.792000000,"retriedCpuTime":0.0,"wallTime":67.795000000,"waitingForPrerequisitesTime":0.001000000,"queuedTime":0.0,"waitingForResourcesTime":0.0,"semanticAnalyzingTime":0.309000000,"columnAccessPermissionCheckingTime":0.0,"dispatchingTime":0.309000000,"planningTime":0.633000000,"analysisTime":0.630000000,"executionTime":67.485000000,"peakRunningTasks":20,"peakUserMemoryBytes":98161684,"peakTotalNonRevocableMemoryBytes":1098226989,"peakTaskUserMemory":10433657,"peakTaskTotalMemory":202274048,"peakNodeTotalMemory":222073173,"totalBytes":26975446031,"totalRows":2880361048,"outputBytes":4801,"outputRows":100,"writtenOutputBytes":0,"writtenOutputRows":0,"writtenIntermediateBytes":0,"spilledBytes":0,"cumulativeMemory":5.359635941788949E12,"cumulativeTotalMemory":8.514280417681293E12,"completedSplits":7776,"complete":true,"runtimeStats":{"S2-taskBlockedTimeNanos":{"name":"S2-taskBlockedTimeNanos","unit":"NANO","sum":27989185194024,"count":8,"max":3515909036612,"min":3487641694410},"S4-storageReadTimeNanos":{"name":"S4-storageReadTimeNanos","unit":"NANO","sum":518096,"count":3,"max":321762,"min":34511},"S2-taskQueuedTimeNanos":{"name":"S2-taskQueuedTimeNanos","unit":"NANO","sum":48801138,"count":8,"max":6738803,"min":5766974},"S0-taskScheduledTimeNanos":{"name":"S0-taskScheduledTimeNanos","unit":"NANO","sum":209784883,"count":1,"max":209784883,"min":209784883},"getColumnMetadataTimeNanos":{"name":"getColumnMetadataTimeNanos","unit":"NANO","sum":685074,"count":3,"max":255850,"min":190574},"S3-storageReadDataBytes":{"name":"S3-storageReadDataBytes","unit":"BYTE","sum":2742381,"count":4,"max":1200554,"min":382716},"S2-storageReadTimeNanos":{"name":"S2-storageReadTimeNanos","unit":"NANO","sum":829910814599,"count":4608,"max":1151544516,"min":414646},"S3-taskElapsedTimeNanos":{"name":"S3-taskElapsedTimeNanos","unit":"NANO","sum":132470280278,"count":2,"max":66235150310,"min":66235129968},"S4-taskQueuedTimeNanos":{"name":"S4-taskQueuedTimeNanos","unit":"NANO","sum":5974912,"count":1,"max":5974912,"min":5974912},"getLayoutTimeNanos":{"name":"getLayoutTimeNanos","unit":"NANO","sum":738167,"count":10,"max":136117,"min":19686},"S3-taskScheduledTimeNanos":{"name":"S3-taskScheduledTimeNanos","unit":"NANO","sum":32721491,"count":2,"max":31909077,"min":812414},"S0-taskBlockedTimeNanos":{"name":"S0-taskBlockedTimeNanos","unit":"NANO","sum":2205311948416,"count":1,"max":2205311948416,"min":2205311948416},"S0-taskQueuedTimeNanos":{"name":"S0-taskQueuedTimeNanos","unit":"NANO","sum":660178,"count":1,"max":660178,"min":660178},"S0-taskElapsedTimeNanos":{"name":"S0-taskElapsedTimeNanos","unit":"NANO","sum":66847822748,"count":1,"max":66847822748,"min":66847822748},"getColumnHandleTimeNanos":{"name":"getColumnHandleTimeNanos","unit":"NANO","sum":263464,"count":3,"max":93747,"min":80925},"S3-taskQueuedTimeNanos":{"name":"S3-taskQueuedTimeNanos","unit":"NANO","sum":8419576,"count":2,"max":4525473,"min":3894103},"S1-taskScheduledTimeNanos":{"name":"S1-taskScheduledTimeNanos","unit":"NANO","sum":7055786357,"count":8,"max":1001603591,"min":758190450},"S3-storageReadTimeNanos":{"name":"S3-storageReadTimeNanos","unit":"NANO","sum":15021541,"count":4,"max":6508053,"min":1594166},"S1-taskBlockedTimeNanos":{"name":"S1-taskBlockedTimeNanos","unit":"NANO","sum":33706157940138,"count":8,"max":4226899584281,"min":4195314506513},"S4-taskElapsedTimeNanos":{"name":"S4-taskElapsedTimeNanos","unit":"NANO","sum":66262362546,"count":1,"max":66262362546,"min":66262362546},"S4-storageReadDataBytes":{"name":"S4-storageReadDataBytes","unit":"BYTE","sum":304148,"count":3,"max":292344,"min":1636},"optimizerTimeNanos":{"name":"optimizerTimeNanos","unit":"NANO","sum":619408022,"count":1,"max":619408022,"min":619408022},"S2-taskElapsedTimeNanos":{"name":"S2-taskElapsedTimeNanos","unit":"NANO","sum":437475196767,"count":8,"max":54987529654,"min":54501142322},"getMaterializedViewTimeNanos":{"name":"getMaterializedViewTimeNanos","unit":"NANO","sum":159733,"count":3,"max":79028,"min":7812},"S4-taskScheduledTimeNanos":{"name":"S4-taskScheduledTimeNanos","unit":"NANO","sum":8503489,"count":1,"max":8503489,"min":8503489},"S1-driverCountPerTask":{"name":"S1-driverCountPerTask","unit":"NONE","sum":512,"count":8,"max":64,"min":64},"S2-driverCountPerTask":{"name":"S2-driverCountPerTask","unit":"NONE","sum":7228,"count":8,"max":905,"min":902},"S2-storageReadDataBytes":{"name":"S2-storageReadDataBytes","unit":"BYTE","sum":26985884944,"count":4608,"max":7544493,"min":832389},"S3-driverCountPerTask":{"name":"S3-driverCountPerTask","unit":"NONE","sum":2,"count":2,"max":1,"min":1},"S1-taskElapsedTimeNanos":{"name":"S1-taskElapsedTimeNanos","unit":"NANO","sum":534676732151,"count":8,"max":66844299925,"min":66826526146},"fragmentPlanTimeNanos":{"name":"fragmentPlanTimeNanos","unit":"NANO","sum":6255917,"count":1,"max":6255917,"min":6255917},"S2-taskScheduledTimeNanos":{"name":"S2-taskScheduledTimeNanos","unit":"NANO","sum":1247769073084,"count":8,"max":179266206602,"min":142363558183},"S4-driverCountPerTask":{"name":"S4-driverCountPerTask","unit":"NONE","sum":1,"count":1,"max":1,"min":1},"logicalPlannerTimeNanos":{"name":"logicalPlannerTimeNanos","unit":"NANO","sum":3338980,"count":1,"max":3338980,"min":3338980},"S0-driverCountPerTask":{"name":"S0-driverCountPerTask","unit":"NONE","sum":33,"count":1,"max":33,"min":33},"S2-getSplitsTimeNanos":{"name":"S2-getSplitsTimeNanos","unit":"NANO","sum":66297590722,"count":1,"max":66297590722,"min":66297590722},"S1-taskQueuedTimeNanos":{"name":"S1-taskQueuedTimeNanos","unit":"NANO","sum":6907098,"count":8,"max":1063968,"min":715634},"getViewTimeNanos":{"name":"getViewTimeNanos","unit":"NANO","sum":306013241,"count":3,"max":166087194,"min":237082},"getTableHandleTimeNanos":{"name":"getTableHandleTimeNanos","unit":"NANO","sum":116566,"count":3,"max":41279,"min":34847},"S3-getSplitsTimeNanos":{"name":"S3-getSplitsTimeNanos","unit":"NANO","sum":82612539,"count":1,"max":82612539,"min":82612539},"S4-getSplitsTimeNanos":{"name":"S4-getSplitsTimeNanos","unit":"NANO","sum":56516072,"count":1,"max":56516072,"min":56516072}}},"context":{"user":"presto","principal":"presto","remoteClientAddress":"10.128.18.54","userAgent":"Presto JDBC Driver/0.285-SNAPSHOT-9d121df","clientInfo":null,"clientTags":[],"source":"presto-jdbc","queryActionType":null,"catalog":"glue","schema":"tpcds_sf1000_parquet_delta","resourceGroupId":["global"],"sessionProperties":{"query_max_scan_raw_input_bytes":"10TB","query_max_memory_per_node":"32985348833B","query_max_run_time":"2h","pushdown_subfields_enabled":"true","query_max_total_memory_per_node":"43980465110B","query_max_execution_time":"3h","query_max_output_size":"500GB","task_concurrency":"32","join_distribution_type":"AUTOMATIC","join_reordering_strategy":"AUTOMATIC","query_max_stage_count":"300","resource_overcommit":"true","query_max_memory":"2TB"},"resourceEstimates":{"executionTime":null,"cpuTime":null,"peakMemory":null,"peakTaskMemory":null},"serverAddress":"10.128.58.157","serverVersion":"0.282","environment":"reetikaalluxio"},"ioMetadata":{"inputs":[{"catalogName":"glue","schema":"tpcds_sf1000_parquet_delta","table":"store_sales","columns":["ss_item_sk","ss_sold_date_sk","ss_net_profit"],"connectorInfo":{"partitionIds":[""],"truncated":false},"statistics":{"rowCount":{"value":2.879987999E9,"unknown":false},"totalSize":{"value":"NaN","unknown":true},"columnStatistics":{"ss_item_sk:int:2:REGULAR":{"nullsFraction":{"value":0.0,"unknown":false},"distinctValuesCount":{"value":304649.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":1.0,"max":300000.0}},"ss_sold_date_sk:bigint:0:REGULAR":{"nullsFraction":{"value":0.04500048022595944,"unknown":false},"distinctValuesCount":{"value":1820.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":2450816.0,"max":2452642.0}},"ss_net_profit:decimal(7,2):22:REGULAR":{"nullsFraction":{"value":0.044990789213354636,"unknown":false},"distinctValuesCount":{"value":1499246.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":-10000.0,"max":9986.0}}}},"serializedCommitOutput":""},{"catalogName":"glue","schema":"tpcds_sf1000_parquet_delta","table":"item","columns":["i_manufact_id","i_brand_id","i_brand","i_item_sk"],"connectorInfo":{"partitionIds":[""],"truncated":false},"statistics":{"rowCount":{"value":300000.0,"unknown":false},"totalSize":{"value":"NaN","unknown":true},"columnStatistics":{"i_manufact_id:int:13:REGULAR":{"nullsFraction":{"value":0.0025366666666666667,"unknown":false},"distinctValuesCount":{"value":977.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":1.0,"max":1000.0}},"i_brand_id:int:7:REGULAR":{"nullsFraction":{"value":0.002543333333333333,"unknown":false},"distinctValuesCount":{"value":958.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":1001001.0,"max":1.0016017E7}},"i_brand:varchar(50):8:REGULAR":{"nullsFraction":{"value":0.0025533333333333332,"unknown":false},"distinctValuesCount":{"value":723.0,"unknown":false},"dataSize":{"value":6031853.0,"unknown":false},"range":null},"i_item_sk:int:0:REGULAR":{"nullsFraction":{"value":0.0,"unknown":false},"distinctValuesCount":{"value":300000.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":1.0,"max":300000.0}}}},"serializedCommitOutput":""},{"catalogName":"glue","schema":"tpcds_sf1000_parquet_delta","table":"date_dim","columns":["d_date_sk","d_moy","d_year"],"connectorInfo":{"partitionIds":[""],"truncated":false},"statistics":{"rowCount":{"value":73049.0,"unknown":false},"totalSize":{"value":"NaN","unknown":true},"columnStatistics":{"d_moy:int:8:REGULAR":{"nullsFraction":{"value":0.0,"unknown":false},"distinctValuesCount":{"value":12.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":1.0,"max":12.0}},"d_date_sk:int:0:REGULAR":{"nullsFraction":{"value":0.0,"unknown":false},"distinctValuesCount":{"value":69971.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":2415022.0,"max":2488070.0}},"d_year:int:6:REGULAR":{"nullsFraction":{"value":0.0,"unknown":false},"distinctValuesCount":{"value":201.0,"unknown":false},"dataSize":{"value":"NaN","unknown":true},"range":{"min":1900.0,"max":2100.0}}}},"serializedCommitOutput":""}],"output":null},"failureInfo":null,"warnings":[],"queryType":"SELECT","failedTasks":[],"stageStatistics":[{"stageId":0,"stageExecutionId":0,"tasks":1,"totalScheduledTime":"209.78ms","totalCpuTime":"19.16ms","retriedCpuTime":"0.00ns","totalBlockedTime":"36.76m","rawInputDataSize":"40.50kB","processedInputDataSize":"23.84kB","physicalWrittenDataSize":"0B","gcStatistics":{"stageId":0,"stageExecutionId":0,"tasks":1,"fullGcTasks":0,"minFullGcSec":0,"maxFullGcSec":0,"totalFullGcSec":0,"averageFullGcSec":0},"cpuDistribution":{"p25":19,"p50":19,"p75":19,"p90":19,"p95":19,"p99":19,"min":19,"max":19,"total":19,"average":19.0},"memoryDistribution":{"p25":0,"p50":0,"p75":0,"p90":0,"p95":0,"p99":0,"min":0,"max":0,"total":0,"average":0.0}},{"stageId":1,"stageExecutionId":0,"tasks":8,"totalScheduledTime":"7.06s","totalCpuTime":"5.21s","retriedCpuTime":"0.00ns","totalBlockedTime":"9.36h","rawInputDataSize":"20.22MB","processedInputDataSize":"20.36MB","physicalWrittenDataSize":"0B","gcStatistics":{"stageId":1,"stageExecutionId":0,"tasks":8,"fullGcTasks":0,"minFullGcSec":0,"maxFullGcSec":0,"totalFullGcSec":0,"averageFullGcSec":0},"cpuDistribution":{"p25":644,"p50":667,"p75":724,"p90":732,"p95":732,"p99":732,"min":537,"max":732,"total":5201,"average":650.125},"memoryDistribution":{"p25":3524986,"p50":3734633,"p75":3862624,"p90":4039274,"p95":4039274,"p99":4039274,"min":3255311,"max":4039274,"total":29209950,"average":3651243.75}},{"stageId":2,"stageExecutionId":0,"tasks":8,"totalScheduledTime":"20.80m","totalCpuTime":"6.11m","retriedCpuTime":"0.00ns","totalBlockedTime":"7.77h","rawInputDataSize":"25.13GB","processedInputDataSize":"25.13GB","physicalWrittenDataSize":"0B","gcStatistics":{"stageId":2,"stageExecutionId":0,"tasks":8,"fullGcTasks":0,"minFullGcSec":0,"maxFullGcSec":0,"totalFullGcSec":0,"averageFullGcSec":0},"cpuDistribution":{"p25":43793,"p50":44210,"p75":50229,"p90":51156,"p95":51156,"p99":51156,"min":41436,"max":51156,"total":366538,"average":45817.25},"memoryDistribution":{"p25":187623459,"p50":194675349,"p75":202396048,"p90":209791544,"p95":209791544,"p99":209791544,"min":156494783,"max":209791544,"total":1534324295,"average":1.91790536875E8}},{"stageId":3,"stageExecutionId":0,"tasks":2,"totalScheduledTime":"32.72ms","totalCpuTime":"18.69ms","retriedCpuTime":"0.00ns","totalBlockedTime":"0.00ns","rawInputDataSize":"2.62MB","processedInputDataSize":"2.62MB","physicalWrittenDataSize":"0B","gcStatistics":{"stageId":3,"stageExecutionId":0,"tasks":2,"fullGcTasks":0,"minFullGcSec":0,"maxFullGcSec":0,"totalFullGcSec":0,"averageFullGcSec":0},"cpuDistribution":{"p25":0,"p50":17,"p75":17,"p90":17,"p95":17,"p99":17,"min":0,"max":17,"total":17,"average":8.5},"memoryDistribution":{"p25":0,"p50":10607,"p75":10607,"p90":10607,"p95":10607,"p99":10607,"min":0,"max":10607,"total":10607,"average":5303.5}},{"stageId":4,"stageExecutionId":0,"tasks":1,"totalScheduledTime":"8.50ms","totalCpuTime":"8.11ms","retriedCpuTime":"0.00ns","totalBlockedTime":"0.00ns","rawInputDataSize":"297.02kB","processedInputDataSize":"297.02kB","physicalWrittenDataSize":"0B","gcStatistics":{"stageId":4,"stageExecutionId":0,"tasks":1,"fullGcTasks":0,"minFullGcSec":0,"maxFullGcSec":0,"totalFullGcSec":0,"averageFullGcSec":0},"cpuDistribution":{"p25":8,"p50":8,"p75":8,"p90":8,"p95":8,"p99":8,"min":8,"max":8,"total":8,"average":8.0},"memoryDistribution":{"p25":125128,"p50":125128,"p75":125128,"p90":125128,"p95":125128,"p99":125128,"min":125128,"max":125128,"total":125128,"average":125128.0}}],"operatorStatistics":[{"stageId":0,"stageExecutionId":0,"pipelineId":0,"operatorId":0,"planNodeId":"637","operatorType":"ExchangeOperator","totalDrivers":32,"addInputCalls":0,"addInputWall":"0.00ns","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"40.50kB","rawInputPositions":508,"inputDataSize":"23.84kB","inputPositions":508,"sumSquaredInputPositions":10434.0,"getOutputCalls":234,"getOutputWall":"40.51ms","getOutputCpu":"2.92ms","getOutputAllocation":"0B","outputDataSize":"23.84kB","outputPositions":508,"physicalWrittenDataSize":"0B","blockedWall":"35.64m","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":"{\"@type\":\"exchangeClientStatus\",\"bufferedBytes\":0,\"maxBufferedBytes\":2000,\"averageBytesPerRequest\":126,\"successfulRequestsCount\":18496,\"bufferedPages\":0,\"noMoreLocations\":true,\"pageBufferClientStatuses\":[]}","runtimeStats":{}},{"stageId":0,"stageExecutionId":0,"pipelineId":1,"operatorId":0,"planNodeId":"723","operatorType":"LocalExchangeSourceOperator","totalDrivers":1,"addInputCalls":0,"addInputWall":"0.00ns","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"23.84kB","inputPositions":508,"sumSquaredInputPositions":258064.0,"getOutputCalls":217,"getOutputWall":"660.37us","getOutputCpu":"626.04us","getOutputAllocation":"0B","outputDataSize":"23.84kB","outputPositions":508,"physicalWrittenDataSize":"0B","blockedWall":"1.11m","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":0,"stageExecutionId":0,"pipelineId":0,"operatorId":1,"planNodeId":"723","operatorType":"LocalExchangeSinkOperator","totalDrivers":32,"addInputCalls":217,"addInputWall":"1.67ms","addInputCpu":"1.63ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"23.84kB","inputPositions":508,"sumSquaredInputPositions":10434.0,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"23.84kB","outputPositions":508,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":32,"finishWall":"103.92us","finishCpu":"90.84us","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":0,"stageExecutionId":0,"pipelineId":1,"operatorId":1,"planNodeId":"523","operatorType":"TopNOperator","totalDrivers":1,"addInputCalls":217,"addInputWall":"1.52ms","addInputCpu":"1.50ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"23.84kB","inputPositions":508,"sumSquaredInputPositions":258064.0,"getOutputCalls":309,"getOutputWall":"607.95us","getOutputCpu":"617.25us","getOutputAllocation":"0B","outputDataSize":"4.69kB","outputPositions":100,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":3,"finishWall":"15.97us","finishCpu":"15.96us","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"95.92kB","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"95.92kB","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":0,"stageExecutionId":0,"pipelineId":1,"operatorId":2,"planNodeId":"16","operatorType":"TaskOutputOperator","totalDrivers":1,"addInputCalls":1,"addInputWall":"97.68us","addInputCpu":"97.66us","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"4.69kB","inputPositions":100,"sumSquaredInputPositions":10000.0,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"4.69kB","outputPositions":100,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":1,"finishWall":"1.78us","finishCpu":"1.85us","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":1,"stageExecutionId":0,"pipelineId":0,"operatorId":0,"planNodeId":"755","operatorType":"ExchangeOperator","totalDrivers":256,"addInputCalls":0,"addInputWall":"0.00ns","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"20.21MB","rawInputPositions":310747,"inputDataSize":"20.36MB","inputPositions":310747,"sumSquaredInputPositions":2.408627899E9,"getOutputCalls":12576,"getOutputWall":"306.18ms","getOutputCpu":"165.80ms","getOutputAllocation":"0B","outputDataSize":"20.36MB","outputPositions":310747,"physicalWrittenDataSize":"0B","blockedWall":"4.63h","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":"{\"@type\":\"exchangeClientStatus\",\"bufferedBytes\":0,\"maxBufferedBytes\":116386,\"averageBytesPerRequest\":1526,\"successfulRequestsCount\":456800,\"bufferedPages\":0,\"noMoreLocations\":true,\"pageBufferClientStatuses\":[]}","runtimeStats":{}},{"stageId":1,"stageExecutionId":0,"pipelineId":1,"operatorId":0,"planNodeId":"749","operatorType":"LocalExchangeSourceOperator","totalDrivers":256,"addInputCalls":0,"addInputWall":"0.00ns","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"20.36MB","inputPositions":310747,"sumSquaredInputPositions":6.17057147E8,"getOutputCalls":211697,"getOutputWall":"793.90ms","getOutputCpu":"695.42ms","getOutputAllocation":"0B","outputDataSize":"20.36MB","outputPositions":310747,"physicalWrittenDataSize":"0B","blockedWall":"4.72h","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":1,"stageExecutionId":0,"pipelineId":0,"operatorId":1,"planNodeId":"749","operatorType":"LocalExchangeSinkOperator","totalDrivers":256,"addInputCalls":12288,"addInputWall":"2.14s","addInputCpu":"1.35s","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"20.36MB","inputPositions":310747,"sumSquaredInputPositions":2.408627899E9,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"20.36MB","outputPositions":310747,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":256,"finishWall":"2.62ms","finishCpu":"2.40ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":1,"stageExecutionId":0,"pipelineId":1,"operatorId":1,"planNodeId":"8","operatorType":"HashAggregationOperator","totalDrivers":256,"addInputCalls":211697,"addInputWall":"619.83ms","addInputCpu":"605.29ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"20.36MB","inputPositions":310747,"sumSquaredInputPositions":6.17057147E8,"getOutputCalls":388139,"getOutputWall":"460.66ms","getOutputCpu":"456.44ms","getOutputAllocation":"0B","outputDataSize":"28.32kB","outputPositions":508,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":907,"finishWall":"26.50ms","finishCpu":"7.50ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"131.66kB","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"131.66kB","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":1,"stageExecutionId":0,"pipelineId":1,"operatorId":2,"planNodeId":"830","operatorType":"FilterAndProjectOperator","totalDrivers":256,"addInputCalls":217,"addInputWall":"6.42ms","addInputCpu":"1.67ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"28.32kB","inputPositions":508,"sumSquaredInputPositions":1576.0,"getOutputCalls":388100,"getOutputWall":"232.67ms","getOutputCpu":"231.91ms","getOutputAllocation":"0B","outputDataSize":"23.84kB","outputPositions":508,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":256,"finishWall":"172.94us","finishCpu":"163.10us","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"884B","peakTotalMemoryReservation":"884B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":1,"stageExecutionId":0,"pipelineId":1,"operatorId":3,"planNodeId":"522","operatorType":"TopNOperator","totalDrivers":256,"addInputCalls":217,"addInputWall":"6.20ms","addInputCpu":"2.81ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"23.84kB","inputPositions":508,"sumSquaredInputPositions":1576.0,"getOutputCalls":388356,"getOutputWall":"280.82ms","getOutputCpu":"271.32ms","getOutputAllocation":"0B","outputDataSize":"23.84kB","outputPositions":508,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":690,"finishWall":"27.99ms","finishCpu":"6.61ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"35.36kB","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"35.36kB","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":1,"stageExecutionId":0,"pipelineId":1,"operatorId":4,"planNodeId":"522","operatorType":"TaskOutputOperator","totalDrivers":256,"addInputCalls":217,"addInputWall":"4.63ms","addInputCpu":"3.68ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"23.84kB","inputPositions":508,"sumSquaredInputPositions":1576.0,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"23.84kB","outputPositions":508,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":256,"finishWall":"162.03us","finishCpu":"174.68us","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":0,"operatorId":0,"planNodeId":"633","operatorType":"ExchangeOperator","totalDrivers":256,"addInputCalls":0,"addInputWall":"0.00ns","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"81.84kB","rawInputPositions":2288,"inputDataSize":"90.16kB","inputPositions":2288,"sumSquaredInputPositions":654368.0,"getOutputCalls":9,"getOutputWall":"205.95us","getOutputCpu":"201.40us","getOutputAllocation":"0B","outputDataSize":"90.16kB","outputPositions":2288,"physicalWrittenDataSize":"0B","blockedWall":"1.26s","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":"{\"@type\":\"exchangeClientStatus\",\"bufferedBytes\":0,\"maxBufferedBytes\":10607,\"averageBytesPerRequest\":765275,\"successfulRequestsCount\":768,\"bufferedPages\":0,\"noMoreLocations\":true,\"pageBufferClientStatuses\":[]}","runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":1,"operatorId":0,"planNodeId":"720","operatorType":"LocalExchangeSourceOperator","totalDrivers":256,"addInputCalls":0,"addInputWall":"0.00ns","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"90.16kB","inputPositions":2288,"sumSquaredInputPositions":22688.0,"getOutputCalls":256,"getOutputWall":"1.12ms","getOutputCpu":"1.02ms","getOutputAllocation":"0B","outputDataSize":"90.16kB","outputPositions":2288,"physicalWrittenDataSize":"0B","blockedWall":"1.30s","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":0,"operatorId":1,"planNodeId":"720","operatorType":"LocalExchangeSinkOperator","totalDrivers":256,"addInputCalls":8,"addInputWall":"2.18ms","addInputCpu":"1.86ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"90.16kB","inputPositions":2288,"sumSquaredInputPositions":654368.0,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"90.16kB","outputPositions":2288,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":256,"finishWall":"2.05ms","finishCpu":"1.88ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":2,"operatorId":0,"planNodeId":"635","operatorType":"ExchangeOperator","totalDrivers":256,"addInputCalls":0,"addInputWall":"0.00ns","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"971.59kB","rawInputPositions":49600,"inputDataSize":"1.09MB","inputPositions":49600,"sumSquaredInputPositions":8.6706064E7,"getOutputCalls":54,"getOutputWall":"1.18ms","getOutputCpu":"916.88us","getOutputAllocation":"0B","outputDataSize":"1.09MB","outputPositions":49600,"physicalWrittenDataSize":"0B","blockedWall":"858.74ms","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":"{\"@type\":\"exchangeClientStatus\",\"bufferedBytes\":0,\"maxBufferedBytes\":125128,\"averageBytesPerRequest\":860519,\"successfulRequestsCount\":512,\"bufferedPages\":0,\"noMoreLocations\":true,\"pageBufferClientStatuses\":[]}","runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":1,"operatorId":1,"planNodeId":"538","operatorType":"HashBuilderOperator","totalDrivers":256,"addInputCalls":256,"addInputWall":"1.24ms","addInputCpu":"1.23ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"90.16kB","inputPositions":2288,"sumSquaredInputPositions":22688.0,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"90.16kB","outputPositions":2288,"physicalWrittenDataSize":"0B","blockedWall":"3.89h","finishCalls":768,"finishWall":"21.96ms","finishCpu":"17.66ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"161.30kB","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"161.30kB","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":3,"operatorId":0,"planNodeId":"721","operatorType":"LocalExchangeSourceOperator","totalDrivers":256,"addInputCalls":0,"addInputWall":"0.00ns","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"1.09MB","inputPositions":49600,"sumSquaredInputPositions":9650192.0,"getOutputCalls":1536,"getOutputWall":"3.62ms","getOutputCpu":"3.50ms","getOutputAllocation":"0B","outputDataSize":"1.09MB","outputPositions":49600,"physicalWrittenDataSize":"0B","blockedWall":"1.67s","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":2,"operatorId":1,"planNodeId":"721","operatorType":"LocalExchangeSinkOperator","totalDrivers":256,"addInputCalls":48,"addInputWall":"4.68ms","addInputCpu":"4.03ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"1.09MB","inputPositions":49600,"sumSquaredInputPositions":8.6706064E7,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"1.09MB","outputPositions":49600,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":256,"finishWall":"3.92ms","finishCpu":"1.90ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":4,"operatorId":0,"planNodeId":"826","operatorType":"ScanFilterAndProjectOperator","totalDrivers":6204,"addInputCalls":11317212,"addInputWall":"13.84m","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"25.12GB","rawInputPositions":2879987999,"inputDataSize":"25.12GB","inputPositions":2879987999,"sumSquaredInputPositions":5.420460999613131E15,"getOutputCalls":2835551,"getOutputWall":"17.46m","getOutputCpu":"3.32m","getOutputAllocation":"0B","outputDataSize":"85.82GB","outputPositions":2879987999,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"7.37MB","peakTotalMemoryReservation":"7.37MB","spilledDataSize":"0B","info":null,"runtimeStats":{"storageReadDataBytes":{"name":"storageReadDataBytes","unit":"BYTE","sum":26985884944,"count":4608,"max":7544493,"min":832389},"storageReadTimeNanos":{"name":"storageReadTimeNanos","unit":"NANO","sum":829910814599,"count":4608,"max":1151544516,"min":414646}}},{"stageId":2,"stageExecutionId":0,"pipelineId":3,"operatorId":1,"planNodeId":"539","operatorType":"HashBuilderOperator","totalDrivers":256,"addInputCalls":1536,"addInputWall":"12.04ms","addInputCpu":"6.16ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"1.09MB","inputPositions":49600,"sumSquaredInputPositions":9650192.0,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"1.09MB","outputPositions":49600,"physicalWrittenDataSize":"0B","blockedWall":"3.79h","finishCalls":768,"finishWall":"145.23ms","finishCpu":"127.76ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"157.84kB","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"157.84kB","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":4,"operatorId":1,"planNodeId":"538","operatorType":"LookupJoinOperator","totalDrivers":6204,"addInputCalls":2817000,"addInputWall":"17.99s","addInputCpu":"4.01s","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"85.82GB","inputPositions":2879987999,"sumSquaredInputPositions":5.420460999613131E15,"getOutputCalls":2837257,"getOutputWall":"2.70m","getOutputCpu":"2.44m","getOutputAllocation":"0B","outputDataSize":"127.03MB","outputPositions":2713615,"physicalWrittenDataSize":"0B","blockedWall":"962.25ms","finishCalls":9252,"finishWall":"26.57ms","finishCpu":"30.30ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":"{\"@type\":\"joinOperatorInfo\",\"joinType\":\"INNER\",\"logHistogramProbes\":[2877274385,2713614,0,0,0,0,0,0],\"logHistogramOutput\":[0,2713614,0,0,0,0,0,0],\"lookupSourcePositions\":1774344}","runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":4,"operatorId":2,"planNodeId":"827","operatorType":"FilterAndProjectOperator","totalDrivers":6204,"addInputCalls":1741388,"addInputWall":"2.46s","addInputCpu":"2.40s","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"127.03MB","inputPositions":2713615,"sumSquaredInputPositions":4.815164853E9,"getOutputCalls":2832609,"getOutputWall":"8.89s","getOutputCpu":"7.99s","getOutputAllocation":"0B","outputDataSize":"137.54MB","outputPositions":2713615,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":6204,"finishWall":"18.14ms","finishCpu":"18.74ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"121.87kB","peakTotalMemoryReservation":"121.87kB","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":4,"operatorId":3,"planNodeId":"539","operatorType":"LookupJoinOperator","totalDrivers":6204,"addInputCalls":1536,"addInputWall":"26.53ms","addInputCpu":"25.95ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"137.54MB","inputPositions":2713615,"sumSquaredInputPositions":4.815164853E9,"getOutputCalls":2838822,"getOutputWall":"2.18s","getOutputCpu":"1.93s","getOutputAllocation":"0B","outputDataSize":"21.82MB","outputPositions":438836,"physicalWrittenDataSize":"0B","blockedWall":"946.90ms","finishCalls":9285,"finishWall":"41.52ms","finishCpu":"39.96ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":"{\"@type\":\"joinOperatorInfo\",\"joinType\":\"INNER\",\"logHistogramProbes\":[2275037,438578,0,0,0,0,0,0],\"logHistogramOutput\":[0,438578,0,0,0,0,0,0],\"lookupSourcePositions\":37032600}","runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":4,"operatorId":4,"planNodeId":"829","operatorType":"FilterAndProjectOperator","totalDrivers":6204,"addInputCalls":1536,"addInputWall":"14.67ms","addInputCpu":"14.50ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"21.82MB","inputPositions":438836,"sumSquaredInputPositions":1.262957E8,"getOutputCalls":2834471,"getOutputWall":"1.80s","getOutputCpu":"1.68s","getOutputAllocation":"0B","outputDataSize":"20.56MB","outputPositions":438836,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":6204,"finishWall":"8.86ms","finishCpu":"9.27ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"1.04kB","peakTotalMemoryReservation":"1.04kB","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":4,"operatorId":5,"planNodeId":"753","operatorType":"HashAggregationOperator","totalDrivers":6204,"addInputCalls":1536,"addInputWall":"157.79ms","addInputCpu":"153.92ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"20.56MB","inputPositions":438836,"sumSquaredInputPositions":1.262957E8,"getOutputCalls":2842211,"getOutputWall":"2.47s","getOutputCpu":"1.90s","getOutputAllocation":"0B","outputDataSize":"20.36MB","outputPositions":310747,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":10812,"finishWall":"29.48ms","finishCpu":"29.24ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"157.46kB","peakTotalMemoryReservation":"157.46kB","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":2,"stageExecutionId":0,"pipelineId":4,"operatorId":6,"planNodeId":"753","operatorType":"OptimizedPartitionedOutputOperator","totalDrivers":6204,"addInputCalls":1536,"addInputWall":"60.39ms","addInputCpu":"58.74ms","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"20.36MB","inputPositions":310747,"sumSquaredInputPositions":6.3221985E7,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"20.16MB","outputPositions":310747,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":6204,"finishWall":"300.47ms","finishCpu":"126.00ms","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"55.29kB","peakTotalMemoryReservation":"55.29kB","spilledDataSize":"0B","info":"{\"@type\":\"partitionedOutput\",\"rowsAdded\":310747,\"pagesAdded\":12288,\"outputBufferPeakMemoryUsage\":75172}","runtimeStats":{}},{"stageId":3,"stageExecutionId":0,"pipelineId":0,"operatorId":0,"planNodeId":"249","operatorType":"ScanFilterAndProjectOperator","totalDrivers":2,"addInputCalls":1080,"addInputWall":"15.02ms","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"2.62MB","rawInputPositions":300000,"inputDataSize":"2.62MB","inputPositions":300000,"sumSquaredInputPositions":9.0E10,"getOutputCalls":304,"getOutputWall":"32.65ms","getOutputCpu":"18.61ms","getOutputAllocation":"0B","outputDataSize":"11.27kB","outputPositions":286,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"582.79kB","peakTotalMemoryReservation":"582.79kB","spilledDataSize":"0B","info":null,"runtimeStats":{"storageReadDataBytes":{"name":"storageReadDataBytes","unit":"BYTE","sum":2742381,"count":4,"max":1200554,"min":382716},"storageReadTimeNanos":{"name":"storageReadTimeNanos","unit":"NANO","sum":15021541,"count":4,"max":6508053,"min":1594166}}},{"stageId":3,"stageExecutionId":0,"pipelineId":0,"operatorId":1,"planNodeId":"249","operatorType":"TaskOutputOperator","totalDrivers":2,"addInputCalls":1,"addInputWall":"51.16us","addInputCpu":"51.15us","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"11.27kB","inputPositions":286,"sumSquaredInputPositions":81796.0,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"11.27kB","outputPositions":286,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":2,"finishWall":"2.68us","finishCpu":"3.43us","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":4,"stageExecutionId":0,"pipelineId":0,"operatorId":0,"planNodeId":"244","operatorType":"ScanFilterAndProjectOperator","totalDrivers":1,"addInputCalls":313,"addInputWall":"518.10us","addInputCpu":"0.00ns","addInputAllocation":"0B","rawInputDataSize":"297.02kB","rawInputPositions":73049,"inputDataSize":"297.02kB","inputPositions":73049,"sumSquaredInputPositions":5.336156401E9,"getOutputCalls":82,"getOutputWall":"7.22ms","getOutputCpu":"6.82ms","getOutputAllocation":"0B","outputDataSize":"84.77kB","outputPositions":6200,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":0,"finishWall":"0.00ns","finishCpu":"0.00ns","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"522.10kB","peakTotalMemoryReservation":"522.10kB","spilledDataSize":"0B","info":"{\"@type\":\"splitOperator\",\"splitInfo\":{\"path\":\"s3a://presto-workload/tpcds-sf1000-uncompressed-delta/date_dim/part-00000-41ab58d6-d1bf-484a-9518-cf760bf40a03-c000.parquet\",\"start\":\"0\",\"length\":\"3167427\",\"fileSize\":\"3167427\",\"fileModifiedTime\":\"1725348018000\",\"hosts\":\"[]\",\"database\":\"tpcds_sf1000_parquet_delta\",\"table\":\"date_dim\",\"nodeSelectionStrategy\":\"SOFT_AFFINITY_BY_SPLIT\",\"partitionName\":\"\",\"s3SelectPushdownEnabled\":\"false\",\"cacheQuotaRequirement\":\"CacheQuotaRequirement{cacheQuotaScope=GLOBAL, quota=Optional.empty}\"}}","runtimeStats":{"storageReadDataBytes":{"name":"storageReadDataBytes","unit":"BYTE","sum":304148,"count":3,"max":292344,"min":1636},"storageReadTimeNanos":{"name":"storageReadTimeNanos","unit":"NANO","sum":518096,"count":3,"max":321762,"min":34511}}},{"stageId":4,"stageExecutionId":0,"pipelineId":0,"operatorId":1,"planNodeId":"828","operatorType":"FilterAndProjectOperator","totalDrivers":1,"addInputCalls":1,"addInputWall":"3.99us","addInputCpu":"3.97us","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"84.77kB","inputPositions":6200,"sumSquaredInputPositions":3.844E7,"getOutputCalls":87,"getOutputWall":"1.13ms","getOutputCpu":"1.14ms","getOutputAllocation":"0B","outputDataSize":"139.26kB","outputPositions":6200,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":7,"finishWall":"20.23us","finishCpu":"20.44us","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"600B","peakTotalMemoryReservation":"600B","spilledDataSize":"0B","info":null,"runtimeStats":{}},{"stageId":4,"stageExecutionId":0,"pipelineId":0,"operatorId":2,"planNodeId":"828","operatorType":"TaskOutputOperator","totalDrivers":1,"addInputCalls":6,"addInputWall":"116.32us","addInputCpu":"116.46us","addInputAllocation":"0B","rawInputDataSize":"0B","rawInputPositions":0,"inputDataSize":"139.26kB","inputPositions":6200,"sumSquaredInputPositions":3.844E7,"getOutputCalls":0,"getOutputWall":"0.00ns","getOutputCpu":"0.00ns","getOutputAllocation":"0B","outputDataSize":"139.26kB","outputPositions":6200,"physicalWrittenDataSize":"0B","blockedWall":"0.00ns","finishCalls":1,"finishWall":"2.23us","finishCpu":"2.27us","finishAllocation":"0B","userMemoryReservation":"0B","revocableMemoryReservation":"0B","systemMemoryReservation":"0B","peakUserMemoryReservation":"0B","peakSystemMemoryReservation":"0B","peakTotalMemoryReservation":"0B","spilledDataSize":"0B","info":null,"runtimeStats":{}}],"planStatisticsRead":[{"id":"16","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"523","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"723","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"637","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"522","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"830","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"8","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"749","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"755","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"753","planStatistics":{"rowCount":{"value":"NaN","unknown":true},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"829","planStatistics":{"rowCount":{"value":2807994.207349996,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"539","planStatistics":{"rowCount":{"value":2807994.207349996,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"827","planStatistics":{"rowCount":{"value":2940309.5490711736,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"538","planStatistics":{"rowCount":{"value":2940309.5490711736,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"826","planStatistics":{"rowCount":{"value":2.879987999E9,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":1.0},"sourceInfo":{"confident":true}},{"id":"1","planStatistics":{"rowCount":{"value":2.879987999E9,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":1.0},"sourceInfo":{"confident":true}},{"id":"720","planStatistics":{"rowCount":{"value":306.2835209825998,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"633","planStatistics":{"rowCount":{"value":306.2835209825998,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"249","planStatistics":{"rowCount":{"value":306.2835209825998,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"632","planStatistics":{"rowCount":{"value":306.2835209825998,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":1.0},"sourceInfo":{"confident":true}},{"id":"3","planStatistics":{"rowCount":{"value":300000.0,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":1.0},"sourceInfo":{"confident":true}},{"id":"721","planStatistics":{"rowCount":{"value":6087.416666666666,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"635","planStatistics":{"rowCount":{"value":6087.416666666666,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"828","planStatistics":{"rowCount":{"value":6087.416666666666,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"244","planStatistics":{"rowCount":{"value":6087.416666666666,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":0.0},"sourceInfo":{"confident":false}},{"id":"634","planStatistics":{"rowCount":{"value":6087.416666666666,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":1.0},"sourceInfo":{"confident":true}},{"id":"0","planStatistics":{"rowCount":{"value":73049.0,"unknown":false},"outputSize":{"value":"NaN","unknown":true},"confidence":1.0},"sourceInfo":{"confident":true}}],"planStatisticsWritten":[],"createTime":1750063688.759000000,"executionStartTime":1750063689.069000000,"endTime":1750063756.554000000,"expandedQuery":null,"optimizerInformation":[{"optimizerName":"PruneFilterColumns","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"PruneLimitColumns","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"RemoveRedundantIdentityProjections","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"PruneTableScanColumns","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"PruneCrossJoinColumns","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"PruneProjectColumns","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"FilterRowExpressionRewrite","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"RemoveRedundantIdentityProjections","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"PickTableLayoutForPredicate","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"PickTableLayoutWithoutPredicate","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"RemoveRedundantIdentityProjections","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"CreatePartialTopN","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"ReorderJoins","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"PushPartialAggregationThroughExchange","optimizerTriggered":true,"optimizerApplicable":null},{"optimizerName":"RemoveRedundantIdentityProjections","optimizerTriggered":true,"optimizerApplicable":null}],"scalarFunctions":[],"aggregateFunctions":[],"windowsFunctions":[]},"queryOptimiserEvent":null,"splitCompletedEvent":null,"plan":"Fragment 0 [SINGLE]<<>> CPU: 19.16ms, Scheduled: 209.78ms, Input: 508 rows (23.84kB); per task: avg.: 508.00 std.dev.: 0.00, Output: 100 rows (4.69kB)<<>> Output layout: [d_year, i_brand_id, i_brand, sum]<<>> Output partitioning: SINGLE []<<>> Stage Execution Strategy: UNGROUPED_EXECUTION<<>> - Output[d_year, brand_id, brand, sum_agg] => [d_year:integer, i_brand_id:integer, i_brand:varchar(50), sum:decimal(38,2)]<<>> CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Output: 100 rows (4.69kB)<<>> Input avg.: 100.00 rows, Input std.dev.: 0.00%<<>> brand_id := i_brand_id (3:8)<<>> brand := i_brand (4:8)<<>> sum_agg := sum (5:8)<<>> - TopN[100 by (d_year ASC_NULLS_LAST, sum DESC_NULLS_LAST, i_brand_id ASC_NULLS_LAST)] => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, sum:decimal(38,2)]<<>> CPU: 3.00ms (0.00%), Scheduled: 3.00ms (0.00%), Output: 100 rows (4.69kB)<<>> Input avg.: 508.00 rows, Input std.dev.: 0.00%<<>> - LocalExchange[SINGLE] () => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, sum:decimal(38,2)]<<>> CPU: 3.00ms (0.00%), Scheduled: 3.00ms (0.00%), Output: 508 rows (23.84kB)<<>> Input avg.: 15.88 rows, Input std.dev.: 54.21%<<>> - RemoteSource[1] => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, sum:decimal(38,2)]<<>> CPU: 3.00ms (0.00%), Scheduled: 41.00ms (0.00%), Output: 508 rows (23.84kB)<<>> Input avg.: 15.88 rows, Input std.dev.: 54.21%<<>><<>>Fragment 1 [HASH]<<>> CPU: 5.21s, Scheduled: 7.06s, Input: 310747 rows (20.36MB); per task: avg.: 38843.38 std.dev.: 6857.07, Output: 508 rows (23.84kB)<<>> Output layout: [d_year, i_brand, i_brand_id, sum]<<>> Output partitioning: SINGLE []<<>> Stage Execution Strategy: UNGROUPED_EXECUTION<<>> - TopNPartial[100 by (d_year ASC_NULLS_LAST, sum DESC_NULLS_LAST, i_brand_id ASC_NULLS_LAST)] => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, sum:decimal(38,2)]<<>> CPU: 283.00ms (0.08%), Scheduled: 319.00ms (0.02%), Output: 508 rows (23.84kB)<<>> Input avg.: 1.98 rows, Input std.dev.: 75.06%<<>> - Project[projectLocality = LOCAL] => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, sum:decimal(38,2)]<<>> CPU: 234.00ms (0.06%), Scheduled: 238.00ms (0.01%), Output: 508 rows (23.84kB)<<>> Input avg.: 1.98 rows, Input std.dev.: 75.06%<<>> - Aggregate(FINAL)[d_year, i_brand, i_brand_id][$hashvalue] => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, $hashvalue:bigint, sum:decimal(38,2)]<<>> CPU: 1.07s (0.29%), Scheduled: 1.11s (0.05%), Output: 508 rows (28.32kB)<<>> Input avg.: 1213.86 rows, Input std.dev.: 79.74%<<>> sum := \"presto.default.sum\"((sum_27)) (5:8)<<>> - LocalExchange[HASH][$hashvalue] (d_year, i_brand, i_brand_id) => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, sum_27:varbinary, $hashvalue:bigint]<<>> CPU: 2.04s (0.55%), Scheduled: 2.94s (0.14%), Output: 310747 rows (20.36MB)<<>> Input avg.: 1213.86 rows, Input std.dev.: 232.07%<<>> - RemoteSource[2] => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, sum_27:varbinary, $hashvalue_28:bigint]<<>> CPU: 166.00ms (0.04%), Scheduled: 306.00ms (0.01%), Output: 310747 rows (20.36MB)<<>> Input avg.: 1213.86 rows, Input std.dev.: 232.07%<<>><<>>Fragment 2 [SOURCE]<<>> CPU: 6.11m, Scheduled: 20.80m, Input: 2880039887 rows (25.13GB); per task: avg.: 360004985.88 std.dev.: 2360903.52, Output: 310747 rows (20.17MB)<<>> Output layout: [d_year, i_brand, i_brand_id, sum_27, $hashvalue_37]<<>> Output partitioning: HASH [d_year, i_brand, i_brand_id][$hashvalue_37]<<>> Stage Execution Strategy: UNGROUPED_EXECUTION<<>> - Aggregate(PARTIAL)[d_year, i_brand, i_brand_id][$hashvalue_37] => [d_year:integer, i_brand:varchar(50), i_brand_id:integer, $hashvalue_37:bigint, sum_27:varbinary]<<>> CPU: 2.27s (0.61%), Scheduled: 3.02s (0.14%), Output: 310747 rows (20.16MB)<<>> Input avg.: 70.73 rows, Input std.dev.: 175.18%<<>> sum_27 := \"presto.default.sum\"((ss_net_profit)) (5:8)<<>> - Project[projectLocality = LOCAL] => [ss_net_profit:decimal(7,2), i_brand_id:integer, i_brand:varchar(50), d_year:integer, $hashvalue_37:bigint]<<>> Estimates: {rows: 2807994 (265.45MB), cpu: 245109888183.85, memory: 1228207.22, network: 1228207.22}<<>> CPU: 1.70s (0.46%), Scheduled: 1.82s (0.09%), Output: 438836 rows (20.56MB)<<>> Input avg.: 70.73 rows, Input std.dev.: 175.18%<<>> $hashvalue_37 := combine_hash(combine_hash(combine_hash(BIGINT'0', COALESCE($operator$hash_code(d_year), BIGINT'0')), COALESCE($operator$hash_code(i_brand), BIGINT'0')), COALESCE($operator$hash_code(i_brand_id), BIGINT'0')) (13:10)<<>> - InnerJoin[(\"ss_sold_date_sk\" = \"cast\")][$hashvalue_33, $hashvalue_34] => [ss_net_profit:decimal(7,2), i_brand_id:integer, i_brand:varchar(50), d_year:integer]<<>> Estimates: {rows: 2807994 (265.45MB), cpu: 244961719039.14, memory: 1228207.22, network: 1228207.22}<<>> CPU: 2.14s (0.58%), Scheduled: 2.41s (0.12%), Output: 438836 rows (21.82MB)<<>> Left (probe) Input avg.: 437.40 rows, Input std.dev.: 174.84%<<>> Right (build) Input avg.: 193.75 rows, Input std.dev.: 6.47%<<>> Distribution: REPLICATED<<>> - Project[projectLocality = LOCAL] => [ss_sold_date_sk:bigint, ss_net_profit:decimal(7,2), i_brand_id:integer, i_brand:varchar(50), $hashvalue_33:bigint]<<>> Estimates: {rows: 2940310 (277.96MB), cpu: 244668311238.19, memory: 108122.55, network: 108122.55}<<>> CPU: 10.41s (2.81%), Scheduled: 11.37s (0.55%), Output: 2713615 rows (137.54MB)<<>> Input avg.: 437.40 rows, Input std.dev.: 174.84%<<>> $hashvalue_33 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(ss_sold_date_sk), BIGINT'0')) (7:7)<<>> - InnerJoin[(\"ss_item_sk\" = \"i_item_sk\")][$hashvalue_29, $hashvalue_30] => [ss_sold_date_sk:bigint, ss_net_profit:decimal(7,2), i_brand_id:integer, i_brand:varchar(50)]<<>> Estimates: {rows: 2940310 (277.96MB), cpu: 244502457507.84, memory: 108122.55, network: 108122.55}<<>> CPU: 2.51m (40.69%), Scheduled: 3.00m (8.65%), Output: 2713615 rows (127.03MB)<<>> Left (probe) Input avg.: 464214.70 rows, Input std.dev.: 174.77%<<>> Right (build) Input avg.: 8.94 rows, Input std.dev.: 33.09%<<>> Distribution: REPLICATED<<>> - ScanProject[table = TableHandle {connectorId='glue', connectorHandle='HiveTableHandle{schemaName=tpcds_sf1000_parquet_delta, tableName=store_sales, analyzePartitionValues=Optional.empty}', layout='Optional[tpcds_sf1000_parquet_delta.store_sales{}]'}, grouped = false, projectLocality = LOCAL] => [ss_sold_date_sk:bigint, ss_item_sk:integer, ss_net_profit:decimal(7,2), $hashvalue_29:bigint]<<>> Estimates: {rows: 2879987999 (346.00GB), cpu: 64166333769.00, memory: 0.00, network: 0.00}/{rows: 2879987999 (346.00GB), cpu: 154252559529.00, memory: 0.00, network: 0.00}<<>> CPU: 3.32m (53.81%), Scheduled: 31.30m (90.22%), Output: 2879987999 rows (85.82GB)<<>> Input avg.: 464214.70 rows, Input std.dev.: 174.77%<<>> $hashvalue_29 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(ss_item_sk), BIGINT'0')) (7:7)<<>> LAYOUT: tpcds_sf1000_parquet_delta.store_sales{}<<>> ss_item_sk := ss_item_sk:int:2:REGULAR (7:6)<<>> ss_sold_date_sk := ss_sold_date_sk:bigint:0:REGULAR (7:6)<<>> ss_net_profit := ss_net_profit:decimal(7,2):22:REGULAR (7:6)<<>> Input: 2879987999 rows (25.12GB), Filtered: 0.00%<<>> - LocalExchange[HASH][$hashvalue_30] (i_item_sk) => [i_item_sk:integer, i_brand_id:integer, i_brand:varchar(50), $hashvalue_30:bigint]<<>> Estimates: {rows: 306 (29.65kB), cpu: 24078544.64, memory: 0.00, network: 108122.55}<<>> CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Output: 2288 rows (90.16kB)<<>> Input avg.: 8.94 rows, Input std.dev.: 556.78%<<>> - RemoteSource[3] => [i_item_sk:integer, i_brand_id:integer, i_brand:varchar(50), $hashvalue_31:bigint]<<>> CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Output: 2288 rows (90.16kB)<<>> Input avg.: 8.94 rows, Input std.dev.: 556.78%<<>> - LocalExchange[HASH][$hashvalue_34] (cast) => [d_year:integer, cast:bigint, $hashvalue_34:bigint]<<>> Estimates: {rows: 6087 (766.87kB), cpu: 2556715.00, memory: 0.00, network: 1120084.67}<<>> CPU: 4.00ms (0.00%), Scheduled: 9.00ms (0.00%), Output: 49600 rows (1.09MB)<<>> Input avg.: 193.75 rows, Input std.dev.: 283.24%<<>> - RemoteSource[4] => [d_year:integer, cast:bigint, $hashvalue_35:bigint]<<>> CPU: 0.00ns (0.00%), Scheduled: 0.00ns (0.00%), Output: 49600 rows (1.09MB)<<>> Input avg.: 193.75 rows, Input std.dev.: 283.24%<<>><<>>Fragment 3 [SOURCE]<<>> CPU: 18.69ms, Scheduled: 32.72ms, Input: 300000 rows (2.62MB); per task: avg.: 150000.00 std.dev.: 150000.00, Output: 286 rows (11.27kB)<<>> Output layout: [i_item_sk, i_brand_id, i_brand, $hashvalue_32]<<>> Output partitioning: BROADCAST []<<>> Stage Execution Strategy: UNGROUPED_EXECUTION<<>> - ScanFilterProject[table = TableHandle {connectorId='glue', connectorHandle='HiveTableHandle{schemaName=tpcds_sf1000_parquet_delta, tableName=item, analyzePartitionValues=Optional.empty}', layout='Optional[tpcds_sf1000_parquet_delta.item{domains={i_manufact_id=[ [[\"445\"]] ]}}]'}, grouped = false, filterPredicate = (i_manufact_id) = (INTEGER'445'), projectLocality = LOCAL] => [i_item_sk:integer, i_brand_id:integer, i_brand:varchar(50), $hashvalue_32:bigint]<<>> Estimates: {rows: 300000 (12.62MB), cpu: 12025757.00, memory: 0.00, network: 0.00}/{rows: 306 (13.20kB), cpu: 24051514.00, memory: 0.00, network: 0.00}/{rows: 306 (13.20kB), cpu: 24065029.32, memory: 0.00, network: 0.00}<<>> CPU: 19.00ms (0.01%), Scheduled: 48.00ms (0.00%), Output: 286 rows (11.27kB)<<>> Input avg.: 150000.00 rows, Input std.dev.: 100.00%<<>> $hashvalue_32 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(i_item_sk), BIGINT'0')) (8:6)<<>> LAYOUT: tpcds_sf1000_parquet_delta.item{domains={i_manufact_id=[ [[\"445\"]] ]}}<<>> i_manufact_id := i_manufact_id:int:13:REGULAR (8:6)<<>> i_brand_id := i_brand_id:int:7:REGULAR (8:6)<<>> i_brand := i_brand:varchar(50):8:REGULAR (8:6)<<>> i_item_sk := i_item_sk:int:0:REGULAR (8:6)<<>> Input: 300000 rows (2.62MB), Filtered: 99.90%<<>><<>>Fragment 4 [SOURCE]<<>> CPU: 8.11ms, Scheduled: 8.50ms, Input: 73049 rows (297.02kB); per task: avg.: 73049.00 std.dev.: 0.00, Output: 6200 rows (139.26kB)<<>> Output layout: [d_year, cast, $hashvalue_36]<<>> Output partitioning: BROADCAST []<<>> Stage Execution Strategy: UNGROUPED_EXECUTION<<>> - Project[projectLocality = LOCAL] => [d_year:integer, cast:bigint, $hashvalue_36:bigint]<<>> Estimates: {rows: 6087 (136.73kB), cpu: 2416704.42, memory: 0.00, network: 0.00}<<>> CPU: 1.00ms (0.00%), Scheduled: 1.00ms (0.00%), Output: 6200 rows (139.26kB)<<>> Input avg.: 6200.00 rows, Input std.dev.: 0.00%<<>> $hashvalue_36 := combine_hash(BIGINT'0', COALESCE($operator$hash_code(cast), BIGINT'0')) (6:7)<<>> - ScanFilterProject[table = TableHandle {connectorId='glue', connectorHandle='HiveTableHandle{schemaName=tpcds_sf1000_parquet_delta, tableName=date_dim, analyzePartitionValues=Optional.empty}', layout='Optional[tpcds_sf1000_parquet_delta.date_dim{domains={d_moy=[ [[\"12\"]] ]}}]'}, grouped = false, filterPredicate = (d_moy) = (INTEGER'12'), projectLocality = LOCAL] => [d_year:integer, cast:bigint]<<>> Estimates: {rows: 73049 (1.60MB), cpu: 1095735.00, memory: 0.00, network: 0.00}/{rows: 6087 (136.73kB), cpu: 2191470.00, memory: 0.00, network: 0.00}/{rows: 6087 (136.73kB), cpu: 2276693.83, memory: 0.00, network: 0.00}<<>> CPU: 7.00ms (0.00%), Scheduled: 8.00ms (0.00%), Output: 6200 rows (84.77kB)<<>> Input avg.: 73049.00 rows, Input std.dev.: 0.00%<<>> cast := CAST(d_date_sk AS bigint) (6:7)<<>> LAYOUT: tpcds_sf1000_parquet_delta.date_dim{domains={d_moy=[ [[\"12\"]] ]}}<<>> d_moy := d_moy:int:8:REGULAR (6:6)<<>> d_date_sk := d_date_sk:int:0:REGULAR (6:6)<<>> d_year := d_year:int:6:REGULAR (6:6)<<>> Input: 73049 rows (297.02kB), Filtered: 91.51%<<>><<>>","cpuTimeMillis":371792,"retriedCpuTimeMillis":0,"wallTimeMillis":67795,"queuedTimeMillis":0,"analysisTimeMillis":630} diff --git a/cmd/loadeljson/202506160854_0.ndjson b/cmd/loadeljson/202506160854_0.ndjson new file mode 100644 index 00000000..83cafb44 --- /dev/null +++ b/cmd/loadeljson/202506160854_0.ndjson @@ -0,0 +1,2 @@ +{"instanceId":"978ecaae-0fe1-4997-8bfc-78b04e54662a","clusterName":"cluster1","queryCreatedEvent":null,"queryCompletedEvent":{"metadata":{"queryId":"20250616_085426_00030_ux3a6","transactionId":"9c48a888-c021-48e0-9374-df86b05c765d","tracingId":"noop_dummy_id","query":"SELECT * FROM jmx.current.\"com.facebook.alluxio:name=client.cachehitrate,type=gauges\"","queryHash":"b1b008c1baff725a","preparedQuery":null,"queryState":"FAILED","uri":"http://10.128.58.157:8585/v1/query/20250616_085426_00030_ux3a6","plan":"","jsonPlan":null,"graphvizPlan":null,"payload":"","runtimeOptimizedStages":[],"planNodeRuntimeStats":null},"statistics":{"cpuTime":0.0,"retriedCpuTime":0.0,"wallTime":0.0,"waitingForPrerequisitesTime":0.001000000,"queuedTime":0.0,"waitingForResourcesTime":0.0,"semanticAnalyzingTime":0.0,"columnAccessPermissionCheckingTime":0.0,"dispatchingTime":0.0,"planningTime":0.0,"analysisTime":null,"executionTime":0.0,"peakRunningTasks":0,"peakUserMemoryBytes":0,"peakTotalNonRevocableMemoryBytes":0,"peakTaskUserMemory":0,"peakTaskTotalMemory":0,"peakNodeTotalMemory":0,"totalBytes":0,"totalRows":0,"outputBytes":0,"outputRows":0,"writtenOutputBytes":0,"writtenOutputRows":0,"writtenIntermediateBytes":0,"spilledBytes":0,"cumulativeMemory":0.0,"cumulativeTotalMemory":0.0,"completedSplits":0,"complete":true,"runtimeStats":{}},"context":{"user":"presto","principal":"presto","remoteClientAddress":"10.128.13.249","userAgent":"StatementClientV1/0.286","clientInfo":null,"clientTags":[],"source":"presto-cli","queryActionType":null,"catalog":"glue","schema":"tpcds_sf1000_partitioned_delta","resourceGroupId":["global"],"sessionProperties":{"query_max_scan_raw_input_bytes":"10TB","query_max_memory_per_node":"32985348833B","query_max_run_time":"2h","pushdown_subfields_enabled":"true","query_max_total_memory_per_node":"43980465110B","query_max_execution_time":"3h","query_max_output_size":"500GB","task_concurrency":"32","join_distribution_type":"AUTOMATIC","join_reordering_strategy":"AUTOMATIC","query_max_stage_count":"300","resource_overcommit":"true","query_max_memory":"2TB"},"resourceEstimates":{"executionTime":null,"cpuTime":null,"peakMemory":null,"peakTaskMemory":null},"serverAddress":"10.128.58.157","serverVersion":"0.282","environment":"reetikaalluxio"},"ioMetadata":{"inputs":[],"output":null},"failureInfo":{"errorCode":{"code":1,"name":"SYNTAX_ERROR","type":"USER_ERROR","retriable":false},"failureType":"com.facebook.presto.sql.analyzer.SemanticException","failureMessage":"Table jmx.current.com.facebook.alluxio:name=client.cachehitrate,type=gauges does not exist","failureTask":null,"failureHost":null,"failuresJson":"{\"type\":\"com.facebook.presto.sql.analyzer.SemanticException\",\"message\":\"Table jmx.current.com.facebook.alluxio:name=client.cachehitrate,type=gauges does not exist\",\"suppressed\":[],\"stack\":[\"com.facebook.presto.util.MetadataUtils.getTableColumnMetadata(MetadataUtils.java:92)\",\"com.facebook.presto.util.MetadataUtils.getTableColumnsMetadata(MetadataUtils.java:54)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitTable(StatementAnalyzer.java:1354)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitTable(StatementAnalyzer.java:367)\",\"com.facebook.presto.sql.tree.Table.accept(Table.java:60)\",\"com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:381)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.analyzeFrom(StatementAnalyzer.java:2801)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuerySpecification(StatementAnalyzer.java:1771)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuerySpecification(StatementAnalyzer.java:367)\",\"com.facebook.presto.sql.tree.QuerySpecification.accept(QuerySpecification.java:138)\",\"com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:381)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:389)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuery(StatementAnalyzer.java:1188)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.visitQuery(StatementAnalyzer.java:367)\",\"com.facebook.presto.sql.tree.Query.accept(Query.java:105)\",\"com.facebook.presto.sql.tree.AstVisitor.process(AstVisitor.java:27)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer$Visitor.process(StatementAnalyzer.java:381)\",\"com.facebook.presto.sql.analyzer.StatementAnalyzer.analyze(StatementAnalyzer.java:359)\",\"com.facebook.presto.sql.analyzer.Analyzer.analyzeSemantic(Analyzer.java:122)\",\"com.facebook.presto.sql.analyzer.BuiltInQueryAnalyzer.analyze(BuiltInQueryAnalyzer.java:100)\",\"com.facebook.presto.execution.SqlQueryExecution.(SqlQueryExecution.java:207)\",\"com.facebook.presto.execution.SqlQueryExecution.(SqlQueryExecution.java:109)\",\"com.facebook.presto.execution.SqlQueryExecution$SqlQueryExecutionFactory.createQueryExecution(SqlQueryExecution.java:954)\",\"com.facebook.presto.dispatcher.LocalDispatchQueryFactory.lambda$createDispatchQuery$0(LocalDispatchQueryFactory.java:171)\",\"com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)\",\"com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:75)\",\"com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)\",\"java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\",\"java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\",\"java.base/java.lang.Thread.run(Thread.java:829)\"],\"errorCode\":{\"code\":1,\"name\":\"SYNTAX_ERROR\",\"type\":\"USER_ERROR\",\"retriable\":false},\"errorCause\":\"UNKNOWN\"}"},"warnings":[],"queryType":"SELECT","failedTasks":[],"stageStatistics":[],"operatorStatistics":[],"planStatisticsRead":[],"planStatisticsWritten":[],"createTime":1750064067.198000000,"executionStartTime":1750064067.208000000,"endTime":1750064067.208000000,"expandedQuery":null,"optimizerInformation":[],"scalarFunctions":[],"aggregateFunctions":[],"windowsFunctions":[]},"queryOptimiserEvent":null,"splitCompletedEvent":null,"plan":"null","cpuTimeMillis":0,"retriedCpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":0,"analysisTimeMillis":0} +{"instanceId":"978ecaae-0fe1-4997-8bfc-78b04e54662a","clusterName":"cluster1","queryCreatedEvent":null,"queryCompletedEvent":{"metadata":{"queryId":"20250616_085452_00031_ux3a6","transactionId":null,"tracingId":"noop_dummy_id","query":"show tables from FROM jmx.current like '%alluxio%'","queryHash":"8bf8b4b9e84fccf1","preparedQuery":null,"queryState":"FAILED","uri":"http://10.128.58.157:8585/v1/query/20250616_085452_00031_ux3a6","plan":"","jsonPlan":null,"graphvizPlan":null,"payload":"","runtimeOptimizedStages":[],"planNodeRuntimeStats":null},"statistics":{"cpuTime":0.0,"retriedCpuTime":0.0,"wallTime":0.0,"waitingForPrerequisitesTime":0.0,"queuedTime":0.0,"waitingForResourcesTime":0.0,"semanticAnalyzingTime":0.0,"columnAccessPermissionCheckingTime":0.0,"dispatchingTime":0.0,"planningTime":0.0,"analysisTime":null,"executionTime":0.0,"peakRunningTasks":0,"peakUserMemoryBytes":0,"peakTotalNonRevocableMemoryBytes":0,"peakTaskUserMemory":0,"peakTaskTotalMemory":0,"peakNodeTotalMemory":0,"totalBytes":0,"totalRows":0,"outputBytes":0,"outputRows":0,"writtenOutputBytes":0,"writtenOutputRows":0,"writtenIntermediateBytes":0,"spilledBytes":0,"cumulativeMemory":0.0,"cumulativeTotalMemory":0.0,"completedSplits":0,"complete":true,"runtimeStats":{}},"context":{"user":"presto","principal":"presto","remoteClientAddress":"10.128.13.249","userAgent":"StatementClientV1/0.286","clientInfo":null,"clientTags":[],"source":"presto-cli","queryActionType":null,"catalog":"glue","schema":"tpcds_sf1000_partitioned_delta","resourceGroupId":null,"sessionProperties":{},"resourceEstimates":{"executionTime":null,"cpuTime":null,"peakMemory":null,"peakTaskMemory":null},"serverAddress":"10.128.58.157","serverVersion":"0.282","environment":"reetikaalluxio"},"ioMetadata":{"inputs":[],"output":null},"failureInfo":{"errorCode":{"code":1,"name":"SYNTAX_ERROR","type":"USER_ERROR","retriable":false},"failureType":"com.facebook.presto.sql.parser.ParsingException","failureMessage":"line 1:18: mismatched input 'FROM'. Expecting: ","failureTask":null,"failureHost":null,"failuresJson":"{\"type\":\"com.facebook.presto.sql.parser.ParsingException\",\"message\":\"line 1:18: mismatched input 'FROM'. Expecting: \",\"suppressed\":[],\"stack\":[\"com.facebook.presto.sql.parser.ErrorHandler.syntaxError(ErrorHandler.java:109)\",\"org.antlr.v4.runtime.ProxyErrorListener.syntaxError(ProxyErrorListener.java:41)\",\"org.antlr.v4.runtime.Parser.notifyErrorListeners(Parser.java:544)\",\"org.antlr.v4.runtime.DefaultErrorStrategy.reportUnwantedToken(DefaultErrorStrategy.java:377)\",\"org.antlr.v4.runtime.DefaultErrorStrategy.singleTokenDeletion(DefaultErrorStrategy.java:548)\",\"org.antlr.v4.runtime.DefaultErrorStrategy.sync(DefaultErrorStrategy.java:266)\",\"com.facebook.presto.sql.parser.SqlBaseParser.identifier(SqlBaseParser.java:13855)\",\"com.facebook.presto.sql.parser.SqlBaseParser.qualifiedName(SqlBaseParser.java:13443)\",\"com.facebook.presto.sql.parser.SqlBaseParser.statement(SqlBaseParser.java:3811)\",\"com.facebook.presto.sql.parser.SqlBaseParser.singleStatement(SqlBaseParser.java:257)\",\"com.facebook.presto.sql.parser.SqlParser.invokeParser(SqlParser.java:173)\",\"com.facebook.presto.sql.parser.SqlParser.createStatement(SqlParser.java:108)\",\"com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.prepareQuery(BuiltInQueryPreparer.java:70)\",\"com.facebook.presto.sql.analyzer.BuiltInQueryPreparer.prepareQuery(BuiltInQueryPreparer.java:56)\",\"com.facebook.presto.dispatcher.DispatchManager.createQueryInternal(DispatchManager.java:330)\",\"com.facebook.presto.dispatcher.DispatchManager.lambda$createQuery$0(DispatchManager.java:279)\",\"com.facebook.airlift.concurrent.BoundedExecutor.drainQueue(BoundedExecutor.java:78)\",\"java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\",\"java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\",\"java.base/java.lang.Thread.run(Thread.java:829)\"],\"errorLocation\":{\"lineNumber\":1,\"columnNumber\":18},\"errorCode\":{\"code\":1,\"name\":\"SYNTAX_ERROR\",\"type\":\"USER_ERROR\",\"retriable\":false},\"errorCause\":\"UNKNOWN\"}"},"warnings":[],"queryType":null,"failedTasks":[],"stageStatistics":[],"operatorStatistics":[],"planStatisticsRead":[],"planStatisticsWritten":[],"createTime":1750064092.783000000,"executionStartTime":1750064092.783000000,"endTime":1750064092.783000000,"expandedQuery":null,"optimizerInformation":[],"scalarFunctions":[],"aggregateFunctions":[],"windowsFunctions":[]},"queryOptimiserEvent":null,"splitCompletedEvent":null,"plan":"null","cpuTimeMillis":0,"retriedCpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":0,"analysisTimeMillis":0} diff --git a/cmd/loadeljson/db.go b/cmd/loadeljson/db.go new file mode 100644 index 00000000..b0123ae6 --- /dev/null +++ b/cmd/loadeljson/db.go @@ -0,0 +1,385 @@ +package loadeljson + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "pbench/log" +) + +// insertEventListenerData inserts data into multiple tables to mirror Java implementation in MySQLWriter.post() +func insertEventListenerData(ctx context.Context, db *sql.DB, qce *QueryCompletedEvent, queryId string) error { + // 1. Insert into presto_query_creation_info + if err := insertQueryCreationInfo(ctx, db, qce, queryId); err != nil { + return fmt.Errorf("failed to insert query creation info: %w", err) + } + + // 2. Insert into presto_query_plans + if err := insertQueryPlans(ctx, db, qce, queryId); err != nil { + return fmt.Errorf("failed to insert query plans: %w", err) + } + + // 3. Insert into presto_query_stage_stats + if err := insertQueryStageStats(ctx, db, qce, queryId); err != nil { + return fmt.Errorf("failed to insert query stage stats: %w", err) + } + + // 4. Insert into presto_query_operator_stats + if err := insertQueryOperatorStats(ctx, db, qce, queryId); err != nil { + return fmt.Errorf("failed to insert query operator stats: %w", err) + } + + // 5. Insert into presto_query_statistics + if err := insertQueryStatistics(ctx, db, qce, queryId); err != nil { + return fmt.Errorf("failed to insert query statistics: %w", err) + } + + return nil +} + +func insertQueryCreationInfo(ctx context.Context, db *sql.DB, qce *QueryCompletedEvent, queryId string) error { + query := `REPLACE INTO presto_query_creation_info( + query_id, query, create_time, schema_name, catalog_name, environment, + user, remote_client_address, source, user_agent, uri, + session_properties_json, server_version, client_info, resource_group_name, + principal, transaction_id, client_tags, resource_estimates, dt + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + + var schema, catalog, remoteAddr, source, userAgent, clientInfo, principal, resourceGroupId string + if qce.Context.Schema != nil { + schema = *qce.Context.Schema + } + if qce.Context.Catalog != nil { + catalog = *qce.Context.Catalog + } + if qce.Context.RemoteClientAddress != nil { + remoteAddr = *qce.Context.RemoteClientAddress + } + if qce.Context.Source != nil { + source = *qce.Context.Source + } + if qce.Context.UserAgent != nil { + userAgent = *qce.Context.UserAgent + } + if qce.Context.ClientInfo != nil { + clientInfo = *qce.Context.ClientInfo + } + if qce.Context.Principal != nil { + principal = *qce.Context.Principal + } + if qce.Context.ResourceGroupId != nil { + resourceGroupId = qce.Context.ResourceGroupId.Value + } + + // Handle transaction_id + var transactionId string + if qce.Metadata.TransactionId != nil { + transactionId = *qce.Metadata.TransactionId + } + + // Marshal client_tags + clientTagsJson, err := json.Marshal(qce.Context.ClientTags) + if err != nil { + log.Error().Err(err).Str("query_id", queryId).Msg("failed to marshal client tags") + clientTagsJson = []byte("[]") + } + + // Marshal resource_estimates + resourceEstimatesJson, err := json.Marshal(qce.Context.ResourceEstimates) + if err != nil { + log.Error().Err(err).Str("query_id", queryId).Msg("failed to marshal resource estimates") + resourceEstimatesJson = []byte("{}") + } + + sessionPropsJson, err := json.Marshal(qce.Context.SessionProperties) + if err != nil { + log.Error().Err(err).Str("query_id", queryId).Msg("failed to marshal session properties") + sessionPropsJson = []byte("{}") + } + + dt := qce.CreateTime.Time.Format("2006-01-02 15:04:05") + + _, err = db.ExecContext(ctx, query, + queryId, qce.Metadata.Query, + qce.CreateTime.Time.Format("2006-01-02 15:04:05"), + schema, catalog, qce.Context.Environment, + qce.Context.User, remoteAddr, source, userAgent, qce.Metadata.Uri, + string(sessionPropsJson), qce.Context.ServerVersion, clientInfo, resourceGroupId, + principal, transactionId, string(clientTagsJson), string(resourceEstimatesJson), dt, + ) + + return err +} + +func insertQueryPlans(ctx context.Context, db *sql.DB, qce *QueryCompletedEvent, queryId string) error { + query := `REPLACE INTO presto_query_plans( + query_id, query, plan, json_plan, environment, dt + ) VALUES (?, ?, ?, ?, ?, ?)` + + var plan, jsonPlan string + if qce.Metadata.Plan != nil { + plan = *qce.Metadata.Plan + } + if qce.Metadata.JsonPlan != nil { + jsonPlan = *qce.Metadata.JsonPlan + } + + dt := qce.CreateTime.Time.Format("2006-01-02 15:04:05") + + _, err := db.ExecContext(ctx, query, + queryId, qce.Metadata.Query, plan, jsonPlan, qce.Context.Environment, dt, + ) + + return err +} + +func insertQueryStageStats(ctx context.Context, db *sql.DB, qce *QueryCompletedEvent, queryId string) error { + if len(qce.StageStatistics) == 0 { + return nil + } + + query := `REPLACE INTO presto_query_stage_stats( + query_id, stage_id, stage_execution_id, tasks, + total_scheduled_time_ms, total_cpu_time_ms, retried_cpu_time_ms, total_blocked_time_ms, + raw_input_data_size_bytes, processed_input_data_size_bytes, physical_written_data_size_bytes, + gc_statistics, cpu_distribution, memory_distribution, dt + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + + dt := qce.CreateTime.Time.Format("2006-01-02 15:04:05") + + for _, stage := range qce.StageStatistics { + // These fields are not available in the event listener JSON, so we use empty JSON objects + gcStatsJson := []byte("{}") + cpuDistJson := []byte("{}") + memDistJson := []byte("{}") + + _, err := db.ExecContext(ctx, query, + queryId, stage.StageId, stage.StageExecutionId, stage.Tasks, + stage.TotalScheduledTime.Milliseconds(), + stage.TotalCpuTime.Milliseconds(), + stage.RetriedCpuTime.Milliseconds(), + stage.TotalBlockedTime.Milliseconds(), + stage.RawInputDataSize.Bytes, + stage.ProcessedInputDataSize.Bytes, + stage.PhysicalWrittenDataSize.Bytes, + string(gcStatsJson), string(cpuDistJson), string(memDistJson), dt, + ) + + if err != nil { + return fmt.Errorf("failed to insert stage %d: %w", stage.StageId, err) + } + } + + return nil +} + +func insertQueryOperatorStats(ctx context.Context, db *sql.DB, qce *QueryCompletedEvent, queryId string) error { + if len(qce.OperatorStatistics) == 0 { + return nil + } + + query := `REPLACE INTO presto_query_operator_stats( + query_id, stage_id, stage_execution_id, pipeline_id, operator_id, + plan_node_id, operator_type, total_drivers, + add_input_calls, add_input_wall_ms, add_input_cpu_ms, add_input_allocation_bytes, + raw_input_data_size_bytes, raw_input_positions, input_data_size_bytes, input_positions, + sum_squared_input_positions, get_output_calls, get_output_wall_ms, get_output_cpu_ms, + get_output_allocation_bytes, output_data_size_bytes, output_positions, + physical_written_data_size_bytes, blocked_wall_ms, finish_calls, finish_wall_ms, + finish_cpu_ms, finish_allocation_bytes, user_memory_reservation_bytes, + revocable_memory_reservation_bytes, system_memory_reservation_bytes, + peak_user_memory_reservation_bytes, peak_system_memory_reservation_bytes, + peak_total_memory_reservation_bytes, spilled_data_size_bytes, info, dt + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + + dt := qce.CreateTime.Time.Format("2006-01-02 15:04:05") + + for _, op := range qce.OperatorStatistics { + var info string + if op.Info != nil { + info = *op.Info + } + + _, err := db.ExecContext(ctx, query, + queryId, op.StageId, op.StageExecutionId, op.PipelineId, op.OperatorId, + op.PlanNodeId, op.OperatorType, op.TotalDrivers, + op.AddInputCalls, op.AddInputWall.Milliseconds(), op.AddInputCpu.Milliseconds(), + op.AddInputAllocation.Bytes, op.RawInputDataSize.Bytes, op.RawInputPositions, + op.InputDataSize.Bytes, op.InputPositions, 0.0, // sumSquaredInputPositions not available + op.GetOutputCalls, op.GetOutputWall.Milliseconds(), op.GetOutputCpu.Milliseconds(), + op.GetOutputAllocation.Bytes, op.OutputDataSize.Bytes, op.OutputPositions, + op.PhysicalWrittenDataSize.Bytes, op.BlockedWall.Milliseconds(), op.FinishCalls, + op.FinishWall.Milliseconds(), op.FinishCpu.Milliseconds(), op.FinishAllocation.Bytes, + op.UserMemoryReservation.Bytes, op.RevocableMemoryReservation.Bytes, + op.SystemMemoryReservation.Bytes, op.PeakUserMemoryReservation.Bytes, + op.PeakSystemMemoryReservation.Bytes, op.PeakTotalMemoryReservation.Bytes, + op.SpilledDataSize.Bytes, info, dt, + ) + + if err != nil { + return fmt.Errorf("failed to insert operator %d: %w", op.OperatorId, err) + } + } + + return nil +} + +func insertQueryStatistics(ctx context.Context, db *sql.DB, qce *QueryCompletedEvent, queryId string) error { + // Insert into presto_query_statistics table + // This mirrors the Java implementation in PrestoQueryStatsDao.insertQueryStatistics + + query := `REPLACE INTO presto_query_statistics( + query_id, query, query_type, schema_name, catalog_name, environment, + user, remote_client_address, source, user_agent, uri, + session_properties_json, server_version, client_info, resource_group_name, + principal, transaction_id, client_tags, resource_estimates, + create_time, end_time, execution_start_time, query_state, + failure_message, failure_type, failures_json, failure_task, failure_host, + error_code, error_code_name, error_category, warnings_json, + splits, analysis_time_ms, queued_time_ms, query_wall_time_ms, + query_execution_time_ms, bytes_per_cpu_sec, bytes_per_sec, rows_per_cpu_sec, + total_bytes, total_rows, output_rows, output_bytes, + written_rows, written_bytes, cumulative_memory, peak_user_memory_bytes, + peak_total_memory_bytes, peak_task_total_memory, peak_task_user_memory, + written_intermediate_bytes, peak_node_total_memory, total_split_cpu_time_ms, + stage_count, cumulative_total_memory, dt + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + + var queryType string + if qce.QueryType != nil { + queryType = *qce.QueryType + } + + var schema, catalog, remoteAddr, source, userAgent, clientInfo, principal, resourceGroupId string + if qce.Context.Schema != nil { + schema = *qce.Context.Schema + } + if qce.Context.Catalog != nil { + catalog = *qce.Context.Catalog + } + if qce.Context.RemoteClientAddress != nil { + remoteAddr = *qce.Context.RemoteClientAddress + } + if qce.Context.Source != nil { + source = *qce.Context.Source + } + if qce.Context.UserAgent != nil { + userAgent = *qce.Context.UserAgent + } + if qce.Context.ClientInfo != nil { + clientInfo = *qce.Context.ClientInfo + } + if qce.Context.Principal != nil { + principal = *qce.Context.Principal + } + if qce.Context.ResourceGroupId != nil { + resourceGroupId = qce.Context.ResourceGroupId.Value + } + + var failureMsg, failureType, failureTask, failureHost, failuresJson string + var errorCode int + var errorCodeName, errorCategory string + + if qce.FailureInfo != nil { + if qce.FailureInfo.FailureMessage != nil { + failureMsg = *qce.FailureInfo.FailureMessage + } + if qce.FailureInfo.FailureType != nil { + failureType = *qce.FailureInfo.FailureType + } + if qce.FailureInfo.FailureTask != nil { + failureTask = *qce.FailureInfo.FailureTask + } + if qce.FailureInfo.FailureHost != nil { + failureHost = *qce.FailureInfo.FailureHost + } + failuresJson = qce.FailureInfo.FailuresJson + errorCode = qce.FailureInfo.ErrorCode.Code + errorCodeName = qce.FailureInfo.ErrorCode.Name + errorCategory = qce.FailureInfo.ErrorCode.Type + } + + // Handle transaction_id + var transactionId string + if qce.Metadata.TransactionId != nil { + transactionId = *qce.Metadata.TransactionId + } + + // Marshal client_tags + clientTagsJson, err := json.Marshal(qce.Context.ClientTags) + if err != nil { + log.Error().Err(err).Str("query_id", queryId).Msg("failed to marshal client tags") + clientTagsJson = []byte("[]") + } + + // Marshal resource_estimates + resourceEstimatesJson, err := json.Marshal(qce.Context.ResourceEstimates) + if err != nil { + log.Error().Err(err).Str("query_id", queryId).Msg("failed to marshal resource estimates") + resourceEstimatesJson = []byte("{}") + } + + sessionPropsJson, err := json.Marshal(qce.Context.SessionProperties) + if err != nil { + log.Error().Err(err).Str("query_id", queryId).Msg("failed to marshal session properties") + sessionPropsJson = []byte("{}") + } + warningsJson, err := json.Marshal(qce.Warnings) + if err != nil { + log.Error().Err(err).Str("query_id", queryId).Msg("failed to marshal warnings") + warningsJson = []byte("[]") + } + + var analysisTimeMs int64 + if qce.Statistics.AnalysisTime != nil { + analysisTimeMs = qce.Statistics.AnalysisTime.Milliseconds() + } + + // Calculate query_execution_time_ms from ResourceEstimates + var queryExecutionTimeMs int64 + if qce.Context.ResourceEstimates.ExecutionTime != nil { + queryExecutionTimeMs = qce.Context.ResourceEstimates.ExecutionTime.Milliseconds() + } + + cpuTimeMs := qce.Statistics.CpuTime.Milliseconds() + var bytesPerCpuSec, bytesPerSec, rowsPerCpuSec int64 + if cpuTimeMs > 0 { + bytesPerCpuSec = qce.Statistics.TotalBytes / cpuTimeMs + rowsPerCpuSec = qce.Statistics.TotalRows / cpuTimeMs + } + // Calculate bytes_per_sec from ResourceEstimates.ExecutionTime + if queryExecutionTimeMs > 0 { + bytesPerSec = qce.Statistics.TotalBytes / queryExecutionTimeMs + } + + dt := qce.CreateTime.Time.Format("2006-01-02 15:04:05") + + _, err = db.ExecContext(ctx, query, + queryId, qce.Metadata.Query, queryType, schema, catalog, qce.Context.Environment, + qce.Context.User, remoteAddr, source, userAgent, qce.Metadata.Uri, + string(sessionPropsJson), qce.Context.ServerVersion, clientInfo, resourceGroupId, + principal, transactionId, string(clientTagsJson), string(resourceEstimatesJson), + qce.CreateTime.Time.Format("2006-01-02 15:04:05"), + qce.EndTime.Time.Format("2006-01-02 15:04:05"), + qce.ExecutionStartTime.Time.Format("2006-01-02 15:04:05"), + qce.Metadata.QueryState, + failureMsg, failureType, failuresJson, failureTask, failureHost, + errorCode, errorCodeName, errorCategory, string(warningsJson), + qce.Statistics.CompletedSplits, analysisTimeMs, + qce.Statistics.QueuedTime.Milliseconds(), + qce.Statistics.WallTime.Milliseconds(), + queryExecutionTimeMs, + bytesPerCpuSec, bytesPerSec, rowsPerCpuSec, + qce.Statistics.TotalBytes, qce.Statistics.TotalRows, + qce.Statistics.OutputPositions, qce.Statistics.OutputBytes, + qce.Statistics.WrittenOutputRows, qce.Statistics.WrittenOutputBytes, qce.Statistics.CumulativeMemory, + qce.Statistics.PeakUserMemoryBytes, + qce.Statistics.PeakTotalNonRevocableMemoryBytes, qce.Statistics.PeakTaskTotalMemory, qce.Statistics.PeakTaskUserMemory, + qce.Statistics.WrittenOutputBytes, qce.Statistics.PeakNodeTotalMemory, + cpuTimeMs, + len(qce.StageStatistics), qce.Statistics.CumulativeTotalMemory, dt, + ) + + return err +} diff --git a/cmd/loadeljson/db_test.go b/cmd/loadeljson/db_test.go new file mode 100644 index 00000000..e176eea1 --- /dev/null +++ b/cmd/loadeljson/db_test.go @@ -0,0 +1,441 @@ +package loadeljson + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInsertQueryCreationInfo(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + + qce := &QueryCompletedEvent{ + Metadata: QueryMetadata{ + QueryId: "20250616_085426_00030_ux3a6", + Query: "SELECT * FROM test", + QueryState: "FINISHED", + Uri: "http://example.com", + }, + Context: QueryContext{ + User: "presto", + ServerVersion: "0.282", + Environment: "test", + ClientTags: []string{}, + SessionProperties: map[string]string{}, + ResourceEstimates: ResourceEstimates{}, + }, + CreateTime: PrestoTime{Time: time.Date(2025, 6, 16, 8, 48, 8, 0, time.UTC)}, + } + + mock.ExpectExec("REPLACE INTO presto_query_creation_info"). + WithArgs( + queryId, + qce.Metadata.Query, + sqlmock.AnyArg(), // create_time + "", "", // schema, catalog + qce.Context.Environment, + qce.Context.User, + "", "", "", // remote_client_address, source, user_agent + qce.Metadata.Uri, + sqlmock.AnyArg(), // session_properties_json + qce.Context.ServerVersion, + "", "", // client_info, resource_group_name + "", "", // principal, transaction_id + sqlmock.AnyArg(), // client_tags + sqlmock.AnyArg(), // resource_estimates + sqlmock.AnyArg(), // dt + ). + WillReturnResult(sqlmock.NewResult(1, 1)) + + err = insertQueryCreationInfo(ctx, db, qce, queryId) + assert.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestInsertQueryPlans(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + plan := "test plan" + jsonPlan := "{}" + + qce := &QueryCompletedEvent{ + Metadata: QueryMetadata{ + QueryId: "test_id", + Query: "SELECT 1", + Plan: &plan, + JsonPlan: &jsonPlan, + }, + Context: QueryContext{ + Environment: "test", + }, + CreateTime: PrestoTime{Time: time.Date(2025, 6, 16, 8, 48, 8, 0, time.UTC)}, + } + + mock.ExpectExec("REPLACE INTO presto_query_plans"). + WithArgs( + queryId, + qce.Metadata.Query, + plan, + jsonPlan, + qce.Context.Environment, + sqlmock.AnyArg(), // dt + ). + WillReturnResult(sqlmock.NewResult(1, 1)) + + err = insertQueryPlans(ctx, db, qce, queryId) + assert.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestInsertQueryStageStats(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + + qce := &QueryCompletedEvent{ + StageStatistics: []StageStatistics{ + { + StageId: 0, + StageExecutionId: 0, + Tasks: 1, + TotalScheduledTime: Duration{Duration: 209 * time.Millisecond}, + TotalCpuTime: Duration{Duration: 19 * time.Millisecond}, + RetriedCpuTime: Duration{Duration: 0}, + TotalBlockedTime: Duration{Duration: 36*time.Minute + 46*time.Second}, + RawInputDataSize: DataSize{Bytes: 40500}, + ProcessedInputDataSize: DataSize{Bytes: 23840}, + PhysicalWrittenDataSize: DataSize{Bytes: 0}, + }, + }, + CreateTime: PrestoTime{Time: time.Date(2025, 6, 16, 8, 48, 8, 0, time.UTC)}, + } + + mock.ExpectExec("REPLACE INTO presto_query_stage_stats"). + WithArgs( + queryId, + 0, 0, 1, // stage_id, stage_execution_id, tasks + int64(209), int64(19), int64(0), // scheduled, cpu, retried cpu + int64(2206000), // blocked time in ms + int64(40500), int64(23840), int64(0), // data sizes + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), // gc, cpu dist, mem dist + sqlmock.AnyArg(), // dt + ). + WillReturnResult(sqlmock.NewResult(1, 1)) + + err = insertQueryStageStats(ctx, db, qce, queryId) + assert.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestInsertQueryStageStats_Empty(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + + qce := &QueryCompletedEvent{ + StageStatistics: []StageStatistics{}, + CreateTime: PrestoTime{Time: time.Now()}, + } + + // Should not execute any queries when there are no stage statistics + err = insertQueryStageStats(ctx, db, qce, queryId) + assert.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestInsertQueryOperatorStats(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + info := "test info" + + qce := &QueryCompletedEvent{ + OperatorStatistics: []OperatorStatistics{ + { + StageId: 0, + StageExecutionId: 0, + PipelineId: 0, + OperatorId: 0, + PlanNodeId: "637", + OperatorType: "ExchangeOperator", + TotalDrivers: 32, + AddInputCalls: 0, + AddInputWall: Duration{Duration: 0}, + AddInputCpu: Duration{Duration: 0}, + Info: &info, + }, + }, + CreateTime: PrestoTime{Time: time.Date(2025, 6, 16, 8, 48, 8, 0, time.UTC)}, + } + + mock.ExpectExec("REPLACE INTO presto_query_operator_stats"). + WithArgs( + queryId, + 0, 0, 0, 0, // stage_id, stage_execution_id, pipeline_id, operator_id + "637", "ExchangeOperator", int64(32), // plan_node_id, operator_type, total_drivers + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), // add_input_calls, add_input_wall_ms, add_input_cpu_ms, add_input_allocation_bytes + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), // raw_input_data_size_bytes, raw_input_positions, input_data_size_bytes, input_positions + sqlmock.AnyArg(), // sum_squared_input_positions + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), // get_output_calls, get_output_wall_ms, get_output_cpu_ms, get_output_allocation_bytes + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), // output_data_size_bytes, output_positions, physical_written_data_size_bytes + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), // blocked_wall_ms, finish_calls, finish_wall_ms + sqlmock.AnyArg(), sqlmock.AnyArg(), // finish_cpu_ms, finish_allocation_bytes + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), // user_memory_reservation_bytes, revocable_memory_reservation_bytes, system_memory_reservation_bytes + sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), // peak_user_memory_reservation_bytes, peak_system_memory_reservation_bytes, peak_total_memory_reservation_bytes + sqlmock.AnyArg(), info, sqlmock.AnyArg(), // spilled_data_size_bytes, info, dt + ). + WillReturnResult(sqlmock.NewResult(1, 1)) + + err = insertQueryOperatorStats(ctx, db, qce, queryId) + assert.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestInsertQueryStatistics(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + queryType := "SELECT" + + qce := &QueryCompletedEvent{ + Metadata: QueryMetadata{ + QueryId: "test_id", + Query: "SELECT 1", + QueryState: "FINISHED", + Uri: "http://example.com", + }, + Statistics: QueryStatistics{ + CpuTime: Duration{Duration: 371792 * time.Millisecond}, + WallTime: Duration{Duration: 67795 * time.Millisecond}, + QueuedTime: Duration{Duration: 0}, + PeakUserMemoryBytes: 98161684, + PeakTotalNonRevocableMemoryBytes: 1098226989, + PeakTaskUserMemory: 10433657, + PeakTaskTotalMemory: 202274048, + PeakNodeTotalMemory: 222073173, + TotalBytes: 26975446031, + TotalRows: 2880361048, + OutputPositions: 100, + OutputBytes: 4801, + WrittenOutputRows: 0, + WrittenOutputBytes: 0, + CumulativeMemory: 5.359635941788949e12, + CumulativeTotalMemory: 8.514280417681293e12, + CompletedSplits: 7776, + }, + Context: QueryContext{ + User: "presto", + ServerVersion: "0.282", + Environment: "test", + ClientTags: []string{}, + SessionProperties: map[string]string{}, + ResourceEstimates: ResourceEstimates{}, + }, + QueryType: &queryType, + Warnings: []interface{}{}, + CreateTime: PrestoTime{Time: time.Date(2025, 6, 16, 8, 48, 8, 759000000, time.UTC)}, + ExecutionStartTime: PrestoTime{Time: time.Date(2025, 6, 16, 8, 48, 9, 69000000, time.UTC)}, + EndTime: PrestoTime{Time: time.Date(2025, 6, 16, 8, 49, 16, 554000000, time.UTC)}, + StageStatistics: []StageStatistics{}, + } + + mock.ExpectExec("REPLACE INTO presto_query_statistics"). + WillReturnResult(sqlmock.NewResult(1, 1)) + + err = insertQueryStatistics(ctx, db, qce, queryId) + assert.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestInsertQueryStatistics_WithFailure(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + failureMsg := "Syntax error" + failureType := "com.facebook.presto.sql.parser.ParsingException" + + qce := &QueryCompletedEvent{ + Metadata: QueryMetadata{ + QueryId: "test_id", + Query: "SELECT * FROM nonexistent", + QueryState: "FAILED", + Uri: "http://example.com", + }, + Statistics: QueryStatistics{ + CpuTime: Duration{Duration: 0}, + WallTime: Duration{Duration: 0}, + QueuedTime: Duration{Duration: 0}, + PeakUserMemoryBytes: 0, + PeakTotalNonRevocableMemoryBytes: 0, + PeakTaskUserMemory: 0, + PeakTaskTotalMemory: 0, + PeakNodeTotalMemory: 0, + TotalBytes: 0, + TotalRows: 0, + OutputPositions: 0, + OutputBytes: 0, + WrittenOutputRows: 0, + WrittenOutputBytes: 0, + CumulativeMemory: 0, + CumulativeTotalMemory: 0, + CompletedSplits: 0, + }, + Context: QueryContext{ + User: "test", + ServerVersion: "0.282", + Environment: "test", + ClientTags: []string{}, + SessionProperties: map[string]string{}, + ResourceEstimates: ResourceEstimates{}, + }, + FailureInfo: &QueryFailureInfo{ + ErrorCode: ErrorCode{ + Code: 1, + Name: "SYNTAX_ERROR", + Type: "USER_ERROR", + }, + FailureMessage: &failureMsg, + FailureType: &failureType, + FailuresJson: "{}", + }, + Warnings: []interface{}{}, + CreateTime: PrestoTime{Time: time.Now()}, + ExecutionStartTime: PrestoTime{Time: time.Now()}, + EndTime: PrestoTime{Time: time.Now()}, + StageStatistics: []StageStatistics{}, + } + + mock.ExpectExec("REPLACE INTO presto_query_statistics"). + WillReturnResult(sqlmock.NewResult(1, 1)) + + err = insertQueryStatistics(ctx, db, qce, queryId) + assert.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestInsertEventListenerData(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + + qce := &QueryCompletedEvent{ + Metadata: QueryMetadata{ + QueryId: "test_id", + Query: "SELECT 1", + QueryState: "FINISHED", + Uri: "http://example.com", + }, + Statistics: QueryStatistics{ + CpuTime: Duration{Duration: 0}, + WallTime: Duration{Duration: 0}, + QueuedTime: Duration{Duration: 0}, + PeakUserMemoryBytes: 0, + PeakTotalNonRevocableMemoryBytes: 0, + PeakTaskUserMemory: 0, + PeakTaskTotalMemory: 0, + PeakNodeTotalMemory: 0, + TotalBytes: 0, + TotalRows: 0, + OutputPositions: 0, + OutputBytes: 0, + WrittenOutputRows: 0, + WrittenOutputBytes: 0, + CumulativeMemory: 0, + CumulativeTotalMemory: 0, + CompletedSplits: 0, + }, + Context: QueryContext{ + User: "test", + ServerVersion: "0.282", + Environment: "test", + ClientTags: []string{}, + SessionProperties: map[string]string{}, + ResourceEstimates: ResourceEstimates{}, + }, + Warnings: []interface{}{}, + CreateTime: PrestoTime{Time: time.Now()}, + ExecutionStartTime: PrestoTime{Time: time.Now()}, + EndTime: PrestoTime{Time: time.Now()}, + StageStatistics: []StageStatistics{}, + OperatorStatistics: []OperatorStatistics{}, + } + + // Expect all 5 insert operations + mock.ExpectExec("REPLACE INTO presto_query_creation_info"). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("REPLACE INTO presto_query_plans"). + WillReturnResult(sqlmock.NewResult(1, 1)) + // Stage stats and operator stats are empty, so no expectations for them + mock.ExpectExec("REPLACE INTO presto_query_statistics"). + WillReturnResult(sqlmock.NewResult(1, 1)) + + err = insertEventListenerData(ctx, db, qce, queryId) + assert.NoError(t, err) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestInsertEventListenerData_DatabaseError(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + ctx := context.Background() + queryId := "test_query_id" + + qce := &QueryCompletedEvent{ + Metadata: QueryMetadata{ + QueryId: "test_id", + Query: "SELECT 1", + QueryState: "FINISHED", + Uri: "http://example.com", + }, + Statistics: QueryStatistics{}, + Context: QueryContext{User: "test", ServerVersion: "0.282", Environment: "test", ClientTags: []string{}, SessionProperties: map[string]string{}, ResourceEstimates: ResourceEstimates{}}, + Warnings: []interface{}{}, + CreateTime: PrestoTime{Time: time.Now()}, + ExecutionStartTime: PrestoTime{Time: time.Now()}, + EndTime: PrestoTime{Time: time.Now()}, + StageStatistics: []StageStatistics{}, + OperatorStatistics: []OperatorStatistics{}, + } + + // Simulate a database error + mock.ExpectExec("REPLACE INTO presto_query_creation_info"). + WillReturnError(sql.ErrConnDone) + + err = insertEventListenerData(ctx, db, qce, queryId) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to insert query creation info") +} diff --git a/cmd/loadeljson/main.go b/cmd/loadeljson/main.go new file mode 100644 index 00000000..5a5db1c8 --- /dev/null +++ b/cmd/loadeljson/main.go @@ -0,0 +1,362 @@ +package loadeljson + +import ( + "bufio" + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "os/signal" + "path/filepath" + "pbench/log" + "pbench/stage" + "pbench/utils" + "reflect" + "sync" + "syscall" + "time" + + "github.com/spf13/cobra" +) + +var ( + RunName string + Comment string + OutputPath string + RecordRun bool + MySQLCfgPath string + InfluxCfgPath string + Parallelism int + IsNDJSON bool + + runRecorders = make([]stage.RunRecorder, 0, 3) + queryResults = make([]*stage.QueryResult, 0, 8) + runStartTime, runEndTime = newSyncedTime(time.Now()), newSyncedTime(time.UnixMilli(0)) + mysqlDb *sql.DB + pseudoStage *stage.Stage + + parallelismGuard chan struct{} + resultChan = make(chan *stage.QueryResult) + runningTasks sync.WaitGroup +) + +func Run(_ *cobra.Command, args []string) { + OutputPath = filepath.Join(OutputPath, RunName) + utils.PrepareOutputDirectory(OutputPath) + + // also start to write logs to the output directory from this point on. + logPath := filepath.Join(OutputPath, "loadeljson.log") + flushLog := utils.InitLogFile(logPath) + defer flushLog() + + // Any error run recorder initialization will make the run recorder a noop. + // The program will continue with corresponding error logs. + mysqlDb = utils.InitMySQLConnFromCfg(MySQLCfgPath) + if RecordRun { + registerRunRecorder(stage.NewFileBasedRunRecorder()) + registerRunRecorder(stage.NewInfluxRunRecorder(InfluxCfgPath)) + registerRunRecorder(stage.NewMySQLRunRecorderWithDb(mysqlDb)) + } + + log.Info().Int("parallelism", Parallelism).Send() + ctx, cancel := context.WithCancel(context.Background()) + timeToExit := make(chan os.Signal, 1) + signal.Notify(timeToExit, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + // Handle SIGINT, SIGTERM, and SIGQUIT. When ctx is canceled, in-progress MySQL transactions and InfluxDB operations will roll back. + go func() { + sig := <-timeToExit + if sig != nil { + log.Info().Msg("abort loading") + cancel() + } + }() + + // To reuse the `pbench run` code, especially run recorders, we create a pseudo main stage. + pseudoStage = &stage.Stage{ + Id: "load_el_json", + ColdRuns: &stage.RunsValueOne, + States: &stage.SharedStageStates{ + RunName: RunName, + Comment: Comment, + OutputPath: OutputPath, + RunStartTime: time.Now(), + }, + } + + // Kick off preparation work for all the run recorders + for _, recorder := range runRecorders { + if err := recorder.Start(ctx, pseudoStage); err != nil { + log.Fatal().Err(err).Msgf("failed to prepare %s", reflect.TypeOf(recorder).Name()) + } + } + + // Use this to make sure there will be no more than Parallelism goroutines. + parallelismGuard = make(chan struct{}, Parallelism) + + // This is the task scheduler go routine. It feeds files to task runners with back pressure. + go func() { + for _, path := range args { + if ctx.Err() != nil { + break + } + if err := processPath(ctx, path); err != nil { + // This whole command is not abort-on-error. We only log errors. + log.Error().Str("path", path).Err(err).Msg("failed to process path") + } + } + // Keep the main thread waiting for queryResults until all task runner finishes. + runningTasks.Wait() + close(resultChan) + }() + + for qr := range resultChan { + queryResults = append(queryResults, qr) + } + + pseudoStage.States.RunStartTime = runStartTime.GetTime() + pseudoStage.States.RunFinishTime = runEndTime.GetTime() + for _, r := range runRecorders { + rCtx, rCancel := utils.GetCtxWithTimeout(time.Second * 5) + r.RecordRun(rCtx, pseudoStage, queryResults) + rCancel() + } + + log.Info().Int("file_loaded", len(queryResults)).Send() + // This causes the signal handler to exit. + close(timeToExit) +} + +func scheduleFile(ctx context.Context, path string) { + parallelismGuard <- struct{}{} + runningTasks.Add(1) + if IsNDJSON { + go processNDJSONFile(ctx, path) + } else { + go processFile(ctx, path) + } +} + +func processFile(ctx context.Context, path string) { + defer func() { + // Allow another task runner to start. + <-parallelismGuard + runningTasks.Done() + }() + + bytes, ioErr := os.ReadFile(path) + if ioErr != nil { + log.Error().Err(ioErr).Str("path", path).Msg("failed to read file") + return + } + + processJSONBytes(ctx, path, bytes, 0) +} + +func processNDJSONFile(ctx context.Context, path string) { + defer func() { + // Allow another task runner to start. + <-parallelismGuard + runningTasks.Done() + }() + + file, err := os.Open(path) + if err != nil { + log.Error().Err(err).Str("path", path).Msg("failed to open NDJSON file") + return + } + defer file.Close() + + scanner := bufio.NewScanner(file) + // Increase buffer size to handle large JSON lines (default is 64KB, we set to 10MB) + const maxCapacity = 10 * 1024 * 1024 // 10MB + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + lineNum := 0 + for scanner.Scan() { + lineNum++ + if ctx.Err() != nil { + log.Info().Str("path", path).Msg("abort processing NDJSON file") + break + } + line := scanner.Bytes() + if len(line) == 0 { + continue + } + processJSONBytes(ctx, path, line, lineNum) + } + + if err := scanner.Err(); err != nil { + log.Error().Err(err).Str("path", path).Msg("error reading NDJSON file") + } +} + +func processJSONBytes(ctx context.Context, path string, jsonBytes []byte, lineNum int) { + queryEvent := new(QueryEvent) + // Note that this step can succeed with any valid JSON file. But we need to do some additional validation to skip + // invalid event listener JSON files. + if unmarshalErr := json.Unmarshal(jsonBytes, queryEvent); unmarshalErr != nil { + if lineNum > 0 { + log.Error().Err(unmarshalErr).Str("path", path).Int("line", lineNum).Msg("failed to unmarshal JSON") + } else { + log.Error().Err(unmarshalErr).Str("path", path).Msg("failed to unmarshal JSON") + } + return + } + + // Validate that this is a QueryCompletedEvent + if queryEvent.QueryCompletedEvent == nil { + if lineNum > 0 { + log.Error().Str("path", path).Int("line", lineNum).Msg("no QueryCompletedEvent found") + } else { + log.Error().Str("path", path).Msg("no QueryCompletedEvent found in file") + } + return + } + + qce := queryEvent.QueryCompletedEvent + if qce.Metadata.QueryId == "" || qce.CreateTime.Time.IsZero() { + if lineNum > 0 { + log.Error().Str("path", path).Int("line", lineNum).Msg("invalid QueryCompletedEvent: missing queryId or createTime") + } else { + log.Error().Str("path", path).Msg("invalid QueryCompletedEvent: missing queryId or createTime") + } + return + } + + // Copy the Plan from QueryEvent to QueryMetadata + qce.Metadata.Plan = &queryEvent.Plan + + if lineNum > 0 { + log.Info().Str("path", path).Int("line", lineNum).Msg("start to process event listener line") + } else { + log.Info().Str("path", path).Msg("start to process event listener file") + } + + queryId := qce.Metadata.QueryId + if RecordRun { + queryId = RunName + "_" + queryId + } + + fileName := filepath.Base(path) + queryResult := &stage.QueryResult{ + StageId: pseudoStage.Id, + Query: &stage.Query{ + Text: qce.Metadata.Query, + File: &fileName, + ColdRun: true, + ExpectedRowCount: -1, // means disabled + }, + QueryId: queryId, + InfoUrl: qce.Metadata.Uri, + RowCount: int(qce.Statistics.OutputPositions), + StartTime: qce.CreateTime.Time, + EndTime: &qce.EndTime.Time, + } + + if qce.FailureInfo != nil { + // Need to set this so the run recorders will mark this query as failed. + queryResult.QueryError = fmt.Errorf("%s", qce.FailureInfo.ErrorCode.Name) + } + + // Unlike benchmarks run by pbench, we do not know when did the run start and finish when loading them from files. + // We infer that the whole run starts at min(queryStartTime) and ends at max(queryEndTime). + runStartTime.Synchronized(func(st *syncedTime) { + if queryResult.StartTime.Before(st.t) { + st.t = queryResult.StartTime + // Changes to the pseudoStage will be synced to the database by the run recorder. + pseudoStage.States.RunStartTime = queryResult.StartTime + } + }) + + if queryResult.EndTime != nil { + dur := queryResult.EndTime.Sub(queryResult.StartTime) + queryResult.Duration = &dur + runEndTime.Synchronized(func(st *syncedTime) { + if queryResult.EndTime.After(st.t) { + st.t = *queryResult.EndTime + } + }) + } + + // Insert into MySQL if configured + if mysqlDb != nil { + if err := insertEventListenerData(ctx, mysqlDb, qce, queryId); err != nil { + if lineNum > 0 { + log.Error().Err(err).Str("path", path).Int("line", lineNum).Msg("failed to insert event listener record") + } else { + log.Error().Err(err).Str("path", path).Msg("failed to insert event listener record") + } + return + } + } + + for _, r := range runRecorders { + rCtx, rCancel := utils.GetCtxWithTimeout(time.Second * 5) + r.RecordQuery(rCtx, pseudoStage, queryResult) + rCancel() + } + + if lineNum > 0 { + log.Info().Str("path", path).Int("line", lineNum).Str("query_id", queryId).Msg("success") + } else { + log.Info().Str("path", path).Str("query_id", queryId).Msg("success") + } + resultChan <- queryResult +} + +func processPath(ctx context.Context, path string) error { + utils.ExpandHomeDirectory(&path) + stat, err := os.Stat(path) + if err != nil { + return err + } + if !stat.IsDir() { + // Skip dot files + if filepath.Base(path)[0] == '.' { + log.Debug().Str("path", path).Msg("skipping dot file") + return nil + } + scheduleFile(ctx, path) + return nil + } + + // Process directory recursively + return filepath.WalkDir(path, func(filePath string, d os.DirEntry, err error) error { + if err != nil { + log.Error().Err(err).Str("path", filePath).Msg("error walking directory") + return nil // Continue walking despite errors + } + + if ctx.Err() != nil { + log.Info().Msg("abort task scheduling") + return filepath.SkipAll + } + + // Skip dot files and dot directories + name := d.Name() + if name[0] == '.' { + if d.IsDir() { + log.Debug().Str("path", filePath).Msg("skipping dot directory") + return filepath.SkipDir + } + log.Debug().Str("path", filePath).Msg("skipping dot file") + return nil + } + + // Only schedule regular files + if !d.IsDir() { + scheduleFile(ctx, filePath) + } + + return nil + }) +} + +func registerRunRecorder(r stage.RunRecorder) { + if r == nil || reflect.ValueOf(r).IsNil() { + return + } + runRecorders = append(runRecorders, r) +} diff --git a/cmd/loadeljson/main_test.go b/cmd/loadeljson/main_test.go new file mode 100644 index 00000000..5665b7f4 --- /dev/null +++ b/cmd/loadeljson/main_test.go @@ -0,0 +1,272 @@ +package loadeljson + +import ( + "embed" + "encoding/json" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Embed test JSON files +// +//go:embed *.ndjson +var testFiles embed.FS + +func TestProcessJSONBytes_ValidEvent(t *testing.T) { + jsonData := `{ + "instanceId":"test-instance", + "clusterName":"test-cluster", + "queryCompletedEvent":{ + "metadata":{ + "queryId":"20250616_085426_00030_ux3a6", + "query":"SELECT * FROM test", + "queryState":"FINISHED", + "uri":"http://example.com/query/123" + }, + "statistics":{ + "cpuTime":371792.0, + "wallTime":67795.0, + "queuedTime":0.0, + "peakUserMemoryBytes":98161684, + "peakTotalNonRevocableMemoryBytes":1098226989, + "peakTaskUserMemory":10433657, + "peakTaskTotalMemory":202274048, + "peakNodeTotalMemory":222073173, + "totalBytes":26975446031, + "totalRows":2880361048, + "outputPositions":100, + "outputBytes":4801, + "writtenOutputRows":0, + "writtenOutputBytes":0, + "cumulativeMemory":5.359635941788949E12, + "cumulativeTotalMemory":8.514280417681293E12, + "completedSplits":7776 + }, + "context":{ + "user":"presto", + "serverVersion":"0.282", + "environment":"test", + "clientTags":[], + "sessionProperties":{}, + "resourceEstimates":{} + }, + "ioMetadata":{"inputs":[]}, + "warnings":[], + "failedTasks":[], + "createTime":1750063688.759, + "executionStartTime":1750063689.069, + "endTime":1750063756.554, + "stageStatistics":[], + "operatorStatistics":[], + "planStatisticsRead":[], + "planStatisticsWritten":[], + "optimizerInformation":[], + "scalarFunctions":[], + "aggregateFunctions":[], + "windowsFunctions":[] + }, + "plan":"test plan", + "cpuTimeMillis":371792, + "retriedCpuTimeMillis":0, + "wallTimeMillis":67795, + "queuedTimeMillis":0, + "analysisTimeMillis":630 + }` + + var qe QueryEvent + err := json.Unmarshal([]byte(jsonData), &qe) + require.NoError(t, err) + + // Validate the parsed event + assert.NotNil(t, qe.QueryCompletedEvent) + assert.Equal(t, "20250616_085426_00030_ux3a6", qe.QueryCompletedEvent.Metadata.QueryId) + assert.Equal(t, "FINISHED", qe.QueryCompletedEvent.Metadata.QueryState) + assert.False(t, qe.QueryCompletedEvent.CreateTime.Time.IsZero()) + assert.Equal(t, int64(100), qe.QueryCompletedEvent.Statistics.OutputPositions) +} + +func TestProcessJSONBytes_InvalidJSON(t *testing.T) { + invalidJSON := `{"invalid": json}` + + var qe QueryEvent + err := json.Unmarshal([]byte(invalidJSON), &qe) + assert.Error(t, err) +} + +func TestProcessJSONBytes_MissingQueryCompletedEvent(t *testing.T) { + jsonData := `{ + "instanceId":"test-instance", + "clusterName":"test-cluster", + "plan":"test plan" + }` + + var qe QueryEvent + err := json.Unmarshal([]byte(jsonData), &qe) + require.NoError(t, err) + + // Should have no QueryCompletedEvent + assert.Nil(t, qe.QueryCompletedEvent) +} + +func TestProcessJSONBytes_FailedQuery(t *testing.T) { + jsonData := `{ + "instanceId":"test-instance", + "clusterName":"test-cluster", + "queryCompletedEvent":{ + "metadata":{ + "queryId":"failed_query_id", + "query":"SELECT * FROM nonexistent", + "queryState":"FAILED", + "uri":"http://example.com" + }, + "statistics":{ + "cpuTime":0.0, + "wallTime":0.0, + "queuedTime":0.0, + "peakUserMemoryBytes":0, + "peakTotalNonRevocableMemoryBytes":0, + "peakTaskUserMemory":0, + "peakTaskTotalMemory":0, + "peakNodeTotalMemory":0, + "totalBytes":0, + "totalRows":0, + "outputPositions":0, + "outputBytes":0, + "writtenOutputRows":0, + "writtenOutputBytes":0, + "cumulativeMemory":0.0, + "cumulativeTotalMemory":0.0, + "completedSplits":0 + }, + "context":{ + "user":"test", + "serverVersion":"0.282", + "environment":"test", + "clientTags":[], + "sessionProperties":{}, + "resourceEstimates":{} + }, + "ioMetadata":{"inputs":[]}, + "failureInfo":{ + "errorCode":{ + "code":1, + "name":"SYNTAX_ERROR", + "type":"USER_ERROR" + }, + "failureType":"com.facebook.presto.sql.analyzer.SemanticException", + "failureMessage":"Table does not exist", + "failuresJson":"{}" + }, + "warnings":[], + "failedTasks":[], + "createTime":1750064067.198, + "executionStartTime":1750064067.208, + "endTime":1750064067.208, + "stageStatistics":[], + "operatorStatistics":[], + "planStatisticsRead":[], + "planStatisticsWritten":[], + "optimizerInformation":[], + "scalarFunctions":[], + "aggregateFunctions":[], + "windowsFunctions":[] + }, + "plan":"", + "cpuTimeMillis":0, + "retriedCpuTimeMillis":0, + "wallTimeMillis":0, + "queuedTimeMillis":0, + "analysisTimeMillis":0 + }` + + var qe QueryEvent + err := json.Unmarshal([]byte(jsonData), &qe) + require.NoError(t, err) + + assert.NotNil(t, qe.QueryCompletedEvent.FailureInfo) + assert.Equal(t, "SYNTAX_ERROR", qe.QueryCompletedEvent.FailureInfo.ErrorCode.Name) + assert.Equal(t, "FAILED", qe.QueryCompletedEvent.Metadata.QueryState) +} + +// Note: processPath tests are skipped as they require full initialization +// of global variables (mysqlDb, runRecorders, pseudoStage, etc.) which +// makes them integration tests rather than unit tests. These would be better +// suited for end-to-end testing with proper setup/teardown. + +func TestEmbeddedNDJSONFiles(t *testing.T) { + // Test that we can read the embedded NDJSON files + files, err := testFiles.ReadDir(".") + require.NoError(t, err) + + ndjsonCount := 0 + for _, file := range files { + if filepath.Ext(file.Name()) == ".ndjson" { + ndjsonCount++ + + // Try to read and parse the file + data, err := testFiles.ReadFile(file.Name()) + require.NoError(t, err, "Failed to read %s", file.Name()) + + // For NDJSON, we should be able to parse each line + lines := 0 + for _, line := range []byte(string(data)) { + if line == '\n' { + lines++ + } + } + t.Logf("File %s has content of length %d", file.Name(), len(data)) + } + } + + assert.Greater(t, ndjsonCount, 0, "Should have at least one NDJSON test file") +} + +func TestRunStartEndTimeTracking(t *testing.T) { + // Test the min/max time tracking logic used in processJSONBytes + startTime := newSyncedTime(time.Now()) + endTime := newSyncedTime(time.UnixMilli(0)) + + times := []struct { + start time.Time + end time.Time + }{ + { + start: time.Date(2025, 6, 16, 8, 48, 8, 0, time.UTC), + end: time.Date(2025, 6, 16, 8, 50, 0, 0, time.UTC), + }, + { + start: time.Date(2025, 6, 16, 8, 45, 0, 0, time.UTC), + end: time.Date(2025, 6, 16, 8, 55, 0, 0, time.UTC), + }, + { + start: time.Date(2025, 6, 16, 9, 0, 0, 0, time.UTC), + end: time.Date(2025, 6, 16, 9, 10, 0, 0, time.UTC), + }, + } + + for _, tt := range times { + // Update start time (find minimum) + startTime.Synchronized(func(st *syncedTime) { + if tt.start.Before(st.t) { + st.t = tt.start + } + }) + + // Update end time (find maximum) + endTime.Synchronized(func(st *syncedTime) { + if tt.end.After(st.t) { + st.t = tt.end + } + }) + } + + expectedStart := time.Date(2025, 6, 16, 8, 45, 0, 0, time.UTC) + expectedEnd := time.Date(2025, 6, 16, 9, 10, 0, 0, time.UTC) + + assert.True(t, startTime.GetTime().Equal(expectedStart)) + assert.True(t, endTime.GetTime().Equal(expectedEnd)) +} diff --git a/cmd/loadeljson/synced_time.go b/cmd/loadeljson/synced_time.go new file mode 100644 index 00000000..eef8bcde --- /dev/null +++ b/cmd/loadeljson/synced_time.go @@ -0,0 +1,29 @@ +package loadeljson + +import ( + "sync" + "time" +) + +type syncedTime struct { + t time.Time + m sync.Mutex +} + +func newSyncedTime(t time.Time) *syncedTime { + return &syncedTime{ + t: t, + } +} + +func (st *syncedTime) Synchronized(f func(st *syncedTime)) { + st.m.Lock() + defer st.m.Unlock() + f(st) +} + +func (st *syncedTime) GetTime() time.Time { + st.m.Lock() + defer st.m.Unlock() + return st.t +} diff --git a/cmd/loadeljson/synced_time_test.go b/cmd/loadeljson/synced_time_test.go new file mode 100644 index 00000000..ea5b27c9 --- /dev/null +++ b/cmd/loadeljson/synced_time_test.go @@ -0,0 +1,103 @@ +package loadeljson + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestSyncedTime_GetTime(t *testing.T) { + now := time.Now() + st := newSyncedTime(now) + + retrieved := st.GetTime() + assert.True(t, retrieved.Equal(now)) +} + +func TestSyncedTime_Synchronized(t *testing.T) { + initialTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + st := newSyncedTime(initialTime) + + newTime := time.Date(2025, 6, 16, 8, 48, 8, 0, time.UTC) + + st.Synchronized(func(st *syncedTime) { + st.t = newTime + }) + + retrieved := st.GetTime() + assert.True(t, retrieved.Equal(newTime)) +} + +func TestSyncedTime_ConcurrentAccess(t *testing.T) { + initialTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + st := newSyncedTime(initialTime) + + var wg sync.WaitGroup + iterations := 100 + + // Concurrent writes + for i := 0; i < iterations; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + newTime := initialTime.Add(time.Duration(idx) * time.Second) + st.Synchronized(func(st *syncedTime) { + // Simulate some work + time.Sleep(time.Microsecond) + st.t = newTime + }) + }(i) + } + + // Concurrent reads + for i := 0; i < iterations; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _ = st.GetTime() + }() + } + + wg.Wait() + + // Verify that the final time is one of the expected values + finalTime := st.GetTime() + assert.True(t, finalTime.After(initialTime) || finalTime.Equal(initialTime)) +} + +func TestSyncedTime_MinMaxTracking(t *testing.T) { + // Simulate the use case in loadeljson where we track min start time and max end time + minTime := newSyncedTime(time.Now()) + maxTime := newSyncedTime(time.UnixMilli(0)) + + times := []time.Time{ + time.Date(2025, 6, 16, 8, 48, 8, 0, time.UTC), + time.Date(2025, 6, 16, 8, 50, 0, 0, time.UTC), + time.Date(2025, 6, 16, 8, 45, 0, 0, time.UTC), + time.Date(2025, 6, 16, 9, 0, 0, 0, time.UTC), + } + + for _, t := range times { + // Update min time + minTime.Synchronized(func(st *syncedTime) { + if t.Before(st.t) { + st.t = t + } + }) + + // Update max time + maxTime.Synchronized(func(st *syncedTime) { + if t.After(st.t) { + st.t = t + } + }) + } + + expectedMin := time.Date(2025, 6, 16, 8, 45, 0, 0, time.UTC) + expectedMax := time.Date(2025, 6, 16, 9, 0, 0, 0, time.UTC) + + assert.True(t, minTime.GetTime().Equal(expectedMin)) + assert.True(t, maxTime.GetTime().Equal(expectedMax)) +} diff --git a/cmd/loadeljson/test_error.ndjson b/cmd/loadeljson/test_error.ndjson new file mode 100644 index 00000000..29b64e59 --- /dev/null +++ b/cmd/loadeljson/test_error.ndjson @@ -0,0 +1,2 @@ +{"instanceId":"978ecaae-0fe1-4997-8bfc-78b04e54662a","clusterName":"cluster1","queryCompletedEvent":{"metadata":{"queryId":"20250616_085426_00030_ux3a6","transactionId":"9c48a888-c021-48e0-9374-df86b05c765d","query":"SELECT * FROM jmx.current.\"com.facebook.alluxio:name=client.cachehitrate,type=gauges\"","queryHash":"b1b008c1baff725a","queryState":"FAILED","uri":"http://10.128.58.157:8585/v1/query/20250616_085426_00030_ux3a6"},"statistics":{"cpuTime":0.0,"retriedCpuTime":0.0,"wallTime":0.0,"queuedTime":0.0,"peakUserMemoryBytes":0,"peakTotalNonRevocableMemoryBytes":0,"peakTaskUserMemory":0,"peakTaskTotalMemory":0,"peakNodeTotalMemory":0,"totalBytes":0,"totalRows":0,"outputPositions":0,"outputBytes":0,"writtenOutputRows":0,"writtenOutputBytes":0,"cumulativeMemory":0.0,"cumulativeTotalMemory":0.0,"completedSplits":0},"context":{"user":"presto","principal":"presto","remoteClientAddress":"10.128.13.249","userAgent":"StatementClientV1/0.286","source":"presto-cli","catalog":"glue","schema":"tpcds_sf1000_partitioned_delta","resourceGroupId":["global"],"sessionProperties":{},"serverVersion":"0.282","environment":"test","clientTags":[],"resourceEstimates":{}},"ioMetadata":{"inputs":[]},"failureInfo":{"errorCode":{"code":1,"name":"SYNTAX_ERROR","type":"USER_ERROR","retriable":false},"failureType":"com.facebook.presto.sql.analyzer.SemanticException","failureMessage":"Table jmx.current.com.facebook.alluxio:name=client.cachehitrate,type=gauges does not exist","failuresJson":"{}"},"warnings":[],"failedTasks":[],"createTime":1750064067.198,"executionStartTime":1750064067.208,"endTime":1750064067.208,"stageStatistics":[],"operatorStatistics":[],"planStatisticsRead":[],"planStatisticsWritten":[],"optimizerInformation":[],"scalarFunctions":[],"aggregateFunctions":[],"windowsFunctions":[]},"plan":"","cpuTimeMillis":0,"retriedCpuTimeMillis":0,"wallTimeMillis":0,"queuedTimeMillis":0,"analysisTimeMillis":0} + diff --git a/cmd/loadeljson/test_success.ndjson b/cmd/loadeljson/test_success.ndjson new file mode 100644 index 00000000..078e2b07 --- /dev/null +++ b/cmd/loadeljson/test_success.ndjson @@ -0,0 +1,3 @@ +{"instanceId":"978ecaae-0fe1-4997-8bfc-78b04e54662a","clusterName":"cluster1","queryCompletedEvent":{"metadata":{"queryId":"20250616_084808_00027_ux3a6","transactionId":"55e64835-74b8-49c3-8316-e3bdbbee6e04","query":"SELECT * FROM test_table","queryHash":"eb3623724f2f753c","queryState":"FINISHED","uri":"http://10.128.58.157:8585/v1/query/20250616_084808_00027_ux3a6"},"statistics":{"cpuTime":371792.0,"retriedCpuTime":0.0,"wallTime":67795.0,"queuedTime":0.0,"peakUserMemoryBytes":98161684,"peakTotalNonRevocableMemoryBytes":1098226989,"peakTaskUserMemory":10433657,"peakTaskTotalMemory":202274048,"peakNodeTotalMemory":222073173,"totalBytes":26975446031,"totalRows":2880361048,"outputPositions":100,"outputBytes":4801,"writtenOutputRows":0,"writtenOutputBytes":0,"cumulativeMemory":5.359635941788949E12,"cumulativeTotalMemory":8.514280417681293E12,"completedSplits":7776},"context":{"user":"presto","principal":"presto","serverVersion":"0.282","environment":"test","clientTags":[],"sessionProperties":{},"resourceEstimates":{}},"ioMetadata":{"inputs":[]},"warnings":[],"failedTasks":[],"createTime":1750063688.759,"executionStartTime":1750063689.069,"endTime":1750063756.554,"stageStatistics":[],"operatorStatistics":[],"planStatisticsRead":[],"planStatisticsWritten":[],"optimizerInformation":[],"scalarFunctions":[],"aggregateFunctions":[],"windowsFunctions":[]},"plan":"test plan","cpuTimeMillis":371792,"retriedCpuTimeMillis":0,"wallTimeMillis":67795,"queuedTimeMillis":0,"analysisTimeMillis":630} +{"instanceId":"978ecaae-0fe1-4997-8bfc-78b04e54662a","clusterName":"cluster1","queryCompletedEvent":{"metadata":{"queryId":"20250616_084809_00028_ux3a6","query":"SELECT COUNT(*) FROM test_table","queryState":"FINISHED","uri":"http://10.128.58.157:8585/v1/query/20250616_084809_00028_ux3a6"},"statistics":{"cpuTime":1000.0,"wallTime":2000.0,"queuedTime":0.0,"peakUserMemoryBytes":1000000,"peakTotalNonRevocableMemoryBytes":2000000,"peakTaskUserMemory":500000,"peakTaskTotalMemory":1000000,"peakNodeTotalMemory":1500000,"totalBytes":1000,"totalRows":1,"outputPositions":1,"outputBytes":100,"writtenOutputRows":0,"writtenOutputBytes":0,"cumulativeMemory":1000000.0,"cumulativeTotalMemory":2000000.0,"completedSplits":10},"context":{"user":"presto","serverVersion":"0.282","environment":"test","clientTags":[],"sessionProperties":{},"resourceEstimates":{}},"ioMetadata":{"inputs":[]},"warnings":[],"failedTasks":[],"createTime":1750063690.0,"executionStartTime":1750063690.5,"endTime":1750063692.0,"stageStatistics":[],"operatorStatistics":[],"planStatisticsRead":[],"planStatisticsWritten":[],"optimizerInformation":[],"scalarFunctions":[],"aggregateFunctions":[],"windowsFunctions":[]},"plan":"","cpuTimeMillis":1000,"retriedCpuTimeMillis":0,"wallTimeMillis":2000,"queuedTimeMillis":0,"analysisTimeMillis":10} + diff --git a/cmd/loadeljson/types.go b/cmd/loadeljson/types.go new file mode 100644 index 00000000..49e5b6ce --- /dev/null +++ b/cmd/loadeljson/types.go @@ -0,0 +1,327 @@ +package loadeljson + +import ( + "encoding/json" + "fmt" + "time" +) + +// PrestoTime is a custom time type that can unmarshal from both string and numeric formats +type PrestoTime struct { + time.Time +} + +// ResourceGroupId is a custom type that can handle both string and array formats +type ResourceGroupId struct { + Value string +} + +func (pt *PrestoTime) UnmarshalJSON(b []byte) error { + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + + switch value := v.(type) { + case string: + // Try parsing as RFC3339 format + t, err := time.Parse(time.RFC3339, value) + if err != nil { + // Try parsing as RFC3339Nano format + t, err = time.Parse(time.RFC3339Nano, value) + if err != nil { + return err + } + } + pt.Time = t + return nil + case float64: + // Assume it's seconds since epoch (Unix timestamp) + pt.Time = time.Unix(int64(value), int64((value-float64(int64(value)))*1e9)).UTC() + return nil + default: + return fmt.Errorf("invalid time type: %T", value) + } +} + +func (rg *ResourceGroupId) UnmarshalJSON(b []byte) error { + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + + switch value := v.(type) { + case string: + rg.Value = value + return nil + case []interface{}: + // If it's an array, join the elements with "." + parts := make([]string, 0, len(value)) + for _, item := range value { + if str, ok := item.(string); ok { + parts = append(parts, str) + } + } + if len(parts) > 0 { + rg.Value = parts[0] + for i := 1; i < len(parts); i++ { + rg.Value += "." + parts[i] + } + } + return nil + default: + return fmt.Errorf("invalid resource group id type: %T", value) + } +} + +// QueryEvent represents the event listener JSON structure +type QueryEvent struct { + InstanceId string `json:"instanceId"` + ClusterName string `json:"clusterName"` + QueryCreatedEvent *QueryCreatedEvent `json:"queryCreatedEvent,omitempty"` + QueryCompletedEvent *QueryCompletedEvent `json:"queryCompletedEvent,omitempty"` + SplitCompletedEvent interface{} `json:"splitCompletedEvent,omitempty"` + Plan string `json:"plan"` + CpuTimeMillis int64 `json:"cpuTimeMillis"` + RetriedCpuTimeMillis int64 `json:"retriedCpuTimeMillis"` + WallTimeMillis int64 `json:"wallTimeMillis"` + QueuedTimeMillis int64 `json:"queuedTimeMillis"` + AnalysisTimeMillis int64 `json:"analysisTimeMillis"` +} + +type QueryCreatedEvent struct { + CreateTime PrestoTime `json:"createTime"` + Context QueryContext `json:"context"` + Metadata QueryMetadata `json:"metadata"` +} + +type ResourceEstimates struct { + ExecutionTime *Duration `json:"executionTime,omitempty"` + CpuTime *Duration `json:"cpuTime,omitempty"` + PeakMemory *int64 `json:"peakMemory,omitempty"` + PeakTaskMemory *int64 `json:"peakTaskMemory,omitempty"` +} + +type QueryCompletedEvent struct { + Metadata QueryMetadata `json:"metadata"` + Statistics QueryStatistics `json:"statistics"` + Context QueryContext `json:"context"` + IoMetadata QueryIOMetadata `json:"ioMetadata"` + FailureInfo *QueryFailureInfo `json:"failureInfo,omitempty"` + Warnings []interface{} `json:"warnings"` + QueryType *string `json:"queryType,omitempty"` + FailedTasks []string `json:"failedTasks"` + CreateTime PrestoTime `json:"createTime"` + ExecutionStartTime PrestoTime `json:"executionStartTime"` + EndTime PrestoTime `json:"endTime"` + StageStatistics []StageStatistics `json:"stageStatistics"` + OperatorStatistics []OperatorStatistics `json:"operatorStatistics"` + PlanStatisticsRead []interface{} `json:"planStatisticsRead"` + PlanStatisticsWritten []interface{} `json:"planStatisticsWritten"` + PlanNodeHash interface{} `json:"planNodeHash"` + CanonicalPlan interface{} `json:"canonicalPlan"` + StatsEquivalentPlan *string `json:"statsEquivalentPlan,omitempty"` + ExpandedQuery *string `json:"expandedQuery,omitempty"` + OptimizerInformation []interface{} `json:"optimizerInformation"` + CteInformationList []interface{} `json:"cteInformationList"` + ScalarFunctions []string `json:"scalarFunctions"` + AggregateFunctions []string `json:"aggregateFunctions"` + WindowFunctions []string `json:"windowFunctions"` +} + +type QueryMetadata struct { + QueryId string `json:"queryId"` + TransactionId *string `json:"transactionId,omitempty"` + Query string `json:"query"` + QueryState string `json:"queryState"` + Uri string `json:"uri"` + Plan *string `json:"plan,omitempty"` + JsonPlan *string `json:"jsonPlan,omitempty"` + QueryHash *string `json:"queryHash,omitempty"` +} + +type QueryStatistics struct { + CpuTime Duration `json:"cpuTime"` + WallTime Duration `json:"wallTime"` + QueuedTime Duration `json:"queuedTime"` + AnalysisTime *Duration `json:"analysisTime,omitempty"` + PeakUserMemoryBytes int64 `json:"peakUserMemoryBytes"` + PeakTotalNonRevocableMemoryBytes int64 `json:"peakTotalNonRevocableMemoryBytes"` + PeakTaskUserMemory int64 `json:"peakTaskUserMemory"` + PeakTaskTotalMemory int64 `json:"peakTaskTotalMemory"` + PeakNodeTotalMemory int64 `json:"peakNodeTotalMemory"` + TotalBytes int64 `json:"totalBytes"` + TotalRows int64 `json:"totalRows"` + OutputPositions int64 `json:"outputPositions"` + OutputBytes int64 `json:"outputBytes"` + WrittenOutputRows int64 `json:"writtenOutputRows"` + WrittenOutputBytes int64 `json:"writtenOutputBytes"` + CumulativeMemory float64 `json:"cumulativeMemory"` + CumulativeTotalMemory float64 `json:"cumulativeTotalMemory"` + CompletedSplits int `json:"completedSplits"` +} + +type QueryContext struct { + User string `json:"user"` + Principal *string `json:"principal,omitempty"` + RemoteClientAddress *string `json:"remoteClientAddress,omitempty"` + UserAgent *string `json:"userAgent,omitempty"` + ClientInfo *string `json:"clientInfo,omitempty"` + Source *string `json:"source,omitempty"` + Catalog *string `json:"catalog,omitempty"` + Schema *string `json:"schema,omitempty"` + ResourceGroupId *ResourceGroupId `json:"resourceGroupId,omitempty"` + SessionProperties map[string]string `json:"sessionProperties"` + ServerVersion string `json:"serverVersion"` + Environment string `json:"environment"` + ClientTags []string `json:"clientTags"` + ResourceEstimates ResourceEstimates `json:"resourceEstimates"` +} + +type QueryIOMetadata struct { + Inputs []interface{} `json:"inputs"` + Output interface{} `json:"output,omitempty"` +} + +type QueryFailureInfo struct { + ErrorCode ErrorCode `json:"errorCode"` + FailureType *string `json:"failureType,omitempty"` + FailureMessage *string `json:"failureMessage,omitempty"` + FailureTask *string `json:"failureTask,omitempty"` + FailureHost *string `json:"failureHost,omitempty"` + FailuresJson string `json:"failuresJson"` +} + +type ErrorCode struct { + Code int `json:"code"` + Name string `json:"name"` + Type string `json:"type"` +} + +type StageStatistics struct { + StageId int `json:"stageId"` + StageExecutionId int `json:"stageExecutionId"` + Tasks int `json:"tasks"` + TotalScheduledTime Duration `json:"totalScheduledTime"` + TotalCpuTime Duration `json:"totalCpuTime"` + RetriedCpuTime Duration `json:"retriedCpuTime"` + TotalBlockedTime Duration `json:"totalBlockedTime"` + RawInputDataSize DataSize `json:"rawInputDataSize"` + ProcessedInputDataSize DataSize `json:"processedInputDataSize"` + PhysicalWrittenDataSize DataSize `json:"physicalWrittenDataSize"` +} + +type OperatorStatistics struct { + StageId int `json:"stageId"` + StageExecutionId int `json:"stageExecutionId"` + PipelineId int `json:"pipelineId"` + OperatorId int `json:"operatorId"` + PlanNodeId string `json:"planNodeId"` + OperatorType string `json:"operatorType"` + TotalDrivers int64 `json:"totalDrivers"` + AddInputCalls int64 `json:"addInputCalls"` + AddInputWall Duration `json:"addInputWall"` + AddInputCpu Duration `json:"addInputCpu"` + AddInputAllocation DataSize `json:"addInputAllocation"` + RawInputDataSize DataSize `json:"rawInputDataSize"` + RawInputPositions int64 `json:"rawInputPositions"` + InputDataSize DataSize `json:"inputDataSize"` + InputPositions int64 `json:"inputPositions"` + GetOutputCalls int64 `json:"getOutputCalls"` + GetOutputWall Duration `json:"getOutputWall"` + GetOutputCpu Duration `json:"getOutputCpu"` + GetOutputAllocation DataSize `json:"getOutputAllocation"` + OutputDataSize DataSize `json:"outputDataSize"` + OutputPositions int64 `json:"outputPositions"` + PhysicalWrittenDataSize DataSize `json:"physicalWrittenDataSize"` + BlockedWall Duration `json:"blockedWall"` + FinishCalls int64 `json:"finishCalls"` + FinishWall Duration `json:"finishWall"` + FinishCpu Duration `json:"finishCpu"` + FinishAllocation DataSize `json:"finishAllocation"` + UserMemoryReservation DataSize `json:"userMemoryReservation"` + RevocableMemoryReservation DataSize `json:"revocableMemoryReservation"` + SystemMemoryReservation DataSize `json:"systemMemoryReservation"` + PeakUserMemoryReservation DataSize `json:"peakUserMemoryReservation"` + PeakSystemMemoryReservation DataSize `json:"peakSystemMemoryReservation"` + PeakTotalMemoryReservation DataSize `json:"peakTotalMemoryReservation"` + SpilledDataSize DataSize `json:"spilledDataSize"` + Info *string `json:"info,omitempty"` +} + +// Duration represents a time duration in the format used by Presto +type Duration struct { + time.Duration +} + +func (d *Duration) UnmarshalJSON(b []byte) error { + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + switch value := v.(type) { + case float64: + d.Duration = time.Duration(value) * time.Millisecond + return nil + case string: + // Parse Presto duration format: "43.99d", "1.5h", "30m", "45s", "100ms", etc. + var amount float64 + var unit string + n, err := fmt.Sscanf(value, "%f%s", &amount, &unit) + if err != nil || n != 2 { + // Try standard Go duration format as fallback + d.Duration, err = time.ParseDuration(value) + return err + } + + // Convert Presto units to Go duration + switch unit { + case "d": + d.Duration = time.Duration(amount * float64(24*time.Hour)) + case "h": + d.Duration = time.Duration(amount * float64(time.Hour)) + case "m": + d.Duration = time.Duration(amount * float64(time.Minute)) + case "s": + d.Duration = time.Duration(amount * float64(time.Second)) + case "ms": + d.Duration = time.Duration(amount * float64(time.Millisecond)) + case "us": + d.Duration = time.Duration(amount * float64(time.Microsecond)) + case "ns": + d.Duration = time.Duration(amount * float64(time.Nanosecond)) + default: + return fmt.Errorf("unknown duration unit: %s", unit) + } + return nil + default: + return fmt.Errorf("invalid duration type") + } +} + +// DataSize represents a data size in bytes +type DataSize struct { + Bytes int64 +} + +func (ds *DataSize) UnmarshalJSON(b []byte) error { + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + switch value := v.(type) { + case float64: + ds.Bytes = int64(value) + return nil + case string: + // Parse string like "1.5MB", "100KB", etc. + // For simplicity, we'll just try to extract the number + var size float64 + var unit string + fmt.Sscanf(value, "%f%s", &size, &unit) + ds.Bytes = int64(size) + return nil + default: + return fmt.Errorf("invalid data size type") + } +} diff --git a/cmd/loadeljson/types_test.go b/cmd/loadeljson/types_test.go new file mode 100644 index 00000000..26fadf43 --- /dev/null +++ b/cmd/loadeljson/types_test.go @@ -0,0 +1,414 @@ +package loadeljson + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrestoTime_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + expected time.Time + wantErr bool + }{ + { + name: "RFC3339 format", + input: `"2025-06-16T08:48:08Z"`, + expected: time.Date(2025, 6, 16, 8, 48, 8, 0, time.UTC), + wantErr: false, + }, + { + name: "RFC3339Nano format", + input: `"2025-06-16T08:48:08.759Z"`, + expected: time.Date(2025, 6, 16, 8, 48, 8, 759000000, time.UTC), + wantErr: false, + }, + { + name: "Unix timestamp as float", + input: `1750063688.759`, + expected: time.Unix(1750063688, 759000000).UTC(), + wantErr: false, + }, + { + name: "Unix timestamp as integer", + input: `1750063688`, + expected: time.Unix(1750063688, 0), + wantErr: false, + }, + { + name: "Invalid format", + input: `"invalid-time"`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var pt PrestoTime + err := json.Unmarshal([]byte(tt.input), &pt) + + if tt.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + // For float timestamps, allow small precision differences (< 1 microsecond) + diff := pt.Time.Sub(tt.expected) + if diff < 0 { + diff = -diff + } + assert.True(t, diff < time.Microsecond, + "Expected %v, got %v (diff: %v)", tt.expected, pt.Time, diff) + } + }) + } +} + +func TestResourceGroupId_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + expected string + wantErr bool + }{ + { + name: "String format", + input: `"global"`, + expected: "global", + wantErr: false, + }, + { + name: "Array format single element", + input: `["global"]`, + expected: "global", + wantErr: false, + }, + { + name: "Array format multiple elements", + input: `["global", "adhoc", "user"]`, + expected: "global.adhoc.user", + wantErr: false, + }, + { + name: "Empty array", + input: `[]`, + expected: "", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var rg ResourceGroupId + err := json.Unmarshal([]byte(tt.input), &rg) + + if tt.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expected, rg.Value) + } + }) + } +} + +func TestDuration_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + expected time.Duration + wantErr bool + }{ + { + name: "Milliseconds as float", + input: `371792.0`, + expected: 371792 * time.Millisecond, + wantErr: false, + }, + { + name: "Days format", + input: `"1.5d"`, + expected: 36 * time.Hour, + wantErr: false, + }, + { + name: "Hours format", + input: `"2.5h"`, + expected: 150 * time.Minute, + wantErr: false, + }, + { + name: "Minutes format", + input: `"30m"`, + expected: 30 * time.Minute, + wantErr: false, + }, + { + name: "Seconds format", + input: `"45s"`, + expected: 45 * time.Second, + wantErr: false, + }, + { + name: "Milliseconds format", + input: `"100ms"`, + expected: 100 * time.Millisecond, + wantErr: false, + }, + { + name: "Microseconds format", + input: `"500us"`, + expected: 500 * time.Microsecond, + wantErr: false, + }, + { + name: "Nanoseconds format", + input: `"1000ns"`, + expected: 1000 * time.Nanosecond, + wantErr: false, + }, + { + name: "Invalid unit", + input: `"10x"`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var d Duration + err := json.Unmarshal([]byte(tt.input), &d) + + if tt.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expected, d.Duration) + } + }) + } +} + +func TestDataSize_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + input string + expected int64 + wantErr bool + }{ + { + name: "Numeric bytes", + input: `1024`, + expected: 1024, + wantErr: false, + }, + { + name: "Float bytes", + input: `1024.5`, + expected: 1024, + wantErr: false, + }, + { + name: "String with unit KB", + input: `"100KB"`, + expected: 100, + wantErr: false, + }, + { + name: "String with unit MB", + input: `"1.5MB"`, + expected: 1, + wantErr: false, + }, + { + name: "String with unit GB", + input: `"2.5GB"`, + expected: 2, + wantErr: false, + }, + { + name: "String bytes only", + input: `"4096B"`, + expected: 4096, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ds DataSize + err := json.Unmarshal([]byte(tt.input), &ds) + + if tt.wantErr { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expected, ds.Bytes) + } + }) + } +} + +func TestQueryEvent_UnmarshalJSON(t *testing.T) { + // Test with the sample NDJSON data from the commit + jsonData := `{ + "instanceId":"978ecaae-0fe1-4997-8bfc-78b04e54662a", + "clusterName":"cluster1", + "queryCompletedEvent":{ + "metadata":{ + "queryId":"20250616_085426_00030_ux3a6", + "query":"SELECT * FROM test", + "queryState":"FINISHED", + "uri":"http://example.com" + }, + "statistics":{ + "cpuTime":371792.0, + "wallTime":67795.0, + "queuedTime":0.0, + "peakUserMemoryBytes":98161684, + "peakTotalNonRevocableMemoryBytes":1098226989, + "peakTaskUserMemory":10433657, + "peakTaskTotalMemory":202274048, + "peakNodeTotalMemory":222073173, + "totalBytes":26975446031, + "totalRows":2880361048, + "outputPositions":100, + "outputBytes":4801, + "writtenOutputRows":0, + "writtenOutputBytes":0, + "cumulativeMemory":5.359635941788949E12, + "cumulativeTotalMemory":8.514280417681293E12, + "completedSplits":7776 + }, + "context":{ + "user":"presto", + "serverVersion":"0.282", + "environment":"test", + "clientTags":[], + "sessionProperties":{}, + "resourceEstimates":{} + }, + "ioMetadata":{ + "inputs":[] + }, + "warnings":[], + "failedTasks":[], + "createTime":1750063688.759, + "executionStartTime":1750063689.069, + "endTime":1750063756.554, + "stageStatistics":[], + "operatorStatistics":[], + "planStatisticsRead":[], + "planStatisticsWritten":[], + "optimizerInformation":[], + "scalarFunctions":[], + "aggregateFunctions":[], + "windowsFunctions":[] + }, + "plan":"test plan", + "cpuTimeMillis":371792, + "retriedCpuTimeMillis":0, + "wallTimeMillis":67795, + "queuedTimeMillis":0, + "analysisTimeMillis":630 + }` + + var qe QueryEvent + err := json.Unmarshal([]byte(jsonData), &qe) + require.NoError(t, err) + + assert.Equal(t, "978ecaae-0fe1-4997-8bfc-78b04e54662a", qe.InstanceId) + assert.Equal(t, "cluster1", qe.ClusterName) + assert.NotNil(t, qe.QueryCompletedEvent) + assert.Equal(t, "20250616_085426_00030_ux3a6", qe.QueryCompletedEvent.Metadata.QueryId) + assert.Equal(t, "FINISHED", qe.QueryCompletedEvent.Metadata.QueryState) + assert.Equal(t, int64(371792), qe.CpuTimeMillis) + assert.Equal(t, int64(100), qe.QueryCompletedEvent.Statistics.OutputPositions) +} + +func TestQueryEvent_WithFailureInfo(t *testing.T) { + jsonData := `{ + "instanceId":"test-instance", + "clusterName":"test-cluster", + "queryCompletedEvent":{ + "metadata":{ + "queryId":"test_query_id", + "query":"SELECT 1", + "queryState":"FAILED", + "uri":"http://example.com" + }, + "statistics":{ + "cpuTime":0.0, + "wallTime":0.0, + "queuedTime":0.0, + "peakUserMemoryBytes":0, + "peakTotalNonRevocableMemoryBytes":0, + "peakTaskUserMemory":0, + "peakTaskTotalMemory":0, + "peakNodeTotalMemory":0, + "totalBytes":0, + "totalRows":0, + "outputPositions":0, + "outputBytes":0, + "writtenOutputRows":0, + "writtenOutputBytes":0, + "cumulativeMemory":0.0, + "cumulativeTotalMemory":0.0, + "completedSplits":0 + }, + "context":{ + "user":"test", + "serverVersion":"0.282", + "environment":"test", + "clientTags":[], + "sessionProperties":{}, + "resourceEstimates":{} + }, + "ioMetadata":{"inputs":[]}, + "failureInfo":{ + "errorCode":{ + "code":1, + "name":"SYNTAX_ERROR", + "type":"USER_ERROR" + }, + "failureType":"com.facebook.presto.sql.parser.ParsingException", + "failureMessage":"Syntax error", + "failuresJson":"{}" + }, + "warnings":[], + "failedTasks":[], + "createTime":1750063688.759, + "executionStartTime":1750063689.069, + "endTime":1750063756.554, + "stageStatistics":[], + "operatorStatistics":[], + "planStatisticsRead":[], + "planStatisticsWritten":[], + "optimizerInformation":[], + "scalarFunctions":[], + "aggregateFunctions":[], + "windowsFunctions":[] + }, + "plan":"", + "cpuTimeMillis":0, + "retriedCpuTimeMillis":0, + "wallTimeMillis":0, + "queuedTimeMillis":0, + "analysisTimeMillis":0 + }` + + var qe QueryEvent + err := json.Unmarshal([]byte(jsonData), &qe) + require.NoError(t, err) + + assert.NotNil(t, qe.QueryCompletedEvent.FailureInfo) + assert.Equal(t, 1, qe.QueryCompletedEvent.FailureInfo.ErrorCode.Code) + assert.Equal(t, "SYNTAX_ERROR", qe.QueryCompletedEvent.FailureInfo.ErrorCode.Name) + assert.Equal(t, "USER_ERROR", qe.QueryCompletedEvent.FailureInfo.ErrorCode.Type) + assert.NotNil(t, qe.QueryCompletedEvent.FailureInfo.FailureMessage) + assert.Equal(t, "Syntax error", *qe.QueryCompletedEvent.FailureInfo.FailureMessage) +} diff --git a/go.mod b/go.mod index 027652b4..c8f834da 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module pbench go 1.25.7 require ( + github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/alecthomas/participle/v2 v2.1.1 github.com/ethanyzhang/presto-go v0.0.0-20260213063102-87041e7e27c8 github.com/go-playground/validator/v10 v10.24.0 diff --git a/go.sum b/go.sum index 49487750..67fd29e6 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0= github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= @@ -43,6 +45,7 @@ github.com/influxdata/influxdb-client-go/v2 v2.13.0/go.mod h1:k+spCbt9hcvqvUiz0s github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/stage/mysql_run_recorder.go b/stage/mysql_run_recorder.go index 5cc9736f..95d4843e 100644 --- a/stage/mysql_run_recorder.go +++ b/stage/mysql_run_recorder.go @@ -4,11 +4,12 @@ import ( "context" "database/sql" _ "embed" - _ "github.com/go-sql-driver/mysql" "pbench/log" "pbench/utils" "sync/atomic" "time" + + _ "github.com/go-sql-driver/mysql" ) var ( @@ -50,7 +51,7 @@ func NewMySQLRunRecorderWithDb(db *sql.DB) *MySQLRunRecorder { } func (m *MySQLRunRecorder) Start(_ context.Context, s *Stage) error { - recordNewRun := `INSERT INTO pbench_runs (run_name, cluster_fqdn, start_time, queries_ran, failed, mismatch, comment) + recordNewRun := `REPLACE INTO pbench_runs (run_name, cluster_fqdn, start_time, queries_ran, failed, mismatch, comment) VALUES (?, ?, ?, 0, 0, 0, ?)` res, err := m.db.Exec(recordNewRun, s.States.RunName, s.States.ServerFQDN, s.States.RunStartTime, s.States.Comment) if err != nil { @@ -71,7 +72,7 @@ VALUES (?, ?, ?, 0, 0, 0, ?)` } func (m *MySQLRunRecorder) RecordQuery(_ context.Context, s *Stage, result *QueryResult) { - recordNewQuery := `INSERT INTO pbench_queries (run_id, stage_id, query_file, query_index, query_id, sequence_no, + recordNewQuery := `REPLACE INTO pbench_queries (run_id, stage_id, query_file, query_index, query_id, sequence_no, cold_run, succeeded, start_time, end_time, row_count, expected_row_count, duration_ms, info_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` var queryFile string if result.Query.File != nil {