From 4bfbe88e65ffcbde9e8186cd6b33c68ba2b3a9c5 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Tue, 26 May 2026 16:26:35 -0700 Subject: [PATCH 1/7] Sync supported ops with RAPIDS plugin Signed-off-by: Partho Sarthi --- .claude/agents/supported-ops-sync.md | 33 +++ .claude/skills/supported-ops-sync/SKILL.md | 234 ++++++++++++++++++ .../supported-ops-sync/agents/openai.yaml | 12 + .../references/pr-checklist.md | 23 ++ .../operatorsScore-databricks-aws-a10G.csv | 4 + .../operatorsScore-databricks-aws-t4.csv | 4 + .../operatorsScore-databricks-azure-t4.csv | 4 + .../operatorsScore-dataproc-gke-l4.csv | 4 + .../operatorsScore-dataproc-gke-t4.csv | 4 + .../resources/operatorsScore-dataproc-l4.csv | 4 + .../operatorsScore-dataproc-serverless-l4.csv | 4 + .../resources/operatorsScore-dataproc-t4.csv | 4 + .../main/resources/operatorsScore-emr-a10.csv | 4 + .../resources/operatorsScore-emr-a10G.csv | 4 + .../main/resources/operatorsScore-emr-t4.csv | 4 + .../resources/operatorsScore-onprem-a100.csv | 4 + .../main/resources/supportedDataSource.csv | 6 +- core/src/main/resources/supportedExecs.csv | 2 +- core/src/main/resources/supportedExprs.csv | 99 ++++---- .../override_supported_configs.json | 143 +++++++++++ 20 files changed, 553 insertions(+), 47 deletions(-) create mode 100644 .claude/agents/supported-ops-sync.md create mode 100644 .claude/skills/supported-ops-sync/SKILL.md create mode 100644 .claude/skills/supported-ops-sync/agents/openai.yaml create mode 100644 .claude/skills/supported-ops-sync/references/pr-checklist.md diff --git a/.claude/agents/supported-ops-sync.md b/.claude/agents/supported-ops-sync.md new file mode 100644 index 000000000..50fa1b4e2 --- /dev/null +++ b/.claude/agents/supported-ops-sync.md @@ -0,0 +1,33 @@ +--- +name: supported-ops-sync +description: Use this agent to prepare or review RAPIDS plugin supported-operator CSV syncs in spark-rapids-tools. +tools: Bash, Read, Grep, Glob, Edit +--- + + + +You are a focused supported-ops sync agent for `spark-rapids-tools`. + +Use the repo-local skill at `.claude/skills/supported-ops-sync/SKILL.md`. + +Core responsibilities: + +- Start from `dev` unless the user requested another base branch. +- Look for a sibling `../spark-rapids` checkout first, update it from the `NVIDIA/spark-rapids` `main` remote, and clone `NVIDIA/spark-rapids` into the parent folder only if missing. +- Use the root `../spark-rapids/tools/generated_files/supported*.csv` files directly by copying them into one temp input subdirectory, then run `scripts/sync_plugin_files/process_supported_files.py` in preview mode before applying changes. +- Review `operators_plugin_sync_report.txt` and `new_operators.txt`. +- Preserve `TNEW` gates unless parser behavior and tests justify promotion. +- Update `core/src/main/resources/supported*.csv`, `operatorsScore*.csv`, and `scripts/sync_plugin_files/override_supported_configs.json` only when applying the sync. +- Add targeted parser or qualification tests for newly supported expressions when needed. +- Commit with `git commit -s`, push the branch, and create a draft PR with `gh pr create --draft` when asked for end-to-end PR preparation. +- Draft a PR summary using `.claude/skills/supported-ops-sync/references/pr-checklist.md`. + +Do not copy plugin root CSVs directly over tools resources. Do not remove tools-preserved rows just because they are absent from plugin generated files without explicit reviewer approval. diff --git a/.claude/skills/supported-ops-sync/SKILL.md b/.claude/skills/supported-ops-sync/SKILL.md new file mode 100644 index 000000000..71e800c51 --- /dev/null +++ b/.claude/skills/supported-ops-sync/SKILL.md @@ -0,0 +1,234 @@ +--- +name: supported-ops-sync +description: Use when syncing spark-rapids-tools supported operator CSVs from NVIDIA/spark-rapids tools/generated_files, investigating supportedExprs.csv or supportedExecs.csv drift, preparing supported-op sync PRs, or reviewing TNEW operator follow-ups. +--- + + + +# Supported Ops Sync + +## Purpose + +Keep `spark-rapids-tools` aligned with the RAPIDS plugin generated support metadata while preserving tools-side review gates. The plugin source is `NVIDIA/spark-rapids:tools/generated_files`. + +## Default Workflow + +Use these defaults unless the user says otherwise: + +- Tools base branch: `dev` +- Plugin source: `NVIDIA/spark-rapids@main` +- PR target: `NVIDIA/spark-rapids-tools`, base `dev` +- PR mode: create the PR as a draft first +- Branch name: `feature/sync-supported-ops-YYYYMMDD` +- Commit style: signed off with `git commit -s` + +Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev` base branch, or an automatic `TNEW -> S` promotion without test evidence. + +1. Start from the tools base branch: + + ```bash + REPO_ROOT=$(pwd) + PLUGIN_REPO="$(cd "$REPO_ROOT/.." && pwd)/spark-rapids" + SYNC_DATE=$(date +%Y%m%d) + PR_DATE=$(date +%Y-%m-%d) + BRANCH=feature/sync-supported-ops-$SYNC_DATE + + git fetch origin dev + git switch dev + git pull --ff-only origin dev + git switch -c "$BRANCH" + ``` + +2. Look for a sibling RAPIDS plugin checkout first, then update from `main`. Clone into the parent folder only if it is missing: + + ```bash + if [ -d "$PLUGIN_REPO/.git" ]; then + if [ -n "$(git -C "$PLUGIN_REPO" status --porcelain)" ]; then + echo "$PLUGIN_REPO has local changes; ask before updating it." + git -C "$PLUGIN_REPO" status --short + exit 1 + fi + PLUGIN_REMOTE=$(git -C "$PLUGIN_REPO" remote -v | \ + awk '/github.com[:/]NVIDIA\/spark-rapids(.git)?/ && $3 == "\(fetch\)" {print $1; exit}') + PLUGIN_REMOTE=${PLUGIN_REMOTE:-origin} + git -C "$PLUGIN_REPO" fetch "$PLUGIN_REMOTE" main + git -C "$PLUGIN_REPO" switch main + git -C "$PLUGIN_REPO" pull --ff-only "$PLUGIN_REMOTE" main + else + gh repo clone NVIDIA/spark-rapids "$PLUGIN_REPO" -- --depth 1 --branch main + fi + ``` + + Capture the plugin commit for the PR: + + ```bash + git -C "$PLUGIN_REPO" rev-parse HEAD + ``` + +3. Build a direct plugin CSV input directory from the root generated CSVs. The root CSVs are already generated by the plugin; wrapping them in one temp subdirectory lets the existing tools script keep its override, `TNEW`, preservation, and report behavior without re-unioning every Spark-version directory: + + ```bash + RUN_DIR=/tmp/supported-ops-sync-$(date +%Y%m%d-%H%M%S) + PLUGIN_INPUT="$RUN_DIR/plugin-root/000" + mkdir -p "$RUN_DIR/preview" "$PLUGIN_INPUT" + cp "$PLUGIN_REPO/tools/generated_files/supportedDataSource.csv" "$PLUGIN_INPUT/" + cp "$PLUGIN_REPO/tools/generated_files/supportedExecs.csv" "$PLUGIN_INPUT/" + cp "$PLUGIN_REPO/tools/generated_files/supportedExprs.csv" "$PLUGIN_INPUT/" + cp scripts/sync_plugin_files/override_supported_configs.json \ + "$RUN_DIR/override_supported_configs.json" + ``` + +4. Run the existing tools sync utility from the temp run directory. Preview first: + + ```bash + + ( + cd "$RUN_DIR" + python "$REPO_ROOT/scripts/sync_plugin_files/process_supported_files.py" \ + "$RUN_DIR/plugin-root" \ + --configs "$RUN_DIR/override_supported_configs.json" \ + --tools-csv "$REPO_ROOT/core/src/main/resources" \ + --output "$RUN_DIR/preview" + ) + ``` + + Running from `RUN_DIR` keeps `operators_plugin_sync_report.txt` and `new_operators.txt` out of repo root during preview. + +5. Read the preview outputs: + + ```bash + sed -n '1,240p' "$RUN_DIR/operators_plugin_sync_report.txt" + sed -n '1,200p' "$RUN_DIR/new_operators.txt" + ``` + +6. Classify changes: + - Pure support/type changes: verify they are plausible from plugin CSVs. + - New execs/expressions: keep as `TNEW` unless tools parsing and tests prove support. + - Removed plugin rows: preserve them unless a reviewer explicitly agrees to delete tools coverage. + - `TNEW -> S`: require parser coverage or a clear existing parser path. + - Spark 4-only rows: call out separately and keep gated unless this branch has matching Spark 4 support. + +7. Apply only after preview review by rerunning the existing utility with repo outputs: + + ```bash + ( + cd "$RUN_DIR" + python "$REPO_ROOT/scripts/sync_plugin_files/process_supported_files.py" \ + "$RUN_DIR/plugin-root" \ + --configs "$REPO_ROOT/scripts/sync_plugin_files/override_supported_configs.json" \ + --tools-csv "$REPO_ROOT/core/src/main/resources" \ + --output "$REPO_ROOT/core/src/main/resources" + ) + ``` + +8. If `new_operators.txt` is non-empty, run the existing operator score utility: + + ```bash + if [ -s "$RUN_DIR/new_operators.txt" ]; then + python scripts/sync_plugin_files/sync_operator_scores.py \ + "$RUN_DIR/new_operators.txt" \ + core/src/main/resources + fi + ``` + +9. Add targeted tests for promoted expressions or parser behavior changes. Prior examples are in `core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala`, whose suite class is `SQLPlanParserSuite`. + +10. Run targeted validation: + + ```bash + mvn -f core/pom.xml test \ + -DwildcardSuites=com.nvidia.spark.rapids.tool.qualification.PluginTypeCheckerSuite + mvn -f core/pom.xml test \ + -DwildcardSuites=com.nvidia.spark.rapids.tool.planparser.SQLPlanParserSuite + ``` + +11. Review, commit, and create the PR: + + ```bash + git diff -- core/src/main/resources scripts/sync_plugin_files core/src/test + git status --short + git add core/src/main/resources scripts/sync_plugin_files core/src/test + git commit -s -m "Sync supported ops with RAPIDS plugin" + git push -u origin "$BRANCH" + gh pr create --draft --base dev --head "$BRANCH" \ + --title "Sync supported ops with RAPIDS plugin as of $PR_DATE" \ + --body-file /tmp/supported-ops-sync-pr-body.md + ``` + +## Files To Expect + +- Source plugin files: root `tools/generated_files/supported*.csv` +- Tools resource outputs: `core/src/main/resources/supportedDataSource.csv`, `supportedExecs.csv`, `supportedExprs.csv` +- Tools overrides: `scripts/sync_plugin_files/override_supported_configs.json` +- Operator scores: `core/src/main/resources/operatorsScore*.csv` +- Sync report: `operators_plugin_sync_report.txt` in the helper run directory +- New operator list: `new_operators.txt` in the helper run directory + +## Review Rules + +- Do not copy plugin root CSVs directly over tools resources. Stage the root plugin CSVs in a temp input directory and run the tools sync utility so it can apply overrides, mark new rows `TNEW`, and preserve removed rows. +- Do not blindly convert `TNEW` to `S`. `TNEW` means plugin metadata knows about the operator, but tools have not validated parser behavior. +- When new operators are accepted into tools, update operator score CSVs using `sync_operator_scores.py` or the helper's default score-sync path. +- If Spark 4-only operators appear, keep them gated as `TNEW` unless the tools branch has the necessary Spark 4 parser/test support. + +## PR Pattern + +Draft `/tmp/supported-ops-sync-pr-body.md` with this shape: + +```markdown +Contributes to #[issue-if-known] + +### Summary + +Syncs supported operator metadata in spark-rapids-tools with RAPIDS plugin generated files. + +Plugin source: NVIDIA/spark-rapids@ + +### Changes + +- Updated supported CSVs: + - core/src/main/resources/supportedDataSource.csv + - core/src/main/resources/supportedExecs.csv + - core/src/main/resources/supportedExprs.csv +- Updated operator score CSVs for new operators, if any. +- Updated scripts/sync_plugin_files/override_supported_configs.json for new TNEW rows, if any. + +### Sync Report Summary + + + +### New Operators / TNEW Follow-Ups + + + +### Tests + +- [ ] mvn test -DwildcardSuites=com.nvidia.spark.rapids.tool.qualification.PluginTypeCheckerSuite +- [ ] mvn test -DwildcardSuites=com.nvidia.spark.rapids.tool.planparser.SQLPlanParserSuite + +### Follow-Ups + + +``` + +Always include: + +- Plugin ref used, usually `NVIDIA/spark-rapids@main` or a specific commit SHA. +- Summary of supported CSV changes from `operators_plugin_sync_report.txt`. +- New operators and their current `TNEW`/tested status. +- Tests added or explicitly not added, with rationale. +- Follow-ups for execs/expressions that remain `TNEW`. + +For historical examples, compare PRs: + +- `NVIDIA/spark-rapids-tools#1478` +- `NVIDIA/spark-rapids-tools#1618` +- `NVIDIA/spark-rapids-tools#2015` diff --git a/.claude/skills/supported-ops-sync/agents/openai.yaml b/.claude/skills/supported-ops-sync/agents/openai.yaml new file mode 100644 index 000000000..510086cd1 --- /dev/null +++ b/.claude/skills/supported-ops-sync/agents/openai.yaml @@ -0,0 +1,12 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +interface: + display_name: "Supported Ops Sync" + short_description: "Sync Spark RAPIDS supported-op CSVs" + default_prompt: "Use $supported-ops-sync to update the tools supported operator CSVs from NVIDIA/spark-rapids generated files." diff --git a/.claude/skills/supported-ops-sync/references/pr-checklist.md b/.claude/skills/supported-ops-sync/references/pr-checklist.md new file mode 100644 index 000000000..deddbc3ac --- /dev/null +++ b/.claude/skills/supported-ops-sync/references/pr-checklist.md @@ -0,0 +1,23 @@ +# Supported Ops Sync PR Checklist + + + +- Mention the plugin ref used for `tools/generated_files`. +- Confirm the PR branch starts from `dev` unless the user requested another base. +- Include the generated report summary for `supportedDataSource.csv`, `supportedExecs.csv`, and `supportedExprs.csv`. +- List new execs and expressions from `new_operators.txt`. +- Explain which new rows remain `TNEW` and why. +- Explain every `TNEW -> S`, `NS -> S`, or `CO -> S` promotion. +- Confirm operator score CSVs were updated when new operators were added. +- Confirm targeted tests run, or state why a test is not applicable. +- Call out Spark 4-only rows separately. +- Commit with `git commit -s`. +- Open a draft PR against `dev` with a title like `Sync supported ops with RAPIDS plugin as of YYYY-MM-DD`. diff --git a/core/src/main/resources/operatorsScore-databricks-aws-a10G.csv b/core/src/main/resources/operatorsScore-databricks-aws-a10G.csv index 5df39777e..e7b244128 100644 --- a/core/src/main/resources/operatorsScore-databricks-aws-a10G.csv +++ b/core/src/main/resources/operatorsScore-databricks-aws-a10G.csv @@ -323,3 +323,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-databricks-aws-t4.csv b/core/src/main/resources/operatorsScore-databricks-aws-t4.csv index 5df39777e..e7b244128 100644 --- a/core/src/main/resources/operatorsScore-databricks-aws-t4.csv +++ b/core/src/main/resources/operatorsScore-databricks-aws-t4.csv @@ -323,3 +323,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-databricks-azure-t4.csv b/core/src/main/resources/operatorsScore-databricks-azure-t4.csv index a0ccc7565..d313c8679 100644 --- a/core/src/main/resources/operatorsScore-databricks-azure-t4.csv +++ b/core/src/main/resources/operatorsScore-databricks-azure-t4.csv @@ -311,3 +311,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv index 9d9af0cfc..5c86bc1fe 100644 --- a/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-gke-l4.csv @@ -305,3 +305,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv b/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv index 6b0619674..369ba910f 100644 --- a/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-gke-t4.csv @@ -305,3 +305,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-dataproc-l4.csv b/core/src/main/resources/operatorsScore-dataproc-l4.csv index 2cf8de9b0..49a9ad1dd 100644 --- a/core/src/main/resources/operatorsScore-dataproc-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-l4.csv @@ -311,3 +311,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv b/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv index 9d7139355..35f676441 100644 --- a/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-serverless-l4.csv @@ -305,3 +305,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-dataproc-t4.csv b/core/src/main/resources/operatorsScore-dataproc-t4.csv index 154e2f1ce..05449a5b4 100644 --- a/core/src/main/resources/operatorsScore-dataproc-t4.csv +++ b/core/src/main/resources/operatorsScore-dataproc-t4.csv @@ -311,3 +311,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-emr-a10.csv b/core/src/main/resources/operatorsScore-emr-a10.csv index 42874ae30..a68387818 100644 --- a/core/src/main/resources/operatorsScore-emr-a10.csv +++ b/core/src/main/resources/operatorsScore-emr-a10.csv @@ -311,3 +311,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-emr-a10G.csv b/core/src/main/resources/operatorsScore-emr-a10G.csv index 42874ae30..a68387818 100644 --- a/core/src/main/resources/operatorsScore-emr-a10G.csv +++ b/core/src/main/resources/operatorsScore-emr-a10G.csv @@ -311,3 +311,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-emr-t4.csv b/core/src/main/resources/operatorsScore-emr-t4.csv index fde3d8ca3..a391945db 100644 --- a/core/src/main/resources/operatorsScore-emr-t4.csv +++ b/core/src/main/resources/operatorsScore-emr-t4.csv @@ -311,3 +311,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/operatorsScore-onprem-a100.csv b/core/src/main/resources/operatorsScore-onprem-a100.csv index d7283564e..acbfcd95c 100644 --- a/core/src/main/resources/operatorsScore-onprem-a100.csv +++ b/core/src/main/resources/operatorsScore-onprem-a100.csv @@ -323,3 +323,7 @@ MapZipWith,1.5 Uuid,1.5 Invoke,1.5 MapFromEntries,1.5 +ArrayAggregate,1.5 +Hex,1.5 +Sha2,1.5 +StringDecode,1.5 diff --git a/core/src/main/resources/supportedDataSource.csv b/core/src/main/resources/supportedDataSource.csv index 96d0c0915..a4aec7b1b 100644 --- a/core/src/main/resources/supportedDataSource.csv +++ b/core/src/main/resources/supportedDataSource.csv @@ -6,13 +6,13 @@ Delta,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S HiveText,read,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS HiveText,write,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS Iceberg,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S -Iceberg,write,TNEW,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO,CO +Iceberg,write,TNEW,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S JSON,read,S,S,S,S,S,S,S,PS,PS,S,S,NA,NS,NA,PS,NS,PS,NS,NA,NA ORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA ORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA -HiveORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA -HiveORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA Parquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S Parquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S +HiveORC,read,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA +HiveORC,write,S,S,S,S,S,S,S,S,PS,S,S,NA,NS,NA,PS,PS,PS,NS,NA,NA HiveParquet,read,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S HiveParquet,write,S,S,S,S,S,S,S,S,PS,S,S,NA,S,NA,PS,PS,PS,NS,S,S diff --git a/core/src/main/resources/supportedExecs.csv b/core/src/main/resources/supportedExecs.csv index 0643b46e1..c069c51f4 100644 --- a/core/src/main/resources/supportedExecs.csv +++ b/core/src/main/resources/supportedExecs.csv @@ -1,7 +1,7 @@ Exec,Supported,Notes,Params,BOOLEAN,BYTE,SHORT,INT,LONG,FLOAT,DOUBLE,DATE,TIMESTAMP,STRING,DECIMAL,NULL,BINARY,CALENDAR,ARRAY,MAP,STRUCT,UDT,DAYTIME,YEARMONTH CoalesceExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S CollectLimitExec,NS,This is disabled by default because Collect Limit replacement can be slower on the GPU; if huge number of rows in a batch it could help by limiting the number of rows transferred from GPU to CPU,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS -ExpandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +ExpandExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS FileSourceScanExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S FilterExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S GenerateExec,S,None,Input/Output,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS diff --git a/core/src/main/resources/supportedExprs.csv b/core/src/main/resources/supportedExprs.csv index a22eb6935..bc72a8fa3 100644 --- a/core/src/main/resources/supportedExprs.csv +++ b/core/src/main/resources/supportedExprs.csv @@ -27,6 +27,11 @@ And,S,`and`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,N And,S,`and`,None,AST,lhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA And,S,`and`,None,AST,rhs,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA And,S,`and`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ArrayAggregate,TNEW,`aggregate`,None,project,zero,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS +ArrayAggregate,TNEW,`aggregate`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS +ArrayAggregate,TNEW,`aggregate`,None,project,finish,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS +ArrayAggregate,TNEW,`aggregate`,None,project,merge,S,S,S,S,S,S,S,S,PS,S,S,NS,NS,NS,NS,NS,NS,NS,NS,NS +ArrayAggregate,TNEW,`aggregate`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA ArrayContains,S,`array_contains`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA ArrayContains,S,`array_contains`,None,project,key,S,S,S,S,S,S,S,S,PS,S,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS ArrayContains,S,`array_contains`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -56,13 +61,13 @@ ArrayPosition,S,`array_position`,None,project,array,NA,NA,NA,NA,NA,NA,NA,NA,NA,N ArrayPosition,S,`array_position`,None,project,key,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,NA,PS,NS,NA,NA ArrayPosition,S,`array_position`,None,project,result,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArrayRemove,S,`array_remove`,None,project,array,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,NS,PS,NS,NS,NS,NS,NS -ArrayRemove,S,`array_remove`,None,project,element,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +ArrayRemove,S,`array_remove`,None,project,element,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NS,PS,NS,NS,NS ArrayRemove,S,`array_remove`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA ArrayRepeat,S,`array_repeat`,None,project,left,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS ArrayRepeat,S,`array_repeat`,None,project,right,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA ArrayRepeat,S,`array_repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA ArrayTransform,S,`transform`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA -ArrayTransform,S,`transform`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +ArrayTransform,S,`transform`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS ArrayTransform,S,`transform`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA ArrayUnion,S,`array_union`,None,project,array1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA ArrayUnion,S,`array_union`,None,project,array2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA @@ -137,12 +142,10 @@ Cbrt,S,`cbrt`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA, Cbrt,S,`cbrt`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Cbrt,S,`cbrt`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Cbrt,S,`cbrt`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Ceil,S,`ceil`; `ceiling`,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA -Ceil,S,`ceil`; `ceiling`,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ceil,S, ,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ceil,S, ,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA CheckOverflow,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA CheckOverflow,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA -CheckOverflowInTableInsert,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S -CheckOverflowInTableInsert,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S Coalesce,S,`coalesce`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S Coalesce,S,`coalesce`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S Concat,S,`concat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,PS,NA,NA,NA,NA,NA @@ -150,7 +153,7 @@ Concat,S,`concat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,P ConcatWs,S,`concat_ws`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,S,NA,NA,NA,NA,NA ConcatWs,S,`concat_ws`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Contains,S, ,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Contains,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Contains,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Contains,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Conv,S,`conv`,None,project,num,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Conv,S,`conv`,None,project,from_base,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -176,15 +179,15 @@ CreateNamedStruct,S,`named_struct`; `struct`,None,project,name,NA,NA,NA,NA,NA,NA CreateNamedStruct,S,`named_struct`; `struct`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS CreateNamedStruct,S,`named_struct`; `struct`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA CurrentRow$,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`; `dateadd`,None,project,startDate,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`; `dateadd`,None,project,days,NA,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`; `dateadd`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`,None,project,startDate,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`,None,project,days,NA,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateAddInterval,S, ,None,project,start,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateAddInterval,S, ,None,project,interval,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA DateAddInterval,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`date_diff`; `datediff`,None,project,lhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`date_diff`; `datediff`,None,project,rhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`date_diff`; `datediff`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`datediff`,None,project,lhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`datediff`,None,project,rhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`datediff`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateFormatClass,S,`date_format`,None,project,timestamp,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateFormatClass,S,`date_format`,None,project,strfmt,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateFormatClass,S,`date_format`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -237,8 +240,8 @@ Expm1,S,`expm1`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Expm1,S,`expm1`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Flatten,S,`flatten`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA Flatten,S,`flatten`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA -Floor,S,`floor`,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA -Floor,S,`floor`,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Floor,S, ,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Floor,S, ,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA FormatNumber,S,`format_number`,None,project,x,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA FormatNumber,S,`format_number`,None,project,d,NA,NA,NA,PS,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA FormatNumber,S,`format_number`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -278,6 +281,8 @@ GreaterThanOrEqual,S,`>=`,None,AST,rhs,S,S,S,S,S,NS,NS,S,PS,S,NS,NS,NS,NS,NS,NA, GreaterThanOrEqual,S,`>=`,None,AST,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Greatest,S,`greatest`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Greatest,S,`greatest`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA +Hex,TNEW,`hex`,None,project,input,NA,NA,NA,NA,S,NA,NA,NA,NA,S,NA,NA,S,NA,NA,NA,NA,NA,NA,NA +Hex,TNEW,`hex`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA HiveHash,S,`hive-hash`,None,project,input,S,S,S,S,S,S,S,S,PS,S,NS,S,NS,NS,PS,NS,PS,NS,NS,NS HiveHash,S,`hive-hash`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Hour,S,`hour`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -325,9 +330,9 @@ Lag,S,`lag`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS Lag,S,`lag`,None,window,offset,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Lag,S,`lag`,None,window,default,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS Lag,S,`lag`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS -LambdaFunction,S, ,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS -LambdaFunction,S, ,None,project,arguments,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS -LambdaFunction,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +LambdaFunction,S, ,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +LambdaFunction,S, ,None,project,arguments,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS +LambdaFunction,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS LastDay,S,`last_day`,None,project,input,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA LastDay,S,`last_day`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Lead,S,`lead`,None,window,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS @@ -336,8 +341,8 @@ Lead,S,`lead`,None,window,default,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS, Lead,S,`lead`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS Least,S,`least`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Least,S,`least`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA -Length,S,`char_length`; `character_length`; `len`; `length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -Length,S,`char_length`; `character_length`; `len`; `length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Length,S,`char_length`; `character_length`; `length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA +Length,S,`char_length`; `character_length`; `length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA LessThan,S,`<`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA LessThan,S,`<`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA LessThan,S,`<`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -388,7 +393,7 @@ MapValues,S,`map_values`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA, MapValues,S,`map_values`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA MapZipWith,S,`map_zip_with`,None,project,argument1,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA MapZipWith,S,`map_zip_with`,None,project,argument2,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA -MapZipWith,S,`map_zip_with`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +MapZipWith,S,`map_zip_with`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS MapZipWith,S,`map_zip_with`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA Md5,S,`md5`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA Md5,S,`md5`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -422,7 +427,7 @@ Murmur3Hash,S,`hash`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,N NaNvl,S,`nanvl`,None,project,lhs,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA NaNvl,S,`nanvl`,None,project,rhs,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA NaNvl,S,`nanvl`,None,project,result,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -NamedLambdaVariable,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +NamedLambdaVariable,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS Not,S,`!`; `not`,None,project,input,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Not,S,`!`; `not`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Not,S,`!`; `not`,None,AST,input,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -525,15 +530,18 @@ Sequence,S,`sequence`,None,project,step,NA,S,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NS,NA Sequence,S,`sequence`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA Sha1,S,`sha1`; `sha`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA Sha1,S,`sha1`; `sha`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftLeft,S,`<<`; `shiftleft`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftLeft,S,`<<`; `shiftleft`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftLeft,S,`<<`; `shiftleft`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`>>`; `shiftright`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`>>`; `shiftright`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`>>`; `shiftright`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`>>>`; `shiftrightunsigned`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`>>>`; `shiftrightunsigned`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`>>>`; `shiftrightunsigned`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sha2,TNEW,`sha2`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA +Sha2,TNEW,`sha2`,None,project,bitLength,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Sha2,TNEW,`sha2`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftLeft,S,`shiftleft`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftLeft,S,`shiftleft`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftLeft,S,`shiftleft`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`shiftright`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`shiftright`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`shiftright`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Signum,S,`sign`; `signum`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Signum,S,`sign`; `signum`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Sin,S,`sin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -569,27 +577,30 @@ Stack,S,`stack`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS StartsWith,S, ,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StartsWith,S, ,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StartsWith,S, ,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringDecode,TNEW, ,None,project,bin,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA +StringDecode,TNEW, ,None,project,charset,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringDecode,TNEW, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringInstr,S,`instr`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringInstr,S,`instr`,None,project,substr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringInstr,S,`instr`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S,`lpad`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S,`lpad`,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S,`lpad`,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S,`lpad`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S, ,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S, ,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S, ,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringLocate,S,`locate`; `position`,None,project,substr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringLocate,S,`locate`; `position`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringLocate,S,`locate`; `position`,None,project,start,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringLocate,S,`locate`; `position`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S,`rpad`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S,`rpad`,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S,`rpad`,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S,`rpad`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S, ,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S, ,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S, ,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringRepeat,S,`repeat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringRepeat,S,`repeat`,None,project,repeatTimes,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringRepeat,S,`repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringReplace,S,`replace`,None,project,src,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringReplace,S,`replace`,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringReplace,S,`replace`,None,project,replace,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringReplace,S,`replace`,None,project,search,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringReplace,S,`replace`,None,project,replace,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringReplace,S,`replace`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringSplit,S,`split`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringSplit,S,`split`,None,project,regexp,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -653,7 +664,7 @@ TransformKeys,S,`transform_keys`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,N TransformKeys,S,`transform_keys`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NS,NS TransformKeys,S,`transform_keys`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA TransformValues,S,`transform_values`,None,project,argument,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA -TransformValues,S,`transform_values`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,PS,PS,NS,NS,NS +TransformValues,S,`transform_values`,None,project,function,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS TransformValues,S,`transform_values`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA TruncDate,S,`trunc`,None,project,date,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA TruncDate,S,`trunc`,None,project,format,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -832,7 +843,7 @@ VarianceSamp,S,`var_samp`; `variance`,None,aggregation,input,NA,NA,NA,NA,NA,NA,S VarianceSamp,S,`var_samp`; `variance`,None,aggregation,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA VarianceSamp,S,`var_samp`; `variance`,None,window,input,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA VarianceSamp,S,`var_samp`; `variance`,None,window,result,NA,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StaticInvoke,TNEW, ,None,project,result,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS +StaticInvoke,TNEW, ,The supported types are not deterministic since it's a dynamic expression,project,result,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS,PS NormalizeNaNAndZero,S, ,None,project,input,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA NormalizeNaNAndZero,S, ,None,project,result,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA InSubqueryExec,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA @@ -842,6 +853,8 @@ HiveGenericUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,N HiveGenericUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS HiveSimpleUDF,S, ,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS HiveSimpleUDF,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,NS,NS,NS +CheckOverflowInTableInsert,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S +CheckOverflowInTableInsert,S, ,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S Empty2Null,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Empty2Null,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA KnownNullable,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,S,PS,PS,PS,S,S,S diff --git a/scripts/sync_plugin_files/override_supported_configs.json b/scripts/sync_plugin_files/override_supported_configs.json index 70cce52c5..c876cd2c1 100644 --- a/scripts/sync_plugin_files/override_supported_configs.json +++ b/scripts/sync_plugin_files/override_supported_configs.json @@ -307,6 +307,149 @@ "value": "TNEW" } ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "zero", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "finish", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "merge", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "argument", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Hex", + "Context": "project", + "Params": "input", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Hex", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "input", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "bitLength", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "bin", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "charset", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] } ], "supportedDataSource.csv": [ From 72978be33f206d5b0eece47f9e967939be19aea0 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Tue, 26 May 2026 16:35:54 -0700 Subject: [PATCH 2/7] Preserve SQL function aliases during supported ops sync Signed-off-by: Partho Sarthi --- .claude/skills/supported-ops-sync/SKILL.md | 2 + core/src/main/resources/supportedExprs.csv | 58 +++++++++---------- .../process_supported_files.py | 31 +++++++++- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/.claude/skills/supported-ops-sync/SKILL.md b/.claude/skills/supported-ops-sync/SKILL.md index 71e800c51..6276e5d84 100644 --- a/.claude/skills/supported-ops-sync/SKILL.md +++ b/.claude/skills/supported-ops-sync/SKILL.md @@ -113,6 +113,7 @@ Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev - Pure support/type changes: verify they are plausible from plugin CSVs. - New execs/expressions: keep as `TNEW` unless tools parsing and tests prove support. - Removed plugin rows: preserve them unless a reviewer explicitly agrees to delete tools coverage. + - `SQL Func` changes: preserve tools-side aliases when plugin generated CSVs are blank or contain a shorter alias list; these aliases are parser lookup keys in tools. - `TNEW -> S`: require parser coverage or a clear existing parser path. - Spark 4-only rows: call out separately and keep gated unless this branch has matching Spark 4 support. @@ -175,6 +176,7 @@ Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev ## Review Rules - Do not copy plugin root CSVs directly over tools resources. Stage the root plugin CSVs in a temp input directory and run the tools sync utility so it can apply overrides, mark new rows `TNEW`, and preserve removed rows. +- Do not accept `SQL Func` alias loss from the plugin root CSV without review. The tools parser uses those aliases to recognize expressions whose physical plans print SQL function names instead of expression class names. - Do not blindly convert `TNEW` to `S`. `TNEW` means plugin metadata knows about the operator, but tools have not validated parser behavior. - When new operators are accepted into tools, update operator score CSVs using `sync_operator_scores.py` or the helper's default score-sync path. - If Spark 4-only operators appear, keep them gated as `TNEW` unless the tools branch has the necessary Spark 4 parser/test support. diff --git a/core/src/main/resources/supportedExprs.csv b/core/src/main/resources/supportedExprs.csv index bc72a8fa3..c42c2e5e2 100644 --- a/core/src/main/resources/supportedExprs.csv +++ b/core/src/main/resources/supportedExprs.csv @@ -142,8 +142,8 @@ Cbrt,S,`cbrt`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA, Cbrt,S,`cbrt`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Cbrt,S,`cbrt`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Cbrt,S,`cbrt`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -Ceil,S, ,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA -Ceil,S, ,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ceil,S,`ceil`; `ceiling`,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Ceil,S,`ceil`; `ceiling`,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA CheckOverflow,S, ,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA CheckOverflow,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA Coalesce,S,`coalesce`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,S,S @@ -179,15 +179,15 @@ CreateNamedStruct,S,`named_struct`; `struct`,None,project,name,NA,NA,NA,NA,NA,NA CreateNamedStruct,S,`named_struct`; `struct`,None,project,value,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS,NS,NS CreateNamedStruct,S,`named_struct`; `struct`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA CurrentRow$,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`,None,project,startDate,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`,None,project,days,NA,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateAdd,S,`date_add`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`; `dateadd`,None,project,startDate,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`; `dateadd`,None,project,days,NA,S,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateAdd,S,`date_add`; `dateadd`,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateAddInterval,S, ,None,project,start,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateAddInterval,S, ,None,project,interval,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA DateAddInterval,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`datediff`,None,project,lhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`datediff`,None,project,rhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -DateDiff,S,`datediff`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`date_diff`; `datediff`,None,project,lhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`date_diff`; `datediff`,None,project,rhs,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +DateDiff,S,`date_diff`; `datediff`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateFormatClass,S,`date_format`,None,project,timestamp,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateFormatClass,S,`date_format`,None,project,strfmt,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA DateFormatClass,S,`date_format`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -240,8 +240,8 @@ Expm1,S,`expm1`,None,AST,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Expm1,S,`expm1`,None,AST,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Flatten,S,`flatten`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA Flatten,S,`flatten`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA -Floor,S, ,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA -Floor,S, ,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Floor,S,`floor`,None,project,input,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA +Floor,S,`floor`,None,project,result,NA,NA,NA,NA,S,NA,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA FormatNumber,S,`format_number`,None,project,x,NA,S,S,S,S,S,S,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA FormatNumber,S,`format_number`,None,project,d,NA,NA,NA,PS,NA,NA,NA,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA FormatNumber,S,`format_number`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -341,8 +341,8 @@ Lead,S,`lead`,None,window,default,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS, Lead,S,`lead`,None,window,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,PS,NS,PS,NS,NS,NS Least,S,`least`,None,project,param,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA Least,S,`least`,None,project,result,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,NS,NS,NA,NA -Length,S,`char_length`; `character_length`; `length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA -Length,S,`char_length`; `character_length`; `length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +Length,S,`char_length`; `character_length`; `len`; `length`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NS,NA,NA,NA,NA,NA,NA,NA +Length,S,`char_length`; `character_length`; `len`; `length`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA LessThan,S,`<`,None,project,lhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA LessThan,S,`<`,None,project,rhs,S,S,S,S,S,S,S,S,PS,S,S,S,NS,NS,NS,NA,PS,NS,NA,NA LessThan,S,`<`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -533,15 +533,15 @@ Sha1,S,`sha1`; `sha`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,N Sha2,TNEW,`sha2`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA Sha2,TNEW,`sha2`,None,project,bitLength,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Sha2,TNEW,`sha2`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftLeft,S,`shiftleft`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftLeft,S,`shiftleft`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftLeft,S,`shiftleft`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`shiftright`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`shiftright`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRight,S,`shiftright`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -ShiftRightUnsigned,S,`shiftrightunsigned`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftLeft,S,`<<`; `shiftleft`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftLeft,S,`<<`; `shiftleft`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftLeft,S,`<<`; `shiftleft`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`>>`; `shiftright`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`>>`; `shiftright`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRight,S,`>>`; `shiftright`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`>>>`; `shiftrightunsigned`,None,project,value,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`>>>`; `shiftrightunsigned`,None,project,amount,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +ShiftRightUnsigned,S,`>>>`; `shiftrightunsigned`,None,project,result,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Signum,S,`sign`; `signum`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Signum,S,`sign`; `signum`,None,project,result,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA Sin,S,`sin`,None,project,input,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA @@ -583,18 +583,18 @@ StringDecode,TNEW, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA StringInstr,S,`instr`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringInstr,S,`instr`,None,project,substr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringInstr,S,`instr`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S, ,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S, ,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S, ,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringLPad,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S,`lpad`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S,`lpad`,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S,`lpad`,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringLPad,S,`lpad`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringLocate,S,`locate`; `position`,None,project,substr,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringLocate,S,`locate`; `position`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringLocate,S,`locate`; `position`,None,project,start,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringLocate,S,`locate`; `position`,None,project,result,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S, ,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S, ,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S, ,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA -StringRPad,S, ,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S,`rpad`,None,project,str,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S,`rpad`,None,project,len,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S,`rpad`,None,project,pad,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA +StringRPad,S,`rpad`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringRepeat,S,`repeat`,None,project,input,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringRepeat,S,`repeat`,None,project,repeatTimes,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA StringRepeat,S,`repeat`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA diff --git a/scripts/sync_plugin_files/process_supported_files.py b/scripts/sync_plugin_files/process_supported_files.py index 9eecf8ac2..44ee8fd68 100644 --- a/scripts/sync_plugin_files/process_supported_files.py +++ b/scripts/sync_plugin_files/process_supported_files.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -83,6 +83,22 @@ def is_greater(elem1, elem2): return len(elem1) > len(elem2) +def should_preserve_tools_string(column_name, tools_value, plugin_value): + """ + Preserve tools-side expression aliases when plugin generated metadata is less informative. + SQL function aliases are parser lookup keys in tools, and the generated plugin CSV can omit + aliases for newer Spark versions even though older-version aliases should still be recognized. + """ + if column_name != "SQL Func": + return False + + tools_str = str(tools_value).strip() + plugin_str = str(plugin_value).strip() + if tools_str == "" or tools_str == "None": + return False + return plugin_str == "" or plugin_str == "None" or len(tools_str) > len(plugin_str) + + def check_df_rows(row1, row2, keys): """ Given two DataFrame rows (pandas Series) and a list of keys (column names), @@ -248,8 +264,17 @@ def compare_csv_file(union_df, tools_df, keys, report_file, override_configs_jso exists_in_tools = True for tools_column in tools_df.columns: if (tools_column in union_row) and (not tools_row[tools_column] == union_row[tools_column]): - report_file.write(f"Row is changed: {', '.join(tools_row.astype(str))}\n " + - f"{tools_column}: {tools_row[tools_column]} -> {union_row[tools_column]}\n") + if should_preserve_tools_string( + tools_column, tools_row[tools_column], union_row[tools_column]): + report_file.write( + f"Row keeps tools value: {', '.join(tools_row.astype(str))}\n " + + f"{tools_column}: preserving {tools_row[tools_column]} instead of " + + f"{union_row[tools_column]}\n") + union_df.at[union_idx, tools_column] = tools_row[tools_column] + else: + report_file.write(f"Row is changed: {', '.join(tools_row.astype(str))}\n " + + f"{tools_column}: {tools_row[tools_column]} -> " + + f"{union_row[tools_column]}\n") if not exists_in_tools: for column_name in union_df.columns: if is_support_level(union_row[column_name]): From 03190647d11086a5ed763d5973dcb092e553891c Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Tue, 26 May 2026 16:50:24 -0700 Subject: [PATCH 3/7] Remove supported ops sync subagent wrapper Signed-off-by: Partho Sarthi --- .claude/agents/supported-ops-sync.md | 33 ---------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .claude/agents/supported-ops-sync.md diff --git a/.claude/agents/supported-ops-sync.md b/.claude/agents/supported-ops-sync.md deleted file mode 100644 index 50fa1b4e2..000000000 --- a/.claude/agents/supported-ops-sync.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: supported-ops-sync -description: Use this agent to prepare or review RAPIDS plugin supported-operator CSV syncs in spark-rapids-tools. -tools: Bash, Read, Grep, Glob, Edit ---- - - - -You are a focused supported-ops sync agent for `spark-rapids-tools`. - -Use the repo-local skill at `.claude/skills/supported-ops-sync/SKILL.md`. - -Core responsibilities: - -- Start from `dev` unless the user requested another base branch. -- Look for a sibling `../spark-rapids` checkout first, update it from the `NVIDIA/spark-rapids` `main` remote, and clone `NVIDIA/spark-rapids` into the parent folder only if missing. -- Use the root `../spark-rapids/tools/generated_files/supported*.csv` files directly by copying them into one temp input subdirectory, then run `scripts/sync_plugin_files/process_supported_files.py` in preview mode before applying changes. -- Review `operators_plugin_sync_report.txt` and `new_operators.txt`. -- Preserve `TNEW` gates unless parser behavior and tests justify promotion. -- Update `core/src/main/resources/supported*.csv`, `operatorsScore*.csv`, and `scripts/sync_plugin_files/override_supported_configs.json` only when applying the sync. -- Add targeted parser or qualification tests for newly supported expressions when needed. -- Commit with `git commit -s`, push the branch, and create a draft PR with `gh pr create --draft` when asked for end-to-end PR preparation. -- Draft a PR summary using `.claude/skills/supported-ops-sync/references/pr-checklist.md`. - -Do not copy plugin root CSVs directly over tools resources. Do not remove tools-preserved rows just because they are absent from plugin generated files without explicit reviewer approval. From 524b6c65434f6289a731848464720ae980e2b8f6 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Tue, 26 May 2026 17:58:40 -0700 Subject: [PATCH 4/7] Address supported ops sync review comments Signed-off-by: Partho Sarthi --- .claude/skills/supported-ops-sync/SKILL.md | 7 ++++--- .../process_supported_files.py | 20 +++++++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.claude/skills/supported-ops-sync/SKILL.md b/.claude/skills/supported-ops-sync/SKILL.md index 6276e5d84..73df28e36 100644 --- a/.claude/skills/supported-ops-sync/SKILL.md +++ b/.claude/skills/supported-ops-sync/SKILL.md @@ -105,15 +105,16 @@ Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev 5. Read the preview outputs: ```bash - sed -n '1,240p' "$RUN_DIR/operators_plugin_sync_report.txt" - sed -n '1,200p' "$RUN_DIR/new_operators.txt" + wc -l "$RUN_DIR/operators_plugin_sync_report.txt" "$RUN_DIR/new_operators.txt" + cat "$RUN_DIR/operators_plugin_sync_report.txt" + cat "$RUN_DIR/new_operators.txt" ``` 6. Classify changes: - Pure support/type changes: verify they are plausible from plugin CSVs. - New execs/expressions: keep as `TNEW` unless tools parsing and tests prove support. - Removed plugin rows: preserve them unless a reviewer explicitly agrees to delete tools coverage. - - `SQL Func` changes: preserve tools-side aliases when plugin generated CSVs are blank or contain a shorter alias list; these aliases are parser lookup keys in tools. + - `SQL Func` changes: preserve tools-side aliases when plugin generated CSVs are blank or contain a strict subset of the tools aliases; these aliases are parser lookup keys in tools. - `TNEW -> S`: require parser coverage or a clear existing parser path. - Spark 4-only rows: call out separately and keep gated unless this branch has matching Spark 4 support. diff --git a/scripts/sync_plugin_files/process_supported_files.py b/scripts/sync_plugin_files/process_supported_files.py index 44ee8fd68..a0f5875b0 100644 --- a/scripts/sync_plugin_files/process_supported_files.py +++ b/scripts/sync_plugin_files/process_supported_files.py @@ -83,20 +83,32 @@ def is_greater(elem1, elem2): return len(elem1) > len(elem2) +def get_sql_func_aliases(sql_func_value): + """ + Return SQL function aliases from a semicolon-separated SQL Func cell. + """ + sql_func_str = str(sql_func_value).strip() + if sql_func_str == "" or sql_func_str == "None": + return set() + return {alias.strip() for alias in sql_func_str.split(";") if alias.strip()} + + def should_preserve_tools_string(column_name, tools_value, plugin_value): """ Preserve tools-side expression aliases when plugin generated metadata is less informative. SQL function aliases are parser lookup keys in tools, and the generated plugin CSV can omit aliases for newer Spark versions even though older-version aliases should still be recognized. + Preserve only when the plugin aliases are blank or a strict subset of the tools aliases, so an + intentional plugin alias rename is accepted instead of being hidden by a string-length heuristic. """ if column_name != "SQL Func": return False - tools_str = str(tools_value).strip() - plugin_str = str(plugin_value).strip() - if tools_str == "" or tools_str == "None": + tools_aliases = get_sql_func_aliases(tools_value) + plugin_aliases = get_sql_func_aliases(plugin_value) + if not tools_aliases: return False - return plugin_str == "" or plugin_str == "None" or len(tools_str) > len(plugin_str) + return not plugin_aliases or plugin_aliases < tools_aliases def check_df_rows(row1, row2, keys): From 798fa5dd3c04dc92ad14b6a2714daaa5a54bf005 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 29 May 2026 12:37:13 -0700 Subject: [PATCH 5/7] Move supported ops sync skill to separate branch Signed-off-by: Partho Sarthi --- .claude/skills/supported-ops-sync/SKILL.md | 237 ------------------ .../supported-ops-sync/agents/openai.yaml | 12 - .../references/pr-checklist.md | 23 -- 3 files changed, 272 deletions(-) delete mode 100644 .claude/skills/supported-ops-sync/SKILL.md delete mode 100644 .claude/skills/supported-ops-sync/agents/openai.yaml delete mode 100644 .claude/skills/supported-ops-sync/references/pr-checklist.md diff --git a/.claude/skills/supported-ops-sync/SKILL.md b/.claude/skills/supported-ops-sync/SKILL.md deleted file mode 100644 index 73df28e36..000000000 --- a/.claude/skills/supported-ops-sync/SKILL.md +++ /dev/null @@ -1,237 +0,0 @@ ---- -name: supported-ops-sync -description: Use when syncing spark-rapids-tools supported operator CSVs from NVIDIA/spark-rapids tools/generated_files, investigating supportedExprs.csv or supportedExecs.csv drift, preparing supported-op sync PRs, or reviewing TNEW operator follow-ups. ---- - - - -# Supported Ops Sync - -## Purpose - -Keep `spark-rapids-tools` aligned with the RAPIDS plugin generated support metadata while preserving tools-side review gates. The plugin source is `NVIDIA/spark-rapids:tools/generated_files`. - -## Default Workflow - -Use these defaults unless the user says otherwise: - -- Tools base branch: `dev` -- Plugin source: `NVIDIA/spark-rapids@main` -- PR target: `NVIDIA/spark-rapids-tools`, base `dev` -- PR mode: create the PR as a draft first -- Branch name: `feature/sync-supported-ops-YYYYMMDD` -- Commit style: signed off with `git commit -s` - -Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev` base branch, or an automatic `TNEW -> S` promotion without test evidence. - -1. Start from the tools base branch: - - ```bash - REPO_ROOT=$(pwd) - PLUGIN_REPO="$(cd "$REPO_ROOT/.." && pwd)/spark-rapids" - SYNC_DATE=$(date +%Y%m%d) - PR_DATE=$(date +%Y-%m-%d) - BRANCH=feature/sync-supported-ops-$SYNC_DATE - - git fetch origin dev - git switch dev - git pull --ff-only origin dev - git switch -c "$BRANCH" - ``` - -2. Look for a sibling RAPIDS plugin checkout first, then update from `main`. Clone into the parent folder only if it is missing: - - ```bash - if [ -d "$PLUGIN_REPO/.git" ]; then - if [ -n "$(git -C "$PLUGIN_REPO" status --porcelain)" ]; then - echo "$PLUGIN_REPO has local changes; ask before updating it." - git -C "$PLUGIN_REPO" status --short - exit 1 - fi - PLUGIN_REMOTE=$(git -C "$PLUGIN_REPO" remote -v | \ - awk '/github.com[:/]NVIDIA\/spark-rapids(.git)?/ && $3 == "\(fetch\)" {print $1; exit}') - PLUGIN_REMOTE=${PLUGIN_REMOTE:-origin} - git -C "$PLUGIN_REPO" fetch "$PLUGIN_REMOTE" main - git -C "$PLUGIN_REPO" switch main - git -C "$PLUGIN_REPO" pull --ff-only "$PLUGIN_REMOTE" main - else - gh repo clone NVIDIA/spark-rapids "$PLUGIN_REPO" -- --depth 1 --branch main - fi - ``` - - Capture the plugin commit for the PR: - - ```bash - git -C "$PLUGIN_REPO" rev-parse HEAD - ``` - -3. Build a direct plugin CSV input directory from the root generated CSVs. The root CSVs are already generated by the plugin; wrapping them in one temp subdirectory lets the existing tools script keep its override, `TNEW`, preservation, and report behavior without re-unioning every Spark-version directory: - - ```bash - RUN_DIR=/tmp/supported-ops-sync-$(date +%Y%m%d-%H%M%S) - PLUGIN_INPUT="$RUN_DIR/plugin-root/000" - mkdir -p "$RUN_DIR/preview" "$PLUGIN_INPUT" - cp "$PLUGIN_REPO/tools/generated_files/supportedDataSource.csv" "$PLUGIN_INPUT/" - cp "$PLUGIN_REPO/tools/generated_files/supportedExecs.csv" "$PLUGIN_INPUT/" - cp "$PLUGIN_REPO/tools/generated_files/supportedExprs.csv" "$PLUGIN_INPUT/" - cp scripts/sync_plugin_files/override_supported_configs.json \ - "$RUN_DIR/override_supported_configs.json" - ``` - -4. Run the existing tools sync utility from the temp run directory. Preview first: - - ```bash - - ( - cd "$RUN_DIR" - python "$REPO_ROOT/scripts/sync_plugin_files/process_supported_files.py" \ - "$RUN_DIR/plugin-root" \ - --configs "$RUN_DIR/override_supported_configs.json" \ - --tools-csv "$REPO_ROOT/core/src/main/resources" \ - --output "$RUN_DIR/preview" - ) - ``` - - Running from `RUN_DIR` keeps `operators_plugin_sync_report.txt` and `new_operators.txt` out of repo root during preview. - -5. Read the preview outputs: - - ```bash - wc -l "$RUN_DIR/operators_plugin_sync_report.txt" "$RUN_DIR/new_operators.txt" - cat "$RUN_DIR/operators_plugin_sync_report.txt" - cat "$RUN_DIR/new_operators.txt" - ``` - -6. Classify changes: - - Pure support/type changes: verify they are plausible from plugin CSVs. - - New execs/expressions: keep as `TNEW` unless tools parsing and tests prove support. - - Removed plugin rows: preserve them unless a reviewer explicitly agrees to delete tools coverage. - - `SQL Func` changes: preserve tools-side aliases when plugin generated CSVs are blank or contain a strict subset of the tools aliases; these aliases are parser lookup keys in tools. - - `TNEW -> S`: require parser coverage or a clear existing parser path. - - Spark 4-only rows: call out separately and keep gated unless this branch has matching Spark 4 support. - -7. Apply only after preview review by rerunning the existing utility with repo outputs: - - ```bash - ( - cd "$RUN_DIR" - python "$REPO_ROOT/scripts/sync_plugin_files/process_supported_files.py" \ - "$RUN_DIR/plugin-root" \ - --configs "$REPO_ROOT/scripts/sync_plugin_files/override_supported_configs.json" \ - --tools-csv "$REPO_ROOT/core/src/main/resources" \ - --output "$REPO_ROOT/core/src/main/resources" - ) - ``` - -8. If `new_operators.txt` is non-empty, run the existing operator score utility: - - ```bash - if [ -s "$RUN_DIR/new_operators.txt" ]; then - python scripts/sync_plugin_files/sync_operator_scores.py \ - "$RUN_DIR/new_operators.txt" \ - core/src/main/resources - fi - ``` - -9. Add targeted tests for promoted expressions or parser behavior changes. Prior examples are in `core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala`, whose suite class is `SQLPlanParserSuite`. - -10. Run targeted validation: - - ```bash - mvn -f core/pom.xml test \ - -DwildcardSuites=com.nvidia.spark.rapids.tool.qualification.PluginTypeCheckerSuite - mvn -f core/pom.xml test \ - -DwildcardSuites=com.nvidia.spark.rapids.tool.planparser.SQLPlanParserSuite - ``` - -11. Review, commit, and create the PR: - - ```bash - git diff -- core/src/main/resources scripts/sync_plugin_files core/src/test - git status --short - git add core/src/main/resources scripts/sync_plugin_files core/src/test - git commit -s -m "Sync supported ops with RAPIDS plugin" - git push -u origin "$BRANCH" - gh pr create --draft --base dev --head "$BRANCH" \ - --title "Sync supported ops with RAPIDS plugin as of $PR_DATE" \ - --body-file /tmp/supported-ops-sync-pr-body.md - ``` - -## Files To Expect - -- Source plugin files: root `tools/generated_files/supported*.csv` -- Tools resource outputs: `core/src/main/resources/supportedDataSource.csv`, `supportedExecs.csv`, `supportedExprs.csv` -- Tools overrides: `scripts/sync_plugin_files/override_supported_configs.json` -- Operator scores: `core/src/main/resources/operatorsScore*.csv` -- Sync report: `operators_plugin_sync_report.txt` in the helper run directory -- New operator list: `new_operators.txt` in the helper run directory - -## Review Rules - -- Do not copy plugin root CSVs directly over tools resources. Stage the root plugin CSVs in a temp input directory and run the tools sync utility so it can apply overrides, mark new rows `TNEW`, and preserve removed rows. -- Do not accept `SQL Func` alias loss from the plugin root CSV without review. The tools parser uses those aliases to recognize expressions whose physical plans print SQL function names instead of expression class names. -- Do not blindly convert `TNEW` to `S`. `TNEW` means plugin metadata knows about the operator, but tools have not validated parser behavior. -- When new operators are accepted into tools, update operator score CSVs using `sync_operator_scores.py` or the helper's default score-sync path. -- If Spark 4-only operators appear, keep them gated as `TNEW` unless the tools branch has the necessary Spark 4 parser/test support. - -## PR Pattern - -Draft `/tmp/supported-ops-sync-pr-body.md` with this shape: - -```markdown -Contributes to #[issue-if-known] - -### Summary - -Syncs supported operator metadata in spark-rapids-tools with RAPIDS plugin generated files. - -Plugin source: NVIDIA/spark-rapids@ - -### Changes - -- Updated supported CSVs: - - core/src/main/resources/supportedDataSource.csv - - core/src/main/resources/supportedExecs.csv - - core/src/main/resources/supportedExprs.csv -- Updated operator score CSVs for new operators, if any. -- Updated scripts/sync_plugin_files/override_supported_configs.json for new TNEW rows, if any. - -### Sync Report Summary - - - -### New Operators / TNEW Follow-Ups - - - -### Tests - -- [ ] mvn test -DwildcardSuites=com.nvidia.spark.rapids.tool.qualification.PluginTypeCheckerSuite -- [ ] mvn test -DwildcardSuites=com.nvidia.spark.rapids.tool.planparser.SQLPlanParserSuite - -### Follow-Ups - - -``` - -Always include: - -- Plugin ref used, usually `NVIDIA/spark-rapids@main` or a specific commit SHA. -- Summary of supported CSV changes from `operators_plugin_sync_report.txt`. -- New operators and their current `TNEW`/tested status. -- Tests added or explicitly not added, with rationale. -- Follow-ups for execs/expressions that remain `TNEW`. - -For historical examples, compare PRs: - -- `NVIDIA/spark-rapids-tools#1478` -- `NVIDIA/spark-rapids-tools#1618` -- `NVIDIA/spark-rapids-tools#2015` diff --git a/.claude/skills/supported-ops-sync/agents/openai.yaml b/.claude/skills/supported-ops-sync/agents/openai.yaml deleted file mode 100644 index 510086cd1..000000000 --- a/.claude/skills/supported-ops-sync/agents/openai.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 - -interface: - display_name: "Supported Ops Sync" - short_description: "Sync Spark RAPIDS supported-op CSVs" - default_prompt: "Use $supported-ops-sync to update the tools supported operator CSVs from NVIDIA/spark-rapids generated files." diff --git a/.claude/skills/supported-ops-sync/references/pr-checklist.md b/.claude/skills/supported-ops-sync/references/pr-checklist.md deleted file mode 100644 index deddbc3ac..000000000 --- a/.claude/skills/supported-ops-sync/references/pr-checklist.md +++ /dev/null @@ -1,23 +0,0 @@ -# Supported Ops Sync PR Checklist - - - -- Mention the plugin ref used for `tools/generated_files`. -- Confirm the PR branch starts from `dev` unless the user requested another base. -- Include the generated report summary for `supportedDataSource.csv`, `supportedExecs.csv`, and `supportedExprs.csv`. -- List new execs and expressions from `new_operators.txt`. -- Explain which new rows remain `TNEW` and why. -- Explain every `TNEW -> S`, `NS -> S`, or `CO -> S` promotion. -- Confirm operator score CSVs were updated when new operators were added. -- Confirm targeted tests run, or state why a test is not applicable. -- Call out Spark 4-only rows separately. -- Commit with `git commit -s`. -- Open a draft PR against `dev` with a title like `Sync supported ops with RAPIDS plugin as of YYYY-MM-DD`. From 546e24b7ce636b17da37f239a67a9ccb4008addf Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 29 May 2026 12:41:05 -0700 Subject: [PATCH 6/7] Remove supported ops sync scripts from data sync Signed-off-by: Partho Sarthi --- .../override_supported_configs.json | 143 ------------------ .../process_supported_files.py | 43 +----- 2 files changed, 3 insertions(+), 183 deletions(-) diff --git a/scripts/sync_plugin_files/override_supported_configs.json b/scripts/sync_plugin_files/override_supported_configs.json index c876cd2c1..70cce52c5 100644 --- a/scripts/sync_plugin_files/override_supported_configs.json +++ b/scripts/sync_plugin_files/override_supported_configs.json @@ -307,149 +307,6 @@ "value": "TNEW" } ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "zero", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "finish", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "merge", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "argument", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Hex", - "Context": "project", - "Params": "input", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Hex", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Sha2", - "Context": "project", - "Params": "input", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Sha2", - "Context": "project", - "Params": "bitLength", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Sha2", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "StringDecode", - "Context": "project", - "Params": "bin", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "StringDecode", - "Context": "project", - "Params": "charset", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "StringDecode", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] } ], "supportedDataSource.csv": [ diff --git a/scripts/sync_plugin_files/process_supported_files.py b/scripts/sync_plugin_files/process_supported_files.py index a0f5875b0..9eecf8ac2 100644 --- a/scripts/sync_plugin_files/process_supported_files.py +++ b/scripts/sync_plugin_files/process_supported_files.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2026, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -83,34 +83,6 @@ def is_greater(elem1, elem2): return len(elem1) > len(elem2) -def get_sql_func_aliases(sql_func_value): - """ - Return SQL function aliases from a semicolon-separated SQL Func cell. - """ - sql_func_str = str(sql_func_value).strip() - if sql_func_str == "" or sql_func_str == "None": - return set() - return {alias.strip() for alias in sql_func_str.split(";") if alias.strip()} - - -def should_preserve_tools_string(column_name, tools_value, plugin_value): - """ - Preserve tools-side expression aliases when plugin generated metadata is less informative. - SQL function aliases are parser lookup keys in tools, and the generated plugin CSV can omit - aliases for newer Spark versions even though older-version aliases should still be recognized. - Preserve only when the plugin aliases are blank or a strict subset of the tools aliases, so an - intentional plugin alias rename is accepted instead of being hidden by a string-length heuristic. - """ - if column_name != "SQL Func": - return False - - tools_aliases = get_sql_func_aliases(tools_value) - plugin_aliases = get_sql_func_aliases(plugin_value) - if not tools_aliases: - return False - return not plugin_aliases or plugin_aliases < tools_aliases - - def check_df_rows(row1, row2, keys): """ Given two DataFrame rows (pandas Series) and a list of keys (column names), @@ -276,17 +248,8 @@ def compare_csv_file(union_df, tools_df, keys, report_file, override_configs_jso exists_in_tools = True for tools_column in tools_df.columns: if (tools_column in union_row) and (not tools_row[tools_column] == union_row[tools_column]): - if should_preserve_tools_string( - tools_column, tools_row[tools_column], union_row[tools_column]): - report_file.write( - f"Row keeps tools value: {', '.join(tools_row.astype(str))}\n " + - f"{tools_column}: preserving {tools_row[tools_column]} instead of " + - f"{union_row[tools_column]}\n") - union_df.at[union_idx, tools_column] = tools_row[tools_column] - else: - report_file.write(f"Row is changed: {', '.join(tools_row.astype(str))}\n " + - f"{tools_column}: {tools_row[tools_column]} -> " + - f"{union_row[tools_column]}\n") + report_file.write(f"Row is changed: {', '.join(tools_row.astype(str))}\n " + + f"{tools_column}: {tools_row[tools_column]} -> {union_row[tools_column]}\n") if not exists_in_tools: for column_name in union_df.columns: if is_support_level(union_row[column_name]): From 386d23bafccdf7a9f8185c37f2a99b2b2b893356 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 29 May 2026 13:34:22 -0700 Subject: [PATCH 7/7] Restore supported ops TNEW overrides to sync Signed-off-by: Partho Sarthi --- .../override_supported_configs.json | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/scripts/sync_plugin_files/override_supported_configs.json b/scripts/sync_plugin_files/override_supported_configs.json index 70cce52c5..c876cd2c1 100644 --- a/scripts/sync_plugin_files/override_supported_configs.json +++ b/scripts/sync_plugin_files/override_supported_configs.json @@ -307,6 +307,149 @@ "value": "TNEW" } ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "zero", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "finish", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "merge", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "argument", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Hex", + "Context": "project", + "Params": "input", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Hex", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "input", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "bitLength", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "bin", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "charset", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] } ], "supportedDataSource.csv": [