From 9a5d998b2d157c971a70286daa6c3a5be00042f4 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 29 May 2026 12:37:13 -0700 Subject: [PATCH 1/5] Add supported ops sync skill workflow Signed-off-by: Partho Sarthi --- .claude/skills/supported-ops-sync/SKILL.md | 237 ++++++++++++++++++ .../supported-ops-sync/agents/openai.yaml | 12 + .../references/pr-checklist.md | 23 ++ 3 files changed, 272 insertions(+) create mode 100644 .claude/skills/supported-ops-sync/SKILL.md create mode 100644 .claude/skills/supported-ops-sync/agents/openai.yaml create mode 100644 .claude/skills/supported-ops-sync/references/pr-checklist.md diff --git a/.claude/skills/supported-ops-sync/SKILL.md b/.claude/skills/supported-ops-sync/SKILL.md new file mode 100644 index 000000000..73df28e36 --- /dev/null +++ b/.claude/skills/supported-ops-sync/SKILL.md @@ -0,0 +1,237 @@ +--- +name: supported-ops-sync +description: Use when syncing spark-rapids-tools supported operator CSVs from NVIDIA/spark-rapids tools/generated_files, investigating supportedExprs.csv or supportedExecs.csv drift, preparing supported-op sync PRs, or reviewing TNEW operator follow-ups. +--- + + + +# Supported Ops Sync + +## Purpose + +Keep `spark-rapids-tools` aligned with the RAPIDS plugin generated support metadata while preserving tools-side review gates. The plugin source is `NVIDIA/spark-rapids:tools/generated_files`. + +## Default Workflow + +Use these defaults unless the user says otherwise: + +- Tools base branch: `dev` +- Plugin source: `NVIDIA/spark-rapids@main` +- PR target: `NVIDIA/spark-rapids-tools`, base `dev` +- PR mode: create the PR as a draft first +- Branch name: `feature/sync-supported-ops-YYYYMMDD` +- Commit style: signed off with `git commit -s` + +Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev` base branch, or an automatic `TNEW -> S` promotion without test evidence. + +1. Start from the tools base branch: + + ```bash + REPO_ROOT=$(pwd) + PLUGIN_REPO="$(cd "$REPO_ROOT/.." && pwd)/spark-rapids" + SYNC_DATE=$(date +%Y%m%d) + PR_DATE=$(date +%Y-%m-%d) + BRANCH=feature/sync-supported-ops-$SYNC_DATE + + git fetch origin dev + git switch dev + git pull --ff-only origin dev + git switch -c "$BRANCH" + ``` + +2. Look for a sibling RAPIDS plugin checkout first, then update from `main`. Clone into the parent folder only if it is missing: + + ```bash + if [ -d "$PLUGIN_REPO/.git" ]; then + if [ -n "$(git -C "$PLUGIN_REPO" status --porcelain)" ]; then + echo "$PLUGIN_REPO has local changes; ask before updating it." + git -C "$PLUGIN_REPO" status --short + exit 1 + fi + PLUGIN_REMOTE=$(git -C "$PLUGIN_REPO" remote -v | \ + awk '/github.com[:/]NVIDIA\/spark-rapids(.git)?/ && $3 == "\(fetch\)" {print $1; exit}') + PLUGIN_REMOTE=${PLUGIN_REMOTE:-origin} + git -C "$PLUGIN_REPO" fetch "$PLUGIN_REMOTE" main + git -C "$PLUGIN_REPO" switch main + git -C "$PLUGIN_REPO" pull --ff-only "$PLUGIN_REMOTE" main + else + gh repo clone NVIDIA/spark-rapids "$PLUGIN_REPO" -- --depth 1 --branch main + fi + ``` + + Capture the plugin commit for the PR: + + ```bash + git -C "$PLUGIN_REPO" rev-parse HEAD + ``` + +3. Build a direct plugin CSV input directory from the root generated CSVs. The root CSVs are already generated by the plugin; wrapping them in one temp subdirectory lets the existing tools script keep its override, `TNEW`, preservation, and report behavior without re-unioning every Spark-version directory: + + ```bash + RUN_DIR=/tmp/supported-ops-sync-$(date +%Y%m%d-%H%M%S) + PLUGIN_INPUT="$RUN_DIR/plugin-root/000" + mkdir -p "$RUN_DIR/preview" "$PLUGIN_INPUT" + cp "$PLUGIN_REPO/tools/generated_files/supportedDataSource.csv" "$PLUGIN_INPUT/" + cp "$PLUGIN_REPO/tools/generated_files/supportedExecs.csv" "$PLUGIN_INPUT/" + cp "$PLUGIN_REPO/tools/generated_files/supportedExprs.csv" "$PLUGIN_INPUT/" + cp scripts/sync_plugin_files/override_supported_configs.json \ + "$RUN_DIR/override_supported_configs.json" + ``` + +4. Run the existing tools sync utility from the temp run directory. Preview first: + + ```bash + + ( + cd "$RUN_DIR" + python "$REPO_ROOT/scripts/sync_plugin_files/process_supported_files.py" \ + "$RUN_DIR/plugin-root" \ + --configs "$RUN_DIR/override_supported_configs.json" \ + --tools-csv "$REPO_ROOT/core/src/main/resources" \ + --output "$RUN_DIR/preview" + ) + ``` + + Running from `RUN_DIR` keeps `operators_plugin_sync_report.txt` and `new_operators.txt` out of repo root during preview. + +5. Read the preview outputs: + + ```bash + wc -l "$RUN_DIR/operators_plugin_sync_report.txt" "$RUN_DIR/new_operators.txt" + cat "$RUN_DIR/operators_plugin_sync_report.txt" + cat "$RUN_DIR/new_operators.txt" + ``` + +6. Classify changes: + - Pure support/type changes: verify they are plausible from plugin CSVs. + - New execs/expressions: keep as `TNEW` unless tools parsing and tests prove support. + - Removed plugin rows: preserve them unless a reviewer explicitly agrees to delete tools coverage. + - `SQL Func` changes: preserve tools-side aliases when plugin generated CSVs are blank or contain a strict subset of the tools aliases; these aliases are parser lookup keys in tools. + - `TNEW -> S`: require parser coverage or a clear existing parser path. + - Spark 4-only rows: call out separately and keep gated unless this branch has matching Spark 4 support. + +7. Apply only after preview review by rerunning the existing utility with repo outputs: + + ```bash + ( + cd "$RUN_DIR" + python "$REPO_ROOT/scripts/sync_plugin_files/process_supported_files.py" \ + "$RUN_DIR/plugin-root" \ + --configs "$REPO_ROOT/scripts/sync_plugin_files/override_supported_configs.json" \ + --tools-csv "$REPO_ROOT/core/src/main/resources" \ + --output "$REPO_ROOT/core/src/main/resources" + ) + ``` + +8. If `new_operators.txt` is non-empty, run the existing operator score utility: + + ```bash + if [ -s "$RUN_DIR/new_operators.txt" ]; then + python scripts/sync_plugin_files/sync_operator_scores.py \ + "$RUN_DIR/new_operators.txt" \ + core/src/main/resources + fi + ``` + +9. Add targeted tests for promoted expressions or parser behavior changes. Prior examples are in `core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala`, whose suite class is `SQLPlanParserSuite`. + +10. Run targeted validation: + + ```bash + mvn -f core/pom.xml test \ + -DwildcardSuites=com.nvidia.spark.rapids.tool.qualification.PluginTypeCheckerSuite + mvn -f core/pom.xml test \ + -DwildcardSuites=com.nvidia.spark.rapids.tool.planparser.SQLPlanParserSuite + ``` + +11. Review, commit, and create the PR: + + ```bash + git diff -- core/src/main/resources scripts/sync_plugin_files core/src/test + git status --short + git add core/src/main/resources scripts/sync_plugin_files core/src/test + git commit -s -m "Sync supported ops with RAPIDS plugin" + git push -u origin "$BRANCH" + gh pr create --draft --base dev --head "$BRANCH" \ + --title "Sync supported ops with RAPIDS plugin as of $PR_DATE" \ + --body-file /tmp/supported-ops-sync-pr-body.md + ``` + +## Files To Expect + +- Source plugin files: root `tools/generated_files/supported*.csv` +- Tools resource outputs: `core/src/main/resources/supportedDataSource.csv`, `supportedExecs.csv`, `supportedExprs.csv` +- Tools overrides: `scripts/sync_plugin_files/override_supported_configs.json` +- Operator scores: `core/src/main/resources/operatorsScore*.csv` +- Sync report: `operators_plugin_sync_report.txt` in the helper run directory +- New operator list: `new_operators.txt` in the helper run directory + +## Review Rules + +- Do not copy plugin root CSVs directly over tools resources. Stage the root plugin CSVs in a temp input directory and run the tools sync utility so it can apply overrides, mark new rows `TNEW`, and preserve removed rows. +- Do not accept `SQL Func` alias loss from the plugin root CSV without review. The tools parser uses those aliases to recognize expressions whose physical plans print SQL function names instead of expression class names. +- Do not blindly convert `TNEW` to `S`. `TNEW` means plugin metadata knows about the operator, but tools have not validated parser behavior. +- When new operators are accepted into tools, update operator score CSVs using `sync_operator_scores.py` or the helper's default score-sync path. +- If Spark 4-only operators appear, keep them gated as `TNEW` unless the tools branch has the necessary Spark 4 parser/test support. + +## PR Pattern + +Draft `/tmp/supported-ops-sync-pr-body.md` with this shape: + +```markdown +Contributes to #[issue-if-known] + +### Summary + +Syncs supported operator metadata in spark-rapids-tools with RAPIDS plugin generated files. + +Plugin source: NVIDIA/spark-rapids@ + +### Changes + +- Updated supported CSVs: + - core/src/main/resources/supportedDataSource.csv + - core/src/main/resources/supportedExecs.csv + - core/src/main/resources/supportedExprs.csv +- Updated operator score CSVs for new operators, if any. +- Updated scripts/sync_plugin_files/override_supported_configs.json for new TNEW rows, if any. + +### Sync Report Summary + + + +### New Operators / TNEW Follow-Ups + + + +### Tests + +- [ ] mvn test -DwildcardSuites=com.nvidia.spark.rapids.tool.qualification.PluginTypeCheckerSuite +- [ ] mvn test -DwildcardSuites=com.nvidia.spark.rapids.tool.planparser.SQLPlanParserSuite + +### Follow-Ups + + +``` + +Always include: + +- Plugin ref used, usually `NVIDIA/spark-rapids@main` or a specific commit SHA. +- Summary of supported CSV changes from `operators_plugin_sync_report.txt`. +- New operators and their current `TNEW`/tested status. +- Tests added or explicitly not added, with rationale. +- Follow-ups for execs/expressions that remain `TNEW`. + +For historical examples, compare PRs: + +- `NVIDIA/spark-rapids-tools#1478` +- `NVIDIA/spark-rapids-tools#1618` +- `NVIDIA/spark-rapids-tools#2015` diff --git a/.claude/skills/supported-ops-sync/agents/openai.yaml b/.claude/skills/supported-ops-sync/agents/openai.yaml new file mode 100644 index 000000000..510086cd1 --- /dev/null +++ b/.claude/skills/supported-ops-sync/agents/openai.yaml @@ -0,0 +1,12 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +interface: + display_name: "Supported Ops Sync" + short_description: "Sync Spark RAPIDS supported-op CSVs" + default_prompt: "Use $supported-ops-sync to update the tools supported operator CSVs from NVIDIA/spark-rapids generated files." diff --git a/.claude/skills/supported-ops-sync/references/pr-checklist.md b/.claude/skills/supported-ops-sync/references/pr-checklist.md new file mode 100644 index 000000000..deddbc3ac --- /dev/null +++ b/.claude/skills/supported-ops-sync/references/pr-checklist.md @@ -0,0 +1,23 @@ +# Supported Ops Sync PR Checklist + + + +- Mention the plugin ref used for `tools/generated_files`. +- Confirm the PR branch starts from `dev` unless the user requested another base. +- Include the generated report summary for `supportedDataSource.csv`, `supportedExecs.csv`, and `supportedExprs.csv`. +- List new execs and expressions from `new_operators.txt`. +- Explain which new rows remain `TNEW` and why. +- Explain every `TNEW -> S`, `NS -> S`, or `CO -> S` promotion. +- Confirm operator score CSVs were updated when new operators were added. +- Confirm targeted tests run, or state why a test is not applicable. +- Call out Spark 4-only rows separately. +- Commit with `git commit -s`. +- Open a draft PR against `dev` with a title like `Sync supported ops with RAPIDS plugin as of YYYY-MM-DD`. From b7a1bc6204ce8fdfcff7674c0aa24c928a1f09b7 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 29 May 2026 12:41:05 -0700 Subject: [PATCH 2/5] Move supported ops sync scripts to skill branch Signed-off-by: Partho Sarthi --- .../override_supported_configs.json | 143 ++++++++++++++++++ .../process_supported_files.py | 43 +++++- 2 files changed, 183 insertions(+), 3 deletions(-) diff --git a/scripts/sync_plugin_files/override_supported_configs.json b/scripts/sync_plugin_files/override_supported_configs.json index 70cce52c5..c876cd2c1 100644 --- a/scripts/sync_plugin_files/override_supported_configs.json +++ b/scripts/sync_plugin_files/override_supported_configs.json @@ -307,6 +307,149 @@ "value": "TNEW" } ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "zero", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "finish", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "merge", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "ArrayAggregate", + "Context": "project", + "Params": "argument", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Hex", + "Context": "project", + "Params": "input", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Hex", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "input", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "bitLength", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "Sha2", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "bin", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "charset", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] + }, + { + "Expression": "StringDecode", + "Context": "project", + "Params": "result", + "override": [ + { + "key": "Supported", + "value": "TNEW" + } + ] } ], "supportedDataSource.csv": [ diff --git a/scripts/sync_plugin_files/process_supported_files.py b/scripts/sync_plugin_files/process_supported_files.py index 9eecf8ac2..a0f5875b0 100644 --- a/scripts/sync_plugin_files/process_supported_files.py +++ b/scripts/sync_plugin_files/process_supported_files.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -83,6 +83,34 @@ def is_greater(elem1, elem2): return len(elem1) > len(elem2) +def get_sql_func_aliases(sql_func_value): + """ + Return SQL function aliases from a semicolon-separated SQL Func cell. + """ + sql_func_str = str(sql_func_value).strip() + if sql_func_str == "" or sql_func_str == "None": + return set() + return {alias.strip() for alias in sql_func_str.split(";") if alias.strip()} + + +def should_preserve_tools_string(column_name, tools_value, plugin_value): + """ + Preserve tools-side expression aliases when plugin generated metadata is less informative. + SQL function aliases are parser lookup keys in tools, and the generated plugin CSV can omit + aliases for newer Spark versions even though older-version aliases should still be recognized. + Preserve only when the plugin aliases are blank or a strict subset of the tools aliases, so an + intentional plugin alias rename is accepted instead of being hidden by a string-length heuristic. + """ + if column_name != "SQL Func": + return False + + tools_aliases = get_sql_func_aliases(tools_value) + plugin_aliases = get_sql_func_aliases(plugin_value) + if not tools_aliases: + return False + return not plugin_aliases or plugin_aliases < tools_aliases + + def check_df_rows(row1, row2, keys): """ Given two DataFrame rows (pandas Series) and a list of keys (column names), @@ -248,8 +276,17 @@ def compare_csv_file(union_df, tools_df, keys, report_file, override_configs_jso exists_in_tools = True for tools_column in tools_df.columns: if (tools_column in union_row) and (not tools_row[tools_column] == union_row[tools_column]): - report_file.write(f"Row is changed: {', '.join(tools_row.astype(str))}\n " + - f"{tools_column}: {tools_row[tools_column]} -> {union_row[tools_column]}\n") + if should_preserve_tools_string( + tools_column, tools_row[tools_column], union_row[tools_column]): + report_file.write( + f"Row keeps tools value: {', '.join(tools_row.astype(str))}\n " + + f"{tools_column}: preserving {tools_row[tools_column]} instead of " + + f"{union_row[tools_column]}\n") + union_df.at[union_idx, tools_column] = tools_row[tools_column] + else: + report_file.write(f"Row is changed: {', '.join(tools_row.astype(str))}\n " + + f"{tools_column}: {tools_row[tools_column]} -> " + + f"{union_row[tools_column]}\n") if not exists_in_tools: for column_name in union_df.columns: if is_support_level(union_row[column_name]): From b9c5d5e78260c7eeabf1f3d23549e03cc8749f2e Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 29 May 2026 12:49:10 -0700 Subject: [PATCH 3/5] Harden SQL function alias parsing Signed-off-by: Partho Sarthi --- scripts/sync_plugin_files/process_supported_files.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/sync_plugin_files/process_supported_files.py b/scripts/sync_plugin_files/process_supported_files.py index a0f5875b0..533d51419 100644 --- a/scripts/sync_plugin_files/process_supported_files.py +++ b/scripts/sync_plugin_files/process_supported_files.py @@ -87,10 +87,15 @@ def get_sql_func_aliases(sql_func_value): """ Return SQL function aliases from a semicolon-separated SQL Func cell. """ + if pd.isna(sql_func_value): + return set() sql_func_str = str(sql_func_value).strip() - if sql_func_str == "" or sql_func_str == "None": + if sql_func_str == "" or sql_func_str.lower() in ("none", "nan"): return set() - return {alias.strip() for alias in sql_func_str.split(";") if alias.strip()} + return { + alias.strip() for alias in sql_func_str.split(";") + if alias.strip() and alias.strip() not in SupportLevel.__members__ + } def should_preserve_tools_string(column_name, tools_value, plugin_value): From 993c82918e13acf89fe76bdacca9ace7a4d200a5 Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 29 May 2026 13:04:59 -0700 Subject: [PATCH 4/5] Document supported ops sync validation boundaries Signed-off-by: Partho Sarthi --- .claude/skills/supported-ops-sync/SKILL.md | 17 +++++++++++++++++ .../references/pr-checklist.md | 2 ++ 2 files changed, 19 insertions(+) diff --git a/.claude/skills/supported-ops-sync/SKILL.md b/.claude/skills/supported-ops-sync/SKILL.md index 73df28e36..cd40d882c 100644 --- a/.claude/skills/supported-ops-sync/SKILL.md +++ b/.claude/skills/supported-ops-sync/SKILL.md @@ -19,6 +19,17 @@ You may obtain a copy of the License at Keep `spark-rapids-tools` aligned with the RAPIDS plugin generated support metadata while preserving tools-side review gates. The plugin source is `NVIDIA/spark-rapids:tools/generated_files`. +## Automation Boundary + +The CSV sync mechanics are deterministic and should be automated: stage plugin generated CSVs, run the tools sync helper, preserve tools-only aliases, mark new rows as `TNEW`, update score files, and produce a review report. + +Do not treat plugin generated CSVs as the complete source of truth for support promotion. They depend on the plugin change correctly updating generated metadata, and some support depends on plugin code paths, Spark versions, datasource formats, Spark configs, and parser visibility in event logs. + +- Expressions are usually easier to validate from parser output, but still need event-log/parser evidence before promotion. +- Execs and datasources often need manual compatibility review. Iceberg, Delta, merge/write execs, datasource V2 paths, and phased support changes are common high-risk cases. +- If support depends on specific configs, formats, datatypes, or Spark versions, keep rows gated unless the PR includes evidence and targeted tests for those cases. +- If the physical plan or node description does not expose enough information, add or verify parser extraction before promoting support. + ## Default Workflow Use these defaults unless the user says otherwise: @@ -113,9 +124,11 @@ Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev 6. Classify changes: - Pure support/type changes: verify they are plausible from plugin CSVs. - New execs/expressions: keep as `TNEW` unless tools parsing and tests prove support. + - New datasource or exec support: inspect the plugin code path when support depends on format, config, Spark version, or datasource implementation details. - Removed plugin rows: preserve them unless a reviewer explicitly agrees to delete tools coverage. - `SQL Func` changes: preserve tools-side aliases when plugin generated CSVs are blank or contain a strict subset of the tools aliases; these aliases are parser lookup keys in tools. - `TNEW -> S`: require parser coverage or a clear existing parser path. + - `NS -> S`, `CO -> S`, or datasource write/read promotions: require evidence from plugin code and event-log/parser behavior, not only the generated CSV row. - Spark 4-only rows: call out separately and keep gated unless this branch has matching Spark 4 support. 7. Apply only after preview review by rerunning the existing utility with repo outputs: @@ -143,6 +156,8 @@ Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev 9. Add targeted tests for promoted expressions or parser behavior changes. Prior examples are in `core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala`, whose suite class is `SQLPlanParserSuite`. + For exec and datasource promotions, include or cite event logs that exercise the relevant physical plan path and confirm the qualification/profiling output marks the operator as expected. Do not promote based only on generated CSV drift when plugin support is phased, config-gated, or format-specific. + 10. Run targeted validation: ```bash @@ -179,6 +194,7 @@ Ask before continuing only if the user wants a non-`main` plugin ref, a non-`dev - Do not copy plugin root CSVs directly over tools resources. Stage the root plugin CSVs in a temp input directory and run the tools sync utility so it can apply overrides, mark new rows `TNEW`, and preserve removed rows. - Do not accept `SQL Func` alias loss from the plugin root CSV without review. The tools parser uses those aliases to recognize expressions whose physical plans print SQL function names instead of expression class names. - Do not blindly convert `TNEW` to `S`. `TNEW` means plugin metadata knows about the operator, but tools have not validated parser behavior. +- Do not promote exec or datasource support from generated CSVs alone. Check plugin compatibility logic, required Spark configs, supported formats, datatypes, Spark versions, and event-log/parser coverage. - When new operators are accepted into tools, update operator score CSVs using `sync_operator_scores.py` or the helper's default score-sync path. - If Spark 4-only operators appear, keep them gated as `TNEW` unless the tools branch has the necessary Spark 4 parser/test support. @@ -228,6 +244,7 @@ Always include: - Summary of supported CSV changes from `operators_plugin_sync_report.txt`. - New operators and their current `TNEW`/tested status. - Tests added or explicitly not added, with rationale. +- Evidence for any exec or datasource support promotion, including plugin code path and event-log/parser validation when applicable. - Follow-ups for execs/expressions that remain `TNEW`. For historical examples, compare PRs: diff --git a/.claude/skills/supported-ops-sync/references/pr-checklist.md b/.claude/skills/supported-ops-sync/references/pr-checklist.md index deddbc3ac..25f4107e6 100644 --- a/.claude/skills/supported-ops-sync/references/pr-checklist.md +++ b/.claude/skills/supported-ops-sync/references/pr-checklist.md @@ -16,6 +16,8 @@ You may obtain a copy of the License at - List new execs and expressions from `new_operators.txt`. - Explain which new rows remain `TNEW` and why. - Explain every `TNEW -> S`, `NS -> S`, or `CO -> S` promotion. +- For exec or datasource promotions, cite plugin code-path review and event-log/parser evidence. +- Call out config-gated, format-specific, datatype-specific, or Spark-version-specific support. - Confirm operator score CSVs were updated when new operators were added. - Confirm targeted tests run, or state why a test is not applicable. - Call out Spark 4-only rows separately. From 8611943ee6fd9b24ccd339f1c1828251479e6f6e Mon Sep 17 00:00:00 2001 From: Partho Sarthi Date: Fri, 29 May 2026 13:34:22 -0700 Subject: [PATCH 5/5] Drop sync override config from skill PR Signed-off-by: Partho Sarthi --- .../override_supported_configs.json | 143 ------------------ 1 file changed, 143 deletions(-) diff --git a/scripts/sync_plugin_files/override_supported_configs.json b/scripts/sync_plugin_files/override_supported_configs.json index c876cd2c1..70cce52c5 100644 --- a/scripts/sync_plugin_files/override_supported_configs.json +++ b/scripts/sync_plugin_files/override_supported_configs.json @@ -307,149 +307,6 @@ "value": "TNEW" } ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "zero", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "finish", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "merge", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "ArrayAggregate", - "Context": "project", - "Params": "argument", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Hex", - "Context": "project", - "Params": "input", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Hex", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Sha2", - "Context": "project", - "Params": "input", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Sha2", - "Context": "project", - "Params": "bitLength", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "Sha2", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "StringDecode", - "Context": "project", - "Params": "bin", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "StringDecode", - "Context": "project", - "Params": "charset", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] - }, - { - "Expression": "StringDecode", - "Context": "project", - "Params": "result", - "override": [ - { - "key": "Supported", - "value": "TNEW" - } - ] } ], "supportedDataSource.csv": [