[202412] Backport GCU container framework (#4310) + perf enhancements (#4476, #4478, #4554)

azure-pipelines.yml

@@ -175,3 +175,139 @@ stages:
     - publish: '$(System.DefaultWorkingDirectory)/dist/'
       artifact: wheels
       displayName: "Publish Python wheels"
+
+    - script: |
+        set -e
+        pushd generic_config_updater
+        python3 -m build -n
+      displayName: 'Build Python 3 wheel for GCU'
+
+    - publish: '$(System.DefaultWorkingDirectory)/generic_config_updater/dist/'
+      artifact: gcu_wheels
+      displayName: "Publish Python wheels for GCU"
+
+- stage: BuildData
+
+  jobs:
+  - job:
+    displayName: "Build sonic-utilities-data"
+    pool:
+      vmImage: ubuntu-24.04
+
+    container:
+      image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bookworm:$(BUILD_BRANCH)
+
+    steps:
+    - script: |
+        set -ex
+        sudo apt-get update
+        sudo apt-get install -y python3-pip
+        sudo apt-get install -y python3-protobuf
+      displayName: "Install dependencies"
+
+    - script: |
+        sourceBranch=$(Build.SourceBranchName)
+        if [[ "$(Build.Reason)" == "PullRequest" ]];then
+          sourceBranch=$(System.PullRequest.TargetBranch)
+        fi
+        echo "Download artifact branch: $sourceBranch"
+        echo "##vso[task.setvariable variable=sourceBranch]$sourceBranch"
+      displayName: "Get correct artifact downloading branch"
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        source: specific
+        project: build
+        pipeline: 142
+        artifact: sonic-buildimage.vs
+        runVersion: 'latestFromBranch'
+        runBranch: 'refs/heads/$(sourceBranch)'
+        patterns: |
+          **/*.deb
+          **/*.whl
+      displayName: "Download artifacts from latest sonic-buildimage build"
+
+    - script: |
+        set -xe
+        sudo apt-get -y purge libnl-3-dev libnl-route-3-dev || true
+        sudo dpkg -i libnl-3-200_*.deb
+        sudo dpkg -i libnl-genl-3-200_*.deb
+        sudo dpkg -i libnl-route-3-200_*.deb
+        sudo dpkg -i libnl-nf-3-200_*.deb
+        sudo dpkg -i libyang_1.0.73_amd64.deb
+        sudo dpkg -i libyang-cpp_1.0.73_amd64.deb
+        sudo dpkg -i python3-yang_1.0.73_amd64.deb
+      workingDirectory: $(Pipeline.Workspace)/target/debs/bookworm/
+      displayName: 'Install Debian dependencies'
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        source: specific
+        project: build
+        pipeline: 9
+        artifact: sonic-swss-common-bookworm
+        runVersion: 'latestFromBranch'
+        runBranch: 'refs/heads/$(sourceBranch)'
+      displayName: "Download sonic swss common deb packages"
+
+    - script: |
+        set -xe
+        sudo dpkg -i libswsscommon_1.0.0_amd64.deb
+        sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb
+      workingDirectory: $(Pipeline.Workspace)/
+      displayName: 'Install swss-common dependencies'
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        source: specific
+        project: build
+        pipeline: sonic-net.sonic-dash-api
+        artifact: sonic-dash-api
+        runVersion: 'latestFromBranch'
+        runBranch: 'refs/heads/$(BUILD_BRANCH)'
+        path: $(Build.ArtifactStagingDirectory)/download
+        patterns: |
+          libdashapi*.deb
+      displayName: "Download dash api"
+
+    - script: |
+        set -xe
+        sudo apt-get update
+        sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/libdashapi_*.deb
+      workingDirectory: $(Pipeline.Workspace)/
+      displayName: 'Install libdashapi libraries'
+
+    - script: |
+        set -xe
+        sudo pip3 install swsssdk-2.0.1-py3-none-any.whl
+        sudo pip3 install sonic_py_common-1.0-py3-none-any.whl
+        sudo pip3 install sonic_yang_mgmt-1.0-py3-none-any.whl
+        sudo pip3 install sonic_yang_models-1.0-py3-none-any.whl
+        sudo pip3 install sonic_config_engine-1.0-py3-none-any.whl
+        sudo pip3 install sonic_platform_common-1.0-py3-none-any.whl
+      workingDirectory: $(Pipeline.Workspace)/target/python-wheels/bookworm/
+      displayName: 'Install Python dependencies'
+
+    - task: DownloadPipelineArtifact@2
+      inputs:
+        artifact: wheels
+        path: $(Build.ArtifactStagingDirectory)/download
+      displayName: "Download pre-stage built sonic-utilities"
+
+    - script: |
+        sudo pip3 install sonic_utilities-1.2-py3-none-any.whl
+      workingDirectory: $(Build.ArtifactStagingDirectory)/download
+      displayName: 'Install sonic-utilities wheel'
+
+    - script: |
+        set -xe
+        cd sonic-utilities-data
+        dpkg-buildpackage -b
+        cd ..
+        mkdir -p dist
+        mv sonic-utilities-data_*.deb dist/
+      displayName: 'Build sonic-utilities-data package'
+
+    - publish: '$(System.DefaultWorkingDirectory)/dist/'
+      artifact: sonic_utilities_data
+      displayName: "Publish sonic-utilities-data package"

config/main.py

@@ -1,12 +1,9 @@
 #!/usr/sbin/env python
 
-import threading
 import click
-import concurrent.futures
 import datetime
 import ipaddress
 import json
-import jsonpatch
 import netaddr
 import netifaces
 import os
@@ -22,16 +19,18 @@
 from jsonpatch import JsonPatchConflict
 from jsonpointer import JsonPointerException
 from collections import OrderedDict
-from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, extract_scope
-from generic_config_updater.gu_common import HOST_NAMESPACE, GenericConfigUpdaterError
+from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat
+from generic_config_updater.gu_common import HOST_NAMESPACE
+from generic_config_updater.main import (
+    apply_patch_from_file as _gcu_apply_patch_from_file
+)
 from minigraph import parse_device_desc_xml, minigraph_encoder
 from natsort import natsorted
 from portconfig import get_child_ports
 from socket import AF_INET, AF_INET6
 from sonic_py_common import device_info, multi_asic
 from sonic_py_common.general import getstatusoutput_noshell
 from sonic_py_common.interface import get_interface_table_name, get_port_table_name, get_intf_longname
-from sonic_yang_cfg_generator import SonicYangCfgDbGenerator
 from utilities_common import util_base
 from swsscommon import swsscommon
 from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, ConfigDBPipeConnector, \
@@ -141,6 +140,10 @@
 # Helper functions
 #
 
+
+# get_all_running_config is imported from generic_config_updater.main
+
+
 # Sort nested dict
 def sort_dict(data):
     """ Sort of 1st level and 2nd level dict of data naturally by its key
@@ -1236,64 +1239,6 @@ def multiasic_save_to_singlefile(db, filename):
         json.dump(all_current_config, file, indent=4)
 
 
-def apply_patch_wrapper(args):
-    return apply_patch_for_scope(*args)
-
-
-# Function to apply patch for a single ASIC.
-def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path):
-    scope, changes = scope_changes
-    # Replace localhost to DEFAULT_NAMESPACE which is db definition of Host
-    if scope.lower() == HOST_NAMESPACE or scope == "":
-        scope = multi_asic.DEFAULT_NAMESPACE
-
-    scope_for_log = scope if scope else HOST_NAMESPACE
-    thread_id = threading.get_ident()
-    log.log_notice(f"apply_patch_for_scope started for {scope_for_log} by {changes} in thread:{thread_id}")
-
-    try:
-        # Call apply_patch with the ASIC-specific changes and predefined parameters
-        GenericUpdater(scope=scope).apply_patch(jsonpatch.JsonPatch(changes),
-                                                config_format,
-                                                verbose,
-                                                dry_run,
-                                                ignore_non_yang_tables,
-                                                ignore_path)
-        results[scope_for_log] = {"success": True, "message": "Success"}
-        log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes} in thread:{thread_id}")
-    except Exception as e:
-        results[scope_for_log] = {"success": False, "message": str(e)}
-        log.log_error(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}")
-
-
-def validate_patch(patch):
-    try:
-        command = ["show", "runningconfiguration", "all"]
-        proc = subprocess.Popen(command, text=True, stdout=subprocess.PIPE)
-        all_running_config, returncode = proc.communicate()
-        if returncode:
-            log.log_notice(f"Fetch all runningconfiguration failed as output:{all_running_config}")
-            return False
-
-        # Structure validation and simulate apply patch.
-        all_target_config = patch.apply(json.loads(all_running_config))
-
-        # Verify target config by YANG models
-        target_config = all_target_config.pop(HOST_NAMESPACE) if multi_asic.is_multi_asic() else all_target_config
-        target_config.pop("bgpraw", None)
-        if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
-            return False
-
-        if multi_asic.is_multi_asic():
-            for asic in multi_asic.get_namespace_list():
-                target_config = all_target_config.pop(asic)
-                target_config.pop("bgpraw", None)
-                if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
-                    return False
-
-        return True
-    except Exception as e:
-        raise GenericConfigUpdaterError(f"Validate json patch: {patch} failed due to:{e}")
 
 
 def multiasic_validate_single_file(filename):
@@ -1603,6 +1548,7 @@ def print_dry_run_message(dry_run):
     if dry_run:
         click.secho("** DRY RUN EXECUTION **", fg="yellow", underline=True)
 
+
 @config.command('apply-patch')
 @click.argument('patch-file-path', type=str, required=True)
 @click.option('-f', '--format', type=click.Choice([e.name for e in ConfigFormat]),
@@ -1622,74 +1568,21 @@ def apply_patch(ctx, patch_file_path, format, dry_run, parallel, ignore_non_yang
        format or SonicYang format.
 
        <patch-file-path>: Path to the patch file on the file-system."""
-    try:
-        print_dry_run_message(dry_run)
 
-        with open(patch_file_path, 'r') as fh:
-            text = fh.read()
-            patch_as_json = json.loads(text)
-            patch = jsonpatch.JsonPatch(patch_as_json)
+    print_dry_run_message(dry_run)
 
-        if not validate_patch(patch):
-            raise GenericConfigUpdaterError(f"Failed validating patch:{patch}")
-
-        results = {}
-        config_format = ConfigFormat[format.upper()]
-        # Initialize a dictionary to hold changes categorized by scope
-        changes_by_scope = {}
-
-        # Iterate over each change in the JSON Patch
-        for change in patch:
-            scope, modified_path = extract_scope(change["path"])
-
-            # Modify the 'path' in the change to remove the scope
-            change["path"] = modified_path
-
-            # Check if the scope is already in our dictionary, if not, initialize it
-            if scope not in changes_by_scope:
-                changes_by_scope[scope] = []
-
-            # Add the modified change to the appropriate list based on scope
-            changes_by_scope[scope].append(change)
+    try:
+        _gcu_apply_patch_from_file(
+            patch_file_path,
+            config_format_name=format,
+            verbose=verbose,
+            dry_run=dry_run,
+            parallel=parallel,
+            ignore_non_yang_tables=ignore_non_yang_tables,
+            ignore_path=ignore_path,
+        )
 
-        # Empty case to force validate YANG model.
-        if not changes_by_scope:
-            asic_list = [multi_asic.DEFAULT_NAMESPACE]
-            if multi_asic.is_multi_asic():
-                asic_list.extend(multi_asic.get_namespace_list())
-            for asic in asic_list:
-                changes_by_scope[asic] = []
-
-        # Apply changes for each scope
-        if parallel:
-            with concurrent.futures.ThreadPoolExecutor() as executor:
-                # Prepare the argument tuples
-                arguments = [(scope_changes, results, config_format,
-                              verbose, dry_run, ignore_non_yang_tables, ignore_path)
-                             for scope_changes in changes_by_scope.items()]
-
-                # Submit all tasks and wait for them to complete
-                futures = [executor.submit(apply_patch_wrapper, args) for args in arguments]
-
-                # Wait for all tasks to complete
-                concurrent.futures.wait(futures)
-        else:
-            for scope_changes in changes_by_scope.items():
-                apply_patch_for_scope(scope_changes,
-                                      results,
-                                      config_format,
-                                      verbose, dry_run,
-                                      ignore_non_yang_tables,
-                                      ignore_path)
-
-        # Check if any updates failed
-        failures = [scope for scope, result in results.items() if not result['success']]
-
-        if failures:
-            failure_messages = '\n'.join([f"- {failed_scope}: {results[failed_scope]['message']}" for failed_scope in failures])
-            raise GenericConfigUpdaterError(f"Failed to apply patch on the following scopes:\n{failure_messages}")
-
-        log.log_notice(f"Patch applied successfully for {patch}.")
+        log.log_notice("Patch applied successfully.")
         click.secho("Patch applied successfully.", fg="cyan", underline=True)
     except Exception as ex:
         click.secho("Failed to apply patch due to: {}".format(ex), fg="red", underline=True, err=True)

generic_config_updater/.coveragerc

@@ -0,0 +1,6 @@
+[run]
+branch = True
+source = generic_config_updater
+omit =
+    .eggs/*
+    tests/*

generic_config_updater/field_operation_validators.py

@@ -6,7 +6,12 @@
 from sonic_py_common import device_info
 from .gu_common import GenericConfigUpdaterError
 from swsscommon import swsscommon
-from utilities_common.constants import DEFAULT_SUPPORTED_FECS_LIST
+
+# Default FEC modes when STATE_DB does not advertise supported_fecs for a port.
+# Kept local to avoid pulling utilities_common into the GCU wheel.
+# NOTE: A duplicate of this list exists in utilities_common/constants.py.
+# If you update this list, update that copy too.
+DEFAULT_SUPPORTED_FECS_LIST = ['rs', 'fc', 'none', 'auto']
 
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 GCU_TABLE_MOD_CONF_FILE = f"{SCRIPT_DIR}/gcu_field_operation_validators.conf.json"