From a835805e29603e8781bb324dabab3345527c7bf5 Mon Sep 17 00:00:00 2001 From: mazora Date: Tue, 4 Jun 2024 11:03:25 +0300 Subject: [PATCH 1/5] Added Scheduled Config Show Commands: - show time-range-configurations - show scheduled-configurations --- show/main.py | 153 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/show/main.py b/show/main.py index cfdf30d3c61..33df931abc1 100755 --- a/show/main.py +++ b/show/main.py @@ -2081,6 +2081,159 @@ def ztp(status, verbose): cmd += ["--verbose"] run_command(cmd, display_cmd=verbose) +# +# 'time range' command ("show time-range-configurations") +# +@cli.command("time-range-configurations") +@click.argument('time_range_name', required=False) +@clicommon.pass_db +def time_range_configurations(db: ConfigDBConnector, time_range_name): + """Show time range configurations""" + def get_time_range_statuses(db): + """Get time range statuses from STATE_DB""" + status_dict = {} + status_keys = db.keys(db.STATE_DB, "TIME_RANGE_STATUS_TABLE|*") + + if status_keys is not None: + for key in status_keys: + key_values = key.split('|') + time_range_name = key_values[1] + values = db.get_all(db.STATE_DB, key) + if values and "status" in values: + status_dict[time_range_name] = values["status"] + + return status_dict + + def append_scheduled_configurations(config_db, time_range_table): + """Append scheduled configurations from config_db to time range table""" + scheduled_configs = config_db.get_table('SCHEDULED_CONFIGURATIONS') + + for config_name, config in scheduled_configs.items(): + time_range = config.get('time_range') + if time_range in time_range_table: + if 'configurations' not in time_range_table[time_range]: + time_range_table[time_range]['configurations'] = [] + time_range_table[time_range]['configurations'].append(config_name) + + + def print_all_time_range_configurations(configurations, statuses_dict): + """Print all time range configurations in tabular format""" + # Prepare data for tabulate + table_data = [] + for name, config in configurations.items(): + status = statuses_dict.get(name, "Unknown") + years = config.get('years', 'N/A') + first_conf = config['configurations'][0] if 'configurations' in config and config['configurations'] else '' + base_row = [name, status, config['start'], config['end'], years, first_conf] + table_data.append(base_row) + + # Add additional configurations if any + if 'configurations' in config: + for conf in config['configurations'][1:]: + table_data.append(["", "", "", "", "", conf]) + + headers = ["Time Range", "Status", "Start Schedule", "End Schedule", "Years", "Configurations"] + click.echo(tabulate(table_data, headers=headers)) + + def print_time_range_configuration(time_range_name, time_range_table, statuses_dict): + """Print details of a specific time range configuration""" + if time_range_name not in time_range_table: + click.echo(f"Time range configuration '{time_range_name}' not found") + return + + config = time_range_table[time_range_name] + status = statuses_dict.get(time_range_name, "Unknown") + click.echo(f"Time Range: {time_range_name}") + click.echo(f"Status: {status}") + click.echo(f"Start Schedule: {config['start']}") + click.echo(f"End Schedule: {config['end']}") + click.echo(f"Years: {config.get('years', 'N/A')}") + click.echo("Configurations:") + for conf in config.get('configurations', []): + click.echo(f" {conf}") + + config_db = db.cfgdb + time_range_table = config_db.get_table('TIME_RANGE') + time_range_status_dict = get_time_range_statuses(db.db) + append_scheduled_configurations(config_db, time_range_table) + + if time_range_name: + # Show details for the specific time range name + print_time_range_configuration(time_range_name, time_range_table, time_range_status_dict) + else: + # Show all time range configurations + print_all_time_range_configurations(time_range_table, time_range_status_dict) + + +# +# 'scheduled configurations' command ("show scheduled-configurations") +# +@cli.command("scheduled-configurations") +@click.argument('scheduled_configuration_name', required=False) +@clicommon.pass_db +def scheduled_configurations(db: ConfigDBConnector, scheduled_configuration_name): + """Show time range configurations""" + def get_time_range_statuses(db): + """Get time range statuses from STATE_DB""" + status_dict = {} + status_keys = db.keys(db.STATE_DB, "TIME_RANGE_STATUS_TABLE|*") + + if status_keys is not None: + for key in status_keys: + key_values = key.split('|') + time_range_name = key_values[1] + values = db.get_all(db.STATE_DB, key) + if values and "status" in values: + status_dict[time_range_name] = values["status"] + + return status_dict + + def append_time_range_status_to_scheduled_configurations(scheduled_config_table, time_range_status_dict): + """Append time range status to scheduled configurations""" + for _, config in scheduled_config_table.items(): + time_range = config.get('time_range') + status = time_range_status_dict.get(time_range, "Unbound") + config['status'] = status + + + + def print_all_scheduled_configurations(sched_config_table): + """Print all time range configurations in tabular format""" + # Prepare data for tabulate + table_data = [] + for name, config in sched_config_table.items(): + status = config.get('status', "Unbound") + table_data.append([name, status, config['time_range']]) + + alphabetical_order_data = sorted(table_data, key=lambda x: x[0]) + headers = ["Configuration Name", "Status", "Time Range"] + click.echo(tabulate(alphabetical_order_data, headers=headers)) + + def print_scheduled_configuration(scheduled_configuration_name, scheduled_config_table): + """Print details of a specific time range configuration""" + if scheduled_configuration_name not in scheduled_config_table: + click.echo(f"Scheduled configuration '{scheduled_configuration_name}' not found") + return + import json + + config = scheduled_config_table[scheduled_configuration_name] + click.echo(f"Configuration Name: {scheduled_configuration_name}") + click.echo(f"Status: {config.get('status', 'Unbound')}") + click.echo(f"Time Range: {config['time_range']}") + click.echo("Configuration:") + click.echo(json.dumps(json.loads(config.get('configuration').replace('\'', '\"')), sort_keys=True, indent=4)) + + config_db = db.cfgdb + scheduled_config_table = config_db.get_table('SCHEDULED_CONFIGURATIONS') + time_range_status_dict = get_time_range_statuses(db.db) + append_time_range_status_to_scheduled_configurations(scheduled_config_table, time_range_status_dict) + + if scheduled_configuration_name: + # Show details for the specific time range name + print_scheduled_configuration(scheduled_configuration_name, scheduled_config_table) + else: + # Show all time range configurations + print_all_scheduled_configurations(scheduled_config_table) # # 'bfd' group ("show bfd ...") From b71a48f079a714b5d4363e0a7f44deacbcb9f779 Mon Sep 17 00:00:00 2001 From: mazora Date: Tue, 4 Jun 2024 11:40:42 +0300 Subject: [PATCH 2/5] Fixed Comments and Removed Unnecessary Import --- show/main.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/show/main.py b/show/main.py index 33df931abc1..0063b398294 100755 --- a/show/main.py +++ b/show/main.py @@ -2151,7 +2151,8 @@ def print_time_range_configuration(time_range_name, time_range_table, statuses_d click.echo("Configurations:") for conf in config.get('configurations', []): click.echo(f" {conf}") - + + #### main code #### config_db = db.cfgdb time_range_table = config_db.get_table('TIME_RANGE') time_range_status_dict = get_time_range_statuses(db.db) @@ -2164,7 +2165,6 @@ def print_time_range_configuration(time_range_name, time_range_table, statuses_d # Show all time range configurations print_all_time_range_configurations(time_range_table, time_range_status_dict) - # # 'scheduled configurations' command ("show scheduled-configurations") # @@ -2172,7 +2172,7 @@ def print_time_range_configuration(time_range_name, time_range_table, statuses_d @click.argument('scheduled_configuration_name', required=False) @clicommon.pass_db def scheduled_configurations(db: ConfigDBConnector, scheduled_configuration_name): - """Show time range configurations""" + """Show scheduled configurations""" def get_time_range_statuses(db): """Get time range statuses from STATE_DB""" status_dict = {} @@ -2192,13 +2192,13 @@ def append_time_range_status_to_scheduled_configurations(scheduled_config_table, """Append time range status to scheduled configurations""" for _, config in scheduled_config_table.items(): time_range = config.get('time_range') - status = time_range_status_dict.get(time_range, "Unbound") + status = time_range_status_dict.get(time_range) config['status'] = status def print_all_scheduled_configurations(sched_config_table): - """Print all time range configurations in tabular format""" + """Print all scheduled configurations in tabular format""" # Prepare data for tabulate table_data = [] for name, config in sched_config_table.items(): @@ -2210,29 +2210,29 @@ def print_all_scheduled_configurations(sched_config_table): click.echo(tabulate(alphabetical_order_data, headers=headers)) def print_scheduled_configuration(scheduled_configuration_name, scheduled_config_table): - """Print details of a specific time range configuration""" + """Print details of a specific scheduled configuration""" if scheduled_configuration_name not in scheduled_config_table: click.echo(f"Scheduled configuration '{scheduled_configuration_name}' not found") return - import json - + config = scheduled_config_table[scheduled_configuration_name] click.echo(f"Configuration Name: {scheduled_configuration_name}") click.echo(f"Status: {config.get('status', 'Unbound')}") click.echo(f"Time Range: {config['time_range']}") click.echo("Configuration:") click.echo(json.dumps(json.loads(config.get('configuration').replace('\'', '\"')), sort_keys=True, indent=4)) - + + #### main code #### config_db = db.cfgdb scheduled_config_table = config_db.get_table('SCHEDULED_CONFIGURATIONS') time_range_status_dict = get_time_range_statuses(db.db) append_time_range_status_to_scheduled_configurations(scheduled_config_table, time_range_status_dict) - + if scheduled_configuration_name: - # Show details for the specific time range name + # Show details for the specific configuration name print_scheduled_configuration(scheduled_configuration_name, scheduled_config_table) else: - # Show all time range configurations + # Show all scheduled configurations print_all_scheduled_configurations(scheduled_config_table) # From 585419194373b777b7660fe19399f2a4ecd6f1a5 Mon Sep 17 00:00:00 2001 From: mazora Date: Tue, 4 Jun 2024 16:40:06 +0300 Subject: [PATCH 3/5] Implemented Years in show time-range-configuration command --- show/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/show/main.py b/show/main.py index 0063b398294..38b4668419e 100755 --- a/show/main.py +++ b/show/main.py @@ -2122,7 +2122,9 @@ def print_all_time_range_configurations(configurations, statuses_dict): table_data = [] for name, config in configurations.items(): status = statuses_dict.get(name, "Unknown") - years = config.get('years', 'N/A') + start_year = config.get('start_year', '') + end_year = config.get('end_year', '') + years = f"{start_year}{'-' if start_year or end_year else ''}{end_year}" if start_year or end_year else "N/A" first_conf = config['configurations'][0] if 'configurations' in config and config['configurations'] else '' base_row = [name, status, config['start'], config['end'], years, first_conf] table_data.append(base_row) From 37fb35d37042635754394e3c9e3d6f91c0bc3712 Mon Sep 17 00:00:00 2001 From: mazora Date: Tue, 23 Jul 2024 10:27:02 +0300 Subject: [PATCH 4/5] add scheduled-configuration config command --- config/config_mgmt.py | 142 +++++++++++++++++++++++++++++++++++++++++- config/main.py | 42 +++++++++---- 2 files changed, 170 insertions(+), 14 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 4e3115bd356..71537878d23 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -3,6 +3,8 @@ Port Breakout. ''' +from enum import Enum, auto +from logging import config import os import re import shutil @@ -25,7 +27,7 @@ # Globals YANG_DIR = "/usr/local/yang-models" -CONFIG_DB_JSON_FILE = '/etc/sonic/confib_db.json' +CONFIG_DB_JSON_FILE = '/etc/sonic/config_db.json' # TODO: Find a place for it on sonic switch. DEFAULT_CONFIG_DB_JSON_FILE = '/etc/sonic/port_breakout_config_db.json' @@ -983,3 +985,141 @@ def readJsonFile(fileName): raise Exception(e) return result + +class ConfigMgmtScheduledConfig(ConfigMgmt): + ''' + Config MGMT class for Scheduled Configurations. This is derived from ConfigMgmt. + ''' + + def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): + ''' + Initialise the class + + Parameters: + source (str): source for input config, default configDb else file. + debug (bool): verbose mode. + allowTablesWithoutYang (bool): allow tables without yang model in + config or not. + + Returns: + void + ''' + try: + super().__init__(source=source, debug=debug, allowTablesWithoutYang=allowTablesWithoutYang) + + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + raise Exception('ConfigMgmtScheduledConfig Class creation failed') + + def validateConfigData(self): + ''' + Validate current config data Tree, including scheduled configurations. + + Parameters: + void + + Returns: + bool + ''' + # First validate the base configuration + if not super().validateConfigData(): + return False + + # Validate the scheduled configurations + try: + self._validateScheduledConfigurations() + except Exception as e: + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=f'Scheduled Config Data Validation Failed: {str(e)}') + return False + + self.sysLog(msg='Scheduled Config Data Validation successful', doPrint=True) + return True + + def _validateScheduledConfigurations(self): + ''' + Validate the scheduled configurations within the data tree. + + Parameters: + void + + Returns: + void + ''' + # Get the scheduled configurations from the loaded data + scheduled_configs = self.sy.root.find_path("/SCHEDULED_CONFIGURATIONS") + + if scheduled_configs is None: + return + + for scheduled_config in scheduled_configs.data(): + config_name = scheduled_config.name() + self.sysLog(msg=f'Validating Scheduled Configuration: {config_name}', doPrint=True) + + # Validate the 'configuration' field + config_node = scheduled_config.child().find_path("configuration") + if config_node: + self._validate_node(config_node) + + # Validate the 'deactivation_configuration' field + deactivation_node = scheduled_config.child().find_path("deactivation_configuration") + if deactivation_node: + self._validate_node(deactivation_node) + + def _validate_node(self, node): + ''' + Recursively validate a configuration node. + + Parameters: + node (lyd_node): The node to validate. + + Returns: + void + ''' + try: + node.validate(ly.LYD_OPT_CONFIG, self.sy.ctx) + except Exception as e: + self.fail(f'Validation failed for node {node.path()}: {str(e)}') + + def writeConfigDB(self, configJson = ""): + ''' + Write the validated config to Config DB. + + Parameters: + configJson (dict): config to push in config DB. + + Returns: + void + ''' + if configJson == "": + configJson = self.configdbJsonIn + + self.sysLog(doPrint=True, msg='Writing in Config DB') + data = dict() + if self.configdb is None: + configdb = ConfigDBConnector() + configdb.connect(False) + else: + configdb = self.configdb + sonic_cfggen.deep_update(data, configJson) + self.sysLog(msg="Write in DB: {}".format(data)) + configdb.mod_config(sonic_cfggen.FormatConverter.output_to_db(data)) + + + +class ConfigMgmtType(Enum): + BASIC = auto() + DPB = auto() + SCHEDULED_CONFIG = auto() + + +def load_ConfigMgmt(type: ConfigMgmtType = ConfigMgmtType.BASIC, source: str = "", verbose: bool =False, allowTablesWithoutYang: bool = True, sonicYangOptions: int = 0, configdb=None) -> ConfigMgmt: + """ Load config for the commands which are capable of change in config DB. """ + try: + if type == ConfigMgmtType.DPB: + return ConfigMgmtDPB(source=source, debug=verbose, allowTablesWithoutYang=allowTablesWithoutYang) + elif type == ConfigMgmtType.SCHEDULED_CONFIG: + return ConfigMgmtScheduledConfig(source=source, debug=verbose, allowTablesWithoutYang=allowTablesWithoutYang) + else: # ConfigMgmtType.BASIC + return ConfigMgmt(source=source, debug=verbose, allowTablesWithoutYang=allowTablesWithoutYang, sonicYangOptions=sonicYangOptions, configdb=configdb) + except Exception as e: + raise Exception("Failed to load the config. Error: {}".format(str(e))) \ No newline at end of file diff --git a/config/main.py b/config/main.py index b750b49820c..db6c85eb61b 100644 --- a/config/main.py +++ b/config/main.py @@ -56,7 +56,7 @@ from . import vlan from . import vxlan from . import plugins -from .config_mgmt import ConfigMgmtDPB, ConfigMgmt +from .config_mgmt import ConfigMgmtDPB, ConfigMgmt, ConfigMgmtType, load_ConfigMgmt, ConfigMgmtScheduledConfig from . import mclag from . import syslog from . import switchport @@ -207,14 +207,6 @@ def _validate_interface_mode(ctx, breakout_cfg_file, interface_name, target_brko sys.exit(0) return True -def load_ConfigMgmt(verbose): - """ Load config for the commands which are capable of change in config DB. """ - try: - cm = ConfigMgmtDPB(debug=verbose) - return cm - except Exception as e: - raise Exception("Failed to load the config. Error: {}".format(str(e))) - def breakout_warnUser_extraTables(cm, final_delPorts, confirm=True): """ Function to warn user about extra tables while Dynamic Port Breakout(DPB). @@ -2086,7 +2078,7 @@ def override_config_table(db, input_config_db, dry_run): # The ConfigMgmt will load YANG and running # config during initialization. try: - cm = ConfigMgmt(configdb=config_db) + cm: ConfigMgmt = load_ConfigMgmt(configdb=config_db) cm.validateConfigData() except Exception as ex: click.secho("Failed to validate running config. Error: {}".format(ex), fg="magenta") @@ -2100,7 +2092,7 @@ def override_config_table(db, input_config_db, dry_run): cm = None try: # YANG validate of config minigraph generated - cm = ConfigMgmt(configdb=config_db) + cm: ConfigMgmt= load_ConfigMgmt(configdb=config_db) cm.validateConfigData() except Exception as ex: log.log_warning("Failed to validate running config. Alerting: {}".format(ex)) @@ -4552,7 +4544,7 @@ def breakout(ctx, interface_name, mode, verbose, force_remove_dependencies, load # Start Interation with Dy Port BreakOut Config Mgmt try: """ Load config for the commands which are capable of change in config DB """ - cm = load_ConfigMgmt(verbose) + cm: ConfigMgmtDPB = load_ConfigMgmt(type=ConfigMgmtType.DPB, verbose=verbose) """ Delete all ports if forced else print dependencies using ConfigMgmt API """ final_delPorts = [intf for intf in del_intf_dict] @@ -7630,7 +7622,6 @@ def date(date, time): date_time = f'{date} {time}' clicommon.run_command(['timedatectl', 'set-time', date_time]) - # # 'asic-sdk-health-event' group ('config asic-sdk-health-event ...') # @@ -7749,6 +7740,31 @@ def warning(db, category_list, max_events, namespace): def notice(db, category_list, max_events, namespace): handle_asic_sdk_health_suppress(db, 'notice', category_list, max_events, namespace) +@config.command("scheduled-configuration") +@click.argument('json_file_name', metavar='', required=True) +@click.option('-d', '--dry-run', is_flag=True, help="Validate configuration without applying to config db") +def scheduled_configuration(json_file_name, dry_run): + """ config scheduled configuration """ + + verbose = False + + try: + cm: ConfigMgmtScheduledConfig = load_ConfigMgmt(type=ConfigMgmtType.SCHEDULED_CONFIG, source= json_file_name, verbose=verbose) + except Exception as e: + error_msg = "Failed to load ConfigMgmtScheduledConfig: {}".format(str(e)) + click.secho(error_msg, fg='red') + + if not cm.validateConfigData(): + error_msg = "Failed to validate data" + click.secho(error_msg, fg='red') + + if dry_run: + cm.print_data() + return + + cm.writeConfigDB() + + if __name__ == '__main__': config() From 68d7c5be76e502d00b6fc762d3ff2dd365e0170e Mon Sep 17 00:00:00 2001 From: mazora Date: Wed, 7 Aug 2024 11:03:49 +0300 Subject: [PATCH 5/5] Updated YANG validation for scheduled-config CLI command --- config/config_mgmt.py | 33 +++++++++++++++++++-------------- config/main.py | 13 +++++++------ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 71537878d23..d7b38097047 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -1046,26 +1046,32 @@ def _validateScheduledConfigurations(self): void ''' # Get the scheduled configurations from the loaded data - scheduled_configs = self.sy.root.find_path("/SCHEDULED_CONFIGURATIONS") + scheduled_configs = self.configdbJsonIn.get("SCHEDULED_CONFIGURATIONS") - if scheduled_configs is None: + if not scheduled_configs: return - for scheduled_config in scheduled_configs.data(): - config_name = scheduled_config.name() + for config_name, scheduled_config in scheduled_configs.items(): self.sysLog(msg=f'Validating Scheduled Configuration: {config_name}', doPrint=True) # Validate the 'configuration' field - config_node = scheduled_config.child().find_path("configuration") + config_node = scheduled_config.get("configuration") if config_node: - self._validate_node(config_node) + self._validate_configuration(config_node) + else: + raise Exception(f'Missing "configuration" field in scheduled configuration: {config_name}') # Validate the 'deactivation_configuration' field - deactivation_node = scheduled_config.child().find_path("deactivation_configuration") + deactivation_node = scheduled_config.get("deactivation_configuration") if deactivation_node: - self._validate_node(deactivation_node) + if isinstance(deactivation_node, str) and deactivation_node.lower() == "remove": + continue + else: + self._validate_configuration(deactivation_node) + else: + raise Exception(f'Missing "deactivation_configuration" field in scheduled configuration: {config_name}') - def _validate_node(self, node): + def _validate_configuration(self, node): ''' Recursively validate a configuration node. @@ -1075,11 +1081,10 @@ def _validate_node(self, node): Returns: void ''' - try: - node.validate(ly.LYD_OPT_CONFIG, self.sy.ctx) - except Exception as e: - self.fail(f'Validation failed for node {node.path()}: {str(e)}') - + self.loadData(node) + super().validateConfigData() + + def writeConfigDB(self, configJson = ""): ''' Write the validated config to Config DB. diff --git a/config/main.py b/config/main.py index db6c85eb61b..1509bd10331 100644 --- a/config/main.py +++ b/config/main.py @@ -7744,27 +7744,28 @@ def notice(db, category_list, max_events, namespace): @click.argument('json_file_name', metavar='', required=True) @click.option('-d', '--dry-run', is_flag=True, help="Validate configuration without applying to config db") def scheduled_configuration(json_file_name, dry_run): - """ config scheduled configuration """ + """ config scheduled-configuration """ verbose = False - + cm: ConfigMgmtScheduledConfig try: - cm: ConfigMgmtScheduledConfig = load_ConfigMgmt(type=ConfigMgmtType.SCHEDULED_CONFIG, source= json_file_name, verbose=verbose) + cm = load_ConfigMgmt(ConfigMgmtType.SCHEDULED_CONFIG, source= json_file_name, verbose=verbose) except Exception as e: error_msg = "Failed to load ConfigMgmtScheduledConfig: {}".format(str(e)) click.secho(error_msg, fg='red') + return if not cm.validateConfigData(): error_msg = "Failed to validate data" click.secho(error_msg, fg='red') + return if dry_run: - cm.print_data() + click.echo("Successfully validated configuration data, dry-run mode enabled, no changes will be applied") return cm.writeConfigDB() - - + if __name__ == '__main__': config()