diff --git a/requirements.txt b/requirements.txt index db39be82..7bf62d7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -xmlschema[codegen]~=4.1.0 +xmlschema[codegen]~=4.3.1 diff --git a/src/cbexigen/SchemaAnalyzer.py b/src/cbexigen/SchemaAnalyzer.py index 9de9384c..d7afdd28 100644 --- a/src/cbexigen/SchemaAnalyzer.py +++ b/src/cbexigen/SchemaAnalyzer.py @@ -329,6 +329,12 @@ def __get_particle_from_attribute(self, attribute: XsdAttribute): particle.name = attribute.local_name + ns = tools.extract_namespace_uri(getattr(attribute, 'name', '')) + if ns: + particle.namespace = ns + elif hasattr(attribute, 'default_namespace') and attribute.default_namespace: + particle.namespace = attribute.default_namespace + particle.type = self.__get_type_name(attribute) particle.type_short = self.__get_type_name_short(attribute) particle.base_type = self.__get_base_type_name(attribute) @@ -393,6 +399,12 @@ def __get_particle(self, element: XsdElement): particle.name = element.local_name + ns = tools.extract_namespace_uri(element.name) + if ns: + particle.namespace = ns + elif hasattr(element, 'default_namespace') and element.default_namespace: + particle.namespace = element.default_namespace + particle.type = self.__get_type_name(element) particle.type_short = self.__get_type_name_short(element) particle.base_type = self.__get_base_type_name(element) @@ -456,6 +468,12 @@ def __get_abstract_particle(self, element: XsdElement, substitute: XsdElement): particle.name = substitute.local_name + ns = tools.extract_namespace_uri(substitute.name) + if ns: + particle.namespace = ns + elif hasattr(substitute, 'default_namespace') and substitute.default_namespace: + particle.namespace = substitute.default_namespace + particle.type = self.__get_type_name(substitute) particle.type_short = self.__get_type_name_short(substitute) particle.base_type = self.__get_base_type_name(substitute) @@ -572,7 +590,7 @@ def __get_particle_list(self, element: XsdElement, subst_list): if self.__is_abstract(child): # get substituted particles for child qname = self.__get_name(child) - subst_group = self.__current_schema.substitution_groups._target_dict.get(qname) + subst_group = self.__current_schema.maps.substitution_groups.get(qname) if subst_group: for elem in subst_group: particle = self.__get_abstract_particle(child, elem) @@ -590,7 +608,7 @@ def __get_particle_list(self, element: XsdElement, subst_list): if self.__is_abstract_type(element): # get substituted particles for element qname = self.__get_name(element) - subst_group = self.__current_schema.substitution_groups._target_dict.get(qname) + subst_group = self.__current_schema.maps.substitution_groups.get(qname) if subst_group: for elem in subst_group: particle = self.__get_abstract_particle(element, elem) @@ -850,7 +868,7 @@ def __get_child_tree(self, element: XsdElement, level, recursive=True): msg_write((level + 1) * " " + "ABSTRACT TYPE is extension") qname = self.__get_name(child) - sg = self.__current_schema.substitution_groups._target_dict.get(qname) + sg = self.__current_schema.maps.substitution_groups.get(qname) if sg: for substitute in sg: substitute_type_name = self.__get_type_name(substitute) @@ -925,7 +943,7 @@ def analyze_schema_elements(self): self.__build_schema_builtin_types_list() if self.__is_iso20: - for element in self.__current_schema.elements._target_dict.values(): + for element in self.__current_schema.maps.elements.values(): if element.prefixed_name.startswith('xs:'): continue @@ -1010,7 +1028,7 @@ def analyze_schema_elements(self): def __build_schema_builtin_types_list(self): xs_namespace = self.__current_schema.namespaces['xs'] - for value in self.__current_schema.types._target_dict.values(): + for value in self.__current_schema.maps.types.values(): if value.target_namespace == xs_namespace: if value.__class__.__name__ == 'XsdAtomicBuiltin': if value.simple_type.base_type is not None: @@ -1063,7 +1081,7 @@ def __print_child_recursive(element_list, child_element: XsdElement): self.__known_fragments.clear() fragments = {} - for element in self.__current_schema.elements._target_dict.values(): + for element in self.__current_schema.maps.elements.values(): if element.default_namespace: if element.name not in fragments.keys(): fragments[element.name] = __get_fragment(element) @@ -1109,7 +1127,7 @@ def __build_namespace_element_lists(self): current_namespace = self.__current_schema.get_schema('') for ele in current_namespace.elements.values(): items = [] - for value in current_namespace.elements._target_dict.values(): + for value in current_namespace.maps.elements.values(): if value.default_namespace: name = self.__get_type_name_short(value) if name == '' or name in ['AnonType', 'string']: @@ -1143,6 +1161,14 @@ def __build_namespace_element_lists(self): for gen_elem in self.__generate_elements: if gen_elem.type == '{' + imp.default_namespace + '}' + name: + # Only replace particles for empty/abstract container types. + # Concrete types (with their own fields) must keep their particles. + if gen_elem.content_type != 'empty' and len(gen_elem.particles) > 0: + continue + # Skip substitution group heads – their particles come from + # the substitution group expansion, not from namespace imports. + if self.__current_schema.maps.substitution_groups.get(gen_elem.name): + continue gen_elem.particles = items gen_elem.is_in_namespace_elements = True self.__namespace_elements[name] = items @@ -1151,7 +1177,7 @@ def __build_namespace_element_lists(self): def __build_generate_elements_types_list(self): xs_namespace = self.__current_schema.namespaces['xs'] type_list = [] - for value in self.__current_schema.types._target_dict.values(): + for value in self.__current_schema.maps.types.values(): if value.target_namespace != xs_namespace and value.content_type_label == 'element-only': type_list.append(value.local_name) @@ -1244,7 +1270,7 @@ def __replace_particle_list_in_parent(parent_element: ElementData, particle_list with the sorted particles from replacement_list. """ # the replacements need to be sorted alphabetically - replacement_list.sort(key=lambda particle_key: particle_key.name) + replacement_list.sort(key=lambda p: (p.name or '', p.namespace or '')) # the particles need to be sorted by index, for proper removal particle_list.sort(key=lambda x: x[0]) @@ -1259,8 +1285,10 @@ def __replace_particle_list_in_parent(parent_element: ElementData, particle_list # drop all the particles listed in particle_list p_index: int for p_index, p in reversed(particle_list): - if parent_element.particles[p_index] != p: - log_write(f"particle '{p.name}' not found in '{parent_element.name_short}'") + if p_index >= len(parent_element.particles) or parent_element.particles[p_index] is not p: + raise RuntimeError( + f"particle '{p.name}' at index {p_index} does not match in " + f"'{parent_element.name_short}' -- particle list is stale") del parent_element.particles[p_index] # insert the replacements at the original position, assuming the lowest original particle @@ -1301,19 +1329,25 @@ def __copy_particles_from_empty_content_elements(self, element: ElementData, par else: log_write(f' Add to list and remove particle {particle.name}.') replacement_list.append(particle) - if particle in parent.particles: - particles_to_remove.append((parent.particles.index(particle), particle)) + # Use name-based lookup (not object identity) to find the + # matching particle in parent for removal. + for idx, pp in enumerate(parent.particles): + if pp.name == particle.name and (idx, pp) not in particles_to_remove: + particles_to_remove.append((idx, pp)) + break - for particle in parent.particles: + for idx, particle in enumerate(parent.particles): if particle.name != element.name_short: continue + if (idx, particle) in particles_to_remove: + continue log_write(f' Add to list and remove particle {particle.name}.') p_min_occurs = particle.min_occurs p_max_occurs = particle.max_occurs particle.min_occurs = 0 replacement_list.append(particle) - particles_to_remove.append((parent.particles.index(particle), particle)) + particles_to_remove.append((idx, particle)) if len(replacement_list) > 0: self.__replace_particle_list_in_parent(parent, particles_to_remove, replacement_list, @@ -1355,6 +1389,9 @@ def __copy_particles_from_empty_content_elements_particle(self, element: Element abstract=True, min_occurs=0, max_occurs=1) + ns = tools.extract_namespace_uri(element.name) + if ns: + part_new.namespace = ns replacement_list.append(part_new) self.__replace_particle_list_in_parent(parent, particles_to_remove, replacement_list, @@ -1440,10 +1477,13 @@ def __scan_elements_for_empty_content(self): type_short=element.type_short, min_occurs=0, max_occurs=1) + ns = tools.extract_namespace_uri(element.name) + if ns: + part.namespace = ns re_list.append(part) if len(re_list) > 0: - re_list.sort(key=lambda particle_key: particle_key.name) + re_list.sort(key=lambda p: (p.name or '', p.namespace or '')) abstract_seq = [] for part in re_list: log_write(f' Add particle from list {part.name}.') @@ -1457,19 +1497,15 @@ def __scan_for_derived_and_extended_elements(self): log_write('') log_write('Scan for derived and extended elements') - def find_base_type(base_type_name): - result = None - for item in self.__current_schema.elements._target_dict.values(): - if item.prefixed_name.startswith('xs:'): - continue - if item.type.base_type is None: - continue - - if item.type.base_type.local_name == base_type_name: - result = item - break - - return result + # Pre-build lookup: base_type_name -> [elements derived from it] + base_type_map = {} + for item in self.__current_schema.maps.elements.values(): + if item.prefixed_name.startswith('xs:'): + continue + if item.type.base_type is None: + continue + key = item.type.base_type.local_name + base_type_map.setdefault(key, []).append(item) element: ElementData particle: Particle @@ -1483,20 +1519,20 @@ def find_base_type(base_type_name): list_with_missing.append(particle) log_write(f' Adding abstract particle {particle.name} to missing list.') - # get the base type, add it as particle - missing_element = find_base_type(particle.type_short) - if missing_element is not None: - part = self.__get_particle(missing_element) - + # get all elements derived from this type, add them as particles + missing_elements = base_type_map.get(particle.type_short, []) + if len(missing_elements) > 0: particle.min_occurs_old = particle.min_occurs particle.min_occurs = 0 list_with_missing.append(particle) log_write(f' Adding particle {particle.name} to missing list.') - part.min_occurs_old = part.min_occurs - part.min_occurs = 0 - list_with_missing.append(part) - log_write(f' Adding missing particle {part.name} to missing list.') + for missing_element in missing_elements: + part = self.__get_particle(missing_element) + part.min_occurs_old = part.min_occurs + part.min_occurs = 0 + list_with_missing.append(part) + log_write(f' Adding missing particle {part.name} to missing list.') if len(list_with_missing) > 0: first = -1 diff --git a/src/cbexigen/base_coder_classes.py b/src/cbexigen/base_coder_classes.py index bd4d9c24..649439e4 100644 --- a/src/cbexigen/base_coder_classes.py +++ b/src/cbexigen/base_coder_classes.py @@ -444,7 +444,7 @@ def generate_element_grammars(self, element: ElementData): choice_options = self.ChoiceOptions(element, particle) combined_min_occurs_from_choice = \ choice_options.min_occurs if choice_options.particles else particle.min_occurs - if combined_min_occurs_from_choice == 1: + if combined_min_occurs_from_choice >= 1: index_last_nonoptional_particle = particle_index def _particle_is_in_choice(element: ElementData, particle: Particle): @@ -631,11 +631,10 @@ def _add_particle_or_choice_list_to_details( if m < _max: # flag=Grammar.LOOP indicates that the _next_ grammar will be the same as this one # - # Note: We do not loop on the mandatory elements of an array, it would require extra - # logic, and we have no min_occurs larger than 2 (i.e. no large repetition of mandatory - # elements) anyway in our existing standards. - - if m > 1 and m > part.min_occurs - 1 and (part.max_occurs_old is None or m < part.max_occurs_old): + # Do not enter the self-loop until all mandatory occurrences have been consumed. + # Before minOccurs is reached, the grammar must only accept another START event; + # allowing END or LOOP too early changes the event-code width and misaligns EXI. + if m > 1 and m > part.min_occurs and (part.max_occurs_old is None or m < part.max_occurs_old): flag = GrammarFlag.LOOP skip_to_end = True else: diff --git a/src/cbexigen/datatype_classes.py b/src/cbexigen/datatype_classes.py index f1f59bc1..e669a8db 100644 --- a/src/cbexigen/datatype_classes.py +++ b/src/cbexigen/datatype_classes.py @@ -135,7 +135,7 @@ def __generate_functions_enum(self): comment = '// enum for function numbers' enum_type = self.parameters['prefix'] + 'generatedFunctionNumbersType' items = [] - for value in self.scheme.elements._target_dict.values(): + for value in self.scheme.maps.elements.values(): if value.default_namespace: items.append(self.parameters['prefix'] + value.local_name) items.sort() @@ -538,7 +538,7 @@ def generate_file(self): else: element_type = self.scheme.types.get(element.type_short) if element_type is None: - element_type = self.scheme.types._target_dict.get(element.type) + element_type = self.scheme.maps.types.get(element.type) if element_type is None: continue @@ -580,7 +580,7 @@ def generate_file(self): if skip_element: curr_idx += 1 - if curr_idx > len(self.__generate): + if curr_idx >= len(self.__generate): log_write_error('Module datatypes: Generator loop aborted! Index larger than existing elements.') break else: diff --git a/src/cbexigen/decoder_classes.py b/src/cbexigen/decoder_classes.py index e2dabc8d..1d08f552 100644 --- a/src/cbexigen/decoder_classes.py +++ b/src/cbexigen/decoder_classes.py @@ -973,7 +973,7 @@ def generate_file(self): if skip_element: curr_idx += 1 - if curr_idx > len(self.elements_to_generate): + if curr_idx >= len(self.elements_to_generate): log_write_error('Module decoder: Generator loop aborted! Index larger than existing elements.') break else: diff --git a/src/cbexigen/elementData.py b/src/cbexigen/elementData.py index 15c737d0..7c19b5f7 100644 --- a/src/cbexigen/elementData.py +++ b/src/cbexigen/elementData.py @@ -54,6 +54,7 @@ class Particle: integer_bit_size: int = -1 integer_base_type: str = None integer_is_unsigned: bool = False + namespace: str = '' # additional info for the anyType particle process_content: str = None diff --git a/src/cbexigen/encoder_classes.py b/src/cbexigen/encoder_classes.py index 2f107619..6f08c089 100644 --- a/src/cbexigen/encoder_classes.py +++ b/src/cbexigen/encoder_classes.py @@ -969,7 +969,7 @@ def generate_file(self): if skip_element: curr_idx += 1 - if curr_idx > len(self.elements_to_generate): + if curr_idx >= len(self.elements_to_generate): log_write_error('Module encoder: Generator loop aborted! Index larger than existing elements.') break else: diff --git a/src/cbexigen/tools.py b/src/cbexigen/tools.py index f82a9e5f..c280c3e7 100644 --- a/src/cbexigen/tools.py +++ b/src/cbexigen/tools.py @@ -76,6 +76,13 @@ def get_indent(level: int = 1): return level * (' ' * CONFIG_PARAMS['c_code_indent_chars']) +def extract_namespace_uri(qualified_name: str) -> str: + """Extract namespace URI from XSD qualified name like '{uri}localName'.""" + if qualified_name and qualified_name.startswith('{') and '}' in qualified_name: + return qualified_name[1:qualified_name.index('}')] + return '' + + ''' generator tools ''' diff --git a/src/cbexigen/tools_config.py b/src/cbexigen/tools_config.py index c34ae7de..48834014 100644 --- a/src/cbexigen/tools_config.py +++ b/src/cbexigen/tools_config.py @@ -4,12 +4,13 @@ """ Tools for the Exi Codegenerator config """ import importlib +import io +import urllib.request +import zipfile from typing import Union, Dict from pathlib import Path -import urllib.request - CONFIG_ARGS: Dict[str, Union[str, Path]] = { 'program_dir': '', 'config_file': '', @@ -189,6 +190,34 @@ def process_config_parameters(): ISO2_SCHEMAS_URL = "https://standards.iso.org/iso/15118/-2/ed-2/en/" ISO20_SCHEMAS_URL = "https://standards.iso.org/iso/15118/-20/ed-1/en/" +ISO20_AMD1_SCHEMAS_URL = "https://standards.iso.org/iso/15118/-20/ed-1/en/Amd/1/" +ISO20_AMD1_SCHEMAS_ZIP = "AMD1_xsdSchema.zip" + + +def _validate_https_url(url: str) -> str: + if not url.startswith("https://"): + raise ValueError(f"Only https URLs are allowed, got: {url}") + return url + + +def _download_schema_files(base_url: str, schema_names: list, target_path: Path, + label: str) -> bool: + """Download schema files. Returns False if any download failed.""" + for schema in schema_names: + schema_file_path = target_path / schema + if schema_file_path.exists(): + print(f"{label} schema {schema} is already there. Skipping it.") + continue + print(f"{label} schema {schema} not found! Downloading it...") + try: + urllib.request.urlretrieve( + _validate_https_url(base_url + schema), schema_file_path.absolute().as_posix()) + except Exception as err: + print(f"Download failed for {schema}: {err=}, {type(err)=}") + if schema_file_path.exists(): + schema_file_path.unlink() + return False + return True def download_schemas(): @@ -202,11 +231,8 @@ def download_schemas(): CONFIG_ARGS['schema_base_dir'], config_module.c_files_to_generate['iso20_CommonMessages_Datatypes']['schema']) iso20_schema_path = iso20_schema_full_name.parent.resolve() - if not iso2_schema_path.exists(): - iso2_schema_path.mkdir(parents=True, exist_ok=True) - - if not iso20_schema_path.exists(): - iso20_schema_path.mkdir(parents=True, exist_ok=True) + iso2_schema_path.mkdir(parents=True, exist_ok=True) + iso20_schema_path.mkdir(parents=True, exist_ok=True) iso2_schema_files_names = ['V2G_CI_AppProtocol.xsd', 'V2G_CI_MsgDef.xsd', 'V2G_CI_MsgBody.xsd', 'V2G_CI_MsgDataTypes.xsd', 'V2G_CI_MsgHeader.xsd', 'xmldsig-core-schema.xsd'] @@ -214,26 +240,29 @@ def download_schemas(): iso20_schema_files_names = ['V2G_CI_AC.xsd', 'V2G_CI_ACDP.xsd', 'V2G_CI_AppProtocol.xsd', 'V2G_CI_CommonMessages.xsd', 'V2G_CI_CommonTypes.xsd', 'V2G_CI_DC.xsd', 'V2G_CI_WPT.xsd', 'xmldsig-core-schema.xsd'] - for schema in iso2_schema_files_names: - schema_file_path = iso2_schema_path / schema - if not schema_file_path.exists(): - print(f"ISO15118-2 schema {schema} not found! Downloading it...") - try: - urllib.request.urlretrieve( - ISO2_SCHEMAS_URL + schema, schema_file_path.absolute().as_posix()) - except Exception as err: - print(f"Error during downloading: {err=}, {type(err)=}") - else: - print(f"ISO15118-2 schema {schema} is already there. Skipping it.") - - for schema in iso20_schema_files_names: - schema_file_path = iso20_schema_path / schema - if not schema_file_path.exists(): - print(f"ISO15118-20 schema {schema} not found! Downloading it...") - try: - urllib.request.urlretrieve( - ISO20_SCHEMAS_URL + schema, schema_file_path.absolute().as_posix()) - except Exception as err: - print(f"Error during downloading: {err=}, {type(err)=}") - else: - print(f"ISO15118-20 schema {schema} is already there. Skipping it.") + _download_schema_files(ISO2_SCHEMAS_URL, iso2_schema_files_names, iso2_schema_path, "ISO15118-2") + _download_schema_files(ISO20_SCHEMAS_URL, iso20_schema_files_names, iso20_schema_path, "ISO15118-20") + + iso20_amd1_schema_files_names = ['V2G_CI_AC_DER_IEC.xsd', 'V2G_CI_AC_DER_SAE.xsd'] + if not _download_schema_files(ISO20_AMD1_SCHEMAS_URL, iso20_amd1_schema_files_names, + iso20_schema_path, "ISO15118-20 Amd1"): + print(f"Trying zip fallback: downloading {ISO20_AMD1_SCHEMAS_ZIP}...") + try: + with urllib.request.urlopen( # nosec B310 # nosemgrep + _validate_https_url(ISO20_AMD1_SCHEMAS_URL + ISO20_AMD1_SCHEMAS_ZIP)) as response: + zip_data = io.BytesIO(response.read()) + with zipfile.ZipFile(zip_data) as zf: + for schema in iso20_amd1_schema_files_names: + schema_file_path = iso20_schema_path / schema + if schema_file_path.exists(): + print(f"ISO15118-20 Amd1 schema {schema} is already there. Skipping it.") + continue + matching = [n for n in zf.namelist() if n.endswith(schema)] + if matching: + with zf.open(matching[0]) as src, open(schema_file_path, 'wb') as dst: + dst.write(src.read()) + print(f"Extracted {schema} from {ISO20_AMD1_SCHEMAS_ZIP}.") + else: + print(f"Error: {schema} not found in {ISO20_AMD1_SCHEMAS_ZIP}.") + except Exception as err: + print(f"Error during zip download: {err=}, {type(err)=}") diff --git a/src/config.py b/src/config.py index 65884f4b..79968910 100644 --- a/src/config.py +++ b/src/config.py @@ -120,6 +120,10 @@ 'SignedInfo', 'DC_ChargeParameterDiscoveryRes', ] +iso20_ac_der_iec_fragments = [ + 'SignedInfo', + "AC_ChargeParameterDiscoveryRes", +] # general C code style c_code_indent_chars = 4 @@ -726,4 +730,61 @@ 'iso20_ACDP_Datatypes.h', 'iso20_ACDP_Encoder.h'] } }, + 'iso20_AC_DER_IEC_Datatypes': { + 'schema': 'ISO_15118-20/FDIS/V2G_CI_AC_DER_IEC.xsd', + 'prefix': 'iso20_ac_der_iec_', + 'type': 'converter', + 'folder': 'iso-20', + 'h': { + 'filename': 'iso20_AC_DER_IEC_Datatypes.h', + 'identifier': 'ISO20_AC_DER_IEC_DATATYPES_H', + 'include_std_lib': ['stdint.h'], + 'include_other': ['exi_basetypes.h'] + }, + 'c': { + 'filename': 'iso20_AC_DER_IEC_Datatypes.c', + 'identifier': 'ISO20_AC_DER_IEC_DATATYPES_C', + 'include_std_lib': [], + 'include_other': ['iso20_AC_DER_IEC_Datatypes.h'] + } + }, + 'iso20_AC_DER_IEC_Decoder': { + 'schema': 'ISO_15118-20/FDIS/V2G_CI_AC_DER_IEC.xsd', + 'prefix': 'iso20_ac_der_iec_', + 'type': 'decoder', + 'folder': 'iso-20', + 'h': { + 'filename': 'iso20_AC_DER_IEC_Decoder.h', + 'identifier': 'ISO20_AC_DER_IEC_DECODER_H', + 'include_std_lib': [], + 'include_other': ['exi_bitstream.h', 'iso20_AC_DER_IEC_Datatypes.h'] + }, + 'c': { + 'filename': 'iso20_AC_DER_IEC_Decoder.c', + 'identifier': 'ISO20_AC_DER_IEC_DECODER_C', + 'include_std_lib': ['stdint.h'], + 'include_other': ['exi_basetypes.h', 'exi_types_decoder.h', 'exi_basetypes_decoder.h', + 'exi_error_codes.h', 'exi_header.h', 'iso20_AC_DER_IEC_Datatypes.h', + 'iso20_AC_DER_IEC_Decoder.h'] + } + }, + 'iso20_AC_DER_IEC_Encoder': { + 'schema': 'ISO_15118-20/FDIS/V2G_CI_AC_DER_IEC.xsd', + 'prefix': 'iso20_ac_der_iec_', + 'type': 'encoder', + 'folder': 'iso-20', + 'h': { + 'filename': 'iso20_AC_DER_IEC_Encoder.h', + 'identifier': 'ISO20_AC_DER_IEC_ENCODER_H', + 'include_std_lib': [], + 'include_other': ['exi_bitstream.h', 'iso20_AC_DER_IEC_Datatypes.h'] + }, + 'c': { + 'filename': 'iso20_AC_DER_IEC_Encoder.c', + 'identifier': 'ISO20_AC_DER_IEC_ENCODER_C', + 'include_std_lib': ['stdint.h'], + 'include_other': ['exi_basetypes.h', 'exi_basetypes_encoder.h', 'exi_error_codes.h', 'exi_header.h', + 'iso20_AC_DER_IEC_Datatypes.h', 'iso20_AC_DER_IEC_Encoder.h'] + } + }, }