From a93c4d6df3897a8ea7a60ddd41d5e2e26936feb6 Mon Sep 17 00:00:00 2001 From: Christian Andersen Date: Thu, 26 Mar 2026 21:54:06 +0100 Subject: [PATCH 1/8] fix: upgrade xmlschema to 4.3.1 and fix particle dedup Replace _target_dict with maps access for xmlschema 4.3.1 compat. Fix particle dedup in empty content scan to match by name instead of object identity. Fix off-by-one in generator loops and add bounds guard in particle deletion. Signed-off-by: Christian Andersen --- requirements.txt | 2 +- src/cbexigen/SchemaAnalyzer.py | 38 +++++++++++++++++++------------- src/cbexigen/datatype_classes.py | 6 ++--- src/cbexigen/decoder_classes.py | 2 +- src/cbexigen/encoder_classes.py | 2 +- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index db39be82..7bf62d7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -xmlschema[codegen]~=4.1.0 +xmlschema[codegen]~=4.3.1 diff --git a/src/cbexigen/SchemaAnalyzer.py b/src/cbexigen/SchemaAnalyzer.py index 9de9384c..daaf8552 100644 --- a/src/cbexigen/SchemaAnalyzer.py +++ b/src/cbexigen/SchemaAnalyzer.py @@ -572,7 +572,7 @@ def __get_particle_list(self, element: XsdElement, subst_list): if self.__is_abstract(child): # get substituted particles for child qname = self.__get_name(child) - subst_group = self.__current_schema.substitution_groups._target_dict.get(qname) + subst_group = self.__current_schema.maps.substitution_groups.get(qname) if subst_group: for elem in subst_group: particle = self.__get_abstract_particle(child, elem) @@ -590,7 +590,7 @@ def __get_particle_list(self, element: XsdElement, subst_list): if self.__is_abstract_type(element): # get substituted particles for element qname = self.__get_name(element) - subst_group = self.__current_schema.substitution_groups._target_dict.get(qname) + subst_group = self.__current_schema.maps.substitution_groups.get(qname) if subst_group: for elem in subst_group: particle = self.__get_abstract_particle(element, elem) @@ -850,7 +850,7 @@ def __get_child_tree(self, element: XsdElement, level, recursive=True): msg_write((level + 1) * " " + "ABSTRACT TYPE is extension") qname = self.__get_name(child) - sg = self.__current_schema.substitution_groups._target_dict.get(qname) + sg = self.__current_schema.maps.substitution_groups.get(qname) if sg: for substitute in sg: substitute_type_name = self.__get_type_name(substitute) @@ -925,7 +925,7 @@ def analyze_schema_elements(self): self.__build_schema_builtin_types_list() if self.__is_iso20: - for element in self.__current_schema.elements._target_dict.values(): + for element in self.__current_schema.maps.elements.values(): if element.prefixed_name.startswith('xs:'): continue @@ -1010,7 +1010,7 @@ def analyze_schema_elements(self): def __build_schema_builtin_types_list(self): xs_namespace = self.__current_schema.namespaces['xs'] - for value in self.__current_schema.types._target_dict.values(): + for value in self.__current_schema.maps.types.values(): if value.target_namespace == xs_namespace: if value.__class__.__name__ == 'XsdAtomicBuiltin': if value.simple_type.base_type is not None: @@ -1063,7 +1063,7 @@ def __print_child_recursive(element_list, child_element: XsdElement): self.__known_fragments.clear() fragments = {} - for element in self.__current_schema.elements._target_dict.values(): + for element in self.__current_schema.maps.elements.values(): if element.default_namespace: if element.name not in fragments.keys(): fragments[element.name] = __get_fragment(element) @@ -1109,7 +1109,7 @@ def __build_namespace_element_lists(self): current_namespace = self.__current_schema.get_schema('') for ele in current_namespace.elements.values(): items = [] - for value in current_namespace.elements._target_dict.values(): + for value in current_namespace.maps.elements.values(): if value.default_namespace: name = self.__get_type_name_short(value) if name == '' or name in ['AnonType', 'string']: @@ -1151,7 +1151,7 @@ def __build_namespace_element_lists(self): def __build_generate_elements_types_list(self): xs_namespace = self.__current_schema.namespaces['xs'] type_list = [] - for value in self.__current_schema.types._target_dict.values(): + for value in self.__current_schema.maps.types.values(): if value.target_namespace != xs_namespace and value.content_type_label == 'element-only': type_list.append(value.local_name) @@ -1259,8 +1259,10 @@ def __replace_particle_list_in_parent(parent_element: ElementData, particle_list # drop all the particles listed in particle_list p_index: int for p_index, p in reversed(particle_list): - if parent_element.particles[p_index] != p: - log_write(f"particle '{p.name}' not found in '{parent_element.name_short}'") + if p_index >= len(parent_element.particles) or parent_element.particles[p_index] is not p: + raise RuntimeError( + f"particle '{p.name}' at index {p_index} does not match in " + f"'{parent_element.name_short}' -- particle list is stale") del parent_element.particles[p_index] # insert the replacements at the original position, assuming the lowest original particle @@ -1301,19 +1303,25 @@ def __copy_particles_from_empty_content_elements(self, element: ElementData, par else: log_write(f' Add to list and remove particle {particle.name}.') replacement_list.append(particle) - if particle in parent.particles: - particles_to_remove.append((parent.particles.index(particle), particle)) + # Use name-based lookup (not object identity) to find the + # matching particle in parent for removal. + for idx, pp in enumerate(parent.particles): + if pp.name == particle.name and (idx, pp) not in particles_to_remove: + particles_to_remove.append((idx, pp)) + break - for particle in parent.particles: + for idx, particle in enumerate(parent.particles): if particle.name != element.name_short: continue + if (idx, particle) in particles_to_remove: + continue log_write(f' Add to list and remove particle {particle.name}.') p_min_occurs = particle.min_occurs p_max_occurs = particle.max_occurs particle.min_occurs = 0 replacement_list.append(particle) - particles_to_remove.append((parent.particles.index(particle), particle)) + particles_to_remove.append((idx, particle)) if len(replacement_list) > 0: self.__replace_particle_list_in_parent(parent, particles_to_remove, replacement_list, @@ -1459,7 +1467,7 @@ def __scan_for_derived_and_extended_elements(self): def find_base_type(base_type_name): result = None - for item in self.__current_schema.elements._target_dict.values(): + for item in self.__current_schema.maps.elements.values(): if item.prefixed_name.startswith('xs:'): continue if item.type.base_type is None: diff --git a/src/cbexigen/datatype_classes.py b/src/cbexigen/datatype_classes.py index f1f59bc1..e669a8db 100644 --- a/src/cbexigen/datatype_classes.py +++ b/src/cbexigen/datatype_classes.py @@ -135,7 +135,7 @@ def __generate_functions_enum(self): comment = '// enum for function numbers' enum_type = self.parameters['prefix'] + 'generatedFunctionNumbersType' items = [] - for value in self.scheme.elements._target_dict.values(): + for value in self.scheme.maps.elements.values(): if value.default_namespace: items.append(self.parameters['prefix'] + value.local_name) items.sort() @@ -538,7 +538,7 @@ def generate_file(self): else: element_type = self.scheme.types.get(element.type_short) if element_type is None: - element_type = self.scheme.types._target_dict.get(element.type) + element_type = self.scheme.maps.types.get(element.type) if element_type is None: continue @@ -580,7 +580,7 @@ def generate_file(self): if skip_element: curr_idx += 1 - if curr_idx > len(self.__generate): + if curr_idx >= len(self.__generate): log_write_error('Module datatypes: Generator loop aborted! Index larger than existing elements.') break else: diff --git a/src/cbexigen/decoder_classes.py b/src/cbexigen/decoder_classes.py index e2dabc8d..1d08f552 100644 --- a/src/cbexigen/decoder_classes.py +++ b/src/cbexigen/decoder_classes.py @@ -973,7 +973,7 @@ def generate_file(self): if skip_element: curr_idx += 1 - if curr_idx > len(self.elements_to_generate): + if curr_idx >= len(self.elements_to_generate): log_write_error('Module decoder: Generator loop aborted! Index larger than existing elements.') break else: diff --git a/src/cbexigen/encoder_classes.py b/src/cbexigen/encoder_classes.py index 2f107619..6f08c089 100644 --- a/src/cbexigen/encoder_classes.py +++ b/src/cbexigen/encoder_classes.py @@ -969,7 +969,7 @@ def generate_file(self): if skip_element: curr_idx += 1 - if curr_idx > len(self.elements_to_generate): + if curr_idx >= len(self.elements_to_generate): log_write_error('Module encoder: Generator loop aborted! Index larger than existing elements.') break else: From d8b8857cce8f247484eaa6a8dc6b4218d58391da Mon Sep 17 00:00:00 2001 From: Christian Andersen Date: Sun, 22 Mar 2026 18:40:17 +0100 Subject: [PATCH 2/8] feat: add ISO-20 Amd1 and AC_DER_IEC schema support Signed-off-by: Christian Andersen --- src/cbexigen/tools_config.py | 49 +++++++++++++++++++++++++++-- src/config.py | 61 ++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/src/cbexigen/tools_config.py b/src/cbexigen/tools_config.py index c34ae7de..9f7570c1 100644 --- a/src/cbexigen/tools_config.py +++ b/src/cbexigen/tools_config.py @@ -4,12 +4,13 @@ """ Tools for the Exi Codegenerator config """ import importlib +import io +import urllib.request +import zipfile from typing import Union, Dict from pathlib import Path -import urllib.request - CONFIG_ARGS: Dict[str, Union[str, Path]] = { 'program_dir': '', 'config_file': '', @@ -189,6 +190,8 @@ def process_config_parameters(): ISO2_SCHEMAS_URL = "https://standards.iso.org/iso/15118/-2/ed-2/en/" ISO20_SCHEMAS_URL = "https://standards.iso.org/iso/15118/-20/ed-1/en/" +ISO20_AMD1_SCHEMAS_URL = "https://standards.iso.org/iso/15118/-20/ed-1/en/Amd/1/" +ISO20_AMD1_SCHEMAS_ZIP = "AMD1_xsdSchema.zip" def download_schemas(): @@ -237,3 +240,45 @@ def download_schemas(): print(f"Error during downloading: {err=}, {type(err)=}") else: print(f"ISO15118-20 schema {schema} is already there. Skipping it.") + + iso20_amd1_schema_files_names = ['V2G_CI_AC_DER_IEC.xsd', 'V2G_CI_AC_DER_SAE.xsd'] + amd1_needs_zip_fallback = False + + for schema in iso20_amd1_schema_files_names: + schema_file_path = iso20_schema_path / schema + if not schema_file_path.exists(): + print(f"ISO15118-20 Amd1 schema {schema} not found! Downloading it...") + try: + urllib.request.urlretrieve( + ISO20_AMD1_SCHEMAS_URL + schema, schema_file_path.absolute().as_posix()) + except Exception as err: + print(f"Direct download failed for {schema}: {err=}, {type(err)=}") + if schema_file_path.exists(): + schema_file_path.unlink() + amd1_needs_zip_fallback = True + break + else: + print(f"ISO15118-20 Amd1 schema {schema} is already there. Skipping it.") + + if amd1_needs_zip_fallback: + print(f"Trying zip fallback: downloading {ISO20_AMD1_SCHEMAS_ZIP}...") + try: + zip_url = ISO20_AMD1_SCHEMAS_URL + ISO20_AMD1_SCHEMAS_ZIP + with urllib.request.urlopen(zip_url) as response: + zip_data = io.BytesIO(response.read()) + with zipfile.ZipFile(zip_data) as zf: + for schema in iso20_amd1_schema_files_names: + schema_file_path = iso20_schema_path / schema + if schema_file_path.exists(): + print(f"ISO15118-20 Amd1 schema {schema} is already there. Skipping it.") + continue + # find the file in the zip (may be in a subdirectory) + matching = [n for n in zf.namelist() if n.endswith(schema)] + if matching: + with zf.open(matching[0]) as src, open(schema_file_path, 'wb') as dst: + dst.write(src.read()) + print(f"Extracted {schema} from {ISO20_AMD1_SCHEMAS_ZIP}.") + else: + print(f"Error: {schema} not found in {ISO20_AMD1_SCHEMAS_ZIP}.") + except Exception as err: + print(f"Error during zip download: {err=}, {type(err)=}") diff --git a/src/config.py b/src/config.py index 65884f4b..79968910 100644 --- a/src/config.py +++ b/src/config.py @@ -120,6 +120,10 @@ 'SignedInfo', 'DC_ChargeParameterDiscoveryRes', ] +iso20_ac_der_iec_fragments = [ + 'SignedInfo', + "AC_ChargeParameterDiscoveryRes", +] # general C code style c_code_indent_chars = 4 @@ -726,4 +730,61 @@ 'iso20_ACDP_Datatypes.h', 'iso20_ACDP_Encoder.h'] } }, + 'iso20_AC_DER_IEC_Datatypes': { + 'schema': 'ISO_15118-20/FDIS/V2G_CI_AC_DER_IEC.xsd', + 'prefix': 'iso20_ac_der_iec_', + 'type': 'converter', + 'folder': 'iso-20', + 'h': { + 'filename': 'iso20_AC_DER_IEC_Datatypes.h', + 'identifier': 'ISO20_AC_DER_IEC_DATATYPES_H', + 'include_std_lib': ['stdint.h'], + 'include_other': ['exi_basetypes.h'] + }, + 'c': { + 'filename': 'iso20_AC_DER_IEC_Datatypes.c', + 'identifier': 'ISO20_AC_DER_IEC_DATATYPES_C', + 'include_std_lib': [], + 'include_other': ['iso20_AC_DER_IEC_Datatypes.h'] + } + }, + 'iso20_AC_DER_IEC_Decoder': { + 'schema': 'ISO_15118-20/FDIS/V2G_CI_AC_DER_IEC.xsd', + 'prefix': 'iso20_ac_der_iec_', + 'type': 'decoder', + 'folder': 'iso-20', + 'h': { + 'filename': 'iso20_AC_DER_IEC_Decoder.h', + 'identifier': 'ISO20_AC_DER_IEC_DECODER_H', + 'include_std_lib': [], + 'include_other': ['exi_bitstream.h', 'iso20_AC_DER_IEC_Datatypes.h'] + }, + 'c': { + 'filename': 'iso20_AC_DER_IEC_Decoder.c', + 'identifier': 'ISO20_AC_DER_IEC_DECODER_C', + 'include_std_lib': ['stdint.h'], + 'include_other': ['exi_basetypes.h', 'exi_types_decoder.h', 'exi_basetypes_decoder.h', + 'exi_error_codes.h', 'exi_header.h', 'iso20_AC_DER_IEC_Datatypes.h', + 'iso20_AC_DER_IEC_Decoder.h'] + } + }, + 'iso20_AC_DER_IEC_Encoder': { + 'schema': 'ISO_15118-20/FDIS/V2G_CI_AC_DER_IEC.xsd', + 'prefix': 'iso20_ac_der_iec_', + 'type': 'encoder', + 'folder': 'iso-20', + 'h': { + 'filename': 'iso20_AC_DER_IEC_Encoder.h', + 'identifier': 'ISO20_AC_DER_IEC_ENCODER_H', + 'include_std_lib': [], + 'include_other': ['exi_bitstream.h', 'iso20_AC_DER_IEC_Datatypes.h'] + }, + 'c': { + 'filename': 'iso20_AC_DER_IEC_Encoder.c', + 'identifier': 'ISO20_AC_DER_IEC_ENCODER_C', + 'include_std_lib': ['stdint.h'], + 'include_other': ['exi_basetypes.h', 'exi_basetypes_encoder.h', 'exi_error_codes.h', 'exi_header.h', + 'iso20_AC_DER_IEC_Datatypes.h', 'iso20_AC_DER_IEC_Encoder.h'] + } + }, } From 3c733a33ff954eaf197d5e443dd712334b307fef Mon Sep 17 00:00:00 2001 From: Christian Andersen Date: Wed, 1 Apr 2026 01:04:55 +0200 Subject: [PATCH 3/8] fix: validate URL scheme and deduplicate schema download loops Signed-off-by: Christian Andersen --- src/cbexigen/tools_config.py | 84 +++++++++++++++--------------------- 1 file changed, 34 insertions(+), 50 deletions(-) diff --git a/src/cbexigen/tools_config.py b/src/cbexigen/tools_config.py index 9f7570c1..48834014 100644 --- a/src/cbexigen/tools_config.py +++ b/src/cbexigen/tools_config.py @@ -194,6 +194,32 @@ def process_config_parameters(): ISO20_AMD1_SCHEMAS_ZIP = "AMD1_xsdSchema.zip" +def _validate_https_url(url: str) -> str: + if not url.startswith("https://"): + raise ValueError(f"Only https URLs are allowed, got: {url}") + return url + + +def _download_schema_files(base_url: str, schema_names: list, target_path: Path, + label: str) -> bool: + """Download schema files. Returns False if any download failed.""" + for schema in schema_names: + schema_file_path = target_path / schema + if schema_file_path.exists(): + print(f"{label} schema {schema} is already there. Skipping it.") + continue + print(f"{label} schema {schema} not found! Downloading it...") + try: + urllib.request.urlretrieve( + _validate_https_url(base_url + schema), schema_file_path.absolute().as_posix()) + except Exception as err: + print(f"Download failed for {schema}: {err=}, {type(err)=}") + if schema_file_path.exists(): + schema_file_path.unlink() + return False + return True + + def download_schemas(): config_module = get_config_module() @@ -205,11 +231,8 @@ def download_schemas(): CONFIG_ARGS['schema_base_dir'], config_module.c_files_to_generate['iso20_CommonMessages_Datatypes']['schema']) iso20_schema_path = iso20_schema_full_name.parent.resolve() - if not iso2_schema_path.exists(): - iso2_schema_path.mkdir(parents=True, exist_ok=True) - - if not iso20_schema_path.exists(): - iso20_schema_path.mkdir(parents=True, exist_ok=True) + iso2_schema_path.mkdir(parents=True, exist_ok=True) + iso20_schema_path.mkdir(parents=True, exist_ok=True) iso2_schema_files_names = ['V2G_CI_AppProtocol.xsd', 'V2G_CI_MsgDef.xsd', 'V2G_CI_MsgBody.xsd', 'V2G_CI_MsgDataTypes.xsd', 'V2G_CI_MsgHeader.xsd', 'xmldsig-core-schema.xsd'] @@ -217,54 +240,16 @@ def download_schemas(): iso20_schema_files_names = ['V2G_CI_AC.xsd', 'V2G_CI_ACDP.xsd', 'V2G_CI_AppProtocol.xsd', 'V2G_CI_CommonMessages.xsd', 'V2G_CI_CommonTypes.xsd', 'V2G_CI_DC.xsd', 'V2G_CI_WPT.xsd', 'xmldsig-core-schema.xsd'] - for schema in iso2_schema_files_names: - schema_file_path = iso2_schema_path / schema - if not schema_file_path.exists(): - print(f"ISO15118-2 schema {schema} not found! Downloading it...") - try: - urllib.request.urlretrieve( - ISO2_SCHEMAS_URL + schema, schema_file_path.absolute().as_posix()) - except Exception as err: - print(f"Error during downloading: {err=}, {type(err)=}") - else: - print(f"ISO15118-2 schema {schema} is already there. Skipping it.") - - for schema in iso20_schema_files_names: - schema_file_path = iso20_schema_path / schema - if not schema_file_path.exists(): - print(f"ISO15118-20 schema {schema} not found! Downloading it...") - try: - urllib.request.urlretrieve( - ISO20_SCHEMAS_URL + schema, schema_file_path.absolute().as_posix()) - except Exception as err: - print(f"Error during downloading: {err=}, {type(err)=}") - else: - print(f"ISO15118-20 schema {schema} is already there. Skipping it.") + _download_schema_files(ISO2_SCHEMAS_URL, iso2_schema_files_names, iso2_schema_path, "ISO15118-2") + _download_schema_files(ISO20_SCHEMAS_URL, iso20_schema_files_names, iso20_schema_path, "ISO15118-20") iso20_amd1_schema_files_names = ['V2G_CI_AC_DER_IEC.xsd', 'V2G_CI_AC_DER_SAE.xsd'] - amd1_needs_zip_fallback = False - - for schema in iso20_amd1_schema_files_names: - schema_file_path = iso20_schema_path / schema - if not schema_file_path.exists(): - print(f"ISO15118-20 Amd1 schema {schema} not found! Downloading it...") - try: - urllib.request.urlretrieve( - ISO20_AMD1_SCHEMAS_URL + schema, schema_file_path.absolute().as_posix()) - except Exception as err: - print(f"Direct download failed for {schema}: {err=}, {type(err)=}") - if schema_file_path.exists(): - schema_file_path.unlink() - amd1_needs_zip_fallback = True - break - else: - print(f"ISO15118-20 Amd1 schema {schema} is already there. Skipping it.") - - if amd1_needs_zip_fallback: + if not _download_schema_files(ISO20_AMD1_SCHEMAS_URL, iso20_amd1_schema_files_names, + iso20_schema_path, "ISO15118-20 Amd1"): print(f"Trying zip fallback: downloading {ISO20_AMD1_SCHEMAS_ZIP}...") try: - zip_url = ISO20_AMD1_SCHEMAS_URL + ISO20_AMD1_SCHEMAS_ZIP - with urllib.request.urlopen(zip_url) as response: + with urllib.request.urlopen( # nosec B310 # nosemgrep + _validate_https_url(ISO20_AMD1_SCHEMAS_URL + ISO20_AMD1_SCHEMAS_ZIP)) as response: zip_data = io.BytesIO(response.read()) with zipfile.ZipFile(zip_data) as zf: for schema in iso20_amd1_schema_files_names: @@ -272,7 +257,6 @@ def download_schemas(): if schema_file_path.exists(): print(f"ISO15118-20 Amd1 schema {schema} is already there. Skipping it.") continue - # find the file in the zip (may be in a subdirectory) matching = [n for n in zf.namelist() if n.endswith(schema)] if matching: with zf.open(matching[0]) as src, open(schema_file_path, 'wb') as dst: From 0ee8a0479d9d49e5a2b5a8b774250ca82b99a00a Mon Sep 17 00:00:00 2001 From: Christian Andersen Date: Sun, 22 Mar 2026 04:49:02 +0100 Subject: [PATCH 4/8] fix: namespace import particle assignment for derived/substitution group types Guard namespace particle replacement to only apply for empty/abstract container types per EXI 8.5.4.1.3.2 (concrete types build grammars from their own content model). Collect all derived types for substitution group expansion per EXI 8.5.4.1.6 (set S must include all reachable element declarations, not just the first). Signed-off-by: Christian Andersen --- src/cbexigen/SchemaAnalyzer.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/cbexigen/SchemaAnalyzer.py b/src/cbexigen/SchemaAnalyzer.py index daaf8552..02996c46 100644 --- a/src/cbexigen/SchemaAnalyzer.py +++ b/src/cbexigen/SchemaAnalyzer.py @@ -1143,6 +1143,10 @@ def __build_namespace_element_lists(self): for gen_elem in self.__generate_elements: if gen_elem.type == '{' + imp.default_namespace + '}' + name: + # Only replace particles for empty/abstract container types. + # Concrete types (with their own fields) must keep their particles. + if gen_elem.content_type != 'empty' and len(gen_elem.particles) > 0: + continue gen_elem.particles = items gen_elem.is_in_namespace_elements = True self.__namespace_elements[name] = items @@ -1465,8 +1469,8 @@ def __scan_for_derived_and_extended_elements(self): log_write('') log_write('Scan for derived and extended elements') - def find_base_type(base_type_name): - result = None + def find_base_types(base_type_name): + results = [] for item in self.__current_schema.maps.elements.values(): if item.prefixed_name.startswith('xs:'): continue @@ -1474,10 +1478,9 @@ def find_base_type(base_type_name): continue if item.type.base_type.local_name == base_type_name: - result = item - break + results.append(item) - return result + return results element: ElementData particle: Particle @@ -1491,20 +1494,20 @@ def find_base_type(base_type_name): list_with_missing.append(particle) log_write(f' Adding abstract particle {particle.name} to missing list.') - # get the base type, add it as particle - missing_element = find_base_type(particle.type_short) - if missing_element is not None: - part = self.__get_particle(missing_element) - + # get all elements derived from this type, add them as particles + missing_elements = find_base_types(particle.type_short) + if len(missing_elements) > 0: particle.min_occurs_old = particle.min_occurs particle.min_occurs = 0 list_with_missing.append(particle) log_write(f' Adding particle {particle.name} to missing list.') - part.min_occurs_old = part.min_occurs - part.min_occurs = 0 - list_with_missing.append(part) - log_write(f' Adding missing particle {part.name} to missing list.') + for missing_element in missing_elements: + part = self.__get_particle(missing_element) + part.min_occurs_old = part.min_occurs + part.min_occurs = 0 + list_with_missing.append(part) + log_write(f' Adding missing particle {part.name} to missing list.') if len(list_with_missing) > 0: first = -1 From dfea3a4e0ffdec4bc63606ef8ea0bf305cec526e Mon Sep 17 00:00:00 2001 From: Christian Andersen Date: Thu, 26 Mar 2026 20:41:19 +0100 Subject: [PATCH 5/8] fix: skip substitution group heads in namespace import particle assignment Their grammar productions come from the substitution group closure (EXI 8.5.4.1.6), not from namespace imports. Signed-off-by: Christian Andersen --- src/cbexigen/SchemaAnalyzer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cbexigen/SchemaAnalyzer.py b/src/cbexigen/SchemaAnalyzer.py index 02996c46..5dd7e286 100644 --- a/src/cbexigen/SchemaAnalyzer.py +++ b/src/cbexigen/SchemaAnalyzer.py @@ -1147,6 +1147,10 @@ def __build_namespace_element_lists(self): # Concrete types (with their own fields) must keep their particles. if gen_elem.content_type != 'empty' and len(gen_elem.particles) > 0: continue + # Skip substitution group heads – their particles come from + # the substitution group expansion, not from namespace imports. + if self.__current_schema.maps.substitution_groups.get(gen_elem.name): + continue gen_elem.particles = items gen_elem.is_in_namespace_elements = True self.__namespace_elements[name] = items From b83be728ff8f9367ca4ed58ed3dc0d19319abae2 Mon Sep 17 00:00:00 2001 From: Christian Andersen Date: Thu, 26 Mar 2026 19:52:59 +0100 Subject: [PATCH 6/8] fix: namespace-aware particle sorting per EXI 8.5.4.1.6 Sort replacement particles by (name, namespace) instead of name only. Populate namespace on all particle factories and newly created replacement particles so the sort key is consistent. Signed-off-by: Christian Andersen --- src/cbexigen/SchemaAnalyzer.py | 28 ++++++++++++++++++++++++++-- src/cbexigen/elementData.py | 1 + src/cbexigen/tools.py | 7 +++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/cbexigen/SchemaAnalyzer.py b/src/cbexigen/SchemaAnalyzer.py index 5dd7e286..c1a99d31 100644 --- a/src/cbexigen/SchemaAnalyzer.py +++ b/src/cbexigen/SchemaAnalyzer.py @@ -329,6 +329,12 @@ def __get_particle_from_attribute(self, attribute: XsdAttribute): particle.name = attribute.local_name + ns = tools.extract_namespace_uri(getattr(attribute, 'name', '')) + if ns: + particle.namespace = ns + elif hasattr(attribute, 'default_namespace') and attribute.default_namespace: + particle.namespace = attribute.default_namespace + particle.type = self.__get_type_name(attribute) particle.type_short = self.__get_type_name_short(attribute) particle.base_type = self.__get_base_type_name(attribute) @@ -393,6 +399,12 @@ def __get_particle(self, element: XsdElement): particle.name = element.local_name + ns = tools.extract_namespace_uri(element.name) + if ns: + particle.namespace = ns + elif hasattr(element, 'default_namespace') and element.default_namespace: + particle.namespace = element.default_namespace + particle.type = self.__get_type_name(element) particle.type_short = self.__get_type_name_short(element) particle.base_type = self.__get_base_type_name(element) @@ -456,6 +468,12 @@ def __get_abstract_particle(self, element: XsdElement, substitute: XsdElement): particle.name = substitute.local_name + ns = tools.extract_namespace_uri(substitute.name) + if ns: + particle.namespace = ns + elif hasattr(substitute, 'default_namespace') and substitute.default_namespace: + particle.namespace = substitute.default_namespace + particle.type = self.__get_type_name(substitute) particle.type_short = self.__get_type_name_short(substitute) particle.base_type = self.__get_base_type_name(substitute) @@ -1252,7 +1270,7 @@ def __replace_particle_list_in_parent(parent_element: ElementData, particle_list with the sorted particles from replacement_list. """ # the replacements need to be sorted alphabetically - replacement_list.sort(key=lambda particle_key: particle_key.name) + replacement_list.sort(key=lambda p: (p.name or '', p.namespace or '')) # the particles need to be sorted by index, for proper removal particle_list.sort(key=lambda x: x[0]) @@ -1371,6 +1389,9 @@ def __copy_particles_from_empty_content_elements_particle(self, element: Element abstract=True, min_occurs=0, max_occurs=1) + ns = tools.extract_namespace_uri(element.name) + if ns: + part_new.namespace = ns replacement_list.append(part_new) self.__replace_particle_list_in_parent(parent, particles_to_remove, replacement_list, @@ -1456,10 +1477,13 @@ def __scan_elements_for_empty_content(self): type_short=element.type_short, min_occurs=0, max_occurs=1) + ns = tools.extract_namespace_uri(element.name) + if ns: + part.namespace = ns re_list.append(part) if len(re_list) > 0: - re_list.sort(key=lambda particle_key: particle_key.name) + re_list.sort(key=lambda p: (p.name or '', p.namespace or '')) abstract_seq = [] for part in re_list: log_write(f' Add particle from list {part.name}.') diff --git a/src/cbexigen/elementData.py b/src/cbexigen/elementData.py index 15c737d0..7c19b5f7 100644 --- a/src/cbexigen/elementData.py +++ b/src/cbexigen/elementData.py @@ -54,6 +54,7 @@ class Particle: integer_bit_size: int = -1 integer_base_type: str = None integer_is_unsigned: bool = False + namespace: str = '' # additional info for the anyType particle process_content: str = None diff --git a/src/cbexigen/tools.py b/src/cbexigen/tools.py index f82a9e5f..c280c3e7 100644 --- a/src/cbexigen/tools.py +++ b/src/cbexigen/tools.py @@ -76,6 +76,13 @@ def get_indent(level: int = 1): return level * (' ' * CONFIG_PARAMS['c_code_indent_chars']) +def extract_namespace_uri(qualified_name: str) -> str: + """Extract namespace URI from XSD qualified name like '{uri}localName'.""" + if qualified_name and qualified_name.startswith('{') and '}' in qualified_name: + return qualified_name[1:qualified_name.index('}')] + return '' + + ''' generator tools ''' From a85cf3e3c23a118913a299c2ea79c19276ec20db Mon Sep 17 00:00:00 2001 From: Christian Andersen Date: Thu, 26 Mar 2026 20:41:24 +0100 Subject: [PATCH 7/8] refactor: pre-build base type lookup dict in derived element scan Signed-off-by: Christian Andersen --- src/cbexigen/SchemaAnalyzer.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/cbexigen/SchemaAnalyzer.py b/src/cbexigen/SchemaAnalyzer.py index c1a99d31..d7afdd28 100644 --- a/src/cbexigen/SchemaAnalyzer.py +++ b/src/cbexigen/SchemaAnalyzer.py @@ -1497,18 +1497,15 @@ def __scan_for_derived_and_extended_elements(self): log_write('') log_write('Scan for derived and extended elements') - def find_base_types(base_type_name): - results = [] - for item in self.__current_schema.maps.elements.values(): - if item.prefixed_name.startswith('xs:'): - continue - if item.type.base_type is None: - continue - - if item.type.base_type.local_name == base_type_name: - results.append(item) - - return results + # Pre-build lookup: base_type_name -> [elements derived from it] + base_type_map = {} + for item in self.__current_schema.maps.elements.values(): + if item.prefixed_name.startswith('xs:'): + continue + if item.type.base_type is None: + continue + key = item.type.base_type.local_name + base_type_map.setdefault(key, []).append(item) element: ElementData particle: Particle @@ -1523,7 +1520,7 @@ def find_base_types(base_type_name): log_write(f' Adding abstract particle {particle.name} to missing list.') # get all elements derived from this type, add them as particles - missing_elements = find_base_types(particle.type_short) + missing_elements = base_type_map.get(particle.type_short, []) if len(missing_elements) > 0: particle.min_occurs_old = particle.min_occurs particle.min_occurs = 0 From 68c5ce0e43464bd763d091ffffa4f0dbc6acb3ee Mon Sep 17 00:00:00 2001 From: Christian Andersen Date: Fri, 22 May 2026 13:19:37 +0200 Subject: [PATCH 8/8] fix: handle mandatory repeated particles in decoder grammar Treat particles with minOccurs greater than one as mandatory until all required occurrences are consumed. This prevents generated decoder grammars from allowing END or LOOP events before mandatory array entries, which can misalign EXI event-code decoding. --- src/cbexigen/base_coder_classes.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/cbexigen/base_coder_classes.py b/src/cbexigen/base_coder_classes.py index bd4d9c24..649439e4 100644 --- a/src/cbexigen/base_coder_classes.py +++ b/src/cbexigen/base_coder_classes.py @@ -444,7 +444,7 @@ def generate_element_grammars(self, element: ElementData): choice_options = self.ChoiceOptions(element, particle) combined_min_occurs_from_choice = \ choice_options.min_occurs if choice_options.particles else particle.min_occurs - if combined_min_occurs_from_choice == 1: + if combined_min_occurs_from_choice >= 1: index_last_nonoptional_particle = particle_index def _particle_is_in_choice(element: ElementData, particle: Particle): @@ -631,11 +631,10 @@ def _add_particle_or_choice_list_to_details( if m < _max: # flag=Grammar.LOOP indicates that the _next_ grammar will be the same as this one # - # Note: We do not loop on the mandatory elements of an array, it would require extra - # logic, and we have no min_occurs larger than 2 (i.e. no large repetition of mandatory - # elements) anyway in our existing standards. - - if m > 1 and m > part.min_occurs - 1 and (part.max_occurs_old is None or m < part.max_occurs_old): + # Do not enter the self-loop until all mandatory occurrences have been consumed. + # Before minOccurs is reached, the grammar must only accept another START event; + # allowing END or LOOP too early changes the event-code width and misaligns EXI. + if m > 1 and m > part.min_occurs and (part.max_occurs_old is None or m < part.max_occurs_old): flag = GrammarFlag.LOOP skip_to_end = True else: