diff --git a/cove_ocds/lib/__init__.py b/cove_ocds/lib/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cove_ocds/lib/exceptions.py b/cove_ocds/lib/exceptions.py deleted file mode 100644 index c174861f..00000000 --- a/cove_ocds/lib/exceptions.py +++ /dev/null @@ -1,94 +0,0 @@ -from django.utils.functional import lazy -from django.utils.html import format_html, mark_safe -from django.utils.translation import gettext as _ -from libcove.lib.exceptions import CoveInputDataError - -mark_safe_lazy = lazy(mark_safe, str) - - -def raise_invalid_version_argument(version): - raise CoveInputDataError( - context={ - "sub_title": _("Unrecognised version of the schema"), - "link": "index", - "link_text": _("Try Again"), - "msg": _( - format_html( - "We think you tried to run your data against an unrecognised version of " - 'the schema.\n\n Error message: {} is ' - "not a recognised choice for the schema version", - version, - ) - ), - "error": _("%(version)s is not a known schema version") % {"version": version}, - } - ) - - -def raise_invalid_version_data_with_patch(version): - raise CoveInputDataError( - context={ - "sub_title": _("Version format does not comply with the schema"), - "link": "index", - "link_text": _("Try Again"), - "msg": _( - format_html( - 'The value for the "version" field in your data follows the ' - "major.minor.patch pattern but according to the schema the patch digit " - 'shouldn\'t be included (e.g. "1.1.0" should appear as "1.1" in ' - "your data as this tool always uses the latest patch release for a major.minor " - 'version).\n\nPlease get rid of the patch digit and try again.\n\n Error message: ' - " {} format does not comply with the schema", - version, - ) - ), - "error": _("%(version)s is not a known schema version") % {"version": version}, - } - ) - - -def raise_json_deref_error(error): - raise CoveInputDataError( - context={ - "sub_title": _("JSON reference error"), - "link": "index", - "link_text": _("Try Again"), - "msg": _( - format_html( - "We have detected a JSON reference error in the schema. This may be " - " due to some extension trying to resolve non-existing references. " - '\n\n Error message: {}", - error, - ) - ), - "error": _("%(error)s") % {"error": error}, - } - ) - - -def raise_missing_package_error(): - raise CoveInputDataError( - context={ - "sub_title": _("Missing OCDS package"), - "link": "index", - "link_text": _("Try Again"), - "msg": mark_safe_lazy( - _( - "We could not detect a package structure at the top-level of your data. " - 'OCDS releases and records should be published within a release ' - 'package or record package to provide important meta-' - 'data. For more information, please refer to the ' - "Releases and Records section in the OCDS documentation.\n\n ' - "Error message: Missing OCDS package" - ) - ), - "error": _("Missing OCDS package"), - } - ) diff --git a/cove_ocds/lib/ocds_show_extra.py b/cove_ocds/lib/ocds_show_extra.py deleted file mode 100644 index 9f6ac30b..00000000 --- a/cove_ocds/lib/ocds_show_extra.py +++ /dev/null @@ -1,41 +0,0 @@ -from libcove.lib.common import schema_dict_fields_generator - - -def add_extra_fields(data, deref_release_schema): - all_schema_fields = set(schema_dict_fields_generator(deref_release_schema)) - - if "releases" in data: - for release in data.get("releases", []): - if not isinstance(release, dict): - return - add_extra_fields_to_obj(release, all_schema_fields, "") - elif "records" in data: - for record in data.get("records", []): - if not isinstance(record, dict): - return - for release in record.get("releases", []): - add_extra_fields_to_obj(release, all_schema_fields, "") - - -def add_extra_fields_to_obj(obj, all_schema_fields, current_path): - if not isinstance(obj, dict): - return - obj["__extra"] = {} - - for key, value in list(obj.items()): - if key == "__extra": - continue - - new_path = f"{current_path}/{key}" - if new_path not in all_schema_fields: - obj["__extra"][key] = value - continue - - if isinstance(value, list): - for item in value: - add_extra_fields_to_obj(item, all_schema_fields, new_path) - elif isinstance(value, dict): - add_extra_fields_to_obj(value, all_schema_fields, new_path) - - if not obj["__extra"]: - obj.pop("__extra") diff --git a/cove_ocds/lib/views.py b/cove_ocds/lib/views.py deleted file mode 100644 index 7e90b51b..00000000 --- a/cove_ocds/lib/views.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -from collections import defaultdict - - -def group_validation_errors(validation_errors): - validation_errors_grouped = defaultdict(list) - for error_json, values in validation_errors: - error = json.loads(error_json) - if error["message_type"] == "required": - validation_errors_grouped["required"].append((error_json, values)) - elif error["message_type"] in { - "format", - "pattern", - "number", - "string", - "date-time", - "uri", - "object", - "integer", - "array", - }: - validation_errors_grouped["format"].append((error_json, values)) - else: - validation_errors_grouped["other"].append((error_json, values)) - return validation_errors_grouped diff --git a/cove_ocds/locale/en/LC_MESSAGES/django.po b/cove_ocds/locale/en/LC_MESSAGES/django.po index a358d352..5aa80f75 100644 --- a/cove_ocds/locale/en/LC_MESSAGES/django.po +++ b/cove_ocds/locale/en/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-10-10 00:56+0000\n" +"POT-Creation-Date: 2024-10-19 04:51+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -17,54 +17,6 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -#: cove_ocds/lib/exceptions.py:12 -msgid "Unrecognised version of the schema" -msgstr "" - -#: cove_ocds/lib/exceptions.py:14 cove_ocds/lib/exceptions.py:34 -#: cove_ocds/lib/exceptions.py:57 cove_ocds/lib/exceptions.py:77 -#: cove_ocds/views.py:90 cove_ocds/views.py:108 cove_ocds/views.py:126 -#: cove_ocds/views.py:237 -msgid "Try Again" -msgstr "" - -#: cove_ocds/lib/exceptions.py:24 cove_ocds/lib/exceptions.py:47 -#, python-format -msgid "%(version)s is not a known schema version" -msgstr "" - -#: cove_ocds/lib/exceptions.py:32 -msgid "Version format does not comply with the schema" -msgstr "" - -#: cove_ocds/lib/exceptions.py:55 -msgid "JSON reference error" -msgstr "" - -#: cove_ocds/lib/exceptions.py:67 -#, python-format -msgid "%(error)s" -msgstr "" - -#: cove_ocds/lib/exceptions.py:75 cove_ocds/lib/exceptions.py:92 -msgid "Missing OCDS package" -msgstr "" - -#: cove_ocds/lib/exceptions.py:80 -msgid "" -"We could not detect a package structure at the top-level of your data. OCDS " -"releases and records should be published within a release package or record package to provide important meta-data. For " -"more information, please refer to the Releases and " -"Records section in the OCDS documentation.\n" -"\n" -" Error message: Missing OCDS package" -msgstr "" - #: cove_ocds/templates/cove_ocds/additional_checks_table.html:7 msgid "Check Description" msgstr "" @@ -1403,30 +1355,18 @@ msgid "" "reference/#date\">dates in OCDS." msgstr "" -#: cove_ocds/views.py:50 -msgid "Sorry, the page you are looking for is not available" -msgstr "" - -#: cove_ocds/views.py:52 -msgid "Go to Home page" -msgstr "" - -#: cove_ocds/views.py:55 -#, python-format -msgid "" -"The data you were hoping to explore no longer exists.\n" -"\n" -"This is because all data supplied to this website is automatically deleted " -"after %s days, and therefore the analysis of that data is no longer " -"available." +#: cove_ocds/util.py:20 cove_ocds/util.py:39 cove_ocds/util.py:57 +#: cove_ocds/views.py:112 +msgid "Sorry, we can't process that data" msgstr "" -#: cove_ocds/views.py:88 cove_ocds/views.py:106 cove_ocds/views.py:124 -#: cove_ocds/views.py:235 -msgid "Sorry, we can't process that data" +#: cove_ocds/util.py:23 cove_ocds/util.py:42 cove_ocds/util.py:59 +#: cove_ocds/util.py:85 cove_ocds/util.py:108 cove_ocds/util.py:127 +#: cove_ocds/views.py:114 cove_ocds/views.py:149 +msgid "Try Again" msgstr "" -#: cove_ocds/views.py:93 +#: cove_ocds/util.py:26 msgid "" "The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON " "follows the I-JSON format, which requires UTF-8 encoding. Ensure that your " @@ -1436,7 +1376,7 @@ msgid "" "span> Error message: {}" msgstr "" -#: cove_ocds/views.py:111 +#: cove_ocds/util.py:45 msgid "" "We think you tried to upload a JSON file, but it is not well formed JSON.\n" "\n" @@ -1444,13 +1384,75 @@ msgid "" "span> Error message: {}" msgstr "" -#: cove_ocds/views.py:127 +#: cove_ocds/util.py:60 msgid "" "OCDS JSON should have an object as the top level, the JSON you supplied does " "not." msgstr "" -#: cove_ocds/views.py:240 +#: cove_ocds/util.py:82 cove_ocds/util.py:83 +msgid "Missing OCDS package" +msgstr "" + +#: cove_ocds/util.py:88 +msgid "" +"We could not detect a package structure at the top-level of your data. OCDS " +"releases and records should be published within a release package or record package to provide important meta-data. For " +"more information, please refer to the Releases and " +"Records section in the OCDS documentation.\n" +"\n" +" Error message: Missing OCDS package" +msgstr "" + +#: cove_ocds/util.py:105 +msgid "Unrecognised version of the schema" +msgstr "" + +#: cove_ocds/util.py:106 cove_ocds/util.py:125 +#, python-format +msgid "%(version)s is not a known schema version" +msgstr "" + +#: cove_ocds/util.py:111 +msgid "" +"We think you tried to run your data against an unrecognised version of the " +"schema.\n" +"\n" +" Error message: {} is not a recognised choice " +"for the schema version" +msgstr "" + +#: cove_ocds/util.py:124 +msgid "Version format does not comply with the schema" +msgstr "" + +#: cove_ocds/util.py:130 +msgid "" +"The value for the \"version\" field in your data follows the " +"major.minor.patch pattern but according to the schema the patch " +"digit shouldn't be included (e.g. \"1.1.0\" should appear as " +"\"1.1\" in your data as this tool always uses the latest patch " +"release for a major.minor version).\n" +"\n" +"Please get rid of the patch digit and try again.\n" +"\n" +" Error message: {} format does not comply " +"with the schema" +msgstr "" + +#: cove_ocds/util.py:144 +#, python-format +msgid "%(version)s (not a string)" +msgstr "" + +#: cove_ocds/views.py:118 msgid "" "The table isn't structured correctly. For example, a JSON Pointer " "(tender) can't be both a value (tender), a path to " @@ -1460,3 +1462,12 @@ msgid "" " Error message: {}" msgstr "" + +#: cove_ocds/views.py:147 +msgid "JSON reference error" +msgstr "" + +#: cove_ocds/views.py:159 +#, python-format +msgid "%(error)s" +msgstr "" diff --git a/cove_ocds/locale/es/LC_MESSAGES/django.po b/cove_ocds/locale/es/LC_MESSAGES/django.po index 0e60dad3..3643c53b 100644 --- a/cove_ocds/locale/es/LC_MESSAGES/django.po +++ b/cove_ocds/locale/es/LC_MESSAGES/django.po @@ -15,7 +15,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-10-10 00:56+0000\n" +"POT-Creation-Date: 2024-10-19 04:51+0000\n" "PO-Revision-Date: 2020-09-08 08:53+0000\n" "Last-Translator: James McKinney, 2024\n" "Language-Team: Spanish (https://app.transifex.com/open-contracting-" @@ -27,66 +27,6 @@ msgstr "" "Plural-Forms: nplurals=3; plural=n == 1 ? 0 : n != 0 && n % 1000000 == 0 ? " "1 : 2;\n" -#: cove_ocds/lib/exceptions.py:12 -msgid "Unrecognised version of the schema" -msgstr "Versión del esquema no reconocida" - -#: cove_ocds/lib/exceptions.py:14 cove_ocds/lib/exceptions.py:34 -#: cove_ocds/lib/exceptions.py:57 cove_ocds/lib/exceptions.py:77 -#: cove_ocds/views.py:90 cove_ocds/views.py:108 cove_ocds/views.py:126 -#: cove_ocds/views.py:237 -msgid "Try Again" -msgstr "Inténtelo de nuevo" - -#: cove_ocds/lib/exceptions.py:24 cove_ocds/lib/exceptions.py:47 -#, python-format -msgid "%(version)s is not a known schema version" -msgstr "%(version)s no es una versión conocida del esquema" - -#: cove_ocds/lib/exceptions.py:32 -msgid "Version format does not comply with the schema" -msgstr "El formato de la versión no se ajusta al esquema" - -#: cove_ocds/lib/exceptions.py:55 -msgid "JSON reference error" -msgstr "Error de referencia en JSON" - -#: cove_ocds/lib/exceptions.py:67 -#, python-format -msgid "%(error)s" -msgstr "%(error)s" - -#: cove_ocds/lib/exceptions.py:75 cove_ocds/lib/exceptions.py:92 -msgid "Missing OCDS package" -msgstr "Paquete OCDS no encontrado" - -#: cove_ocds/lib/exceptions.py:80 -msgid "" -"We could not detect a package structure at the top-level of your data. OCDS " -"releases and records should be published within a release package or record package to provide important meta-data. For " -"more information, please refer to the Releases and " -"Records section in the OCDS documentation.\n" -"\n" -" Error message: Missing OCDS package" -msgstr "" -"No pudimos detectar una estructura de paquete en el nivel superior de sus " -"datos. Las versiones y registros de OCDS deben publicarse dentro de un paquete de entrega o paquete de registros " -"para proporcionar metadatos importantes. Para obtener más información, " -"consulte la sección Entregas y registros en la documentación " -"de OCDS.\n" -"\n" -" Mensaje de error: Falta el paquete OCDS" - #: cove_ocds/templates/cove_ocds/additional_checks_table.html:7 msgid "Check Description" msgstr "Compruebe la Descripción" @@ -268,9 +208,8 @@ msgid "" "example/1.1/ocds-213czf-000-00001-02-tender.json" msgstr "" "Para ver cómo funciona la herramienta de revisión de datos, intente subir " -"https://raw.githubusercontent.com/open-" -"contracting/sample-data/main/fictional-example/1.1/ocds-213czf-000-00001-02-" -"tender.json" +"https://raw.githubusercontent.com/open-contracting/sample-data/main/" +"fictional-example/1.1/ocds-213czf-000-00001-02-tender.json" #: cove_ocds/templates/cove_ocds/base.html:153 msgid "Built by" @@ -1660,36 +1599,18 @@ msgstr "" "DDT00:00:00Z. Lea más sobre fechas en OCDS" -#: cove_ocds/views.py:50 -msgid "Sorry, the page you are looking for is not available" -msgstr "Lo sentimos, la página que está buscando no está disponible" - -#: cove_ocds/views.py:52 -msgid "Go to Home page" -msgstr "Ir a la Página de Inicio" - -#: cove_ocds/views.py:55 -#, python-format -msgid "" -"The data you were hoping to explore no longer exists.\n" -"\n" -"This is because all data supplied to this website is automatically deleted " -"after %s days, and therefore the analysis of that data is no longer " -"available." -msgstr "" -"Los datos que usted quería explorar ya no existen.\n" -"\n" -"Esto se debe a que todos los datos suministrados a este sitio web se borran " -"automáticamente después de 7 días, y por lo tanto el análisis de esos datos " -"automáticamente después de %s días, y por lo tanto el análisis de esos datos " -"ya no está disponible." - -#: cove_ocds/views.py:88 cove_ocds/views.py:106 cove_ocds/views.py:124 -#: cove_ocds/views.py:235 +#: cove_ocds/util.py:20 cove_ocds/util.py:39 cove_ocds/util.py:57 +#: cove_ocds/views.py:112 msgid "Sorry, we can't process that data" msgstr "Lo sentimos, no podemos procesar esos datos" -#: cove_ocds/views.py:93 +#: cove_ocds/util.py:23 cove_ocds/util.py:42 cove_ocds/util.py:59 +#: cove_ocds/util.py:85 cove_ocds/util.py:108 cove_ocds/util.py:127 +#: cove_ocds/views.py:114 cove_ocds/views.py:149 +msgid "Try Again" +msgstr "Inténtelo de nuevo" + +#: cove_ocds/util.py:26 msgid "" "The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON " "follows the I-JSON format, which requires UTF-8 encoding. Ensure that your " @@ -1706,7 +1627,7 @@ msgstr "" " Mensaje del error: {}" -#: cove_ocds/views.py:111 +#: cove_ocds/util.py:45 msgid "" "We think you tried to upload a JSON file, but it is not well formed JSON.\n" "\n" @@ -1719,7 +1640,7 @@ msgstr "" " Mensaje del error: {}" -#: cove_ocds/views.py:127 +#: cove_ocds/util.py:60 msgid "" "OCDS JSON should have an object as the top level, the JSON you supplied does " "not." @@ -1727,7 +1648,81 @@ msgstr "" "OCDS JSON debe ser un objeto al nivel más alto pero el JSON que usted ha " "aportado no lo es." -#: cove_ocds/views.py:240 +#: cove_ocds/util.py:82 cove_ocds/util.py:83 +msgid "Missing OCDS package" +msgstr "Paquete OCDS no encontrado" + +#: cove_ocds/util.py:88 +msgid "" +"We could not detect a package structure at the top-level of your data. OCDS " +"releases and records should be published within a release package or record package to provide important meta-data. For " +"more information, please refer to the Releases and " +"Records section in the OCDS documentation.\n" +"\n" +" Error message: Missing OCDS package" +msgstr "" +"No pudimos detectar una estructura de paquete en el nivel superior de sus " +"datos. Las versiones y registros de OCDS deben publicarse dentro de un paquete de entrega o paquete de registros " +"para proporcionar metadatos importantes. Para obtener más información, " +"consulte la sección Entregas y registros en la documentación " +"de OCDS.\n" +"\n" +" Mensaje de error: Falta el paquete OCDS" + +#: cove_ocds/util.py:105 +msgid "Unrecognised version of the schema" +msgstr "Versión del esquema no reconocida" + +#: cove_ocds/util.py:106 cove_ocds/util.py:125 +#, python-format +msgid "%(version)s is not a known schema version" +msgstr "%(version)s no es una versión conocida del esquema" + +#: cove_ocds/util.py:111 +msgid "" +"We think you tried to run your data against an unrecognised version of the " +"schema.\n" +"\n" +" Error message: {} is not a recognised choice " +"for the schema version" +msgstr "" + +#: cove_ocds/util.py:124 +msgid "Version format does not comply with the schema" +msgstr "El formato de la versión no se ajusta al esquema" + +#: cove_ocds/util.py:130 +msgid "" +"The value for the \"version\" field in your data follows the " +"major.minor.patch pattern but according to the schema the patch " +"digit shouldn't be included (e.g. \"1.1.0\" should appear as " +"\"1.1\" in your data as this tool always uses the latest patch " +"release for a major.minor version).\n" +"\n" +"Please get rid of the patch digit and try again.\n" +"\n" +" Error message: {} format does not comply " +"with the schema" +msgstr "" + +#: cove_ocds/util.py:144 +#, python-format +msgid "%(version)s (not a string)" +msgstr "" + +#: cove_ocds/views.py:118 msgid "" "The table isn't structured correctly. For example, a JSON Pointer " "(tender) can't be both a value (tender), a path to " @@ -1744,3 +1739,33 @@ msgstr "" "\n" " Error message: {}" + +#: cove_ocds/views.py:147 +msgid "JSON reference error" +msgstr "Error de referencia en JSON" + +#: cove_ocds/views.py:159 +#, python-format +msgid "%(error)s" +msgstr "%(error)s" + +#~ msgid "Sorry, the page you are looking for is not available" +#~ msgstr "Lo sentimos, la página que está buscando no está disponible" + +#~ msgid "Go to Home page" +#~ msgstr "Ir a la Página de Inicio" + +#, python-format +#~ msgid "" +#~ "The data you were hoping to explore no longer exists.\n" +#~ "\n" +#~ "This is because all data supplied to this website is automatically " +#~ "deleted after %s days, and therefore the analysis of that data is no " +#~ "longer available." +#~ msgstr "" +#~ "Los datos que usted quería explorar ya no existen.\n" +#~ "\n" +#~ "Esto se debe a que todos los datos suministrados a este sitio web se " +#~ "borran automáticamente después de 7 días, y por lo tanto el análisis de " +#~ "esos datos automáticamente después de %s días, y por lo tanto el análisis " +#~ "de esos datos ya no está disponible." diff --git a/cove_ocds/util.py b/cove_ocds/util.py new file mode 100644 index 00000000..0a51be61 --- /dev/null +++ b/cove_ocds/util.py @@ -0,0 +1,203 @@ +import json +import re +from decimal import Decimal + +from django.utils.html import format_html, mark_safe +from django.utils.translation import gettext as _ +from libcove.lib.common import schema_dict_fields_generator +from libcove.lib.exceptions import CoveInputDataError +from libcoveocds.schema import SchemaOCDS + + +def read_json(path): + # Read as text, because the json module can read binary UTF-16 and UTF-32. + with open(path, encoding="utf-8") as f: + try: + data = json.load(f) + except UnicodeError as err: + raise CoveInputDataError( + context={ + "sub_title": _("Sorry, we can't process that data"), + "error": format(err), + "link": "index", + "link_text": _("Try Again"), + "msg": format_html( + _( + "The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON follows the " + "I-JSON format, which requires UTF-8 encoding. Ensure that your file uses UTF-8 encoding, " + "then try uploading again.\n\n" + ' ' + "Error message: {}" + ), + err, + ), + } + ) from None + except ValueError as err: + raise CoveInputDataError( + context={ + "sub_title": _("Sorry, we can't process that data"), + "error": format(err), + "link": "index", + "link_text": _("Try Again"), + "msg": format_html( + _( + "We think you tried to upload a JSON file, but it is not well formed JSON.\n\n" + ' ' + "Error message: {}" + ), + err, + ), + } + ) from None + + if not isinstance(data, dict): + raise CoveInputDataError( + context={ + "sub_title": _("Sorry, we can't process that data"), + "link": "index", + "link_text": _("Try Again"), + "msg": _("OCDS JSON should have an object as the top level, the JSON you supplied does not."), + } + ) + + return data + + +def get_schema(request, context, supplied_data, lib_cove_ocds_config, package_data): + request_version = request.POST.get("version") + data_version = package_data.get("version") + + schema_ocds = SchemaOCDS( + # This will be the user-requested version, the previously-determined version, or None. + select_version=request_version or supplied_data.schema_version, + package_data=package_data, + lib_cove_ocds_config=lib_cove_ocds_config, + record_pkg="records" in package_data, + ) + + if schema_ocds.missing_package: + raise CoveInputDataError( + context={ + "sub_title": _("Missing OCDS package"), + "error": _("Missing OCDS package"), + "link": "index", + "link_text": _("Try Again"), + "msg": mark_safe( + _( + "We could not detect a package structure at the top-level of your data. OCDS releases and " + 'records should be published within a release package or record package to provide important meta-data. ' + 'For more information, please refer to the Releases and Records section in the OCDS ' + "documentation.\n\n" + ' ' + "Error message: Missing OCDS package" + ) + ), + } + ) + + if schema_ocds.invalid_version_argument: + raise CoveInputDataError( + context={ + "sub_title": _("Unrecognised version of the schema"), + "error": _("%(version)s is not a known schema version") % {"version": request_version}, + "link": "index", + "link_text": _("Try Again"), + "msg": format_html( + _( + "We think you tried to run your data against an unrecognised version of the schema.\n\n" + ' ' + "Error message: {} is not a recognised choice for the schema version", + ), + request_version, + ), + } + ) + + if schema_ocds.invalid_version_data: + if isinstance(data_version, str) and re.compile(r"^\d+\.\d+\.\d+$").match(data_version): + raise CoveInputDataError( + context={ + "sub_title": _("Version format does not comply with the schema"), + "error": _("%(version)s is not a known schema version") % {"version": data_version}, + "link": "index", + "link_text": _("Try Again"), + "msg": format_html( + _( + 'The value for the "version" field in your data follows the major.minor.' + "patch pattern but according to the schema the patch digit shouldn't be included " + '(e.g. "1.1.0" should appear as "1.1" in your data as this tool ' + "always uses the latest patch release for a major.minor version).\n\n" + "Please get rid of the patch digit and try again.\n\n" + ' ' + "Error message: {} format does not comply with the schema", + ), + json.dumps(data_version), + ), + } + ) + + if not isinstance(data_version, str): + data_version = _("%(version)s (not a string)") % {"version": json.dumps(data_version)} + context["unrecognized_version_data"] = data_version + + # Cache the extended schema. + if schema_ocds.extensions: + schema_ocds.create_extended_schema_file(supplied_data.upload_dir(), supplied_data.upload_url()) + + # If the schema is not extended, extended_schema_file is None. + schema_url = schema_ocds.extended_schema_file or schema_ocds.schema_url + + # Regenerate alternative formats if the user requests a different version. + replace = bool(supplied_data.schema_version) and schema_ocds.version != supplied_data.schema_version + + return schema_ocds, schema_url, replace + + +def default(obj): + if isinstance(obj, Decimal): + return float(obj) + return json.JSONEncoder().default(obj) + + +def add_extra_fields(data, deref_release_schema): + all_schema_fields = set(schema_dict_fields_generator(deref_release_schema)) + + if "releases" in data: + for release in data.get("releases", []): + if not isinstance(release, dict): + return + _add_extra_fields_to_obj(release, all_schema_fields, "") + elif "records" in data: + for record in data.get("records", []): + if not isinstance(record, dict): + return + for release in record.get("releases", []): + _add_extra_fields_to_obj(release, all_schema_fields, "") + + +def _add_extra_fields_to_obj(obj, all_schema_fields, current_path): + if not isinstance(obj, dict): + return + obj["__extra"] = {} + + for key, value in list(obj.items()): + if key == "__extra": + continue + + new_path = f"{current_path}/{key}" + if new_path not in all_schema_fields: + obj["__extra"][key] = value + continue + + if isinstance(value, list): + for item in value: + _add_extra_fields_to_obj(item, all_schema_fields, new_path) + elif isinstance(value, dict): + _add_extra_fields_to_obj(value, all_schema_fields, new_path) + + if not obj["__extra"]: + obj.pop("__extra") diff --git a/cove_ocds/views.py b/cove_ocds/views.py index a05c86dc..73498429 100644 --- a/cove_ocds/views.py +++ b/cove_ocds/views.py @@ -4,7 +4,7 @@ import os import re import warnings -from decimal import Decimal +from collections import defaultdict from cove.views import cove_web_input_error, explore_data_context from django.conf import settings @@ -20,212 +20,89 @@ from libcoveocds.config import LibCoveOCDSConfig from libcoveocds.schema import SchemaOCDS -from cove_ocds.lib.views import group_validation_errors - -from .lib import exceptions -from .lib.ocds_show_extra import add_extra_fields +from cove_ocds import util logger = logging.getLogger(__name__) MAXIMUM_RELEASES_OR_RECORDS = 100 -def format_lang(choices, lang): - """Format the urls with `{lang}` contained in a schema_version_choices.""" - formatted_choices = {} - for version, (display, url, tag) in choices.items(): - formatted_choices[version] = (display, url.format(lang=lang), tag) - return formatted_choices - - @cove_web_input_error def explore_ocds(request, pk): - try: - context, db_data, error = explore_data_context(request, pk) - # https://github.com/OpenDataServices/lib-cove-web/pull/145 - except FileNotFoundError: - return render( - request, - "error.html", - { - "sub_title": _("Sorry, the page you are looking for is not available"), - "link": "index", - "link_text": _("Go to Home page"), - "support_email": settings.COVE_CONFIG.get("support_email"), - "msg": _( - "The data you were hoping to explore no longer exists.\n\nThis is because all " - "data supplied to this website is automatically deleted after %s days, and therefore " - "the analysis of that data is no longer available." - ) - % getattr(settings, "DELETE_FILES_AFTER_DAYS", 7), - }, - status=404, - ) + context, supplied_data, error = explore_data_context(request, pk) if error: return error + # Initialize the CoVE configuration. + lib_cove_ocds_config = LibCoveOCDSConfig(settings.COVE_CONFIG) lib_cove_ocds_config.config["current_language"] = translation.get_language() - lib_cove_ocds_config.config["schema_version_choices"] = format_lang( - lib_cove_ocds_config.config["schema_version_choices"], request.LANGUAGE_CODE - ) + # Format the urls with `{lang}` contained in a schema_version_choices. + lib_cove_ocds_config.config["schema_version_choices"] = { + version: (display, url.format(lang=request.LANGUAGE_CODE), tag) + for version, (display, url, tag) in lib_cove_ocds_config.config["schema_version_choices"].items() + } - upload_dir = db_data.upload_dir() - upload_url = db_data.upload_url() - file_name = db_data.original_file.path - file_type = context["file_type"] + # Read the supplied data, and convert to alternative formats (if not done on a previous request). - post_version_choice = request.POST.get("version") - replace = False - validation_errors_path = os.path.join(upload_dir, "validation_errors-3.json") + if context["file_type"] == "json": + package_data = util.read_json(supplied_data.original_file.path) - if file_type == "json": - with open(file_name, encoding="utf-8") as fp: - try: - json_data = json.load(fp) - except UnicodeError as err: - raise CoveInputDataError( - context={ - "sub_title": _("Sorry, we can't process that data"), - "link": "index", - "link_text": _("Try Again"), - "msg": format_html( - _( - "The file that you uploaded doesn't appear to be well formed JSON. OCDS JSON follows " - "the I-JSON format, which requires UTF-8 encoding. Ensure that your file uses UTF-8 " - 'encoding, then try uploading again.\n\n Error message: {}' - ), - err, - ), - "error": format(err), - } - ) from None - except ValueError as err: - raise CoveInputDataError( - context={ - "sub_title": _("Sorry, we can't process that data"), - "link": "index", - "link_text": _("Try Again"), - "msg": format_html( - _( - "We think you tried to upload a JSON file, but it is not well formed JSON." - '\n\n Error message: {}", - ), - err, - ), - "error": format(err), - } - ) from None - - if not isinstance(json_data, dict): - raise CoveInputDataError( - context={ - "sub_title": _("Sorry, we can't process that data"), - "link": "index", - "link_text": _("Try Again"), - "msg": _("OCDS JSON should have an object as the top level, the JSON you supplied does not."), - } - ) - - version_in_data = json_data.get("version") or "" - db_data.data_schema_version = version_in_data - select_version = post_version_choice or db_data.schema_version - schema_ocds = SchemaOCDS( - select_version=select_version, - package_data=json_data, - lib_cove_ocds_config=lib_cove_ocds_config, - record_pkg="records" in json_data, - ) - - if schema_ocds.missing_package: - exceptions.raise_missing_package_error() - if schema_ocds.invalid_version_argument: - exceptions.raise_invalid_version_argument(post_version_choice) - if schema_ocds.invalid_version_data: - if isinstance(version_in_data, str) and re.compile(r"^\d+\.\d+\.\d+$").match(version_in_data): - exceptions.raise_invalid_version_data_with_patch(version_in_data) - else: - if not isinstance(version_in_data, str): - version_in_data = f"{version_in_data} (it must be a string)" - context["unrecognized_version_data"] = version_in_data - - if schema_ocds.version != db_data.schema_version: - replace = True - if schema_ocds.extensions: - schema_ocds.create_extended_schema_file(upload_dir, upload_url) - url = schema_ocds.extended_schema_file or schema_ocds.schema_url + schema_ocds, schema_url, replace = util.get_schema( + request, context, supplied_data, lib_cove_ocds_config, package_data + ) - if "records" in json_data: - context["conversion"] = None - else: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=FlattenToolWarning) + if "records" in package_data: + context["conversion"] = None + else: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=FlattenToolWarning) - convert_json_context = convert_json( - upload_dir, - upload_url, - file_name, - lib_cove_ocds_config, - schema_url=url, + context.update( + convert_json( + upload_dir=supplied_data.upload_dir(), + upload_url=supplied_data.upload_url(), + file_name=supplied_data.original_file.path, + lib_cove_config=lib_cove_ocds_config, + schema_url=schema_url, # Unsure why exists() was added in https://github.com/open-contracting/cove-ocds/commit/d793c49 - replace=replace and os.path.exists(os.path.join(upload_dir, "flattened.xlsx")), + replace=replace and os.path.exists(os.path.join(supplied_data.upload_dir(), "flattened.xlsx")), request=request, flatten=request.POST.get("flatten"), ) - - context.update(convert_json_context) - + ) else: - metatab_schema_url = SchemaOCDS(select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url - with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=FlattenToolWarning) - metatab_data = get_spreadsheet_meta_data(upload_dir, file_name, metatab_schema_url, file_type) + meta_data = get_spreadsheet_meta_data( + supplied_data.upload_dir(), + supplied_data.original_file.path, + SchemaOCDS(select_version="1.1", lib_cove_ocds_config=lib_cove_ocds_config).pkg_schema_url, + context["file_type"], + ) - if "version" not in metatab_data: - metatab_data["version"] = "1.0" - else: - db_data.data_schema_version = metatab_data["version"] + meta_data.setdefault("version", "1.0") + # Make "missing_package" pass. + meta_data["releases"] = {} - select_version = post_version_choice or db_data.schema_version - schema_ocds = SchemaOCDS( - select_version=select_version, - package_data=metatab_data, - lib_cove_ocds_config=lib_cove_ocds_config, + schema_ocds, schema_url, replace = util.get_schema( + request, context, supplied_data, lib_cove_ocds_config, meta_data ) - if schema_ocds.invalid_version_argument: - exceptions.raise_invalid_version_argument(post_version_choice) - if schema_ocds.invalid_version_data: - version_in_data = metatab_data.get("version") - if re.compile(r"^\d+\.\d+\.\d+$").match(version_in_data): - exceptions.raise_invalid_version_data_with_patch(version_in_data) - else: - context["unrecognized_version_data"] = version_in_data - - if db_data.schema_version and schema_ocds.version != db_data.schema_version: # if user changes schema version - replace = True - - if schema_ocds.extensions: - schema_ocds.create_extended_schema_file(upload_dir, upload_url) - url = schema_ocds.extended_schema_file or schema_ocds.schema_url - pkg_url = schema_ocds.pkg_schema_url - with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=FlattenToolWarning) try: context.update( + # __wrapped__ is missing when the function is patched by tests. getattr(convert_spreadsheet, "__wrapped__", convert_spreadsheet)( - upload_dir, - upload_url, - file_name, - file_type, - lib_cove_ocds_config, - schema_url=url, - pkg_schema_url=pkg_url, + supplied_data.upload_dir(), + upload_url=supplied_data.upload_url(), + file_name=supplied_data.original_file.path, + file_type=context["file_type"], + lib_cove_config=lib_cove_ocds_config, + schema_url=schema_url, + pkg_schema_url=schema_ocds.pkg_schema_url, replace=replace, ) ) @@ -235,92 +112,111 @@ def explore_ocds(request, pk): "sub_title": _("Sorry, we can't process that data"), "link": "index", "link_text": _("Try Again"), + "error": format(err), "msg": format_html( _( "The table isn't structured correctly. For example, a JSON Pointer (tender" ") can't be both a value (tender), a path to an object (" - "tender/id) and a path to an array (tender/0/title)." - '\n\n Error message: {}", + "tender/id) and a path to an array (tender/0/title).\n\n" + ' ' + "Error message: {}", ), err, ), - "error": format(err), } ) from None except Exception as err: - logger.exception(extra={"request": request}) + logger.exception("", extra={"request": request}) raise CoveInputDataError(wrapped_err=err) from None - with open(context["converted_path"], encoding="utf-8") as fp: - json_data = json.load(fp) + with open(context["converted_path"], "rb") as f: + package_data = json.load(f) + # Perform the validation. + + # common_checks_ocds() calls libcove.lib.common.common_checks_context(), which writes `validation_errors-3.json`. + validation_errors_path = os.path.join(supplied_data.upload_dir(), "validation_errors-3.json") if replace and os.path.exists(validation_errors_path): os.remove(validation_errors_path) + context = common_checks_ocds(context, supplied_data.upload_dir(), package_data, schema_ocds) - context = common_checks_ocds(context, upload_dir, json_data, schema_ocds) - + # Set by SchemaOCDS.get_schema_obj(deref=True), which, at the latest, is called indirectly by common_checks_ocds(). if schema_ocds.json_deref_error: - exceptions.raise_json_deref_error(schema_ocds.json_deref_error) + raise CoveInputDataError( + context={ + "sub_title": _("JSON reference error"), + "link": "index", + "link_text": _("Try Again"), + "msg": _( + format_html( + "We have detected a JSON reference error in the schema. This may be " + " due to some extension trying to resolve non-existing references. " + '\n\n Error message: {}", + schema_ocds.json_deref_error, + ) + ), + "error": _("%(error)s") % {"error": schema_ocds.json_deref_error}, + } + ) + + # Update the row in the database. + + # The data_schema_version column is NOT NULL. + supplied_data.data_schema_version = package_data.get("version") or "" + supplied_data.schema_version = schema_ocds.version + supplied_data.rendered = True # not relevant to CoVE OCDS + supplied_data.save() + + # Finalize the context and select the template. + + validation_errors_grouped = defaultdict(list) + for error_json, values in context["validation_errors"]: + match json.loads(error_json)["message_type"]: + case "required": + key = "required" + case "format" | "pattern" | "number" | "string" | "date-time" | "uri" | "object" | "integer" | "array": + key = "format" + case _: + key = "other" + validation_errors_grouped[key].append((error_json, values)) context.update( { - "data_schema_version": db_data.data_schema_version, - "first_render": not db_data.rendered, - "validation_errors_grouped": group_validation_errors(context["validation_errors"]), + "data_schema_version": supplied_data.data_schema_version, + "validation_errors_grouped": validation_errors_grouped, } ) for key in ("additional_closed_codelist_values", "additional_open_codelist_values"): - for codelist_info in context[key].values(): - if codelist_info["codelist_url"].startswith(schema_ocds.codelists): - codelist_info["codelist_url"] = ( - f"https://standard.open-contracting.org/{db_data.data_schema_version}/en/schema/codelists/#" - + re.sub(r"([A-Z])", r"-\1", codelist_info["codelist"].split(".")[0]).lower() + for additional_codelist_values in context[key].values(): + if additional_codelist_values["codelist_url"].startswith(schema_ocds.codelists): + additional_codelist_values["codelist_url"] = ( + f"https://standard.open-contracting.org/{supplied_data.data_schema_version}/en/schema/codelists/#" + + re.sub(r"([A-Z])", r"-\1", additional_codelist_values["codelist"].split(".")[0]).lower() ) - schema_version = getattr(schema_ocds, "version", None) - if schema_version: - db_data.schema_version = schema_version - if not db_data.rendered: - db_data.rendered = True - - db_data.save() - - if "records" in json_data: - context["release_or_record"] = "record" - ocds_show_schema = SchemaOCDS(record_pkg=True) - ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True) + has_records = "records" in package_data + if has_records: template = "cove_ocds/explore_record.html" - if hasattr(json_data, "get") and hasattr(json_data.get("records"), "__iter__"): - context["records"] = json_data["records"] - if isinstance(json_data["records"], list) and len(json_data["records"]) < MAXIMUM_RELEASES_OR_RECORDS: - context["ocds_show_data"] = ocds_show_data(json_data, ocds_show_deref_schema) - else: - context["records"] = [] + context["release_or_record"] = "record" + key = "records" else: - context["release_or_record"] = "release" - ocds_show_schema = SchemaOCDS(record_pkg=False) - ocds_show_deref_schema = ocds_show_schema.get_schema_obj(deref=True) template = "cove_ocds/explore_release.html" - if hasattr(json_data, "get") and hasattr(json_data.get("releases"), "__iter__"): - context["releases"] = json_data["releases"] - if isinstance(json_data["releases"], list) and len(json_data["releases"]) < MAXIMUM_RELEASES_OR_RECORDS: - context["ocds_show_data"] = ocds_show_data(json_data, ocds_show_deref_schema) - else: - context["releases"] = [] + context["release_or_record"] = "release" + key = "releases" + + if isinstance(package_data, dict) and isinstance(package_data.get(key), list): + # This is for the "Releases Table" and "Records Table" features. + context[key] = package_data[key] + + # This is for the OCDS Show feature. + # https://github.com/open-contracting/cove-ocds/commit/d8dbf55 + if len(package_data[key]) < MAXIMUM_RELEASES_OR_RECORDS: + new_package_data = copy.deepcopy(package_data) + util.add_extra_fields(new_package_data, SchemaOCDS(record_pkg=has_records).get_schema_obj(deref=True)) + context["ocds_show_data"] = json.dumps(new_package_data, default=util.default) + else: + context[key] = [] return render(request, template, context) - - -# This should only be run when data is small. -def ocds_show_data(json_data, ocds_show_deref_schema): - new_json_data = copy.deepcopy(json_data) - add_extra_fields(new_json_data, ocds_show_deref_schema) - return json.dumps(new_json_data, default=default) - - -def default(self, obj): - if isinstance(obj, Decimal): - return float(obj) - return json.JSONEncoder().default(obj) diff --git a/docs/architecture.rst b/docs/architecture.rst index 6ec97977..cc1ee3a9 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -9,8 +9,6 @@ The OCDS Data Review tool comprises two main parts, which are documented here. cove-ocds --------- -``cove_ocds/lib`` contains OCDS data specific exceptions (errors generated by invalid data or input), as well as additional functions for OCDS Show (the JavaScript data explorer). - ``tests/`` also contains fixtures for testing, and the tests themselves; templates and related static files; code for the CLI version of the DRT; and locale files for translations. ``cove_ocds/views.py`` does most of the heavy lifting of taking an input file from the web interface and carrying out the various validation checks and conversions, then piping the output back to the right templates. diff --git a/requirements.txt b/requirements.txt index ccfd4413..4223a136 100644 --- a/requirements.txt +++ b/requirements.txt @@ -72,7 +72,7 @@ libcove==0.32.1 # -r requirements.in # libcoveocds # libcoveweb -libcoveocds==0.16.0 +libcoveocds==0.16.2 # via -r requirements.in libcoveweb==0.31.0 # via -r requirements.in diff --git a/requirements_dev.txt b/requirements_dev.txt index b7c674ab..0c6d81b5 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -121,7 +121,7 @@ libcove==0.32.1 # -r requirements.txt # libcoveocds # libcoveweb -libcoveocds==0.16.0 +libcoveocds==0.16.2 # via -r requirements.txt libcoveweb==0.31.0 # via -r requirements.txt diff --git a/tests/test_functional.py b/tests/test_functional.py index 74490431..d928a3f6 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -377,7 +377,7 @@ def test_500_error(server_url, browser): ( "tenders_releases_1_release_with_wrong_version_type.json", [ - "Your data specifies a version 1000 (it must be a string) which is not recognised", + "Your data specifies a version 1000 (not a string) which is not recognised", f"checked against OCDS release package schema version {OCDS_DEFAULT_SCHEMA_VERSION}. You can", "Convert to Spreadsheet", ], @@ -388,7 +388,7 @@ def test_500_error(server_url, browser): "tenders_releases_1_release_with_patch_in_version.json", [ '"version" field in your data follows the major.minor.patch pattern', - "100.100.0 format does not comply with the schema", + '"100.100.0" format does not comply with the schema', "Error message", ], ["Convert to Spreadsheet"], diff --git a/tests/test_general.py b/tests/test_general.py index 6238ca82..303d1e7e 100644 --- a/tests/test_general.py +++ b/tests/test_general.py @@ -260,6 +260,7 @@ def test_explore_page_null_version(client): data.current_app = "cove_ocds" resp = client.get(data.get_absolute_url()) assert resp.status_code == 200 + assert b"null (not a string)" in resp.content @pytest.mark.django_db