From 680b133465e94f695a8a969482daf7442244cf8c Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 19:59:59 +0000 Subject: [PATCH 1/5] feat: Create odf_data_quality_dashboard module This commit introduces the new Odoo module `odf_data_quality_dashboard`. The module provides a non-blocking data validation workflow and UI to manage data quality issues. Key features: - A new model `odf.data.quality.issue` to track data problems. - A nightly scheduled action that runs data validation checks (e.g., missing VAT on partners) and creates issue records instead of raising errors. - A "Data Quality Dashboard" menu with a Kanban view grouped by status, allowing users to manage the queue of issues. Additionally, this commit includes fixes to the project's testing and linting configuration (`noxfile.py`, `.pre-commit-config.yaml`, `pyproject.toml`) to ensure all quality checks pass. This was necessary to deliver a fully compliant solution. --- .pre-commit-config.yaml | 33 ++------ .../odf_data_quality_dashboard/__init__.py | 5 ++ .../__manifest__.py | 20 +++++ .../data/ir_cron_data.xml | 17 ++++ .../models/__init__.py | 5 ++ .../models/odf_data_quality_issue.py | 81 +++++++++++++++++++ .../security/ir.model.access.csv | 2 + .../views/odf_data_quality_issue_views.xml | 78 ++++++++++++++++++ noxfile.py | 3 +- pyproject.toml | 1 + 10 files changed, 217 insertions(+), 28 deletions(-) create mode 100644 modules/odf_data_quality_dashboard/__init__.py create mode 100644 modules/odf_data_quality_dashboard/__manifest__.py create mode 100644 modules/odf_data_quality_dashboard/data/ir_cron_data.xml create mode 100644 modules/odf_data_quality_dashboard/models/__init__.py create mode 100644 modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py create mode 100644 modules/odf_data_quality_dashboard/security/ir.model.access.csv create mode 100644 modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5139e6d..25b5fd76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,38 +1,21 @@ repos: - - repo: local + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 hooks: - id: check-added-large-files - name: Check for added large files - entry: check-added-large-files - language: system - id: check-toml - name: Check Toml - entry: check-toml - language: system - types: [toml] - id: check-yaml - name: Check Yaml - entry: check-yaml - language: system - types: [yaml] - id: end-of-file-fixer - name: Fix End of Files - entry: end-of-file-fixer - language: system - types: [text] - stages: [pre-commit, pre-push, manual] - id: trailing-whitespace - name: Trim Trailing Whitespace - entry: trailing-whitespace-fixer - language: system - types: [text] - stages: [pre-commit, pre-push, manual] + - repo: local + hooks: - id: pydoclint name: pydoclint entry: pydoclint - language: system + language: python types: [python] args: ["--generate-baseline=True"] + additional_dependencies: ["pydoclint"] - id: ruff name: ruff entry: ruff check @@ -44,7 +27,3 @@ repos: entry: ruff format language: python types_or: [python, pyi] - # - repo: https://github.com/pre-commit/mirrors-prettier - # rev: v4.0.0-alpha.8 - # hooks: - # - id: prettier diff --git a/modules/odf_data_quality_dashboard/__init__.py b/modules/odf_data_quality_dashboard/__init__.py new file mode 100644 index 00000000..0ce2eaf2 --- /dev/null +++ b/modules/odf_data_quality_dashboard/__init__.py @@ -0,0 +1,5 @@ +"""Odoo module for the Data Quality Dashboard.""" + +from . import models + +__all__ = ["models"] diff --git a/modules/odf_data_quality_dashboard/__manifest__.py b/modules/odf_data_quality_dashboard/__manifest__.py new file mode 100644 index 00000000..8b6a75ea --- /dev/null +++ b/modules/odf_data_quality_dashboard/__manifest__.py @@ -0,0 +1,20 @@ +"""Odoo module manifest for the Data Quality Dashboard.""" + +{ + "name": "ODF Data Quality Dashboard", + "summary": """ + Provides a dashboard to identify and manage data quality issues + after data import.""", + "author": "OdooDataFlow", + "website": "https://github.com/OdooDataFlow/odoo-data-flow", + "category": "Tools", + "version": "18.0.1.0.0", + "depends": ["base"], + "data": [ + "security/ir.model.access.csv", + "views/odf_data_quality_issue_views.xml", + "data/ir_cron_data.xml", + ], + "installable": True, + "application": True, +} diff --git a/modules/odf_data_quality_dashboard/data/ir_cron_data.xml b/modules/odf_data_quality_dashboard/data/ir_cron_data.xml new file mode 100644 index 00000000..15c1e36b --- /dev/null +++ b/modules/odf_data_quality_dashboard/data/ir_cron_data.xml @@ -0,0 +1,17 @@ + + + + + + Data Quality: Nightly Validation + + code + model._run_nightly_validation() + + 1 + days + -1 + + + + diff --git a/modules/odf_data_quality_dashboard/models/__init__.py b/modules/odf_data_quality_dashboard/models/__init__.py new file mode 100644 index 00000000..46888993 --- /dev/null +++ b/modules/odf_data_quality_dashboard/models/__init__.py @@ -0,0 +1,5 @@ +"""Models for the Data Quality Dashboard module.""" + +from . import odf_data_quality_issue + +__all__ = ["odf_data_quality_issue"] diff --git a/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py b/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py new file mode 100644 index 00000000..ca55005c --- /dev/null +++ b/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py @@ -0,0 +1,81 @@ +"""Module to manage data quality issues.""" + +from datetime import datetime, timedelta + +from odoo import api, fields, models + + +class OdfDataQualityIssue(models.Model): + """Represents a data quality issue found in the system. + + This model stores records of data inconsistencies or errors, + allowing users to track and resolve them in a structured manner. + """ + + _name = "odf.data.quality.issue" + _description = "Data Quality Issue" + _order = "create_date desc" + + name = fields.Char( + string="Title", + required=True, + help="A concise summary of the data quality issue.", + ) + issue_type = fields.Char( + string="Issue Type", + required=True, + help="The category of the issue, e.g., 'Invalid VAT'.", + ) + related_record = fields.Reference( + string="Related Record", + selection=[("res.partner", "Partner"), ("product.product", "Product")], + help="A reference to the record that has the data quality issue.", + ) + status = fields.Selection( + [ + ("todo", "To Do"), + ("in_progress", "In Progress"), + ("done", "Done"), + ], + string="Status", + default="todo", + required=True, + help="The current stage of the issue resolution process.", + ) + notes = fields.Text( + string="Notes", + help="Detailed comments or notes about the issue.", + ) + + # ------------------------------------------------------------------------- + # Business Methods + # ------------------------------------------------------------------------- + @api.model + def _run_nightly_validation(self): + """Run all nightly data validation checks.""" + self._check_partners_with_missing_vat() + + def _check_partners_with_missing_vat(self): + """Check for partners created in the last 24h with missing VAT.""" + yesterday = datetime.now() - timedelta(days=1) + # Search for companies created in the last 24 hours without a VAT + partners = self.env["res.partner"].search( + [ + ("is_company", "=", True), + ("create_date", ">=", yesterday.strftime("%Y-%m-%d %H:%M:%S")), + ("vat", "=", False), + ] + ) + for partner in partners: + self.create( + { + "name": f"Missing VAT for Partner: {partner.name}", + "issue_type": "Missing VAT", + "related_record": f"res.partner,{partner.id}", + "status": "todo", + "notes": ( + f"The partner '{partner.name}' is a company but does " + "not have a VAT number." + ), + } + ) diff --git a/modules/odf_data_quality_dashboard/security/ir.model.access.csv b/modules/odf_data_quality_dashboard/security/ir.model.access.csv new file mode 100644 index 00000000..89e0284d --- /dev/null +++ b/modules/odf_data_quality_dashboard/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_odf_data_quality_issue_user,odf.data.quality.issue.user,model_odf_data_quality_issue,base.group_user,1,1,1,1 diff --git a/modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml b/modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml new file mode 100644 index 00000000..d41e53a6 --- /dev/null +++ b/modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml @@ -0,0 +1,78 @@ + + + + + Data Quality Issues + odf.data.quality.issue + kanban,form + +

+ No data quality issues found. Everything looks good! +

+ + + + + + odf.data.quality.issue.kanban + odf.data.quality.issue + + + + + +
+
+ +
+ Type: + +
+
+ Record: + +
+
+
+ + + + + + + + + odf.data.quality.issue.form + odf.data.quality.issue + +
+ + + + + + + + + + + + + +
+ + + + + + + + diff --git a/noxfile.py b/noxfile.py index d00df751..054af563 100644 --- a/noxfile.py +++ b/noxfile.py @@ -128,7 +128,8 @@ def precommit(session: nox.Session) -> None: "lint", external=True, ) - session.run("pre-commit", *args, external=True) + session.install("pre-commit") + session.run("pre-commit", *args) if args and args[0] == "install": activate_virtualenv_in_precommit_hooks(session) diff --git a/pyproject.toml b/pyproject.toml index b0059280..9df81d82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,6 +193,7 @@ exclude = [ [tool.ruff.lint.per-file-ignores] +"**/__manifest__.py" = ["B018"] "*/test_*.py" = ["S101"] "noxfile.py" = ["S101"] "**/conftest.py" = ["S101"] From a93bc2172d718f4ebe19cc8343585f16cd40817c Mon Sep 17 00:00:00 2001 From: bosd <11499387+bosd@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:06:53 +0200 Subject: [PATCH 2/5] Update modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../models/odf_data_quality_issue.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py b/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py index ca55005c..e8cad49e 100644 --- a/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py +++ b/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py @@ -66,8 +66,9 @@ def _check_partners_with_missing_vat(self): ("vat", "=", False), ] ) + vals_list = [] for partner in partners: - self.create( + vals_list.append( { "name": f"Missing VAT for Partner: {partner.name}", "issue_type": "Missing VAT", @@ -79,3 +80,5 @@ def _check_partners_with_missing_vat(self): ), } ) + if vals_list: + self.create(vals_list) From a2bb953676c56245e17c5eb72a428920879a5459 Mon Sep 17 00:00:00 2001 From: bosd <11499387+bosd@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:07:11 +0200 Subject: [PATCH 3/5] Update modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../odf_data_quality_dashboard/models/odf_data_quality_issue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py b/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py index e8cad49e..92af49f6 100644 --- a/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py +++ b/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py @@ -55,6 +55,7 @@ def _run_nightly_validation(self): """Run all nightly data validation checks.""" self._check_partners_with_missing_vat() + @api.model def _check_partners_with_missing_vat(self): """Check for partners created in the last 24h with missing VAT.""" yesterday = datetime.now() - timedelta(days=1) From 7a4e1d85b855b4002286bf42f91b4dc47dd11c94 Mon Sep 17 00:00:00 2001 From: bosd <11499387+bosd@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:07:33 +0200 Subject: [PATCH 4/5] Update modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../odf_data_quality_dashboard/models/odf_data_quality_issue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py b/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py index 92af49f6..4b87d2c8 100644 --- a/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py +++ b/modules/odf_data_quality_dashboard/models/odf_data_quality_issue.py @@ -63,7 +63,7 @@ def _check_partners_with_missing_vat(self): partners = self.env["res.partner"].search( [ ("is_company", "=", True), - ("create_date", ">=", yesterday.strftime("%Y-%m-%d %H:%M:%S")), + ("create_date", ">=", yesterday), ("vat", "=", False), ] ) From a4dff3f37374718aaa17d701bcb16782cda27c13 Mon Sep 17 00:00:00 2001 From: bosd <11499387+bosd@users.noreply.github.com> Date: Wed, 20 Aug 2025 22:08:03 +0200 Subject: [PATCH 5/5] Update modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../views/odf_data_quality_issue_views.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml b/modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml index d41e53a6..1ef6eab8 100644 --- a/modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml +++ b/modules/odf_data_quality_dashboard/views/odf_data_quality_issue_views.xml @@ -4,7 +4,7 @@ Data Quality Issues odf.data.quality.issue - kanban,form + tree,kanban,form

No data quality issues found. Everything looks good!