Express indicators requirements using DSL rules and move towards #62

README.md

@@ -54,12 +54,8 @@ To ease maintenance, the notebooks are made up of reusable components with clear
 - **Environment**: Setup Google Colaboratory in general
   - `environment`: Install requirements, import packages, load extensions and configure the notebook.
 - **Setup**: Setup Google Colaboratory for a data source
-  - `setup_charts`: Install charts requirements, import charts packages and define plot functions.
   - `setup_kingfisher`: Connect to the Kingfisher Process database. Choose the collection(s) and schema to work with.
   - `setup_fieldlist`: Load the field list.
-  - `setup_metadata_from_registry`: Define the functions to list publications and their metadata, including coverage, from the Registry.
-  - `setup_usability`: Define the usability functions.
-  - `setup_red_flags`: Define the red flags functions.
 - **Errors**: Review any issues in loading the data
   - `errors_kingfisher`: Check for data collection (Kingfisher Collect) and processing (Kingfisher Process) errors.
 - **Scope**: Understand the scope of the data

component_check_red_flags_external.py

@@ -2,10 +2,10 @@
 #
 # Generate a list of the fields published:
 
-fields_list = fields_table.iloc[:, 0].tolist()
-
-indicators_dic = get_red_flags_dictionary(fields_list)
-result = redflags_checks(fields_list, indicators_dic)
+fields_list = set(fields_table["path"])
+indicators = load_indicators(prefix="R")
+result = indicator_checks(fields_list, indicators)
+result = result.rename(columns={"id": "R_id", "indicator": "red_flag"})
 
 # ### Export results
 
@@ -25,7 +25,7 @@
 
 # #### Most common fields to indicators
 
-common_fields = most_common_fields_to_calculate_indicators(indicators_dic, fields_table)
+common_fields = most_common_fields_to_calculate_indicators(fields_list, indicators)
 common_fields
 
 # #### Save the table to a spreadsheet

component_check_red_flags_kingfisher.py

@@ -12,11 +12,11 @@
 #     release_type = 'compiled_release'
 # -
 
-fields_list = fields_table.iloc[:, 0].tolist()
-
-indicators_dict = get_red_flags_dictionary(fields_list)
-result = redflags_checks(fields_list, indicators_dict, check_coverage=True)
-result["coverage"] = get_coverage(indicators_dict)
+fields_list = set(fields_table["path"])
+indicators = load_indicators(prefix="R")
+result = indicator_checks(fields_list, indicators)
+result = result.rename(columns={"id": "R_id", "indicator": "red_flag"})
+result["coverage"] = get_coverage(fields_list, indicators)
 
 # ### Export results
 
@@ -32,7 +32,7 @@
 #
 # This table shows the most frequent fields used to calculate indicators and if they are published.  You can use this table to highlight to the publisher the key data gaps.
 
-common_fields = most_common_fields_to_calculate_indicators(indicators_dict, fields_table)
+common_fields = most_common_fields_to_calculate_indicators(fields_list, indicators)
 common_fields
 
 # #### Save tables to spreadsheet

component_check_usability_external.py

@@ -2,16 +2,16 @@
 #
 # Generate a list of the fields published:
 
-fields_list = fields_table.iloc[:, 0].tolist()
-
-indicators_dict = get_indicators_dictionary(fields_list)
-result = usability_checks(fields_list, indicators_dict)
+fields_list = set(fields_table["path"])
+indicators = load_indicators(prefix="U")
+result = indicator_checks(fields_list, indicators)
+result = result.rename(columns={"id": "U_id"})
 
 # ### Export and visualize results
 
 # #### Choose language of the export
 
-lang = get_usability_language_select_box()
+lang = widgets.Dropdown(options=["Spanish", "English"], description="language", style={"description_width": "initial"})
 lang
 
 # #### Load use case indicators spreadsheet

component_check_usability_kingfisher.py

@@ -2,17 +2,17 @@
 #
 # Generate a list of the fields published:
 
-fields_list = fields_table.iloc[:, 0].tolist()
-
-indicators_dict = get_indicators_dictionary(fields_list)
-result = usability_checks(fields_list, indicators_dict)
-result["coverage"] = get_coverage(indicators_dict)
+fields_list = set(fields_table["path"])
+indicators = load_indicators(prefix="U")
+result = indicator_checks(fields_list, indicators)
+result = result.rename(columns={"id": "U_id"})
+result["coverage"] = get_coverage(fields_list, indicators)
 
 # ### Export and visualize results
 
 # #### Choose language of the export
 
-lang = get_usability_language_select_box()
+lang = widgets.Dropdown(options=["Spanish", "English"], description="language", style={"description_width": "initial"})
 lang
 
 # #### Load use case indicators spreadsheet
@@ -27,7 +27,7 @@
 #
 # This table shows the most frequent fields used to calculate indicators and if they are published.  You can use this table to highlight to the publisher the key data gaps.
 
-fields_count = most_common_fields_to_calculate_indicators(indicators_dict, fields_table)
+fields_count = most_common_fields_to_calculate_indicators(fields_list, indicators)
 fields_count
 
 # #### Save tables to spreadsheet

component_environment.py

@@ -2,10 +2,10 @@
 #
 # *You must run the cells in this section each time you connect to a new runtime. For example, when you return to the notebook after an idle timeout, when the runtime crashes, or when you restart or factory reset the runtime.*
 #
-# Install requirements (*Note: ocdskingfishercolab installs google-colab, which expects specific versions of pandas and numpy*):
+# Install requirements:
 
 # ! pip install --upgrade pip > pip.log
-# ! pip install --upgrade ocdskingfishercolab ipywidgets psycopg2-binary >> pip.log
+# ! pip install --upgrade git+https://github.com/open-contracting/kingfisher-colab@95-setup-functions psycopg2-binary >> pip.log
 
 # +
 # @title Import packages and load extensions { display-mode: "form" }
@@ -27,7 +27,6 @@
 from ipywidgets import widgets
 from ocdskingfishercolab import (
     authenticate_gspread,
-    calculate_coverage,
     download_dataframe_as_csv,
     format_thousands,
     render_json,
@@ -36,6 +35,22 @@
     set_dark_mode,
     set_light_mode,
 )
+from ocdskingfishercolab.charts import (
+    plot_objects_per_stage,
+    plot_objects_per_year,
+    plot_release_count,
+    plot_releases_by_month,
+    plot_top_buyers,
+    plot_usability_indicators,
+)
+from ocdskingfishercolab.indicators.indicators import (
+    calculate_coverage,
+    indicator_checks,
+    load_indicators,
+    most_common_fields_to_calculate_indicators,
+)
+from ocdskingfishercolab.registry import format_coverage, get_publication_select_box, get_publications
+from ocdskingfishercolab.usability import check_red_flags_indicators, check_usability_indicators, is_relevant
 
 # Load https://pypi.org/project/ipython-sql/
 # %load_ext sql