From fe0ccb141cc1e15cefc166fdc8f62bb6308969a0 Mon Sep 17 00:00:00 2001 From: Sara Bratsch Date: Tue, 7 Oct 2025 11:50:45 -0500 Subject: [PATCH 1/3] update export reports --- notebooks/Applications API.ipynb | 4 +- notebooks/Export Reports.ipynb | 2 +- notebooks/utils/api_helpers.py | 2 +- notebooks/utils/dataframe_helpers.py | 89 +++++++++++++++------------- 4 files changed, 53 insertions(+), 44 deletions(-) diff --git a/notebooks/Applications API.ipynb b/notebooks/Applications API.ipynb index f03600e..9725540 100644 --- a/notebooks/Applications API.ipynb +++ b/notebooks/Applications API.ipynb @@ -59,8 +59,8 @@ "source": [ "application_route = \"/applications\"\n", "\n", - "# Source must be \"internal\" or \"vendor\"\n", - "# Type must be \"other\", \"genai\", \"classifier\", \"recommender\", \"regression\", or \"unsupervised\"\n", + "# Please see https://api.fairnow.dev/v2/docs for enumerated values for ApplicationSource and ApplicationType\n", + "\n", "request_body = {\n", " \"name\": \"Your Application Name\",\n", " \"source\": \"internal\",\n", diff --git a/notebooks/Export Reports.ipynb b/notebooks/Export Reports.ipynb index 1a9fef9..da7ff27 100644 --- a/notebooks/Export Reports.ipynb +++ b/notebooks/Export Reports.ipynb @@ -147,7 +147,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.3" + "version": "3.13.7" } }, "nbformat": 4, diff --git a/notebooks/utils/api_helpers.py b/notebooks/utils/api_helpers.py index a2fb5e7..d345187 100644 --- a/notebooks/utils/api_helpers.py +++ b/notebooks/utils/api_helpers.py @@ -114,7 +114,7 @@ def get_vendor_data(client): """ Get vendor data from the FairNow API. """ - application_route = "/vendors/" + application_route = "/vendors" response = None try: diff --git a/notebooks/utils/dataframe_helpers.py b/notebooks/utils/dataframe_helpers.py index 12ac3db..0f5b9c6 100644 --- a/notebooks/utils/dataframe_helpers.py +++ b/notebooks/utils/dataframe_helpers.py @@ -164,31 +164,40 @@ def create_inventory_df(client_id): Create a pandas DataFrame from the inventory data. Includes all applications, joining vendor info when available. """ + client = get_client(client_id) # Replace with your Client Id # Retrieve application data from the API - apps_response = get_application_data(client) - - if not apps_response or 'applications' not in apps_response or not apps_response['applications']: - print("No application data found") + applications = get_application_data(client) + + if not applications: + print("ERROR: No applications found") return pd.DataFrame() # Extract fields from response extracted_data = [] - for app in apps_response['applications']: - app_id = app['application_id'] - app_name = app['application_name'] - vendor_id = app.get('vendor_id', '') - risk_metadata = app.get('risk_metadata', {}) or {} - application_risk = risk_metadata.get('risk_framework_level', '') - application_source = app.get('application_source', '') - application_development_status = app.get('application_development_status', '') - application_approval_status = app.get('approval_status', '') + for app in applications: + app_id = app['id'] + app_name = app['name'] + vendor_id = '' + # Extract vendor_id from vendor_links if present + vendor_links = app.get('vendor_links', []) + if vendor_links: + vendor_id = vendor_links[0].get('vendor_id', '') + + application_source = app.get('source', '') + application_development_status = app.get('development_status', '') + + # Extract approval status + approval_statuses = app.get('approval_statuses', []) + application_approval_status = '' + if approval_statuses: + application_approval_status = approval_statuses[0].get('status', '') + extracted_data.append({ 'application_id': app_id, 'application_name': app_name, 'vendor_id': vendor_id, - 'application_risk': application_risk, 'application_source': application_source, 'application_development_status': application_development_status, 'application_approval_status': application_approval_status, @@ -198,28 +207,28 @@ def create_inventory_df(client_id): apps_df = create_df(extracted_data) # Retrieve vendor data from the API - response = get_vendor_data(client) + vendors_response = get_vendor_data(client) - if not response: - print("No vendor data found") + if not vendors_response: + print("WARNING: No vendor data found, adding empty vendor columns") apps_df['vendor_name'] = '' apps_df['vendor_status'] = '' return apps_df # Extract fields from response - extracted_data = [] - for vendor in response: - vendor_id = vendor.get('vendor_id', '') - vendor_name = vendor.get('vendor_name', '') + vendor_data = [] + for vendor in vendors_response: + vendor_id = vendor.get('id', '') + vendor_name = vendor.get('name', '') vendor_status = vendor.get('status', '') - extracted_data.append({ + vendor_data.append({ 'vendor_id': vendor_id, 'vendor_name': vendor_name, 'vendor_status': vendor_status, }) # Convert to DataFrame - vendors_df = create_df(extracted_data) + vendors_df = create_df(vendor_data) # Merge DataFrames merged_df = pd.merge( @@ -228,6 +237,7 @@ def create_inventory_df(client_id): on='vendor_id', how='left' ) + merged_df = merged_df.drop_duplicates() merged_df = merged_df[[ @@ -236,7 +246,6 @@ def create_inventory_df(client_id): 'application_source', 'application_development_status', 'application_approval_status', - 'application_risk', 'vendor_id', 'vendor_name', 'vendor_status' @@ -249,46 +258,47 @@ def create_risks_df(client_id): Create a pandas DataFrame from the risks data. Returns all applications, with null values for those without risk information. """ + client = get_client(client_id) # Replace with your Client Id # Retrieve application data from the API - apps_response = get_application_data(client) + applications = get_application_data(client) - if not apps_response or 'applications' not in apps_response or not apps_response['applications']: - print("No application data found") + if not applications: + print("ERROR: No applications found") return pd.DataFrame() # First create a DataFrame with all applications all_apps_data = [] - for app in apps_response['applications']: - risk_metadata = app.get('risk_metadata', {}) or {} - application_risk_level = risk_metadata.get('risk_framework_level', None) + for app in applications: + assessed_risk_level = app.get('assessed_risk_level', None) all_apps_data.append({ - 'application_id': app['application_id'], - 'application_name': app['application_name'], - 'application_risk_level': application_risk_level + 'application_id': app['id'], + 'application_name': app['name'], + 'assessed_risk_level': assessed_risk_level }) all_apps_df = create_df(all_apps_data) # Create a DataFrame from the risk data risk_data = [] - for app in apps_response['applications']: - app_id = app['application_id'] - risk_items = app.get('risk_item_list', []) or [] + for app in applications: + app_id = app['id'] + risk_items = app.get('assigned_risk_items', []) or [] for risk_item in risk_items: risk_data.append({ 'application_id': app_id, - 'risk_type': risk_item.get('risk_type_label', ''), + 'risk_type': risk_item.get('risk_type', ''), 'severity': risk_item.get('severity', ''), 'probability': risk_item.get('probability', ''), + # 'description': risk_item.get('description', ''), }) # Convert risk data to DataFrame risk_df = create_df(risk_data) if risk_df.empty: - print("No risk data found") - return pd.DataFrame() + print("WARNING: No risk data found") + return all_apps_df # Return apps without risk data # Merge all applications with risk data result = pd.merge( @@ -297,5 +307,4 @@ def create_risks_df(client_id): on='application_id', how='left' ) - return result From 05e15bff2fd09cd69d22de3b8f8783d90a32f247 Mon Sep 17 00:00:00 2001 From: Sara Bratsch Date: Wed, 8 Oct 2025 17:45:35 -0500 Subject: [PATCH 2/3] refactor compliance exports --- .gitignore | 1 + notebooks/Export Reports.ipynb | 19 +- notebooks/utils/api_helpers.py | 70 ++---- notebooks/utils/dataframe_helpers.py | 326 ++++++++++++++++----------- notebooks/utils/fairnow.py | 4 +- 5 files changed, 222 insertions(+), 198 deletions(-) diff --git a/.gitignore b/.gitignore index 1e5022b..da1b582 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ notebooks/.ipynb_checkpoints .venv/ .python-version __pycache__/ +.DS_Store diff --git a/notebooks/Export Reports.ipynb b/notebooks/Export Reports.ipynb index da7ff27..5e94b7f 100644 --- a/notebooks/Export Reports.ipynb +++ b/notebooks/Export Reports.ipynb @@ -38,14 +38,13 @@ "outputs": [], "source": [ "# Add imports\n", - "from utils.dataframe_helpers import create_compliance_df\n", + "from utils.dataframe_helpers import create_application_compliance_df\n", "from utils.fairnow import export_to_tsv\n", "\n", - "# Create Compliance DataFrames for Application\n", - "# Note: DataFrames can be used for graphing or further analysis\n", - "application_compliance_df = create_compliance_df(client_id='client_id', control_type=\"application\") # Replace with your Client Id\n", + "# # Create Compliance DataFrames for Application\n", + "# # Note: DataFrames can be used for graphing or further analysis\n", + "application_compliance_df = create_application_compliance_df(client_id='client_id') # Replace with your Client Id\n", "\n", - "# Export to TSV\n", "export_to_tsv(application_compliance_df, 'application_compliance_export.tsv')" ] }, @@ -60,17 +59,15 @@ { "cell_type": "code", "execution_count": null, - "id": "6d64b2ac", + "id": "1a09ad93", "metadata": {}, "outputs": [], "source": [ - "# Add imports\n", - "from utils.dataframe_helpers import create_compliance_df\n", + "# # Add imports\n", + "from utils.dataframe_helpers import create_company_compliance_df\n", "from utils.fairnow import export_to_tsv\n", "\n", - "# Create Compliance DataFrames for Company\n", - "# Note: DataFrames can be used for graphing or further analysis\n", - "company_compliance_df = create_compliance_df(client_id='client_id', control_type=\"company\") # Replace with your Client Id\n", + "company_compliance_df = create_company_compliance_df(client_id='client_id') # Replace with your Client Id\n", "\n", "# Export to TSV\n", "export_to_tsv(company_compliance_df, 'company_compliance_export.tsv')" diff --git a/notebooks/utils/api_helpers.py b/notebooks/utils/api_helpers.py index d345187..8fe14ce 100644 --- a/notebooks/utils/api_helpers.py +++ b/notebooks/utils/api_helpers.py @@ -1,6 +1,3 @@ -# utils/api_helpers.py -import pandas as pd - def get_application_data(client): """ Get application data from the FairNow API. @@ -20,53 +17,18 @@ def get_application_data(client): print(e) -def get_application_controls(client, application_id, application_version, control_type): - """ - Get framework controls by framework ID from the FairNow API. - Returns a DataFrame directly with essential fields. - """ - application_route = f"/controls/application/" - query_parameters = { - "application_id": application_id, - "application_version": application_version, - "control_type": control_type # application or company - } +def get_application_by_id(client, application_id: str): + route = f"/applications/{application_id}" try: - response = client.get(application_route, params=query_parameters, timeout=None) + response = client.get(route, timeout=None) if response.status_code == 200: - full_response = response.json() - - # Create flattened data for DataFrame - flattened_data = [] - for control in full_response.get('controls', []): - # For controls with frameworks, create a row for each framework - frameworks = control.get('frameworks_in_scope', []) - if frameworks: - for framework in frameworks: - flattened_data.append({ - 'control_id': control.get('control_id'), - 'ready': control.get('ready'), - 'framework': framework, - 'application_id': application_id, - 'application_version': application_version - }) - else: - # For controls without frameworks, create a single row - flattened_data.append({ - 'control_id': control.get('control_id'), - 'ready': control.get('ready'), - 'framework': '', - 'application_id': application_id, - 'application_version': application_version - }) - - # Create DataFrame directly - return pd.DataFrame(flattened_data) + response = response.json() + return response else: print(f"Error: {response.status_code} - {response.text}") return None except Exception as e: - print(e) + print(f"[EXCEPTION] Error fetching application {application_id}: {e}") return None @@ -89,26 +51,24 @@ def get_frameworks(client): print(e) -def get_framework_controls(client, framework_id): +def get_controls(client): """ - Get framework controls by framework ID from the FairNow API. + Get framework data from the FairNow API. """ - application_route = f"/controls/framework/" - query_parameters = {"framework_id": framework_id} - framework_response = None - + application_route = "/controls" + + response = None try: - response = client.get(application_route, params=query_parameters, timeout=None) + response = client.get(application_route, timeout=None) if response.status_code == 200: - framework_response = response.json() - return framework_response + response = response.json() + return response else: print(f"Error: {response.status_code} - {response.text}") return None except Exception as e: print(e) - return None - + def get_vendor_data(client): """ diff --git a/notebooks/utils/dataframe_helpers.py b/notebooks/utils/dataframe_helpers.py index 0f5b9c6..942d790 100644 --- a/notebooks/utils/dataframe_helpers.py +++ b/notebooks/utils/dataframe_helpers.py @@ -1,12 +1,12 @@ -# utils/dataframe_helpers.py - import pandas as pd -from utils.api_helpers import get_application_data, get_application_controls, get_frameworks, get_vendor_data +from utils.api_helpers import get_application_data, get_frameworks, get_vendor_data, get_application_by_id, get_controls from utils.fairnow import get_client +from concurrent.futures import ThreadPoolExecutor, as_completed + def create_df(api_response): """ - Create a pandasDataFrame from a JSON response. + Create a pandas DataFrame from a JSON response. """ # Convert to DataFrame df = pd.DataFrame(api_response) @@ -14,149 +14,215 @@ def create_df(api_response): return pd.DataFrame(df) -def create_compliance_df(client_id, control_type): +def create_application_compliance_df(client_id) -> pd.DataFrame: """ - Create a pandas DataFrame from the compliance data. - If control_type is 'company', returns framework-level aggregation. - If control_type is 'application', returns application + framework-level details. + Create a pandas DataFrame from the application compliance data. """ client = get_client(client_id) - # Retrieve application data from the API - apps_response = get_application_data(client) - - if not apps_response or 'applications' not in apps_response or not apps_response['applications']: - print("No application data found") - return pd.DataFrame() + def build_raw_application_controls(client) -> pd.DataFrame: + applications_list = get_application_data(client) - # Create master list of all applications - master_apps = [] - for app in apps_response['applications']: - app_id = app['application_id'] - application_name = app['application_name'] - application_version = app.get('application_version', '') - risk_assessment = app.get('risk_assessment', {}) - framework_items = risk_assessment.get('framework_assessment_items', []) or [] - for framework in framework_items: - master_apps.append({ - 'application_id': app_id, - 'application_name': application_name, - 'application_version': application_version, - 'framework_id': framework.get('framework_id', ''), - }) - master_apps_df = create_df(master_apps) - - if master_apps_df.empty: - print("No framework data found in applications") - return pd.DataFrame() + if not applications_list: + print("[DEBUG] No applications returned from API.") + return pd.DataFrame() - # Create a list of dictionaries for application_id and application_version - app_version_df = master_apps_df[['application_id', 'application_version']].drop_duplicates().to_dict('records') - - # Collect controls data - controls_list = [] - for app in app_version_df: - controls_df = get_application_controls(client, app['application_id'], app['application_version'], control_type) - if controls_df is not None: - controls_df = controls_df.rename(columns={'framework': 'framework_id'}) - controls_list.append(controls_df) - - if not controls_list: - print("No controls data found") - # Return empty DataFrame with correct columns based on control_type - if control_type == 'company': - return pd.DataFrame(columns=['framework_id', 'framework_name', 'count_controls_ready', 'total_controls']) - else: - return pd.DataFrame(columns=['application_id', 'application_name', 'application_version', - 'framework_id', 'framework_name', 'count_controls_ready', 'total_controls']) - - # Combine all controls - all_controls_df = pd.concat(controls_list, ignore_index=True) - - # Get frameworks names - frameworks_df = create_frameworks_df(client) - if frameworks_df.empty: - print("No frameworks data found") - return pd.DataFrame() - frameworks_df = frameworks_df[['framework_id', 'framework_name']].drop_duplicates() + all_rows = [] + + def process_app(app): + app_id = app.get("id") + app_name = app.get("name") + + app_json = get_application_by_id(client, app_id) + if not app_json: + return [] + + controls_dict = {c.get("id"): c for c in app_json.get("controls", [])} + frameworks = app_json.get("frameworks", []) + + rows = [] + + if not frameworks: + # No frameworks attached: still include a placeholder row for application_id + rows.append({ + "application_id": app_id, + "application_name": app_name, + "framework_id": pd.NA, + "framework_name": pd.NA, + "control_id": pd.NA, + "control_status": pd.NA, + "control_implemented": pd.NA, + }) + return rows + + for fw in frameworks: + fw_id = fw.get("id") + fw_name = fw.get("name") + for req in fw.get("requirements", []): + for ctrl_link in req.get("control_links", []): + ctrl_id = ctrl_link.get("id") + ctrl = controls_dict.get(ctrl_id, {}) + status = ctrl.get("status", {}) + rows.append({ + "application_id": app_id, + "application_name": app_name, + "framework_id": fw_id, + "framework_name": fw_name, + "control_id": ctrl_id, + "control_status": status.get("control_state"), + "control_implemented": status.get("is_complete"), + }) + return rows + + # Parallelize fetching + with ThreadPoolExecutor(max_workers=6) as executor: + futures = {executor.submit(process_app, app): app for app in applications_list} + for future in as_completed(futures): + rows = future.result() + if rows: + all_rows.extend(rows) + + raw_df = pd.DataFrame(all_rows) + + if not raw_df.empty: + raw_df = raw_df.drop_duplicates(subset=["application_id", "framework_id", "control_id"]) + + return raw_df + + + def aggregate_compliance(raw_df: pd.DataFrame) -> pd.DataFrame: + if raw_df.empty: + return pd.DataFrame() - # Get controls for control_type 'company' - if control_type == 'company': - # Get unique controls per framework - unique_controls = all_controls_df.drop_duplicates(['framework_id', 'control_id']) - - # Aggregate at framework level - result = unique_controls.groupby(['framework_id']).agg( - count_controls_ready=('ready', lambda x: sum(x == True)), - total_controls=('ready', 'count') - ).reset_index() - - # Merge with frameworks_df - result = pd.merge( - result, - frameworks_df, - on='framework_id', - how='left' + # Keep only ACTIVE controls + active_df = raw_df[raw_df["control_status"] == "ACTIVE"].copy() + + # Standard aggregation per application + framework + summary_df = ( + active_df.groupby( + ["application_id", "application_name", "framework_id", "framework_name"], + as_index=False + ) + .agg( + count_controls_ready=("control_implemented", lambda x: x.sum(skipna=True)), + count_controls_total=("control_implemented", "count"), + ) ) - # Reorder columns - result = result[['framework_id', - 'framework_name', - 'count_controls_ready', - 'total_controls']] - - # Get controls for control_type 'application' - else: - # Aggregate controls data with application details - result = all_controls_df.groupby(['application_id', 'application_version', 'framework_id']).agg( - count_controls_ready=('ready', lambda x: sum(x == True)), - total_controls=('ready', 'count') - ).reset_index() - - # Map framework_name from frameworks_df - result = pd.merge( - result, - frameworks_df, - on='framework_id', - how='left' - ) + # Find apps that had no frameworks (framework_id is pd.NA) + apps_with_no_frameworks = raw_df[raw_df["framework_id"].isna()][["application_id", "application_name"]].drop_duplicates() + if not apps_with_no_frameworks.empty: + for _, row in apps_with_no_frameworks.iterrows(): + summary_df = pd.concat([ + summary_df, + pd.DataFrame([{ + "application_id": row["application_id"], + "application_name": row["application_name"], + "framework_id": pd.NA, + "framework_name": pd.NA, + "count_controls_ready": pd.NA, + "count_controls_total": pd.NA, + }]) + ], ignore_index=True) - # Join back to master_apps_df to include all applications - result = pd.merge( - master_apps_df[['application_id', 'application_version', 'application_name']].drop_duplicates(), - result, - on=['application_id', 'application_version'], - how='left' - ) + # Ensure integer type for counts where applicable + summary_df["count_controls_ready"] = summary_df["count_controls_ready"].astype("Int64") + summary_df["count_controls_total"] = summary_df["count_controls_total"].astype("Int64") - # Reorder columns - result = result[['application_id', - 'application_name', - 'application_version', - 'framework_id', - 'framework_name', - 'count_controls_ready', - 'total_controls']] + summary_df = summary_df.sort_values(by="application_name", ascending=True).reset_index(drop=True) - result = result.drop_duplicates() - return result + return summary_df + raw_df = build_raw_application_controls(client) + final_df = aggregate_compliance(raw_df) + return final_df -def create_frameworks_df(client): + +def create_company_compliance_df(client_id) -> pd.DataFrame: """ - Create a pandas DataFrame from the frameworks data. + Create a pandas DataFrame summarizing company-level compliance by framework. + Only frameworks with at least one ACTIVE company-level control are included. """ - frameworks_response = get_frameworks(client) + client = get_client(client_id) - if not frameworks_response: - print("No frameworks data received") - return pd.DataFrame() - else: - df = pd.DataFrame(frameworks_response) - if df.empty: - print("No frameworks data found") + def build_raw_company_controls(client) -> pd.DataFrame: + frameworks_list = get_frameworks(client) + controls_list = get_controls(client) + + if not frameworks_list: return pd.DataFrame() - return df[['framework_id', 'framework_name']] + + # Only COMPANY type controls + controls_dict = {c["id"]: c for c in controls_list if c.get("type") == "COMPANY"} + + all_rows = [] + + def process_framework(fw): + fw_id = fw.get("id") + fw_name = fw.get("name") + rows = [] + + for req in fw.get("requirements", []): + for ctrl_link in req.get("control_links", []): + ctrl_id = ctrl_link.get("id") + ctrl = controls_dict.get(ctrl_id) + if not ctrl: + continue + status = ctrl.get("status", {}) + rows.append({ + "framework_id": fw_id, + "framework_name": fw_name, + "control_id": ctrl_id, + "control_status": status.get("control_state"), + "control_implemented": status.get("is_complete"), + }) + + # Only keep frameworks with at least one ACTIVE control + active_rows = [r for r in rows if r.get("control_status") == "ACTIVE"] + if active_rows: + return active_rows + else: + return [] + + # Parallelize per framework + with ThreadPoolExecutor(max_workers=6) as executor: + futures = {executor.submit(process_framework, fw): fw for fw in frameworks_list} + for future in as_completed(futures): + rows = future.result() + if rows: + all_rows.extend(rows) + + raw_df = pd.DataFrame(all_rows) + if not raw_df.empty: + raw_df = raw_df.drop_duplicates(subset=["framework_id", "control_id"]) + + return raw_df + + def aggregate_company_compliance(raw_df: pd.DataFrame) -> pd.DataFrame: + if raw_df.empty: + print("[DEBUG] No raw data to aggregate.") + return pd.DataFrame() + + summary_df = ( + raw_df.groupby( + ["framework_id", "framework_name"], + as_index=False + ) + .agg( + count_controls_ready=("control_implemented", lambda x: x.sum(skipna=True)), + count_controls_total=("control_implemented", "count"), + ) + ) + + summary_df["count_controls_ready"] = summary_df["count_controls_ready"].astype("Int64") + summary_df["count_controls_total"] = summary_df["count_controls_total"].astype("Int64") + + summary_df = summary_df.sort_values(by="framework_name", ascending=True).reset_index(drop=True) + return summary_df + + raw_df = build_raw_company_controls(client) + final_df = aggregate_company_compliance(raw_df) + return final_df def create_inventory_df(client_id): @@ -290,7 +356,7 @@ def create_risks_df(client_id): 'risk_type': risk_item.get('risk_type', ''), 'severity': risk_item.get('severity', ''), 'probability': risk_item.get('probability', ''), - # 'description': risk_item.get('description', ''), + 'description': risk_item.get('description', '') }) # Convert risk data to DataFrame diff --git a/notebooks/utils/fairnow.py b/notebooks/utils/fairnow.py index d79cea6..aacb091 100644 --- a/notebooks/utils/fairnow.py +++ b/notebooks/utils/fairnow.py @@ -30,7 +30,7 @@ def get_client(client_id): The client secret will be prompted for at runtime. """ client_secret = getpass("Client Secret: ") - fairnow_token_endpoint = "https://auth.fairnow.ai/oauth2/token" + fairnow_token_endpoint = "https://auth.fairnow.dev/oauth2/token" auth = OAuth2ClientCredentials( token_url=fairnow_token_endpoint, @@ -38,6 +38,6 @@ def get_client(client_id): client_secret=client_secret, ) - fairnow_base_url = "https://api.fairnow.ai/v2" + fairnow_base_url = "https://api.fairnow.dev/v2" client = httpx.Client(base_url=fairnow_base_url, auth=auth) return client From 1be1e176eaf6885def31e0921a04ce2dfc7d3120 Mon Sep 17 00:00:00 2001 From: Sara Bratsch Date: Thu, 9 Oct 2025 12:00:18 -0500 Subject: [PATCH 3/3] update export reports for new apis --- notebooks/utils/api_helpers.py | 32 ++--- notebooks/utils/dataframe_helpers.py | 181 +++++++++++++-------------- notebooks/utils/fairnow.py | 4 +- 3 files changed, 109 insertions(+), 108 deletions(-) diff --git a/notebooks/utils/api_helpers.py b/notebooks/utils/api_helpers.py index 8fe14ce..2792b3a 100644 --- a/notebooks/utils/api_helpers.py +++ b/notebooks/utils/api_helpers.py @@ -1,12 +1,12 @@ -def get_application_data(client): +def get_applications(client): """ - Get application data from the FairNow API. + Get applications data from the FairNow API. """ - application_route = "/applications" + route = "/applications" response = None try: - response = client.get(application_route, timeout=None) + response = client.get(route, timeout=None) if response.status_code == 200: response = response.json() return response @@ -18,7 +18,12 @@ def get_application_data(client): def get_application_by_id(client, application_id: str): + """ + Get individual application data from the FairNow API. + """ route = f"/applications/{application_id}" + + response = None try: response = client.get(route, timeout=None) if response.status_code == 200: @@ -28,19 +33,18 @@ def get_application_by_id(client, application_id: str): print(f"Error: {response.status_code} - {response.text}") return None except Exception as e: - print(f"[EXCEPTION] Error fetching application {application_id}: {e}") - return None + print(e) def get_frameworks(client): """ Get framework data from the FairNow API. """ - application_route = "/frameworks" + route = "/frameworks" response = None try: - response = client.get(application_route, timeout=None) + response = client.get(route, timeout=None) if response.status_code == 200: response = response.json() return response @@ -53,13 +57,13 @@ def get_frameworks(client): def get_controls(client): """ - Get framework data from the FairNow API. + Get controls data from the FairNow API. """ - application_route = "/controls" + route = "/controls" response = None try: - response = client.get(application_route, timeout=None) + response = client.get(route, timeout=None) if response.status_code == 200: response = response.json() return response @@ -70,15 +74,15 @@ def get_controls(client): print(e) -def get_vendor_data(client): +def get_vendors(client): """ Get vendor data from the FairNow API. """ - application_route = "/vendors" + route = "/vendors" response = None try: - response = client.get(application_route, timeout=None) + response = client.get(route, timeout=None) if response.status_code == 200: response = response.json() return response diff --git a/notebooks/utils/dataframe_helpers.py b/notebooks/utils/dataframe_helpers.py index 942d790..7d5dc4e 100644 --- a/notebooks/utils/dataframe_helpers.py +++ b/notebooks/utils/dataframe_helpers.py @@ -1,10 +1,9 @@ import pandas as pd -from utils.api_helpers import get_application_data, get_frameworks, get_vendor_data, get_application_by_id, get_controls +from utils.api_helpers import get_applications, get_frameworks, get_vendors, get_application_by_id, get_controls from utils.fairnow import get_client -from concurrent.futures import ThreadPoolExecutor, as_completed -def create_df(api_response): +def create_df(api_response) -> pd.DataFrame: """ Create a pandas DataFrame from a JSON response. """ @@ -21,10 +20,8 @@ def create_application_compliance_df(client_id) -> pd.DataFrame: client = get_client(client_id) def build_raw_application_controls(client) -> pd.DataFrame: - applications_list = get_application_data(client) - + applications_list = get_applications(client) if not applications_list: - print("[DEBUG] No applications returned from API.") return pd.DataFrame() all_rows = [] @@ -40,11 +37,9 @@ def process_app(app): controls_dict = {c.get("id"): c for c in app_json.get("controls", [])} frameworks = app_json.get("frameworks", []) - rows = [] - + # Include a placeholder row for applications without frameworks if not frameworks: - # No frameworks attached: still include a placeholder row for application_id - rows.append({ + return [{ "application_id": app_id, "application_name": app_name, "framework_id": pd.NA, @@ -52,9 +47,9 @@ def process_app(app): "control_id": pd.NA, "control_status": pd.NA, "control_implemented": pd.NA, - }) - return rows + }] + rows = [] for fw in frameworks: fw_id = fw.get("id") fw_name = fw.get("name") @@ -74,19 +69,15 @@ def process_app(app): }) return rows - # Parallelize fetching - with ThreadPoolExecutor(max_workers=6) as executor: - futures = {executor.submit(process_app, app): app for app in applications_list} - for future in as_completed(futures): - rows = future.result() - if rows: - all_rows.extend(rows) + # Process applications sequentially + for app in applications_list: + rows = process_app(app) + if rows: + all_rows.extend(rows) raw_df = pd.DataFrame(all_rows) - if not raw_df.empty: raw_df = raw_df.drop_duplicates(subset=["application_id", "framework_id", "control_id"]) - return raw_df @@ -97,7 +88,7 @@ def aggregate_compliance(raw_df: pd.DataFrame) -> pd.DataFrame: # Keep only ACTIVE controls active_df = raw_df[raw_df["control_status"] == "ACTIVE"].copy() - # Standard aggregation per application + framework + # Aggregate per application + framework summary_df = ( active_df.groupby( ["application_id", "application_name", "framework_id", "framework_name"], @@ -109,23 +100,21 @@ def aggregate_compliance(raw_df: pd.DataFrame) -> pd.DataFrame: ) ) - # Find apps that had no frameworks (framework_id is pd.NA) - apps_with_no_frameworks = raw_df[raw_df["framework_id"].isna()][["application_id", "application_name"]].drop_duplicates() - if not apps_with_no_frameworks.empty: - for _, row in apps_with_no_frameworks.iterrows(): - summary_df = pd.concat([ - summary_df, - pd.DataFrame([{ - "application_id": row["application_id"], - "application_name": row["application_name"], - "framework_id": pd.NA, - "framework_name": pd.NA, - "count_controls_ready": pd.NA, - "count_controls_total": pd.NA, - }]) - ], ignore_index=True) - - # Ensure integer type for counts where applicable + # Handle applications without frameworks + apps_no_fw = raw_df[raw_df["framework_id"].isna()][["application_id", "application_name"]].drop_duplicates() + if not apps_no_fw.empty: + placeholder_rows = pd.DataFrame({ + "application_id": apps_no_fw["application_id"], + "application_name": apps_no_fw["application_name"], + "framework_id": pd.NA, + "framework_name": pd.NA, + "count_controls_ready": pd.NA, + "count_controls_total": pd.NA, + }) + summary_df = pd.concat([summary_df, placeholder_rows], ignore_index=True) + + + # Cast to int and sort summary_df["count_controls_ready"] = summary_df["count_controls_ready"].astype("Int64") summary_df["count_controls_total"] = summary_df["count_controls_total"].astype("Int64") @@ -134,8 +123,7 @@ def aggregate_compliance(raw_df: pd.DataFrame) -> pd.DataFrame: return summary_df raw_df = build_raw_application_controls(client) - final_df = aggregate_compliance(raw_df) - return final_df + return aggregate_compliance(raw_df) def create_company_compliance_df(client_id) -> pd.DataFrame: @@ -160,8 +148,8 @@ def build_raw_company_controls(client) -> pd.DataFrame: def process_framework(fw): fw_id = fw.get("id") fw_name = fw.get("name") - rows = [] + rows = [] for req in fw.get("requirements", []): for ctrl_link in req.get("control_links", []): ctrl_id = ctrl_link.get("id") @@ -169,38 +157,31 @@ def process_framework(fw): if not ctrl: continue status = ctrl.get("status", {}) - rows.append({ - "framework_id": fw_id, - "framework_name": fw_name, - "control_id": ctrl_id, - "control_status": status.get("control_state"), - "control_implemented": status.get("is_complete"), - }) - - # Only keep frameworks with at least one ACTIVE control - active_rows = [r for r in rows if r.get("control_status") == "ACTIVE"] - if active_rows: - return active_rows - else: - return [] + if status.get("control_state") == "ACTIVE": # Filter early + rows.append({ + "framework_id": fw_id, + "framework_name": fw_name, + "control_id": ctrl_id, + "control_status": status.get("control_state"), + "control_implemented": status.get("is_complete"), + }) + return rows - # Parallelize per framework - with ThreadPoolExecutor(max_workers=6) as executor: - futures = {executor.submit(process_framework, fw): fw for fw in frameworks_list} - for future in as_completed(futures): - rows = future.result() - if rows: - all_rows.extend(rows) + # Process frameworks sequentially + for fw in frameworks_list: + rows = process_framework(fw) + if rows: + all_rows.extend(rows) - raw_df = pd.DataFrame(all_rows) - if not raw_df.empty: - raw_df = raw_df.drop_duplicates(subset=["framework_id", "control_id"]) + if not all_rows: + return pd.DataFrame() + raw_df = pd.DataFrame(all_rows) + raw_df = raw_df.drop_duplicates(subset=["framework_id", "control_id"]) return raw_df def aggregate_company_compliance(raw_df: pd.DataFrame) -> pd.DataFrame: if raw_df.empty: - print("[DEBUG] No raw data to aggregate.") return pd.DataFrame() summary_df = ( @@ -214,6 +195,7 @@ def aggregate_company_compliance(raw_df: pd.DataFrame) -> pd.DataFrame: ) ) + # Cast to int and sort summary_df["count_controls_ready"] = summary_df["count_controls_ready"].astype("Int64") summary_df["count_controls_total"] = summary_df["count_controls_total"].astype("Int64") @@ -221,8 +203,7 @@ def aggregate_company_compliance(raw_df: pd.DataFrame) -> pd.DataFrame: return summary_df raw_df = build_raw_company_controls(client) - final_df = aggregate_company_compliance(raw_df) - return final_df + return aggregate_company_compliance(raw_df) def create_inventory_df(client_id): @@ -234,32 +215,51 @@ def create_inventory_df(client_id): client = get_client(client_id) # Replace with your Client Id # Retrieve application data from the API - applications = get_application_data(client) + applications = get_applications(client) if not applications: - print("ERROR: No applications found") return pd.DataFrame() # Extract fields from response extracted_data = [] + for app in applications: - app_id = app['id'] - app_name = app['name'] - vendor_id = '' - # Extract vendor_id from vendor_links if present - vendor_links = app.get('vendor_links', []) - if vendor_links: - vendor_id = vendor_links[0].get('vendor_id', '') - - application_source = app.get('source', '') - application_development_status = app.get('development_status', '') - + app_id = app.get("id", "") + app_name = app.get("name", "") + vendor_id = "" + + # Extract vendor_id safely from vendor_links + vendor_links = app.get("vendor_links") or [] + if isinstance(vendor_links, list): + # Prefer the first link with a vendor_id key + vendor = next( + (v for v in vendor_links if v.get("vendor_id")), + None + ) + if vendor: + vendor_id = vendor.get("vendor_id", "") + + application_source = app.get("source", "") + application_development_status = app.get("development_status", "") + # Extract approval status - approval_statuses = app.get('approval_statuses', []) - application_approval_status = '' - if approval_statuses: - application_approval_status = approval_statuses[0].get('status', '') - + approval_statuses = app.get("approval_statuses") or [] + application_approval_status = "" + + if isinstance(approval_statuses, list): + default_status = next( + (a for a in approval_statuses if a.get("approval_type") == "default"), + None, + ) + + if default_status: + application_approval_status = default_status.get("status", "") + elif approval_statuses: + raise ValueError( + f"No 'default' approval_type found for application {app.get('id')}. " + f"Available statuses: {approval_statuses}" + ) + extracted_data.append({ 'application_id': app_id, 'application_name': app_name, @@ -273,10 +273,9 @@ def create_inventory_df(client_id): apps_df = create_df(extracted_data) # Retrieve vendor data from the API - vendors_response = get_vendor_data(client) + vendors_response = get_vendors(client) if not vendors_response: - print("WARNING: No vendor data found, adding empty vendor columns") apps_df['vendor_name'] = '' apps_df['vendor_status'] = '' return apps_df @@ -328,10 +327,9 @@ def create_risks_df(client_id): client = get_client(client_id) # Replace with your Client Id # Retrieve application data from the API - applications = get_application_data(client) + applications = get_applications(client) if not applications: - print("ERROR: No applications found") return pd.DataFrame() # First create a DataFrame with all applications @@ -363,7 +361,6 @@ def create_risks_df(client_id): risk_df = create_df(risk_data) if risk_df.empty: - print("WARNING: No risk data found") return all_apps_df # Return apps without risk data # Merge all applications with risk data diff --git a/notebooks/utils/fairnow.py b/notebooks/utils/fairnow.py index aacb091..d79cea6 100644 --- a/notebooks/utils/fairnow.py +++ b/notebooks/utils/fairnow.py @@ -30,7 +30,7 @@ def get_client(client_id): The client secret will be prompted for at runtime. """ client_secret = getpass("Client Secret: ") - fairnow_token_endpoint = "https://auth.fairnow.dev/oauth2/token" + fairnow_token_endpoint = "https://auth.fairnow.ai/oauth2/token" auth = OAuth2ClientCredentials( token_url=fairnow_token_endpoint, @@ -38,6 +38,6 @@ def get_client(client_id): client_secret=client_secret, ) - fairnow_base_url = "https://api.fairnow.dev/v2" + fairnow_base_url = "https://api.fairnow.ai/v2" client = httpx.Client(base_url=fairnow_base_url, auth=auth) return client