From 788042d1e59c4c5071ad51064bbad6f64aed3bfb Mon Sep 17 00:00:00 2001 From: Sara Bratsch Date: Mon, 6 Oct 2025 12:22:29 -0500 Subject: [PATCH 1/4] update Applications API --- notebooks/Applications API.ipynb | 107 ++++++++++++++++++++++++------- 1 file changed, 84 insertions(+), 23 deletions(-) diff --git a/notebooks/Applications API.ipynb b/notebooks/Applications API.ipynb index 3ce4440..3af36a3 100644 --- a/notebooks/Applications API.ipynb +++ b/notebooks/Applications API.ipynb @@ -47,7 +47,7 @@ "id": "4", "metadata": {}, "source": [ - "#### At a minimum an AI Application requires an Application Name, Source and Type. Valid Sources are `internal` or `vendor`. The response will give you a `application_id` and `application_version`, save these for use in other notebooks." + "#### At a minimum an AI Application requires an Application Name, Source and Type. The response will give you the `application_id`. Save this for use in other notebooks." ] }, { @@ -55,19 +55,48 @@ "execution_count": null, "id": "5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created new Application:\n", + "{\n", + " \"created_at\": \"2025-10-06T16:08:40.449531Z\",\n", + " \"created_by\": \"b2e4aab8-1f11-47df-bd3b-3243a6782130\",\n", + " \"updated_at\": \"2025-10-06T16:08:40.449531Z\",\n", + " \"updated_by\": \"b2e4aab8-1f11-47df-bd3b-3243a6782130\",\n", + " \"id\": \"266a1b05-804c-4ba7-b663-cff73be508c4\",\n", + " \"name\": \"Sara 6 Oct Test API Guides\",\n", + " \"source\": \"internal\",\n", + " \"type\": \"other\",\n", + " \"components\": {},\n", + " \"risk_metadata\": {},\n", + " \"approval_statuses\": [\n", + " {\n", + " \"approval_type\": \"default\",\n", + " \"status\": \"DRAFT\"\n", + " }\n", + " ],\n", + " \"assessed_risk_level\": \"insufficient-data\",\n", + " \"frameworks\": [],\n", + " \"controls\": []\n", + "}\n" + ] + } + ], "source": [ "application_route = \"/applications\"\n", + "\n", + "# Source must be \"internal\" or \"vendor\"\n", + "# Type must be \"other\", \"genai\", \"classifier\", \"recommender\", \"regression\", or \"unsupervised\"\n", "request_body = {\n", - " \"application_name\": \"API Client Test Application\", # Can provide your own name\n", - " \"application_type\": \"Test Type\", # Can provide your own type description\n", - " \"application_source\": \"internal\", # Must be 'internal' or 'vendor'\n", - " \"application_version\": None, # Optional, will default to `1.0`\n", - " \"vendor_id\": None # Optional if 'application_source' is 'internal', required if source is 'vendor'\n", + " \"name\": \"Your Application Name\",\n", + " \"source\": \"internal\",\n", + " \"type\": \"other\"\n", "}\n", "\n", "application_id = None\n", - "application_version = None\n", "\n", "response = client.post(application_route, json=request_body)\n", "\n", @@ -75,7 +104,6 @@ " application = response.json()\n", " \n", " application_id = application.get(\"application_id\")\n", - " application_version = application.get(\"application_version\")\n", " application_name = application.get(\"application_name\")\n", " print(\"Created new Application:\")\n", " print(json.dumps(response.json(), indent=4))\n", @@ -88,7 +116,7 @@ "id": "6", "metadata": {}, "source": [ - "#### With the `application_id` and `application_version` you can now lookup the new Application:" + "#### With the `application_id` you can now lookup the new Application:" ] }, { @@ -96,15 +124,41 @@ "execution_count": null, "id": "7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "API Response:\n", + "{\n", + " \"created_at\": \"2025-10-06T16:08:40.449531Z\",\n", + " \"created_by\": \"b2e4aab8-1f11-47df-bd3b-3243a6782130\",\n", + " \"updated_at\": \"2025-10-06T16:08:40.449531Z\",\n", + " \"updated_by\": \"b2e4aab8-1f11-47df-bd3b-3243a6782130\",\n", + " \"id\": \"266a1b05-804c-4ba7-b663-cff73be508c4\",\n", + " \"name\": \"Sara 6 Oct Test API Guides\",\n", + " \"source\": \"internal\",\n", + " \"type\": \"other\",\n", + " \"components\": {},\n", + " \"risk_metadata\": {},\n", + " \"approval_statuses\": [\n", + " {\n", + " \"approval_type\": \"default\",\n", + " \"status\": \"DRAFT\"\n", + " }\n", + " ],\n", + " \"assessed_risk_level\": \"insufficient-data\",\n", + " \"frameworks\": [],\n", + " \"controls\": []\n", + "}\n" + ] + } + ], "source": [ - "applications_route = \"/applications/\"\n", - "query_parameters = {\n", - " \"application_id\": application_id,\n", - " \"application_version\": application_version,\n", - "}\n", + "application_route = \"/applications\"\n", + "application_id = \"application_id\" # Your application_id from the previous step\n", "\n", - "response = client.get(applications_route, params=query_parameters)\n", + "response = client.get(f\"{application_route}/{application_id}\")\n", "\n", "if response.status_code == 200:\n", " print(\"API Response:\")\n", @@ -118,7 +172,7 @@ "id": "8", "metadata": {}, "source": [ - "#### Save the `application_id` and `application_version` so that you can perform other compliance tasks for the application like approving it for deployment, testing for biases or adding evidence that a control has been met." + "#### Save the `application_id` so that you can perform other compliance tasks for the application like approving it for deployment, testing for biases or adding evidence that a control has been met." ] }, { @@ -126,16 +180,23 @@ "execution_count": null, "id": "9", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Application ID: 266a1b05-804c-4ba7-b663-cff73be508c4\n" + ] + } + ], "source": [ - "print(f\"Application ID: {application_id}\")\n", - "print(f\"Application Version: {application_version}\")" + "print(f\"Application ID: {application_id}\")" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -149,7 +210,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.13.7" } }, "nbformat": 4, From 2541d1978007be5ea51c46a86ea97479bccc8c23 Mon Sep 17 00:00:00 2001 From: Sara Bratsch Date: Mon, 6 Oct 2025 13:21:48 -0500 Subject: [PATCH 2/4] update Generating Synthetic User Bias Test Data --- ...rating Synthetic User Bias Test Data.ipynb | 49 +++++-------------- 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/notebooks/Generating Synthetic User Bias Test Data.ipynb b/notebooks/Generating Synthetic User Bias Test Data.ipynb index 98f7b5e..8129f47 100644 --- a/notebooks/Generating Synthetic User Bias Test Data.ipynb +++ b/notebooks/Generating Synthetic User Bias Test Data.ipynb @@ -8,7 +8,7 @@ "# Generating Synthetic User Bias Test Data\n", "\n", "\n", - "#### This notebook demonstrates how to generate synthetic test data to simulate the evaluation of bias for protected classes. This is optional as may have real scoring data to be evaluated.\n", + "#### This notebook demonstrates how to generate synthetic test data to simulate the evaluation of bias for protected classes. This is optional as you may have real scoring data to be evaluated.\n", "\n", "### Prerequisites:\n", "\n", @@ -44,44 +44,17 @@ "#### The following columns are required:\n", "* `TimeStamp` (ISO8601 Timestamp, e.g `2023-12-14T16:26:05.898156Z`)\n", "* `Score` (a number between 0 and 1)\n", - "* `Each of the Protected Class Columns` (see below.)\n", + "* `Protected Class Columns` (race, gender, disability_status, gender_identity)\n", "\n", "#### Optionally, you can add up to 3 additional columns to use for grouping and filtering the bias results after testing." ] }, - { - "cell_type": "markdown", - "id": "3", - "metadata": {}, - "source": [ - "#### Lookup the Protected Class Column Names\n", - "\n", - "#### The following code can be used the Fairnow API to lookup the column names that correspond to protected classes used to evaluate bias. If you need a different set of column names for your test please contact Fairnow.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "reference_columns_route = \"/reference/protected-class-columns\"\n", - "\n", - "response = client.get(reference_columns_route)\n", - "\n", - "if response.status_code == 200:\n", - " print(json.dumps(response.json(), indent=4))\n", - "else:\n", - " print(f\"Error: {response.status_code} - {response.text}\")" - ] - }, { "cell_type": "markdown", "id": "5", "metadata": {}, "source": [ - "#### Now pull the column names out and generate some random data. In addition to the required columns `Timestamp` and `Score` and the protected class columns the example contains two columns that can be used to filter the results. " + "#### Now generate some random data. In addition to the required columns `Timestamp` and `Score` and the protected class columns the example contains two columns that can be used to filter the results. " ] }, { @@ -91,9 +64,9 @@ "metadata": {}, "outputs": [], "source": [ - "table_data = response.json()\n", - "protected_class_columns = [ column[\"id\"] for column in table_data[\"table_data\"]]\n", - "print(f\"Protected class column names: {protected_class_columns}\")\n", + "# The default protected class columns will be used if none are provided.\n", + "# Update protected class columns by updating the list below:\n", + "protected_class_columns = [\"race\", \"gender\", \"disability_status\", \"gender_identity\"]\n", "\n", "number_of_protected_class_values = 10\n", "number_of_rows = 1000\n", @@ -130,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "8", "metadata": {}, "outputs": [], @@ -169,21 +142,21 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.13.7" } }, "nbformat": 4, From f37e0af7be32ebd9de543c62cfa0b525a342a27f Mon Sep 17 00:00:00 2001 From: Sara Bratsch Date: Mon, 6 Oct 2025 15:05:13 -0500 Subject: [PATCH 3/4] update User Data Testing --- notebooks/User Data Testing.ipynb | 39 ++++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/notebooks/User Data Testing.ipynb b/notebooks/User Data Testing.ipynb index 84ed6eb..6b98394 100644 --- a/notebooks/User Data Testing.ipynb +++ b/notebooks/User Data Testing.ipynb @@ -25,7 +25,7 @@ "\n", "#### To use this notebook, you'll need a `Client ID` and `Client Secret`. These will either have been provided to you, or you can generate from https://app.fairnow.ai and going the the Admin menu. This notebook assumes you have these available to enter when prompted.\n", "\n", - "#### To run the simulation you will need an `application_id` and `application_version` for the specific model you want to test. Details of how to create and lookup AI Applications can be found in the `Applications AI` notebook.\n", + "#### To run the simulation you will need an `application_id` for the specific model you want to test. Details of how to create and lookup AI Applications can be found in the `Applications API` notebook.\n", "\n", "#### Finally, you'll need a `threshold` value, which is the value at which anyone with a score above is considered a passing score.\n", "\n", @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "5", "metadata": {}, "outputs": [], @@ -88,8 +88,9 @@ "metadata": {}, "outputs": [], "source": [ - "application_id = None # Replace with your own Application Id\n", + "application_id = '' # Replace with your own Application Id\n", "threshold = 0.5 # Change to your own threshold setting\n", + "protected_class_columns = ['race', 'gender', 'disability_status', 'gender_identity'] # Default protected class columns\n", "\n", "if not application_id:\n", " raise ValueError(\"Please set application_id\")\n", @@ -97,9 +98,13 @@ "if threshold <= 0.0 or threshold >= 1.0:\n", " raise ValueError(\"Threshold must be between 0.0 and 1.0\")\n", "\n", + "if not protected_class_columns:\n", + " raise ValueError(\"Please set protected_class_columns. Default is ['race', 'gender', 'disability_status', 'gender_identity']\")\n", + "\n", "start_test_route = f\"/applications/{application_id}/tests\"\n", "\n", - "test_type = \"fairness_ml_user\"\n", + "\n", + "test_type = \"fairness_ml_user\" # Do not change test_type\n", "\n", "request_body = {\n", " \"test_name\": \"API Client Testing\",\n", @@ -107,9 +112,9 @@ " \"test_type\": test_type,\n", " \"inputs\": {\n", " \"type\": test_type,\n", - " \"threshold\": threshold\n", - " }\n", - "\n", + " \"threshold\": threshold,\n", + " \"protected_class_columns\": protected_class_columns\n", + " }\n", "}\n", "\n", "response = client.post(start_test_route, json=request_body, timeout=None)\n", @@ -136,6 +141,8 @@ "metadata": {}, "outputs": [], "source": [ + "import httpx\n", + "\n", "response_body = response.json()\n", "test_id = response_body[\"test_id\"]\n", "presigned_upload = response_body[\"presigned_url_scores_upload\"]\n", @@ -169,6 +176,8 @@ "metadata": {}, "outputs": [], "source": [ + "import time\n", + "\n", "test_route = f\"/applications/{application_id}/tests/{test_id}\"\n", "\n", "response = client.get(test_route)\n", @@ -179,7 +188,7 @@ " raise ValueError(f\"API Error Response: {response.status_code} - {response.text}\")\n", "\n", "while current_status not in ['ready', 'error']:\n", - " sleep(15)\n", + " time.sleep(15)\n", " response = client.get(test_route)\n", " if response.status_code == 200:\n", " current_status = response.json()[\"status\"][\"id\"]\n", @@ -224,7 +233,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "15", "metadata": {}, "outputs": [], @@ -254,19 +263,11 @@ "source": [ "!cat results.csv" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -280,7 +281,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.13.7" } }, "nbformat": 4, From cc54425659deff9d0e6c599fc4736f81850b8b11 Mon Sep 17 00:00:00 2001 From: Sara Bratsch Date: Mon, 6 Oct 2025 16:17:23 -0500 Subject: [PATCH 4/4] clean up --- notebooks/Applications API.ipynb | 72 +------------------ ...rating Synthetic User Bias Test Data.ipynb | 35 ++------- notebooks/Getting Started.ipynb | 14 +--- 3 files changed, 10 insertions(+), 111 deletions(-) diff --git a/notebooks/Applications API.ipynb b/notebooks/Applications API.ipynb index 3af36a3..f03600e 100644 --- a/notebooks/Applications API.ipynb +++ b/notebooks/Applications API.ipynb @@ -55,36 +55,7 @@ "execution_count": null, "id": "5", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Created new Application:\n", - "{\n", - " \"created_at\": \"2025-10-06T16:08:40.449531Z\",\n", - " \"created_by\": \"b2e4aab8-1f11-47df-bd3b-3243a6782130\",\n", - " \"updated_at\": \"2025-10-06T16:08:40.449531Z\",\n", - " \"updated_by\": \"b2e4aab8-1f11-47df-bd3b-3243a6782130\",\n", - " \"id\": \"266a1b05-804c-4ba7-b663-cff73be508c4\",\n", - " \"name\": \"Sara 6 Oct Test API Guides\",\n", - " \"source\": \"internal\",\n", - " \"type\": \"other\",\n", - " \"components\": {},\n", - " \"risk_metadata\": {},\n", - " \"approval_statuses\": [\n", - " {\n", - " \"approval_type\": \"default\",\n", - " \"status\": \"DRAFT\"\n", - " }\n", - " ],\n", - " \"assessed_risk_level\": \"insufficient-data\",\n", - " \"frameworks\": [],\n", - " \"controls\": []\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "application_route = \"/applications\"\n", "\n", @@ -124,36 +95,7 @@ "execution_count": null, "id": "7", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "API Response:\n", - "{\n", - " \"created_at\": \"2025-10-06T16:08:40.449531Z\",\n", - " \"created_by\": \"b2e4aab8-1f11-47df-bd3b-3243a6782130\",\n", - " \"updated_at\": \"2025-10-06T16:08:40.449531Z\",\n", - " \"updated_by\": \"b2e4aab8-1f11-47df-bd3b-3243a6782130\",\n", - " \"id\": \"266a1b05-804c-4ba7-b663-cff73be508c4\",\n", - " \"name\": \"Sara 6 Oct Test API Guides\",\n", - " \"source\": \"internal\",\n", - " \"type\": \"other\",\n", - " \"components\": {},\n", - " \"risk_metadata\": {},\n", - " \"approval_statuses\": [\n", - " {\n", - " \"approval_type\": \"default\",\n", - " \"status\": \"DRAFT\"\n", - " }\n", - " ],\n", - " \"assessed_risk_level\": \"insufficient-data\",\n", - " \"frameworks\": [],\n", - " \"controls\": []\n", - "}\n" - ] - } - ], + "outputs": [], "source": [ "application_route = \"/applications\"\n", "application_id = \"application_id\" # Your application_id from the previous step\n", @@ -180,15 +122,7 @@ "execution_count": null, "id": "9", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Application ID: 266a1b05-804c-4ba7-b663-cff73be508c4\n" - ] - } - ], + "outputs": [], "source": [ "print(f\"Application ID: {application_id}\")" ] diff --git a/notebooks/Generating Synthetic User Bias Test Data.ipynb b/notebooks/Generating Synthetic User Bias Test Data.ipynb index 8129f47..d210538 100644 --- a/notebooks/Generating Synthetic User Bias Test Data.ipynb +++ b/notebooks/Generating Synthetic User Bias Test Data.ipynb @@ -10,33 +10,6 @@ "\n", "#### This notebook demonstrates how to generate synthetic test data to simulate the evaluation of bias for protected classes. This is optional as you may have real scoring data to be evaluated.\n", "\n", - "### Prerequisites:\n", - "\n", - "#### To use this notebook, you'll need a `Client ID` and `Client Secret`. These will either have been provided to you, or you can generate from https://app.fairnow.ai and going the the Admin menu. This notebook assumes you have these available to enter when prompted." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import datetime\n", - "import random\n", - "import csv\n", - "\n", - "from utils.fairnow import get_client\n", - "\n", - "client = get_client(client_id='client_id') # Replace with your Client Id" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ "### Data Requirements\n", "\n", "#### The CSV data files require the first row to contain the column names.\n", @@ -59,11 +32,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "6", "metadata": {}, "outputs": [], "source": [ + "import datetime\n", + "import random\n", + "import csv\n", + "\n", "# The default protected class columns will be used if none are provided.\n", "# Update protected class columns by updating the list below:\n", "protected_class_columns = [\"race\", \"gender\", \"disability_status\", \"gender_identity\"]\n", @@ -103,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 8, "id": "8", "metadata": {}, "outputs": [], diff --git a/notebooks/Getting Started.ipynb b/notebooks/Getting Started.ipynb index 0cee8b0..b466be2 100644 --- a/notebooks/Getting Started.ipynb +++ b/notebooks/Getting Started.ipynb @@ -73,18 +73,6 @@ "\n", "#### On the other hand, a `Client authentication failed` exception most likely means the `Client ID` or `Client Secret` were invalid." ] - }, - { - "cell_type": "markdown", - "id": "6", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "id": "7", - "metadata": {}, - "source": [] } ], "metadata": { @@ -103,7 +91,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "3.13.3" + "version": "3.13.7" } }, "nbformat": 4,