Skip to content
This repository was archived by the owner on Jun 24, 2026. It is now read-only.

PLAT-473: Update notebooks post migration #15

Merged
sarab-fairnow merged 4 commits into from
Oct 6, 2025
Merged

PLAT-473: Update notebooks post migration #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
788042d
update Applications API
Oct 6, 2025
2541d19
update Generating Synthetic User Bias Test Data
Oct 6, 2025
f37e0af
update User Data Testing
Oct 6, 2025
cc54425
clean up
Oct 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 15 additions & 20 deletions notebooks/Applications API.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
"id": "4",
"metadata": {},
"source": [
"#### At a minimum an AI Application requires an Application Name, Source and Type. Valid Sources are `internal` or `vendor`. The response will give you a `application_id` and `application_version`, save these for use in other notebooks."
"#### At a minimum an AI Application requires an Application Name, Source and Type. The response will give you the `application_id`. Save this for use in other notebooks."
]
},
{
Expand All @@ -58,24 +58,23 @@
"outputs": [],
"source": [
"application_route = \"/applications\"\n",
"\n",
"# Source must be \"internal\" or \"vendor\"\n",
"# Type must be \"other\", \"genai\", \"classifier\", \"recommender\", \"regression\", or \"unsupervised\"\n",
"request_body = {\n",
" \"application_name\": \"API Client Test Application\", # Can provide your own name\n",
" \"application_type\": \"Test Type\", # Can provide your own type description\n",
" \"application_source\": \"internal\", # Must be 'internal' or 'vendor'\n",
" \"application_version\": None, # Optional, will default to `1.0`\n",
" \"vendor_id\": None # Optional if 'application_source' is 'internal', required if source is 'vendor'\n",
" \"name\": \"Your Application Name\",\n",
" \"source\": \"internal\",\n",
" \"type\": \"other\"\n",
"}\n",
"\n",
"application_id = None\n",
"application_version = None\n",
"\n",
"response = client.post(application_route, json=request_body)\n",
"\n",
"if response.status_code == 200:\n",
" application = response.json()\n",
" \n",
" application_id = application.get(\"application_id\")\n",
" application_version = application.get(\"application_version\")\n",
" application_name = application.get(\"application_name\")\n",
" print(\"Created new Application:\")\n",
" print(json.dumps(response.json(), indent=4))\n",
Expand All @@ -88,7 +87,7 @@
"id": "6",
"metadata": {},
"source": [
"#### With the `application_id` and `application_version` you can now lookup the new Application:"
"#### With the `application_id` you can now lookup the new Application:"
]
},
{
Expand All @@ -98,13 +97,10 @@
"metadata": {},
"outputs": [],
"source": [
"applications_route = \"/applications/\"\n",
"query_parameters = {\n",
" \"application_id\": application_id,\n",
" \"application_version\": application_version,\n",
"}\n",
"application_route = \"/applications\"\n",
"application_id = \"application_id\" # Your application_id from the previous step\n",
"\n",
"response = client.get(applications_route, params=query_parameters)\n",
"response = client.get(f\"{application_route}/{application_id}\")\n",
"\n",
"if response.status_code == 200:\n",
" print(\"API Response:\")\n",
Expand All @@ -118,7 +114,7 @@
"id": "8",
"metadata": {},
"source": [
"#### Save the `application_id` and `application_version` so that you can perform other compliance tasks for the application like approving it for deployment, testing for biases or adding evidence that a control has been met."
"#### Save the `application_id` so that you can perform other compliance tasks for the application like approving it for deployment, testing for biases or adding evidence that a control has been met."
]
},
{
Expand All @@ -128,14 +124,13 @@
"metadata": {},
"outputs": [],
"source": [
"print(f\"Application ID: {application_id}\")\n",
"print(f\"Application Version: {application_version}\")"
"print(f\"Application ID: {application_id}\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
Expand All @@ -149,7 +144,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
"version": "3.13.7"
}
},
"nbformat": 4,
Expand Down
82 changes: 16 additions & 66 deletions notebooks/Generating Synthetic User Bias Test Data.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,92 +8,42 @@
"# Generating Synthetic User Bias Test Data\n",
"\n",
"\n",
"#### This notebook demonstrates how to generate synthetic test data to simulate the evaluation of bias for protected classes. This is optional as may have real scoring data to be evaluated.\n",
"#### This notebook demonstrates how to generate synthetic test data to simulate the evaluation of bias for protected classes. This is optional as you may have real scoring data to be evaluated.\n",
"\n",
"### Prerequisites:\n",
"\n",
"#### To use this notebook, you'll need a `Client ID` and `Client Secret`. These will either have been provided to you, or you can generate from https://app.fairnow.ai and going the the Admin menu. This notebook assumes you have these available to enter when prompted."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import datetime\n",
"import random\n",
"import csv\n",
"\n",
"from utils.fairnow import get_client\n",
"\n",
"client = get_client(client_id='client_id') # Replace with your Client Id"
]
},
{
"cell_type": "markdown",
"id": "2",
"metadata": {},
"source": [
"### Data Requirements\n",
"\n",
"#### The CSV data files require the first row to contain the column names.\n",
"\n",
"#### The following columns are required:\n",
"* `TimeStamp` (ISO8601 Timestamp, e.g `2023-12-14T16:26:05.898156Z`)\n",
"* `Score` (a number between 0 and 1)\n",
"* `Each of the Protected Class Columns` (see below.)\n",
"* `Protected Class Columns` (race, gender, disability_status, gender_identity)\n",
"\n",
"#### Optionally, you can add up to 3 additional columns to use for grouping and filtering the bias results after testing."
]
},
{
"cell_type": "markdown",
"id": "3",
"metadata": {},
"source": [
"#### Lookup the Protected Class Column Names\n",
"\n",
"#### The following code can be used the Fairnow API to lookup the column names that correspond to protected classes used to evaluate bias. If you need a different set of column names for your test please contact Fairnow.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4",
"metadata": {},
"outputs": [],
"source": [
"reference_columns_route = \"/reference/protected-class-columns\"\n",
"\n",
"response = client.get(reference_columns_route)\n",
"\n",
"if response.status_code == 200:\n",
" print(json.dumps(response.json(), indent=4))\n",
"else:\n",
" print(f\"Error: {response.status_code} - {response.text}\")"
]
},
{
"cell_type": "markdown",
"id": "5",
"metadata": {},
"source": [
"#### Now pull the column names out and generate some random data. In addition to the required columns `Timestamp` and `Score` and the protected class columns the example contains two columns that can be used to filter the results. "
"#### Now generate some random data. In addition to the required columns `Timestamp` and `Score` and the protected class columns the example contains two columns that can be used to filter the results. "
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"id": "6",
"metadata": {},
"outputs": [],
"source": [
"table_data = response.json()\n",
"protected_class_columns = [ column[\"id\"] for column in table_data[\"table_data\"]]\n",
"print(f\"Protected class column names: {protected_class_columns}\")\n",
"import datetime\n",
"import random\n",
"import csv\n",
"\n",
"# The default protected class columns will be used if none are provided.\n",
"# Update protected class columns by updating the list below:\n",
"protected_class_columns = [\"race\", \"gender\", \"disability_status\", \"gender_identity\"]\n",
"\n",
"number_of_protected_class_values = 10\n",
"number_of_rows = 1000\n",
Expand Down Expand Up @@ -130,7 +80,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"id": "8",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -169,21 +119,21 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
"pygments_lexer": "ipython3",
"version": "3.13.7"
}
},
"nbformat": 4,
Expand Down
14 changes: 1 addition & 13 deletions notebooks/Getting Started.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,18 +73,6 @@
"\n",
"#### On the other hand, a `Client authentication failed` exception most likely means the `Client ID` or `Client Secret` were invalid."
]
},
{
"cell_type": "markdown",
"id": "6",
"metadata": {},
"source": []
},
{
"cell_type": "markdown",
"id": "7",
"metadata": {},
"source": []
}
],
"metadata": {
Expand All @@ -103,7 +91,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "3.13.3"
"version": "3.13.7"
}
},
"nbformat": 4,
Expand Down
39 changes: 20 additions & 19 deletions notebooks/User Data Testing.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"\n",
"#### To use this notebook, you'll need a `Client ID` and `Client Secret`. These will either have been provided to you, or you can generate from https://app.fairnow.ai and going the the Admin menu. This notebook assumes you have these available to enter when prompted.\n",
"\n",
"#### To run the simulation you will need an `application_id` and `application_version` for the specific model you want to test. Details of how to create and lookup AI Applications can be found in the `Applications AI` notebook.\n",
"#### To run the simulation you will need an `application_id` for the specific model you want to test. Details of how to create and lookup AI Applications can be found in the `Applications API` notebook.\n",
"\n",
"#### Finally, you'll need a `threshold` value, which is the value at which anyone with a score above is considered a passing score.\n",
"\n",
Expand Down Expand Up @@ -65,7 +65,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"id": "5",
"metadata": {},
"outputs": [],
Expand All @@ -88,28 +88,33 @@
"metadata": {},
"outputs": [],
"source": [
"application_id = None # Replace with your own Application Id\n",
"application_id = '' # Replace with your own Application Id\n",
"threshold = 0.5 # Change to your own threshold setting\n",
"protected_class_columns = ['race', 'gender', 'disability_status', 'gender_identity'] # Default protected class columns\n",
"\n",
"if not application_id:\n",
" raise ValueError(\"Please set application_id\")\n",
"\n",
"if threshold <= 0.0 or threshold >= 1.0:\n",
" raise ValueError(\"Threshold must be between 0.0 and 1.0\")\n",
"\n",
"if not protected_class_columns:\n",
" raise ValueError(\"Please set protected_class_columns. Default is ['race', 'gender', 'disability_status', 'gender_identity']\")\n",
"\n",
"start_test_route = f\"/applications/{application_id}/tests\"\n",
"\n",
"test_type = \"fairness_ml_user\"\n",
"\n",
"test_type = \"fairness_ml_user\" # Do not change test_type\n",
"\n",
"request_body = {\n",
" \"test_name\": \"API Client Testing\",\n",
" \"test_description\": \"Testing User Bias\",\n",
" \"test_type\": test_type,\n",
" \"inputs\": {\n",
" \"type\": test_type,\n",
" \"threshold\": threshold\n",
" }\n",
"\n",
" \"threshold\": threshold,\n",
" \"protected_class_columns\": protected_class_columns\n",
" }\n",
"}\n",
"\n",
"response = client.post(start_test_route, json=request_body, timeout=None)\n",
Expand All @@ -136,6 +141,8 @@
"metadata": {},
"outputs": [],
"source": [
"import httpx\n",
"\n",
"response_body = response.json()\n",
"test_id = response_body[\"test_id\"]\n",
"presigned_upload = response_body[\"presigned_url_scores_upload\"]\n",
Expand Down Expand Up @@ -169,6 +176,8 @@
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
"test_route = f\"/applications/{application_id}/tests/{test_id}\"\n",
"\n",
"response = client.get(test_route)\n",
Expand All @@ -179,7 +188,7 @@
" raise ValueError(f\"API Error Response: {response.status_code} - {response.text}\")\n",
"\n",
"while current_status not in ['ready', 'error']:\n",
" sleep(15)\n",
" time.sleep(15)\n",
" response = client.get(test_route)\n",
" if response.status_code == 200:\n",
" current_status = response.json()[\"status\"][\"id\"]\n",
Expand Down Expand Up @@ -224,7 +233,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 17,
"id": "15",
"metadata": {},
"outputs": [],
Expand Down Expand Up @@ -254,19 +263,11 @@
"source": [
"!cat results.csv"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
Expand All @@ -280,7 +281,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
"version": "3.13.7"
}
},
"nbformat": 4,
Expand Down
Loading