diff --git a/docs/images/V1_Silicon.png b/docs/images/V1_Silicon.png index ddf028e4..60dca91d 100644 Binary files a/docs/images/V1_Silicon.png and b/docs/images/V1_Silicon.png differ diff --git a/modules/silicon_design/README.md b/modules/silicon_design/README.md index 8d642dc2..9e4fab94 100644 --- a/modules/silicon_design/README.md +++ b/modules/silicon_design/README.md @@ -10,9 +10,9 @@ This RAD Lab module provides a managed environment for custom silicon design usi ## Samples notebooks -- [Inverter](scripts/build/notebooks/inverter.md) +- [Inverter](scripts/build/notebooks/inverter/experiment.md) -![gds render](scripts/build/notebooks/inverter.svg) +![gds render](scripts/build/notebooks/inverter/layout.svg) ## GCP Products/Services @@ -44,6 +44,7 @@ When deploying in an existing project, ensure the identity has the following per - `roles/resourcemanager.projectIamAdmin` - `roles/iam.serviceAccountAdmin` - `roles/iam.serviceAccountUser` +- `roles/serviceusage.serviceUsageAdmin` NOTE: Additional [permissions](./radlab-launcher/README.md#iam-permissions-prerequisites) are required when deploying the RAD Lab modules via [RAD Lab Launcher](./radlab-launcher) diff --git a/modules/silicon_design/main.tf b/modules/silicon_design/main.tf index 4daa90d4..977e39e4 100644 --- a/modules/silicon_design/main.tf +++ b/modules/silicon_design/main.tf @@ -3,7 +3,7 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at +nnn * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -19,9 +19,16 @@ locals { project = (var.create_project ? try(module.project_radlab_silicon_design.0, null) : try(data.google_project.existing_project.0, null) - ) + ) + project_number = (var.create_project + ? try(module.project_radlab_silicon_design.0.project_number, null) + : try(data.google_project.existing_project.0.number, null) + ) region = join("-", [split("-", var.zone)[0], split("-", var.zone)[1]]) + network_name = var.network_name != null ? var.network_name : "${var.name}-network" + subnet_name = var.subnet_name != null ? var.subnet_name : "${var.name}-subnet" + network = ( var.create_network ? try(module.vpc_ai_notebook.0.network.network, null) @@ -30,7 +37,7 @@ locals { subnet = ( var.create_network - ? try(module.vpc_ai_notebook.0.subnets["${local.region}/${var.subnet_name}"], null) + ? try(module.vpc_ai_notebook.0.subnets["${local.region}/${local.subnet_name}"], null) : try(data.google_compute_subnetwork.default.0, null) ) @@ -39,7 +46,19 @@ locals { "roles/notebooks.admin", "roles/compute.instanceAdmin", "roles/iam.serviceAccountUser", - "roles/storage.objectViewer", + "roles/storage.admin", + "roles/aiplatform.admin", + ] + + cloudbuild_sa_project_roles = [ + "roles/compute.admin", + "roles/storage.admin", + ] + + image_builder_sa_project_roles = [ + "roles/compute.instanceAdmin", + "roles/compute.storageAdmin", + "roles/storage.admin", ] project_services = var.enable_services ? [ @@ -47,9 +66,11 @@ locals { "notebooks.googleapis.com", "cloudbuild.googleapis.com", "artifactregistry.googleapis.com", + "aiplatform.googleapis.com", ] : [] - notebook_names = length(var.notebook_names) > 0 ? var.notebook_names : [for i in range(var.notebook_count): "silicon-design-notebook-${i}"] + notebook_names = length(var.notebook_names) > 0 ? var.notebook_names : [for i in range(var.notebook_count): "${var.name}-nodebook-${i}"] + image_tag = var.image_tag != "" ? var.image_tag : formatdate("YYYYMMDDhhmm", timestamp()) } resource "random_id" "default" { @@ -61,14 +82,14 @@ resource "random_id" "default" { ############################ data "google_project" "existing_project" { - count = var.create_project ? 0 : 1 - project_id = var.project_name + count = var.create_project ? 0 : 1 + project_id = var.project_name } module "project_radlab_silicon_design" { count = var.create_project ? 1 : 0 source = "terraform-google-modules/project-factory/google" - version = "~> 11.0" + version = "~> 13.0" name = format("%s-%s", var.project_name, local.random_id) random_project_id = false @@ -83,8 +104,12 @@ resource "google_project_service" "enabled_services" { for_each = toset(local.project_services) project = local.project.project_id service = each.value - disable_dependent_services = true - disable_on_destroy = true + disable_dependent_services = false + disable_on_destroy = false + + lifecycle { + prevent_destroy = true + } depends_on = [ module.project_radlab_silicon_design @@ -94,29 +119,29 @@ resource "google_project_service" "enabled_services" { data "google_compute_network" "default" { count = var.create_network ? 0 : 1 project = local.project.project_id - name = var.network_name + name = local.network_name } data "google_compute_subnetwork" "default" { count = var.create_network ? 0 : 1 project = local.project.project_id - name = var.subnet_name + name = local.subnet_name region = local.region } module "vpc_ai_notebook" { count = var.create_network ? 1 : 0 source = "terraform-google-modules/network/google" - version = "~> 3.0" + version = "~> 5.0" project_id = local.project.project_id - network_name = var.network_name + network_name = local.network_name routing_mode = "GLOBAL" description = "VPC Network created via Terraform" subnets = [ { - subnet_name = var.subnet_name + subnet_name = local.subnet_name subnet_ip = var.ip_cidr_range subnet_region = local.region description = "Subnetwork inside *vpc-silicon-design* VPC network, created via Terraform" @@ -126,7 +151,7 @@ module "vpc_ai_notebook" { firewall_rules = [ { - name = "fw-silicon-design-notebook-allow-internal" + name = "${var.name}-allow-internal" description = "Firewall rule to allow traffic on all ports inside *vpc-silicon-design* VPC network." priority = 65534 ranges = ["10.0.0.0/8"] @@ -146,7 +171,7 @@ module "vpc_ai_notebook" { resource "google_service_account" "sa_p_notebook" { project = local.project.project_id - account_id = format("sa-p-notebook-%s", local.random_id) + account_id = "${var.name}-n-sa" display_name = "Notebooks in trusted environment" } @@ -157,9 +182,40 @@ resource "google_project_iam_member" "sa_p_notebook_permissions" { role = each.value } +resource "google_project_service_identity" "sa_cloudbuild_identity" { + provider = google-beta + project = local.project.project_id + service = "cloudbuild.googleapis.com" +} + +resource "google_project_iam_member" "sa_cloudbuild_permissions" { + for_each = toset(local.cloudbuild_sa_project_roles) + member = "serviceAccount:${google_project_service_identity.sa_cloudbuild_identity.email}" + project = local.project.project_id + role = each.value +} + +resource "google_service_account_iam_member" "sa_cloudbuild_image_builder_access" { + member = "serviceAccount:${google_project_service_identity.sa_cloudbuild_identity.email}" + role = "roles/iam.serviceAccountUser" + service_account_id = google_service_account.sa_image_builder_identity.id +} + +resource "google_service_account" "sa_image_builder_identity" { + project = local.project.project_id + account_id = "${var.name}-i-sa" +} + +resource "google_project_iam_member" "sa_image_builder_permissions" { + for_each = toset(local.image_builder_sa_project_roles) + project = local.project.project_id + member = "serviceAccount:${google_service_account.sa_image_builder_identity.email}" + role = each.value +} + resource "google_service_account_iam_member" "sa_ai_notebook_user_iam" { for_each = var.trusted_users - member = each.value + member = "user:${each.value}" role = "roles/iam.serviceAccountUser" service_account_id = google_service_account.sa_p_notebook.id } @@ -167,14 +223,14 @@ resource "google_service_account_iam_member" "sa_ai_notebook_user_iam" { resource "google_project_iam_member" "ai_notebook_user_role1" { for_each = var.trusted_users project = local.project.project_id - member = each.value + member = "user:${each.value}" role = "roles/notebooks.admin" } resource "google_project_iam_member" "ai_notebook_user_role2" { for_each = var.trusted_users project = local.project.project_id - member = each.value + member = "user:${each.value}" role = "roles/viewer" } @@ -186,10 +242,10 @@ resource "google_notebooks_instance" "ai_notebook" { machine_type = var.machine_type container_image { - repository = "${google_artifact_registry_repository.containers_repo.location}-docker.pkg.dev/${local.project.project_id}/${google_artifact_registry_repository.containers_repo.repository_id}/openlane-jupyterlab" - tag = "latest" + repository = "${google_artifact_registry_repository.containers_repo.location}-docker.pkg.dev/${local.project.project_id}/${google_artifact_registry_repository.containers_repo.repository_id}/${var.image_name}" + tag = local.image_tag } - + service_account = google_service_account.sa_p_notebook.email install_gpu_driver = false @@ -202,7 +258,7 @@ resource "google_notebooks_instance" "ai_notebook" { network = local.network.self_link subnet = local.subnet.self_link - post_startup_script = "gs://${google_storage_bucket.notebooks_bucket.name}/copy-notebooks.sh" + post_startup_script = "gs://${google_storage_bucket.staging_bucket.name}/copy-notebooks.sh" labels = { module = "silicon-design" @@ -223,7 +279,7 @@ resource "google_artifact_registry_repository" "containers_repo" { project = local.project.project_id location = local.region - repository_id = "containers" + repository_id = "${var.name}-containers" description = "container image repository" format = "DOCKER" @@ -232,9 +288,9 @@ resource "google_artifact_registry_repository" "containers_repo" { ] } -resource "google_storage_bucket" "notebooks_bucket" { +resource "google_storage_bucket" "staging_bucket" { project = local.project.project_id - name = "${local.project.project_id}-silicon-design-notebooks" + name = "${local.project.project_id}-${var.name}-staging" location = local.region force_destroy = true uniform_bucket_level_access = true @@ -245,18 +301,25 @@ resource "google_storage_bucket" "notebooks_bucket" { resource "null_resource" "build_and_push_image" { triggers = { cloudbuild_yaml_sha = filesha1("${path.module}/scripts/build/cloudbuild.yaml") - build_script_sha = filesha1("${path.module}/scripts/build/build.sh") - dockerfile_sha = filesha1("${path.module}/scripts/build/containers/openlane-jupyterlab/Dockerfile") - notebook_sha = filesha1("${path.module}/scripts/build/notebooks/inverter.md") + workflow_sha = filesha1("${path.module}/scripts/build/images/compute_image.wf.json") + dockerfile_sha = filesha1("${path.module}/scripts/build/images/Dockerfile") + environment_sha = filesha1("${path.module}/scripts/build/images/provision/environment.yml") + env_sha = filesha1("${path.module}/scripts/build/images/provision/install.tcl") + profile_sha = filesha1("${path.module}/scripts/build/images/provision/profile.sh") + notebook_sha = filesha1("${path.module}/scripts/build/notebooks/inverter/inverter.md") + image_tag = local.image_tag } provisioner "local-exec" { working_dir = path.module - command = "scripts/build/build.sh ${local.project.project_id} ${google_artifact_registry_repository.containers_repo.location} ${google_artifact_registry_repository.containers_repo.repository_id} ${google_storage_bucket.notebooks_bucket.name}" + command = "gcloud --project=${local.project.project_id} builds submit . --config ./scripts/build/cloudbuild.yaml --substitutions \"_ZONE=${var.zone},_COMPUTE_IMAGE=${var.image_name},_CONTAINER_IMAGE=${google_artifact_registry_repository.containers_repo.location}-docker.pkg.dev/${local.project.project_id}/${google_artifact_registry_repository.containers_repo.repository_id}/${var.image_name},_STAGING_BUCKET=${google_storage_bucket.staging_bucket.name},_COMPUTE_NETWORK=${local.network.id},_COMPUTE_SUBNET=${local.subnet.id},_IMAGE_TAG=${local.image_tag},_CLOUD_BUILD_SA=${google_service_account.sa_image_builder_identity.email}\"" } depends_on = [ google_artifact_registry_repository.containers_repo, - google_storage_bucket.notebooks_bucket, + google_storage_bucket.staging_bucket, + google_project_iam_member.sa_image_builder_permissions, + google_project_iam_member.sa_cloudbuild_permissions, + google_service_account_iam_member.sa_cloudbuild_image_builder_access, ] } diff --git a/modules/silicon_design/outputs.tf b/modules/silicon_design/outputs.tf index 157ca1e0..403b228e 100644 --- a/modules/silicon_design/outputs.tf +++ b/modules/silicon_design/outputs.tf @@ -19,7 +19,7 @@ output "deployment_id" { value = local.random_id } -output "project_radlab_silicon_design_id" { +output "project_id" { description = "Silicon Design RAD Lab Project ID" value = local.project.project_id } @@ -29,11 +29,6 @@ output "notebooks_instance_names" { value = join(", ", google_notebooks_instance.ai_notebook[*].name) } -output "notebooks_bucket_name" { - description = "Notebooks GCS Bucket Name" - value = google_storage_bucket.notebooks_bucket.name -} - output "artifact_registry_repository_id" { description = "Artifact Registry Repository ID" value = google_artifact_registry_repository.containers_repo.repository_id @@ -41,5 +36,15 @@ output "artifact_registry_repository_id" { output "notebooks_container_image" { description = "Container Image URI" - value = "${google_notebooks_instance.ai_notebook[0].container_image[0].repository}:${google_notebooks_instance.ai_notebook[0].container_image[0].tag}" + value = "${google_artifact_registry_repository.containers_repo.location}-docker.pkg.dev/${local.project.project_id}/${google_artifact_registry_repository.containers_repo.repository_id}/${var.image_name}:${local.image_tag}" +} + +output "notebooks_vm_image" { + description = "GCE VM Image Name" + value = "${var.image_name}-${local.image_tag}" +} + +output "notebooks_staging_bucket" { + description = "Noteooks Staging bucket" + value = "${google_storage_bucket.staging_bucket.name}" } diff --git a/modules/silicon_design/scripts/build/cloudbuild.yaml b/modules/silicon_design/scripts/build/cloudbuild.yaml index 93fe61fc..da5b9e26 100644 --- a/modules/silicon_design/scripts/build/cloudbuild.yaml +++ b/modules/silicon_design/scripts/build/cloudbuild.yaml @@ -1,3 +1,4 @@ +# # Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,50 +13,60 @@ # See the License for the specific language governing permissions and # limitations under the License. -timeout: 7200s +timeout: 3600s substitutions: - _OPENLANE_VERSION: 2022.02.01_02.19.58 - _REPOSITORY_LOCATION: $LOCATION - _REPOSITORY_ID: gcr.io - _NOTEBOOKS_BUCKET: $PROJECT_ID-silicon-design-notebooks + _ZONE: 'asia-northeast1-a' + _COMPUTE_IMAGE: 'silicon-design-ubuntu-2004' + _CONTAINER_IMAGE: 'silicon-design-ubuntu-2004' + _IMAGE_TAG: 'default-tag' + _STAGING_BUCKET: 'silicon-design-notebooks' + _COMPUTE_NETWORK: 'global/networks/default' + _COMPUTE_SUBNET: '' + _CLOUD_BUILD_SA: '' options: logging: CLOUD_LOGGING_ONLY steps: -- name: 'python' +- id: 'notebooks-build' + name: 'gcr.io/cloud-builders/gcloud' entrypoint: '/bin/bash' args: - '-c' - |- - python3 -m venv env-jupytext/ - env-jupytext/bin/python -m pip install jupytext - env-jupytext/bin/jupytext --to notebook scripts/build/notebooks/*.md - echo 'gsutil cp gs://$_NOTEBOOKS_BUCKET/*.ipynb /home/jupyter/' > scripts/build/notebooks/copy-notebooks.sh -- name: 'gcr.io/cloud-builders/git' - args: ['clone', '-b', $_OPENLANE_VERSION, 'https://github.com/The-OpenROAD-Project/OpenLane'] -- name: 'gcr.io/cloud-builders/docker' - entrypoint: '/bin/bash' - env: - - EXTERNAL_PDK_INSTALLATION=0 - - NO_PDKS=0 + apt-get update && apt-get install -yq python3.8-venv + python3 -m venv env/ + env/bin/python -m pip install jupytext + env/bin/jupytext --to notebook scripts/build/notebooks/**/*.md + echo "gsutil rsync -r -x '.*\.(sh|json)' gs://$_STAGING_BUCKET/ /home/jupyter/" > scripts/build/notebooks/copy-notebooks.sh + gsutil rsync -r -x '.*\.(md|svg)' scripts/build/notebooks/ gs://$_STAGING_BUCKET/ + waitFor: ['-'] +- id: 'compute-image-build' + name: 'gcr.io/cloud-builders/gcloud' + entrypoint: '/bin/bash' args: - '-c' - |- - apt-get update && apt install -yq python3-venv - python3 -m venv env-openlane/ - env-openlane/bin/python -m pip install pyyaml click - source env-openlane/bin/activate - docker pull $_REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY_ID/openlane-pdk:$_OPENLANE_VERSION || make -C OpenLane OPENLANE_IMAGE_NAME=$_REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY_ID/openlane-pdk:$_OPENLANE_VERSION openlane -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', '$_REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY_ID/openlane-jupyterlab:$BUILD_ID', '--build-arg', 'REPOSITORY_LOCATION=$_REPOSITORY_LOCATION', '--build-arg', 'PROJECT_ID=$PROJECT_ID', '--build-arg', 'REPOSITORY_ID=$_REPOSITORY_ID', '--build-arg', 'OPENLANE_VERSION=$_OPENLANE_VERSION', '-f', './scripts/build/containers/openlane-jupyterlab/Dockerfile', '.'] -- name: 'gcr.io/cloud-builders/docker' - args: ['tag', '$_REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY_ID/openlane-jupyterlab:$BUILD_ID', '$_REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY_ID/openlane-jupyterlab:latest'] + cd scripts/build/images/ + gsutil cp gs://compute-image-tools/release/linux/daisy . + chmod +x daisy + gcloud compute images describe $_COMPUTE_IMAGE-$_IMAGE_TAG || ./daisy -project $PROJECT_ID -zone $_ZONE -variables image_name=$_COMPUTE_IMAGE,image_tag=$_IMAGE_TAG,network=$_COMPUTE_NETWORK,subnet=$_COMPUTE_SUBNET,service_account=$_CLOUD_BUILD_SA compute_image.wf.json + waitFor: ['-'] +- id: 'container-image-pull' + name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['-c', 'docker pull $_CONTAINER_IMAGE:latest || exit 0'] + waitFor: ['-'] +- id: 'container-image-build' + name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', '$_CONTAINER_IMAGE:$_IMAGE_TAG', '--cache-from', '$_CONTAINER_IMAGE:latest', './scripts/build/images'] + waitFor: ['container-image-pull'] +- id: 'container-image-tag' + name: 'gcr.io/cloud-builders/docker' + args: ['tag', '$_CONTAINER_IMAGE:$_IMAGE_TAG', '$_CONTAINER_IMAGE:latest'] + waitFor: ['container-image-build'] +- id: 'container-image-test' + name: 'gcr.io/cloud-builders/docker' + args: ['run', '$_CONTAINER_IMAGE:$_IMAGE_TAG', 'flow.tcl', '-design', 'inverter'] + waitFor: ['container-image-tag'] images: -- '$_REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY_ID/openlane-pdk:$_OPENLANE_VERSION' -- '$_REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY_ID/openlane-jupyterlab:$BUILD_ID' -- '$_REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$_REPOSITORY_ID/openlane-jupyterlab:latest' -artifacts: - objects: - location: gs://$_NOTEBOOKS_BUCKET/ - paths: - - 'scripts/build/notebooks/copy-notebooks.sh' - - 'scripts/build/notebooks/*.ipynb' +- '$_CONTAINER_IMAGE:$_IMAGE_TAG' +- '$_CONTAINER_IMAGE:latest' diff --git a/modules/silicon_design/scripts/build/containers/openlane-jupyterlab/Dockerfile b/modules/silicon_design/scripts/build/containers/openlane-jupyterlab/Dockerfile deleted file mode 100644 index b5b5e070..00000000 --- a/modules/silicon_design/scripts/build/containers/openlane-jupyterlab/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG REPOSITORY_LOCATION -ARG PROJECT_ID -ARG REPOSITORY_ID -ARG OPENLANE_VERSION -FROM $REPOSITORY_LOCATION-docker.pkg.dev/$PROJECT_ID/$REPOSITORY_ID/openlane-pdk:$OPENLANE_VERSION - -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && bash Miniconda3-latest-Linux-x86_64.sh -b -f -p /conda-env && rm Miniconda3-latest-Linux-x86_64.sh - -# install openlane dependencies in conda environment -RUN /conda-env/bin/conda install -c conda-forge -y python pip -# install openlane dependencies in conda environment -RUN /conda-env/bin/python -m pip install click pyyaml matplotlib "jinja2<3.0.0" pandas install XlsxWriter -RUN /conda-env/bin/conda install -c conda-forge -y jupyterlab gdstk iverilog - -RUN groupadd --gid 1001 jupyter -RUN useradd --uid 1000 --gid 1001 jupyter -USER jupyter -EXPOSE 8080 -ENV JUPYTER_PORT 8080 - -WORKDIR /home/jupyter -ENTRYPOINT ["/bin/bash", "-c", "source /conda-env/bin/activate && jupyter lab --ip 0.0.0.0 --allow-root --ServerApp.token='' --ServerApp.allow_origin_pat='^https?://.*\\.notebooks\\.googleusercontent\\.com' --ServerApp.allow_remote_access=True"] diff --git a/modules/silicon_design/scripts/build/images/Dockerfile b/modules/silicon_design/scripts/build/images/Dockerfile new file mode 100644 index 00000000..616c48bc --- /dev/null +++ b/modules/silicon_design/scripts/build/images/Dockerfile @@ -0,0 +1,23 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/deeplearning-platform-release/base-cpu +COPY provision.sh /tmp/provision.sh +COPY provision/ /tmp/provision/ +RUN bash -x /tmp/provision.sh +RUN sed -i -e 's/conda activate base/conda activate base\nconda activate silicon/' /entrypoint.sh +RUN sed -i -e 's@/opt/conda@/opt/conda/envs/silicon@' /run_jupyter.sh +ENV OPENLANE_ROOT=/OpenLane +ENV PATH=$OPENLANE_ROOT:$OPENLANE_ROOT/scripts:$PATH diff --git a/modules/silicon_design/scripts/build/images/compute_image.wf.json b/modules/silicon_design/scripts/build/images/compute_image.wf.json new file mode 100644 index 00000000..2a84f14a --- /dev/null +++ b/modules/silicon_design/scripts/build/images/compute_image.wf.json @@ -0,0 +1,109 @@ +{ + "Name": "silicon-design", + "Project": "${PROJECT}", + "Zone": "${ZONE}", + "Vars": { + "source_image": { + "Description": "source image path", + "Value": "projects/deeplearning-platform-release/global/images/family/common-cpu-ubuntu-2004" + }, + "image_name": { + "Description": "image name prefix", + "Value": "silicon-design-ubuntu-2004" + }, + "image_tag": { + "Description": "image name suffix", + "Value": "${ID}" + }, + "network": { + "Description": "compute network", + "Value": "global/networks/default" + }, + "subnet": { + "Description": "compute subnet", + "Value": "" + }, + "service_account": { + "Description": "compute sevice account", + "Value": "" + } + }, + "Sources": { + "provision": "./provision", + "provision.sh": "./provision.sh" + }, + "Steps": { + "create-disk": { + "CreateDisks": [ + { + "Name": "disk", + "SourceImage": "${source_image}", + "SizeGb": "100", + "Type": "pd-ssd" + } + ] + }, + "create-instance": { + "CreateInstances": [ + { + "Name": "instance", + "Disks": [ + {"Source": "disk"} + ], + "MachineType": "n1-standard-4", + "StartupScript": "provision.sh", + "NetworkInterfaces": [{ + "Network": "${network}", + "Subnetwork": "${subnet}" + }], + "ServiceAccounts": [{ + "email": "${service_account}", + "scopes": ["https://www.googleapis.com/auth/devstorage.read_only"] + }] + } + ] + }, + "wait-for-script": { + "WaitForInstancesSignal": [ + { + "Name": "instance", + "SerialOutput": { + "Port": 1, + "SuccessMatch": "DaisySuccess:", + "FailureMatch": "DaisyFailure:", + "StatusMatch": "DaisyStatus:" + } + } + ], + "Timeout": "45m" + }, + "stop-instance": { + "StopInstances": { + "Instances":["instance"] + } + }, + "create-image": { + "CreateImages": [ + { + "Name": "image", + "SourceDisk": "disk", + "NoCleanup": true, + "RealName": "${image_name}-${image_tag}" + } + ] + }, + "cleanup": { + "DeleteResources": { + "Instances": ["instance"], + "Disks": ["disk"] + } + } + }, + "Dependencies": { + "create-instance": ["create-disk"], + "wait-for-script": ["create-instance"], + "stop-instance": ["wait-for-script"], + "create-image": ["stop-instance"], + "cleanup": ["create-image"] + } +} diff --git a/modules/silicon_design/scripts/build/images/provision.sh b/modules/silicon_design/scripts/build/images/provision.sh new file mode 100644 index 00000000..284b5d36 --- /dev/null +++ b/modules/silicon_design/scripts/build/images/provision.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +trap "echo DaisyFailure: trapped error" ERR + +env +OPENLANE_VERSION=master +OPENROAD_FLOW_VERSION=master +CFU_PLAYGROUND_VERSION=dse_framework +PROVISION_DIR=/tmp/provision + +SYSTEM_NAME=$(dmidecode -s system-product-name || true) + +echo "DaisyStatus: install system dependencies" +apt-get update && apt-get -o DPkg::Lock::Timeout=-1 -yq install locales locales-all time + +if [ -n "$(echo ${SYSTEM_NAME} | grep 'Google Compute Engine')" ]; then +echo "DaisyStatus: fetching provisioning script" +DAISY_SOURCES_PATH=$(curl -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/instance/attributes/daisy-sources-path) +mkdir -p ${PROVISION_DIR} +gsutil -m rsync ${DAISY_SOURCES_PATH}/provision/ ${PROVISION_DIR}/ || true +fi + +echo "DaisyStatus: installing conda-eda environment" +curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -C /usr/local -xvj bin/micromamba +micromamba create --yes -r /opt/conda -n silicon --file ${PROVISION_DIR}/environment.yml + +echo "DaisyStatus: installing OpenLane" +git clone --depth 1 -b ${OPENLANE_VERSION} https://github.com/The-OpenROAD-Project/OpenLane /OpenLane + +echo "DaisyStatus: installing OpenROAD Flow" +git clone --depth 1 -b ${OPENROAD_FLOW_VERSION} https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts /OpenROAD-flow-scripts + +echo "DaisyStatus: installing KLayout Flow" +curl -O https://www.klayout.org/downloads/Ubuntu-20/klayout_0.27.10-1_amd64.deb +dpkg -i klayout_0.27.10-1_amd64.deb || apt-get -f -yq install + +echo "DaisyStatus: patching OpenLane" +cp ${PROVISION_DIR}/install.tcl /OpenLane/configuration/ +echo ' install.tcl' >> /OpenLane/configuration/load_order.txt + +echo "DaisyStatus: adding profile hook" +cp ${PROVISION_DIR}/profile.sh /etc/profile.d/silicon-design-profile.sh + +echo "DaisyStatus: adding papermill launcher" +cp ${PROVISION_DIR}/papermill-launcher /usr/local/bin/ +chmod +x /usr/local/bin/papermill-launcher + +echo "DaisySuccess: done" + +echo "Cloning CFU-Playground repository" +git clone -b ${CFU_PLAYGROUND_VERSION} https://github.com/google/CFU-Playground.git /CFU-Playground + +echo "Running CFU-Playground setup" +cd /CFU-Playground +./scripts/setup +./scripts/setup_vexriscv_build.sh + diff --git a/modules/silicon_design/scripts/build/images/provision/environment.yml b/modules/silicon_design/scripts/build/images/provision/environment.yml new file mode 100644 index 00000000..e4429507 --- /dev/null +++ b/modules/silicon_design/scripts/build/images/provision/environment.yml @@ -0,0 +1,88 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +channels: + - defaults + - litex-hub + - conda-forge +dependencies: + - open_pdks.sky130a + - magic + - openroad + - netgen + - gdstk + - cairosvg + - svgutils + - tcllib + - iverilog + - xls + - pyspice + - pyyaml + - click + - gdsfactory + - pandas + - pymeep=*=mpi_mpich_* + - jupyterlab + - litex-hub::gcc-riscv32-elf-newlib + - litex-hub::openfpgaloader + - litex-hub::dfu-util + - litex-hub::flterm + - litex-hub::openocd + - litex-hub::verilator + - litex-hub::nextpnr-nexus + - litex-hub::nextpnr-ecp5 + - litex-hub::nextpnr-ice40 + - litex-hub::yosys + - litex-hub::iceprog + - litex-hub::prjxray-tools + - litex-hub::prjxray-db + - litex-hub::vtr-optimized + - litex-hub::symbiflow-yosys-plugins + - lxml + - simplejson + - intervaltree + - json-c + - libevent + - python=3.7 + - pip + - pip: + - klayout + - scrapbook[gcs] + - google-cloud-aiplatform + - cloudml-hypertune + - simplejson + - termcolor + - tqdm + - yapf==0.24.0 + - requests + - meson + - Pillow + - intervaltree + - junit-xml + - numpy + - openpyxl + - ordered-set + - parse + - progressbar2 + - pyjson5 + - pytest + - pyyaml + - scipy>=1.2.1 + - sympy + - textx + - python-constraint + - fasm + - https://github.com/chipsalliance/f4pga/archive/8c411eb74e4bb23d1ec243a1515b9bfb48e2cd83.zip#subdirectory=f4pga + - git+https://github.com/f4pga/prjxray.git@ae546d6b7648bf4df9cf63f0b25b2028b5623c43#egg=prjxray + - git+https://github.com/chipsalliance/f4pga-xc-fasm.git@25dc605c9c0896204f0c3425b52a332034cf5e5c#egg=xc-fasm diff --git a/modules/silicon_design/scripts/build/images/provision/install.tcl b/modules/silicon_design/scripts/build/images/provision/install.tcl new file mode 100644 index 00000000..1efb9c49 --- /dev/null +++ b/modules/silicon_design/scripts/build/images/provision/install.tcl @@ -0,0 +1,24 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set ::env(PDK_ROOT) "$::env(CONDA_PREFIX)/share/pdk" +set ::env(TCLLIBPATH) "$::env(CONDA_PREFIX)/opt/conda/lib/tcllib1.20" +set ::env(OL_INSTALL_DIR) "$::env(OPENLANE_ROOT)/install" +set ::env(OPENLANE_LOCAL_INSTALL) 1 +set ::env(TEST_MISMATCHES) none +set ::env(RUN_CVC) 0 +set ::env(RUN_KLAYOUT_XOR) 0 +set ::env(RUN_KLAYOUT_DRC) 0 +set ::env(RUN_KLAYOUT) 0 diff --git a/modules/silicon_design/scripts/build/images/provision/papermill-launcher b/modules/silicon_design/scripts/build/images/provision/papermill-launcher new file mode 100755 index 00000000..c77879a1 --- /dev/null +++ b/modules/silicon_design/scripts/build/images/provision/papermill-launcher @@ -0,0 +1,28 @@ +#!/usr/bin/env python +import os.path +import sys + +import papermill as pm + +def args(param_args): + while len(param_args): + arg = param_args.pop(0).replace('--', '') + if '=' in arg: + yield arg.split('=') + else: + val = params_args.pop(0) + yield arg, val + +def expand_path(p): + return os.path.normpath( + os.path.expandvars(p) + ).replace('gs:/', 'gs://') + +_, input_path, output_path, *params_args = sys.argv +input_path = expand_path(input_path) +output_path = expand_path(output_path) +parameters = dict( + (k, expand_path(v)) + for k, v in args(params_args) +) +pm.execute_notebook(input_path, output_path, parameters=parameters, progress_bar=False) diff --git a/modules/silicon_design/scripts/build/build.sh b/modules/silicon_design/scripts/build/images/provision/profile.sh old mode 100755 new mode 100644 similarity index 59% rename from modules/silicon_design/scripts/build/build.sh rename to modules/silicon_design/scripts/build/images/provision/profile.sh index 4f272242..58b6b8f8 --- a/modules/silicon_design/scripts/build/build.sh +++ b/modules/silicon_design/scripts/build/images/provision/profile.sh @@ -1,5 +1,4 @@ -#!/bin/bash - +# # Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,14 +11,10 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the Lpicense. - -set -ex - -PROJECT_ID=$1 -REPOSITORY_LOCATION=$2 -REPOSITORY_ID=$3 -NOTEBOOKS_BUCKET=$4 +# limitations under the License. -gcloud config set project ${PROJECT_ID} -gcloud builds submit . --config ./scripts/build/cloudbuild.yaml --substitutions "_REPOSITORY_LOCATION=${REPOSITORY_LOCATION},_REPOSITORY_ID=${REPOSITORY_ID},_NOTEBOOKS_BUCKET=${NOTEBOOKS_BUCKET}" +export OPENLANE_ROOT=/OpenLane +export PATH=$OPENLANE_ROOT:$OPENLANE_ROOT/scripts:$PATH +. /opt/conda/etc/profile.d/conda.sh +conda activate base +conda activate silicon diff --git a/modules/silicon_design/scripts/build/notebooks/cfuplayground/cfuplayground.md b/modules/silicon_design/scripts/build/notebooks/cfuplayground/cfuplayground.md new file mode 100644 index 00000000..58f5cb88 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/cfuplayground/cfuplayground.md @@ -0,0 +1,74 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.14.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +# CFU Playground + + +### Design Space Exploration + +```python tags=["parameters"] +# Vexriscv soft core parameters available for tuning +bypass = True +cfu = True +dCacheSize = 2048 +hardwareDiv = True +iCacheSize = 1024 +mulDiv = True +prediction = "none" +safe = True +singleCycleShift = True +singleCycleMulDiv = True +``` + +```python +# Constants +CSR_PLUGIN_CONFIG = "mcycle" +TARGET = "digilent_arty" +``` + +```python +# Change directory to design space exploration project in CFU-Playground +%cd /CFU-Playground/proj/dse_template +``` + +```python tags=[] +import dse_framework + +dse_framework.dse(CSR_PLUGIN_CONFIG, bypass, cfu, dCacheSize, hardwareDiv, iCacheSize, mulDiv, prediction, safe, singleCycleShift, singleCycleMulDiv) +``` + +```python tags=[] +# Obtain metrics and glue to notebook for later use +import scrapbook as sb + +cycles = dse_framework.get_cycle_count() +cells = dse_framework.get_resource_util(TARGET) + +sb.glue('cells', cells) +sb.glue('cycles', cycles) + +print("Cycle Count: " + str(cycles)) +print("Cells Used: " + str (cells)) +``` + +```python +import hypertune + +print('Reporting Metric:', 'Cells^2 + Cycles', (cells*cells) + cycles) +hpt = hypertune.HyperTune() +hpt.report_hyperparameter_tuning_metric( + hyperparameter_metric_tag='cells+cycles', + metric_value=((cells*cells) + cycles), +) +``` diff --git a/modules/silicon_design/scripts/build/notebooks/inverter.md b/modules/silicon_design/scripts/build/notebooks/inverter.md deleted file mode 100644 index d7b78edb..00000000 --- a/modules/silicon_design/scripts/build/notebooks/inverter.md +++ /dev/null @@ -1,82 +0,0 @@ ---- -jupyter: - jupytext: - text_representation: - extension: .md - format_name: markdown - format_version: '1.3' - jupytext_version: 1.13.6 - kernelspec: - display_name: Python 3 (ipykernel) - language: python - name: python3 ---- - -# Inverter Sample - -``` -Copyright 2022 Google LLC. -SPDX-License-Identifier: Apache-2.0 -``` - -This notebook shows how to run a simple inverter design thru an end-to-end RTL to GDSII flow targetting the [SKY130](https://github.com/google/skywater-pdk/) process node. - -## Write verilog - -Invert the `in` input signal and continuously assign it to the `out` output signal. - -```python -%%bash -c 'cat > inverter.v; iverilog inverter.v' -module inverter(input wire in, output wire out); - assign out = !in; -endmodule -``` -## Write OpenLane configuration - -See [OpenLane Variables information](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/configuration/README.md) for the list of available variables. - -```python -%%bash -c 'cat > config.tcl; tclsh config.tcl' -set ::env(DESIGN_NAME) inverter - -set script_dir [file dirname [file normalize [info script]]] -set ::env(VERILOG_FILES) "$script_dir/inverter.v" - -set ::env(CLOCK_TREE_SYNTH) 0 -set ::env(CLOCK_PORT) "" - -set ::env(PL_RANDOM_GLB_PLACEMENT) 1 - -set ::env(FP_SIZING) absolute -set ::env(DIE_AREA) "0 0 34.165 54.885" -set ::env(PL_TARGET_DENSITY) 0.75 - -set ::env(FP_PDN_HORIZONTAL_HALO) 6 -set ::env(FP_PDN_VERTICAL_HALO) 6 - -set ::env(DIODE_INSERTION_STRATEGY) 3 -``` - -## Run OpenLane flow - -[OpenLane](https://github.com/The-OpenROAD-Project/OpenLane) is an automated RTL to GDSII flow based on several components including [OpenROAD](https://github.com/The-OpenROAD-Project/OpenROAD), [Yosys](https://github.com/YosysHQ/yosys), [Magic, Netgen, Fault, CVC, SPEF-Extractor, CU-GR, Klayout and a number of custom scripts for design exploration and optimization. - -```python tags=[] -!flow.tcl -design . -``` - -## Display layout with GDSII Tool Kit - -[Gdstk](https://github.com/heitzmann/gdstk) (GDSII Tool Kit) is a C++/Python library for creation and manipulation of GDSII and OASIS files. - -```python -import pathlib -import gdstk -from IPython.display import SVG - -gds = sorted(pathlib.Path('runs').glob('*/results/final/gds/*.gds'))[0] -library = gdstk.read_gds(gds) -top_cells = library.top_level() -top_cells[0].write_svg('inverter.svg') -SVG('inverter.svg') -``` diff --git a/modules/silicon_design/scripts/build/notebooks/inverter/inverter.md b/modules/silicon_design/scripts/build/notebooks/inverter/inverter.md new file mode 100644 index 00000000..bc4bddc3 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/inverter/inverter.md @@ -0,0 +1,154 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.14.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# Inverter Sample + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to run a simple inverter design thru an end-to-end RTL to GDSII flow targetting the [SKY130](https://github.com/google/skywater-pdk/) process node. + + +## Define flow parameters + +```python tags=["parameters"] +die_width = 50 +target_density = 70 +run_path = 'runs' +``` + +## Write verilog + +Invert the `in` input signal and continuously assign it to the `out` output signal. + +```bash magic_args="-c 'cat > inverter.v; iverilog inverter.v'" +module inverter(input wire in, output wire out); + assign out = !in; +endmodule +``` +## Write flow configuration + +See [OpenLane Variables information](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/configuration/README.md) for the list of available variables. + +```python +%%writefile config.tcl +set ::env(DESIGN_NAME) inverter + +set ::env(VERILOG_FILES) "inverter.v" + +set ::env(FP_SIZING) "absolute" +set ::env(DIE_AREA) "0 0 $::env(DIE_WIDTH) $::env(DIE_WIDTH)" +set ::env(PL_TARGET_DENSITY) [expr {$::env(TARGET_DENSITY) / 100.0}] + +set ::env(CLOCK_TREE_SYNTH) 0 +set ::env(CLOCK_PORT) "" +set ::env(DIODE_INSERTION_STRATEGY) 0 +``` + +## Run OpenLane flow + +[OpenLane](https://github.com/The-OpenROAD-Project/OpenLane) is an automated RTL to GDSII flow based on several components including [OpenROAD](https://github.com/The-OpenROAD-Project/OpenROAD), [Yosys](https://github.com/YosysHQ/yosys), Magic, Netgen, Fault, CVC, SPEF-Extractor, CU-GR, Klayout and a number of custom scripts for design exploration and optimization. + +```python tags=[] +#papermill_description=RunningOpenLaneFlow +%env DIE_WIDTH={die_width} +%env TARGET_DENSITY={target_density} +!flow.tcl -design . -run_path {run_path} +``` + +## Display layout + +Use [GDSII Tool Kit](https://github.com/heitzmann/gdstk) to convert the resulting GDSII file to SVG. + +```python +#papermill_description=RenderingGDS +import pathlib +import gdstk +import IPython.display +import scrapbook as sb + +gds_path = sorted(pathlib.Path(run_path).glob('*/results/final/gds/*.gds'))[-1] +library = gdstk.read_gds(gds_path) +top_cells = library.top_level() +svg_path = pathlib.Path(run_path) / 'layout.svg' +top_cells[0].write_svg(svg_path) +sb.glue('layout', IPython.display.SVG(svg_path), 'display', display=True) +``` + +## Dump flow report + +See [OpenLane Datapoint Definitions](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/regression_results/datapoint_definitions.md) for the description of the report columns. + +```python tags=[] +#papermill_description=DumpingReport +import pandas as pd +import pathlib +import scrapbook as sb + +final_summary_report = sorted(pathlib.Path(run_path).glob('*/reports/metrics.csv'))[-1] +df = pd.read_csv(final_summary_report) +pd.set_option('display.max_rows', None) +sb.glue('summary', df, 'pandas') +df.transpose() +``` + +## Extract power metrics + +Build a pandas dataframe with area, density and power consumption. + +```python tags=[] +#papermill_description=ExtractingMetrics +import scrapbook as sb + +def get_power(sta_power_report): + with sta_power_report.open() as f: + for l in f.readlines(): + if l.startswith('Total'): + return float(l.split(' ')[-2]) + +def area_density_ppa(): + for report in sorted(pathlib.Path(run_path).glob('*/reports')): + sta_power_report = report / 'signoff/28-rcx_mca_sta.power.rpt' + final_summary_report = report / 'metrics.csv' + if final_summary_report.exists() and sta_power_report.exists(): + df = pd.read_csv(final_summary_report) + power = get_power(sta_power_report) + yield (df['DIEAREA_mm^2'][0], df['PL_TARGET_DENSITY'][0], power) + +df = pd.DataFrame(area_density_ppa(), columns=('DIEAREA_mm^2', 'PL_TARGET_DENSITY', 'TOTAL_POWER')) +sb.glue('metrics', df, 'pandas') +(df.style.hide_index() + .format({'area': '{:.8f}', 'density': '{:.2%}', 'power': '{:.8f}'}) + .bar(subset=['TOTAL_POWER'], color='pink') + .background_gradient(subset=['PL_TARGET_DENSITY'], cmap='Greens') + .bar(color='lightblue', vmin=0.001, subset=['DIEAREA_mm^2'])) +``` + +Report metrics for hyper-parameters tuning. + +```python +#papermill_description=ReportingMetrics +import hypertune + +total_power = df['TOTAL_POWER'][0] * 1e6 +print('reporting metric:', 'total_power', total_power) +hpt = hypertune.HyperTune() +hpt.report_hyperparameter_tuning_metric( + hyperparameter_metric_tag='total_power', + metric_value=total_power, +) +``` diff --git a/modules/silicon_design/scripts/build/notebooks/inverter.svg b/modules/silicon_design/scripts/build/notebooks/inverter/layout.svg similarity index 100% rename from modules/silicon_design/scripts/build/notebooks/inverter.svg rename to modules/silicon_design/scripts/build/notebooks/inverter/layout.svg diff --git a/modules/silicon_design/scripts/build/notebooks/lut/lut.md b/modules/silicon_design/scripts/build/notebooks/lut/lut.md new file mode 100644 index 00000000..66a684b2 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/lut/lut.md @@ -0,0 +1,297 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# LUTs exploration + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + + +## Define flow parameters + +```python tags=["parameters"] +fp_core_util = 45 +pl_target_density = 90 +synth_defines='FRACTURABLE' +synth_param_inputs=5 +run_path = 'runs/lut' +``` + +## Get LUT test designs + +```python +!git clone https://github.com/growly/lut-tests.git +``` + +## Write flow configuration + +See [OpenLane Variables information](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/configuration/README.md) for the list of available variables. + +```python +%%writefile config.tcl +# Design +# This is the config used to openlane for synthesis only +set ::env(DESIGN_NAME) "LUT" + +#set ::env(SYNTH_DEFINES) "FRACTURABLE PREDECODE_2" +#set ::env(SYNTH_DEFINES) "" +#set ::env(SYNTH_DEFINES) "FRACTURABLE" +#set ::env(SYNTH_PARAMETERS) "INPUTS=6" + +set script_dir [file dirname [file normalize [info script]]] +set ::env(VERILOG_FILES) [glob $script_dir/lut-tests/src/*.v] +set ::env(CLOCK_TREE_SYNTH) 0 +#set ::env(CLOCK_PORT) "config_clk" +set ::env(CLOCK_PORT) "" +# Design config +set ::env(CLOCK_PERIOD) 30 +#set ::env(CLOCK_PERIOD) "5.21" +set ::env(SYNTH_STRATEGY) "DELAY 1" + +#set ::env(FP_CORE_UTIL) 50 +#set ::env(PL_TARGET_DENSITY) 0.99 +#set ::env(FP_CORE_UTIL) 40 +#set ::env(PL_TARGET_DENSITY) 0.49 + +# "Enable logic verification using yosys, for comparing each netlist at each +# stage of the flow with the previous netlist and verifying that they are +# logically equivalent." Logical equivalence checking? +#set ::env(LEC_ENABLE) "1" +#set ::env(FP_WELLTAP_CELL) "sky130_fd_sc_hd__tap*" + +set ::env(CELL_PAD) "0" +set ::env(TOP_MARGIN_MULT) 1 +set ::env(BOTTOM_MARGIN_MULT) 1 +set ::env(LEFT_MARGIN_MULT) 2 +set ::env(RIGHT_MARGIN_MULT) 2 +#set ::env(FILL_INSERTION) "0" +#set ::env(PL_RESIZER_DESIGN_OPTIMIZATIONS) "0" +#set ::env(PL_RESIZER_TIMING_OPTIMIZATIONS) "0" +#set ::env(GLB_RESIZER_DESIGN_OPTIMIZATIONS) "0" +#set ::env(GLB_RESIZER_TIMING_OPTIMIZATIONS) "0" + +set ::env(RT_MAX_LAYER) "met4" +set ::env(GLB_RT_ALLOW_CONGESTION) "1" + +#set ::env(CELLS_LEF) "$::env(DESIGN_DIR)/cells.lef" +# +#set ::env(DIE_AREA) "0 0 393.76 27.200000000000003" +# +#set ::env(DIODE_INSERTION_STRATEGY) "0" + +set ::env(ROUTING_CORES) 28 + +set ::env(DESIGN_IS_CORE) "0" +set ::env(SYNTH_PARAMETERS) "INPUTS=$::env(SYNTH_PARAM_INPUTS)" + +#set ::env(FP_PDN_CORE_RING) "0" +## +#set ::env(PRODUCTS_PATH) "./build/8x32_DEFAULT/products" +# +#set ::env(INITIAL_NETLIST) "$::env(DESIGN_DIR)/RAM8.nl.v" +#set ::env(INITIAL_DEF) "$::env(DESIGN_DIR)/RAM8.placed.def" +#set ::env(INITIAL_SDC) "$::env(BASE_SDC_FILE)" +# +#set ::env(LVS_CONNECT_BY_LABEL) "1" +# +#set ::env(QUIT_ON_TIMING_VIOLATIONS) "0" +set ::env(TEST_MISMATCHES) none +set ::env(PDN_CFG) "$script_dir/pdn_cfg.tcl" +``` + +```python +%%writefile pdn_cfg.tcl + +set ::env(VDD_NET) $::env(VDD_PIN) +set ::env(GND_NET) $::env(GND_PIN) + + foreach power_pin $::env(STD_CELL_POWER_PINS) { + add_global_connection \ + -net $::env(VDD_NET) \ + -inst_pattern .* \ + -pin_pattern $power_pin \ + -power + } + foreach ground_pin $::env(STD_CELL_GROUND_PINS) { + add_global_connection \ + -net $::env(GND_NET) \ + -inst_pattern .* \ + -pin_pattern $ground_pin \ + -ground + } + +set secondary [] + +foreach net $::env(VDD_NETS) { + if { $net != $::env(VDD_NET)} { + lappend secondary $net + + set db_net [[ord::get_db_block] findNet $net] + if {$db_net == "NULL"} { + set net [odb::dbNet_create [ord::get_db_block] $net] + $net setSpecial + $net setSigType "POWER" + } + } +} + +foreach net $::env(GND_NETS) { + if { $net != $::env(GND_NET)} { + lappend secondary $net + + set db_net [[ord::get_db_block] findNet $net] + if {$db_net == "NULL"} { + set net [odb::dbNet_create [ord::get_db_block] $net] + $net setSpecial + $net setSigType "GROUND" + } + } +} + +set_voltage_domain -name CORE -power $::env(VDD_NET) -ground $::env(GND_NET) \ + -secondary_power $secondary + +define_pdn_grid \ + -name stdcell_grid \ + -starts_with POWER \ + -voltage_domain CORE \ + -pins $::env(FP_PDN_LOWER_LAYER) + +add_pdn_stripe \ + -grid stdcell_grid \ + -layer $::env(FP_PDN_LOWER_LAYER) \ + -width $::env(FP_PDN_VWIDTH) \ + -pitch $::env(FP_PDN_VPITCH) \ + -offset $::env(FP_PDN_VOFFSET) \ + -starts_with POWER + +add_pdn_stripe \ + -grid stdcell_grid \ + -layer $::env(FP_PDN_RAILS_LAYER) \ + -width $::env(FP_PDN_RAIL_WIDTH) \ + -followpins \ + -starts_with POWER + +add_pdn_connect \ + -grid stdcell_grid \ + -layers "$::env(FP_PDN_RAILS_LAYER) $::env(FP_PDN_LOWER_LAYER)" + +define_pdn_grid \ + -macro \ + -name macro \ + -starts_with POWER \ + -halo "$::env(FP_PDN_HORIZONTAL_HALO) $::env(FP_PDN_VERTICAL_HALO)" + +add_pdn_connect \ + -grid macro \ + -layers "$::env(FP_PDN_LOWER_LAYER) $::env(FP_PDN_UPPER_LAYER)" + +``` + +## Run OpenLane flow + +[OpenLane](https://github.com/The-OpenROAD-Project/OpenLane) is an automated RTL to GDSII flow based on several components including [OpenROAD](https://github.com/The-OpenROAD-Project/OpenROAD), [Yosys](https://github.com/YosysHQ/yosys), Magic, Netgen, Fault, CVC, SPEF-Extractor, CU-GR, Klayout and a number of custom scripts for design exploration and optimization. + +```python +#papermill_description=RunningOpenLaneFlow +%env FP_CORE_UTIL={fp_core_util} +%env PL_TARGET_DENSITY={pl_target_density} +%env SYNTH_DEFINES={synth_defines} +%env SYNTH_PARAM_INPUTS={synth_param_inputs} + +!flow.tcl -design . -run_path {run_path} +``` + +## Display layout + +Use [GDSII Tool Kit](https://github.com/heitzmann/gdstk) to convert the resulting GDSII file to SVG. + +```python +!python -m pip install scrapbook +``` + +```python +#papermill_description=RenderingGDS +import pathlib +import gdstk +import IPython.display +import scrapbook as sb + +gds_path = sorted(pathlib.Path(run_path).glob('*/results/final/gds/*.gds'))[-1] +library = gdstk.read_gds(gds_path) +top_cells = library.top_level() +svg_path = pathlib.Path(run_path) / 'adders.svg' +top_cells[0].write_svg(svg_path) +sb.glue('layout', IPython.display.SVG(svg_path), 'display', display=True) +``` + +## Dump flow report + +See [OpenLane Datapoint Definitions](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/regression_results/datapoint_definitions.md) for the description of the report columns. + +```python +#papermill_description=DumpingReport +import pandas as pd +import pathlib +import scrapbook as sb + +final_summary_report = sorted(pathlib.Path(run_path).glob('*/reports/metrics.csv'))[-1] +df = pd.read_csv(final_summary_report) +pd.set_option('display.max_rows', None) +sb.glue('summary', df, 'pandas') +df.transpose() +``` + +## Extract power metrics + +Build a pandas dataframe with area, density and power consumption. + +```python +#papermill_description=ExtractingMetrics +import scrapbook as sb + +def area_density_ppa(): + for report in sorted(pathlib.Path(run_path).glob('*/reports/metrics.csv')): + yield (df['FP_CORE_UTIL'][0], df['PL_TARGET_DENSITY'][0], df['power_typical_switching_uW'][0]) + +df = pd.DataFrame(area_density_ppa(), columns=('DIEAREA_mm^2', 'PL_TARGET_DENSITY', 'power_typical_switching_uW')) +sb.glue('metrics', df, 'pandas') +(df.style.hide_index() + .format({'area': '{:.8f}', 'density': '{:.2%}', 'power': '{:.8f}'}) + .bar(subset=['power_typical_switching_uW'], color='pink') + .background_gradient(subset=['PL_TARGET_DENSITY'], cmap='Greens') + .bar(color='lightblue', vmin=0.001, subset=['DIEAREA_mm^2'])) +``` + +Report metrics for hyper-parameters tuning. + +```python +!python -m pip install cloudml-hypertune +``` + +```python +#papermill_description=ReportingMetrics +import hypertune + +total_power = df['power_typical_switching_uW'][0] * 1e6 +print('reporting metric:', 'power_typical_switching_uW', total_power) +hpt = hypertune.HyperTune() +hpt.report_hyperparameter_tuning_metric( + hyperparameter_metric_tag='power_typical_switching_uW', + metric_value=total_power, +) +``` diff --git a/modules/silicon_design/scripts/build/notebooks/lut/tuning.md b/modules/silicon_design/scripts/build/notebooks/lut/tuning.md new file mode 100644 index 00000000..8e7522ed --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/lut/tuning.md @@ -0,0 +1,207 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# Parameter Tuning Sample + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to leverage [Vertex AI hyperparameter tuning](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) in order to find the right flow parameters value to optimize a given metric. + + +## Define project parameters + +```python tags=["parameters"] +import pathlib + +worker_image = 'us-east4-docker.pkg.dev/catx-demo-radlab/containers/silicon-design-ubuntu-2004:latest' +staging_bucket = 'catx-demo-radlab-staging' +project = 'catx-demo-radlab' +location = 'us-central1' +machine_type = 'n1-standard-8' +notebook = 'lut.ipynb' +prefix = pathlib.Path(notebook).stem +staging_dir = f'{prefix}-tuning' +last_trial_id = 20 +``` + +## Stage the notebook for the experiment + +```python tags=[] +!gsutil cp {notebook} gs://{staging_bucket}/{staging_dir}/{notebook} +``` + +## Create Parameters and Metrics specs + +We want to find the best value for *target density* and *die area* in order optimize *total power* consumption. + +Those keys map to the [parameters](https://papermill.readthedocs.io/en/latest/usage-parameterize.html) and [metrics](https://github.com/GoogleCloudPlatform/cloudml-hypertune) advertised by the notebook. + +```python tags=[] +from google.cloud.aiplatform import hyperparameter_tuning as hpt + +parameter_spec = { + 'pl_target_density': hpt.DoubleParameterSpec(min=0.4, max=0.99, scale='log'), + 'fp_core_util': hpt.DoubleParameterSpec(min=5, max=90, scale='linear'), +} + +metric_spec={'power_typical_switching_uW': 'minimize'} +``` + +## Create Custom Job spec + +```python tags=[] +from google.cloud import aiplatform +import pathlib + +worker_pool_specs = [{ + 'machine_spec': { + 'machine_type': machine_type, + }, + 'replica_count': 1, + 'container_spec': { + 'image_uri': worker_image, + 'args': ['/usr/local/bin/papermill-launcher', + f'gs://{staging_bucket}/{staging_dir}/{notebook}', + f'$AIP_MODEL_DIR/{prefix}_out.ipynb', + '--run_dir=/tmp'] + } +}] +custom_job = aiplatform.CustomJob(display_name=f'{prefix}-custom-job', + worker_pool_specs=worker_pool_specs, + staging_bucket=staging_bucket, + base_output_dir=f'gs://{staging_bucket}/{staging_dir}') +``` + +## Run Hyperparameter tuning job + +```python tags=[] +from google.cloud import aiplatform +parameters_count = len(parameter_spec.keys()) +metrics_count = len(metric_spec.keys()) +max_trial_count = 100 * parameters_count * metrics_count +parallel_trial_count = 20 + +hpt_job = aiplatform.HyperparameterTuningJob( + display_name=f'{prefix}-tuning-job', + custom_job=custom_job, + metric_spec=metric_spec, + parameter_spec=parameter_spec, + max_trial_count=max_trial_count, + parallel_trial_count=parallel_trial_count, + max_failed_trial_count=max_trial_count) +hpt_job.run(sync=False) +``` + +## Fetch notebooks for all study trials + +```python +import pathlib +from google.cloud import storage +import tqdm + +local_dir = pathlib.Path(staging_dir) +local_dir.mkdir(exist_ok=True, parents=True) + +client = storage.Client() +bucket = client.bucket(staging_bucket) +for i in tqdm.tqdm(range(1, last_trial_id+1)): + src = bucket.blob(f'{staging_dir}/{i}/model/{prefix}_out.ipynb') + dst = local_dir / f'{prefix}_out_{i}.ipynb' + with dst.open('wb') as f: + src.download_to_file(f) +``` + +## Extract metrics from notebooks + +```python tags=[] +import scrapbook as sb +books = sb.read_notebooks(str(local_dir)) +``` + +```python tags=[] +import pathlib +import math + +import pandas as pd +import tqdm +def metrics(): + for b in tqdm.tqdm(books): + trial_id = int(pathlib.Path(books[b].filename).stem.split('_')[-1]) + if 'metrics' in books[b].scraps: + metrics = books[b].scraps['metrics'].data + yield trial_id, metrics['FP_CORE_UTIL'][0], metrics['PL_TARGET_DENSITY'][0], metrics['power_typical_switching_uW'][0] + else: + params = books[b].parameters + fp_core_util = float(params.fp_core_util) + pl_target_density = float(params.fp_target_density) + yield trial_id, fp_core_util, pl_target_density, math.nan + +df = pd.DataFrame.from_records(metrics(), columns=['TRIAL_ID', 'FP_CORE_UTIL', 'PL_TARGET_DENSITY', 'power_typical_switching_uW'], index='TRIAL_ID').sort_index() +(df.dropna() + .sort_values(['power_typical_switching_uW'], ascending=[False]).drop_duplicates(['power_typical_switching_uW']) + .style + .format({'FP_CORE_UTIL': '{:.2f}', 'PL_TARGET_DENSITY': '{:.2%}', 'TOTAL_POWER': '{:.6f}'}) + .bar(subset=['power_typical_switching_uW'], color='pink') + .background_gradient(subset=['PL_TARGET_DENSITY'], cmap='Greens') + .bar(color='lightblue', vmin=0.001, subset=['FP_CORE_UTIL'])) +``` + +## Plot experiments + +```python +import matplotlib.colors +from matplotlib import pyplot as plt + +cool = matplotlib.colormaps['cool'] +cool.set_bad(color='none') +ax = df.plot.scatter(x='FP_CORE_UTIL', y='PL_TARGET_DENSITY', c='power_typical_switching_uW', + cmap=cool, s=50, sharex=False, plotnonfinite=True, alpha=1.0, edgecolor='black') +plt.savefig('{prefix}.png') +ax +``` + +```python +from matplotlib import pyplot as plt +from matplotlib import animation +from matplotlib import cm + +from tqdm import tqdm +from IPython.display import Image + +min_total_power = df['power_typical_switching_uW'].min() +max_total_power = df['power_typical_switching_uW'].max() +fig, ax = plt.subplots() +fig.colorbar(cm.ScalarMappable(matplotlib.colors.Normalize(min_total_power, max_total_power), cmap=cool), + label='power_typical_switching_uW', + ax=ax) +ax.set_xlabel('FP_CORE_UTIL') +ax.set_ylabel('PL_TARGET_DENSITY') +plt.close(fig) # hide current figure + +def generate_frames(): + for n in range(50, last_trial_id, 50): + batch = df[0:n] + yield [ax.scatter( + batch['FP_CORE_UTIL'], batch['PL_TARGET_DENSITY'], c=batch['power_typical_switching_uW'], + s=50, vmin=min_total_power, vmax=max_total_power, cmap=cool, plotnonfinite=True, edgecolor='black')] + +frames = list(generate_frames()) +anim = animation.ArtistAnimation(fig, frames) +anim.save('{prefix}.gif', writer=animation.PillowWriter(fps=10)) +Image('{prefix}.gif') +``` diff --git a/modules/silicon_design/scripts/build/notebooks/math/math.md b/modules/silicon_design/scripts/build/notebooks/math/math.md new file mode 100644 index 00000000..94efe54b --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/math/math.md @@ -0,0 +1,52 @@ +--- +jupyter: + jupytext: + formats: ipynb,md + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +## Parameter cell with the tag parameters + +```python tags=["parameters"] +param_1 = 5 +param_2 = 20 +``` + +```python +# Parse string inputs +param_1 = float(param_1) +param_2 = float(param_2) +``` + +```python +# +product = ((param_1-25)**2+10) * ((param_2-25)**2+10) +``` + +```python +import hypertune + +print('reporting metric:', 'product', product) +hpt = hypertune.HyperTune() +hpt.report_hyperparameter_tuning_metric( + hyperparameter_metric_tag = 'product', + metric_value = product, +) +``` + +```python +import pandas as pd +import scrapbook as sb + +df = pd.DataFrame([[param_1, param_2, product]], columns=('param_1', 'param_2', 'product')) +sb.glue('metrics', df, 'pandas') +df +``` diff --git a/modules/silicon_design/scripts/build/notebooks/math/tuning.md b/modules/silicon_design/scripts/build/notebooks/math/tuning.md new file mode 100644 index 00000000..80e48bac --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/math/tuning.md @@ -0,0 +1,175 @@ +--- +jupyter: + jupytext: + formats: ipynb,md + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +## Define project parameters + +```python tags=["parameters"] +import pathlib + +worker_image = 'us-east4-docker.pkg.dev/catx-demo-radlab/containers/silicon-design-ubuntu-2004:latest' +staging_bucket = 'catx-demo-radlab-staging' +project = 'catx-demo-radlab' +location = 'us-central1' +machine_type = 'n1-standard-8' +notebook = 'minimal_job.ipynb' +prefix = pathlib.Path(notebook).stem +staging_dir = f'{prefix}-tuning' +``` + +## Delete data from previous runs and stage the notebook for the experiment + +```python tags=[] +!rm -r {staging_dir} +!gsutil rm -r gs://{staging_bucket}/{staging_dir}/ +!gsutil cp {notebook} gs://{staging_bucket}/{staging_dir}/{notebook} +``` + +## Create Parameters and Metrics specs + +We want to find the best value for *target density* and *die area* in order optimize *total power* consumption. + +Those keys map to the [parameters](https://papermill.readthedocs.io/en/latest/usage-parameterize.html) and [metrics](https://github.com/GoogleCloudPlatform/cloudml-hypertune) advertised by the notebook. + +```python tags=[] +from google.cloud.aiplatform import hyperparameter_tuning as hpt + +parameter_spec = { + 'param_1': hpt.DoubleParameterSpec(min=10, max=100, scale='linear'), + 'param_2': hpt.DoubleParameterSpec(min=10, max=300, scale='linear'), +} + +metric_spec={'product': 'minimize'} +``` + +## Create Custom Job spec + +```python tags=[] +from google.cloud import aiplatform +import pathlib + +worker_pool_specs = [{ + 'machine_spec': { + 'machine_type': machine_type, + }, + 'replica_count': 1, + 'container_spec': { + 'image_uri': worker_image, + 'args': ['/usr/local/bin/papermill-launcher', + f'gs://{staging_bucket}/{staging_dir}/{notebook}', + f'$AIP_MODEL_DIR/{prefix}_out.ipynb', + '--run_dir=/tmp'] + } +}] +custom_job = aiplatform.CustomJob(display_name=f'{prefix}-custom-job', + worker_pool_specs=worker_pool_specs, + staging_bucket=staging_bucket, + base_output_dir=f'gs://{staging_bucket}/{staging_dir}') +``` + +## Run Hyperparameter tuning job + +```python tags=[] +from google.cloud import aiplatform + +parameters_count = len(parameter_spec.keys()) +metrics_count = len(metric_spec.keys()) +max_trial_count = 100 * parameters_count * metrics_count +parallel_trial_count = 20 + +hpt_job = aiplatform.HyperparameterTuningJob( + display_name=f'{prefix}-tuning-job', + custom_job=custom_job, + metric_spec=metric_spec, + parameter_spec=parameter_spec, + max_trial_count=max_trial_count, + parallel_trial_count=parallel_trial_count, + max_failed_trial_count=max_trial_count) +hpt_job.run(sync=False) +``` + +## Check the current state + +```python +[(job.display_name, job.done(), job.state) for job in aiplatform.HyperparameterTuningJob.list(filter=f'display_name={prefix}-tuning-job') if not job.done()] +``` + +## Fetch notebooks for all study trials + +```python +import pathlib +from google.cloud import storage +import tqdm + +local_dir = pathlib.Path(staging_dir) +local_dir.mkdir(exist_ok=True, parents=True) + +client = storage.Client() +bucket = client.bucket(staging_bucket) +for i in tqdm.tqdm(range(1, max_trial_count+1)): + src = bucket.blob(f'{staging_dir}/{i}/model/{prefix}_out.ipynb') + if not src.exists(): + break + dst = local_dir / f'{prefix}_out_{i}.ipynb' + with dst.open('wb') as f: + src.download_to_file(f) +``` + +## Extract metrics from notebooks + +```python tags=[] +import scrapbook as sb +books = sb.read_notebooks(str(local_dir)) +``` + +```python +import pathlib +import math +import pandas as pd +import tqdm + +def metrics(): + for b in tqdm.tqdm(books): + trial_id = int(pathlib.Path(books[b].filename).stem.split('_')[-1]) + param_1 = float(books[b].parameters.param_1) + param_2 = float(books[b].parameters.param_2) + + if 'metrics' in books[b].scraps: + metrics = books[b].scraps['metrics'].data + yield trial_id, param_1, param_2, metrics['product'][0] + else: + yield trial_id, param_1, param_2, math.nan + +df = pd.DataFrame.from_records(metrics(), columns=['TRIAL_ID', 'param_1', 'param_2', 'product'], index='TRIAL_ID').sort_index() + +df.sort_values(['product'], ascending=[True]).style.bar(color='lightblue', vmin=0.001, subset=['product']).background_gradient(subset=['param_1'], cmap='Greens').background_gradient(subset=['param_2'], cmap='Blues') +``` + +## Plot experiments + +```python +import matplotlib.colors +from matplotlib import pyplot as plt + +cool = matplotlib.colormaps['cool'] +cool.set_bad(color='none') +ax = df.plot.scatter(x='param_1', y='param_2', c='product', + cmap=cool, s=50, sharex=False, plotnonfinite=True, alpha=1.0, edgecolor='black') +plt.savefig('{prefix}.png') +ax +``` + +```python + +``` diff --git a/modules/silicon_design/scripts/build/notebooks/openram/openram.md b/modules/silicon_design/scripts/build/notebooks/openram/openram.md new file mode 100644 index 00000000..71ba843d --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/openram/openram.md @@ -0,0 +1,113 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# OpenRAM SKY130 playground + +Generate OpenRAM macros with `open_pdks.sky130a`. + + + +## Install dependencies + +Using conda packages from https://github.com/hdl/conda-eda. + + +```python colab={"base_uri": "https://localhost:8080/"} id="8zaG4mCd4-Ti" outputId="33408422-77f4-4d7b-cc28-086f699bdb87" +!pip install -q condacolab +import condacolab +condacolab.install() +``` + +```python colab={"base_uri": "https://localhost:8080/"} id="twnZMX905E-U" outputId="cc15b371-edb5-4d09-c4cb-26c41c551031" +import condacolab +condacolab.check() +``` + +```python colab={"base_uri": "https://localhost:8080/"} id="emiyv2qr6SnS" outputId="544c3046-62e1-4e4b-f0f8-06a848a3bbcd" +!conda install -c LiteX-Hub -y open_pdks.sky130a magic +!conda install -y gdstk cairosvg +``` + +```python colab={"base_uri": "https://localhost:8080/"} id="Rh8PYcraHuXI" outputId="e2e5ea0a-2403-42e2-8c7b-84acc267b3da" +!conda install https://anaconda.org/LiteX-Hub/netgen/1.5.219_0_ge11dbac/download/linux-64/netgen-1.5.219_0_ge11dbac-20220222_104027.tar.bz2 +``` + + +## Get OpenRAM + +Get latest release and install requirements from PyPI. + + +```python colab={"base_uri": "https://localhost:8080/"} id="jesuQ3pG5NmR" outputId="d219bfb3-b588-4890-dee9-618bcd3be50c" +!git clone -b v1.1.19 https://github.com/VLSIDA/OpenRAM.git +!python -m pip install -r OpenRAM/requirements.txt +``` + +```python colab={"base_uri": "https://localhost:8080/"} id="uNqPoSUB5fOS" outputId="64302d22-4a96-47cf-bee5-763f55cc082b" +%%writefile config.py +""" +Pseudo-dual port (independent read and write ports), 8bit word, 1 kbyte SRAM. +Useful as a byte FIFO between two devices (the reader and the writer). +""" +word_size = 8 # Bits +num_words = 1024 +human_byte_size = "{:.0f}kbytes".format((word_size * num_words)/1024/8) + +# Allow byte writes +#write_size = 8 # Bits + +# Dual port +num_rw_ports = 0 +num_r_ports = 1 +num_w_ports = 1 +ports_human = '1r1w' + +tech_name = "sky130" +nominal_corner_only = True + +# Local wordlines have issues with met3 power routing for now +#local_array_size = 16 + +route_supplies = "ring" +#route_supplies = "left" +check_lvsdrc = True +uniquify = True +#perimeter_pins = False +#netlist_only = True +#analytical_delay = False + +output_name = "sky130_sram_1kbyte_1r1w_8x1024_8" +output_path = "." +``` + +```python colab={"base_uri": "https://localhost:8080/"} id="RT6Zj3BE5nGS" outputId="0626d1d5-e8e0-4786-e4c0-304882b470fa" +%env OPENRAM_HOME=/content/OpenRAM/compiler +%env OPENRAM_TECH=/content/OpenRAM/technology/sky130 +%env PDK_ROOT=/usr/local/share/pdk +%env PYTHONPATH=/env/python:/content/OpenRAM/compiler:/content/OpenRAM/technology:/content/OpenRAM/technology/sky130/modules +!make -C OpenRAM SRAM_GIT_REPO=https://github.com/google/skywater-pdk-libs-sky130_fd_bd_sram.git +!python $OPENRAM_HOME/openram.py config.py +``` + +```python colab={"base_uri": "https://localhost:8080/", "height": 1000} id="NUSqt4xDL4Iu" outputId="f5cf3b6d-3e64-423d-f83e-7a000b57ec63" +import gdstk +library = gdstk.read_gds("sky130_sram_1kbyte_1r1w_8x1024_8.gds") +top_cells = library.top_level() +top_cells[0].write_svg('sky130_sram_1kbyte_1r1w_8x1024_8.svg') +import cairosvg +cairosvg.svg2png(url='sky130_sram_1kbyte_1r1w_8x1024_8.svg', write_to='sky130_sram_1kbyte_1r1w_8x1024_8.png') +from IPython.display import Image +Image('sky130_sram_1kbyte_1r1w_8x1024_8.png') +``` diff --git a/modules/silicon_design/scripts/build/notebooks/serv/subservient.md b/modules/silicon_design/scripts/build/notebooks/serv/subservient.md new file mode 100644 index 00000000..ce9bef33 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/serv/subservient.md @@ -0,0 +1,160 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# Serv Sample + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to run the [SERV](https://github.com/olofk/serv) RISC-V core design thru an end-to-end RTL to GDSII flow targetting the [SKY130](https://github.com/google/skywater-pdk/) process node. + + +## Define flow parameters + +```python tags=["parameters"] +die_width = 200 +target_density = 80 +run_path = 'runs/serv' +``` + +## Get SERV RTL + +```python +!git clone -b serial_dbg_if https://github.com/olofk/subservient +!git clone https://github.com/olofk/serv +``` +## Write flow configuration + +See [OpenLane Variables information](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/configuration/README.md) for the list of available variables. + +```python +%%writefile config.tcl +set ::env(DESIGN_NAME) subservient + +set script_dir [file dirname [file normalize [info script]]] +set ::env(VERILOG_FILES) " + [glob "$script_dir/serv/rtl/*.v"] + [glob "$script_dir/serv/serving/*.v"] + [glob "$script_dir/subservient/rtl/*.v"] +" +set ::env(CLOCK_PERIOD) "10" +set ::env(CLOCK_PORT) "i_clk" +set ::env(DESIGN_IS_CORE) 0 + +set ::env(FP_SIZING) "absolute" +set ::env(DIE_AREA) "0 0 $::env(DIE_WIDTH) $::env(DIE_WIDTH)" +set ::env(PL_TARGET_DENSITY) [expr {$::env(TARGET_DENSITY) / 100.0}] +``` + +## Run OpenLane flow + +[OpenLane](https://github.com/The-OpenROAD-Project/OpenLane) is an automated RTL to GDSII flow based on several components including [OpenROAD](https://github.com/The-OpenROAD-Project/OpenROAD), [Yosys](https://github.com/YosysHQ/yosys), Magic, Netgen, Fault, CVC, SPEF-Extractor, CU-GR, Klayout and a number of custom scripts for design exploration and optimization. + +```python tags=[] +#papermill_description=RunningOpenLaneFlow +%env DIE_WIDTH={die_width} +%env TARGET_DENSITY={target_density} +!flow.tcl -design . -run_path {run_path} -verbose 2 +``` + +## Display layout + +Use [GDSII Tool Kit](https://github.com/heitzmann/gdstk) and [CairoSVG](https://cairosvg.org/) to convert the resulting GDSII file to PNG. + +```python +#papermill_description=RenderingGDS +import pathlib +import gdstk +import cairosvg + +import IPython.display +import scrapbook as sb + +gds_path = sorted(pathlib.Path(run_path).glob('*/results/final/gds/*.gds'))[-1] +library = gdstk.read_gds(gds_path) +top_cells = library.top_level() +svg_path = gds_path.parent / 'subservient.svg' +top_cells[0].write_svg(svg_path) +png_path = gds_path.parent / 'subservient.png' + +cairosvg.svg2png(url=str(svg_path), write_to=str(png_path)) +sb.glue('layout', IPython.display.Image(png_path), 'display', display=True) +``` + +## Dump flow report + +See [OpenLane Datapoint Definitions](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/regression_results/datapoint_definitions.md) for the description of the report columns. + +```python tags=[] +#papermill_description=DumpingReport +import pandas as pd +import pathlib +import scrapbook as sb + +final_summary_report = sorted(pathlib.Path(run_path).glob('*/reports/final_summary_report.csv'))[-1] +df = pd.read_csv(final_summary_report) +pd.set_option('display.max_rows', None) +sb.glue('summary', df, 'pandas') +df.transpose() +``` + +## Extract power metrics + +Build a pandas dataframe with area, density and power consumption. + +```python tags=[] +#papermill_description=ExtractingMetrics +import scrapbook as sb + +def get_power(sta_power_report): + with sta_power_report.open() as f: + for l in f.readlines(): + if l.startswith('Total'): + return float(l.split(' ')[-2]) + +def area_density_ppa(): + for report in sorted(pathlib.Path(run_path).glob('*/reports')): + sta_power_report = report / 'routing/24-parasitics_sta.power.rpt' + final_summary_report = report / 'final_summary_report.csv' + if final_summary_report.exists() and sta_power_report.exists(): + df = pd.read_csv(final_summary_report) + power = get_power(sta_power_report) + yield (df['DIEAREA_mm^2'][0], df['PL_TARGET_DENSITY'][0], power) + +df = pd.DataFrame(area_density_ppa(), columns=('DIEAREA_mm^2', 'PL_TARGET_DENSITY', 'TOTAL_POWER')) +sb.glue('metrics', df, 'pandas') +(df.style.hide_index() + .format({'DIEAREA_mm^2': '{:.8f}', 'PL_TARGET_DENSITY': '{:.2%}', 'TOTAL_POWER': '{:.6f}'}) + .bar(subset=['TOTAL_POWER'], color='pink') + .background_gradient(subset=['PL_TARGET_DENSITY'], cmap='Greens') + .bar(color='lightblue', vmin=0.001, subset=['DIEAREA_mm^2'])) +``` + +Report metrics for hyper-parameters tuning. + +```python +#papermill_description=ReportingMetrics +import hypertune + +total_power = df['TOTAL_POWER'][0] * 1e6 +print('reporting metric:', 'total_power', total_power) +hpt = hypertune.HyperTune() +hpt.report_hyperparameter_tuning_metric( + hyperparameter_metric_tag='total_power', + metric_value=total_power, +) +``` diff --git a/modules/silicon_design/scripts/build/notebooks/serv/tuning.md b/modules/silicon_design/scripts/build/notebooks/serv/tuning.md new file mode 100644 index 00000000..86eb86a2 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/serv/tuning.md @@ -0,0 +1,255 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# Parameter Tuning Sample + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to leverage [Vertex AI hyperparameter tuning](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) in order to find the right flow parameters value to optimize a given metric. + + +## Define project parameters + +```python tags=["parameters"] +import pathlib + +worker_image = 'us-east4-docker.pkg.dev/catx-demo-radlab/containers/silicon-design-ubuntu-2004:latest' +staging_bucket = 'catx-demo-radlab-staging' +project = 'catx-demo-radlab' +location = 'us-central1' +machine_type = 'n1-standard-8' +notebook = 'subservient.ipynb' +prefix = pathlib.Path(notebook).stem +staging_dir = f'{prefix}-tuning' +last_trial_id = 20 +``` + +## Stage the notebook for the experiment + +```python tags=[] +!gsutil cp {notebook} gs://{staging_bucket}/{staging_dir}/{notebook} +``` + +## Create Parameters and Metrics specs + +We want to find the best value for *target density* and *die area* in order optimize *total power* consumption. + +Those keys map to the [parameters](https://papermill.readthedocs.io/en/latest/usage-parameterize.html) and [metrics](https://github.com/GoogleCloudPlatform/cloudml-hypertune) advertised by the notebook. + +```python tags=[] +from google.cloud.aiplatform import hyperparameter_tuning as hpt + +parameter_spec = { + 'target_density': hpt.DoubleParameterSpec(min=10, max=100, scale='log'), + 'die_width': hpt.DoubleParameterSpec(min=10, max=300, scale='linear'), +} + +metric_spec={'total_power': 'minimize'} +``` + +## Create Custom Job spec + +```python tags=[] +from google.cloud import aiplatform +import pathlib + +worker_pool_specs = [{ + 'machine_spec': { + 'machine_type': machine_type, + }, + 'replica_count': 1, + 'container_spec': { + 'image_uri': worker_image, + 'args': ['/usr/local/bin/papermill-launcher', + f'gs://{staging_bucket}/{staging_dir}/{notebook}', + f'$AIP_MODEL_DIR/{prefix}_out.ipynb', + '--run_path=/tmp'] + } +}] +custom_job = aiplatform.CustomJob(display_name=f'{prefix}-custom-job', + worker_pool_specs=worker_pool_specs, + staging_bucket=staging_bucket, + base_output_dir=f'gs://{staging_bucket}/{staging_dir}') +``` + +## Run Hyperparameter tuning job + +```python tags=[] +from google.cloud import aiplatform +parameters_count = len(parameter_spec.keys()) +metrics_count = len(metric_spec.keys()) +max_trial_count = 100 * parameters_count * metrics_count +parallel_trial_count = 20 + +hpt_job = aiplatform.HyperparameterTuningJob( + display_name=f'{prefix}-tuning-job', + custom_job=custom_job, + metric_spec=metric_spec, + parameter_spec=parameter_spec, + max_trial_count=max_trial_count, + parallel_trial_count=parallel_trial_count, + max_failed_trial_count=max_trial_count) +hpt_job.run(sync=False) +``` + +## Fetch notebooks for all study trials + +```python +import pathlib +from google.cloud import storage +import tqdm + +local_dir = pathlib.Path(staging_dir) +local_dir.mkdir(exist_ok=True, parents=True) + +client = storage.Client() +bucket = client.bucket(staging_bucket) +for i in tqdm.tqdm(range(1, last_trial_id+1)): + src = bucket.blob(f'{staging_dir}/{i}/model/{prefix}_out.ipynb') + dst = local_dir / f'{prefix}_out_{i}.ipynb' + with dst.open('wb') as f: + src.download_to_file(f) +``` + +## Extract metrics from notebooks + +```python tags=[] +import scrapbook as sb +books = sb.read_notebooks(str(dst_dir)) +``` + +```python +import pathlib +import math + +import pandas as pd +import tqdm +def metrics(): + for b in tqdm.tqdm(books): + trial_id = int(pathlib.Path(books[b].filename).stem.split('_')[-1]) + if 'metrics' in books[b].scraps: + metrics = books[b].scraps['metrics'].data + yield trial_id, metrics['DIEAREA_mm^2'][0], metrics['PL_TARGET_DENSITY'][0], metrics['TOTAL_POWER'][0] + else: + params = books[b].parameters + die_width_mm = float(params.die_width) / 1000.0 + target_density = float(params.target_density) / 100.0 + yield trial_id, die_width_mm * die_width_mm, target_density, math.nan + +df = pd.DataFrame.from_records(metrics(), columns=['TRIAL_ID', 'DIEAREA_mm^2', 'PL_TARGET_DENSITY', 'TOTAL_POWER'], index='TRIAL_ID').sort_index() +(df.dropna() + .sort_values(['DIEAREA_mm^2', 'TOTAL_POWER'], ascending=[True, True]) + .drop_duplicates(['TOTAL_POWER']) + .style + .format({'DIEAREA_mm^2': '{:.8f}', 'PL_TARGET_DENSITY': '{:.2%}', 'TOTAL_POWER': '{:.6f}'}) + .bar(subset=['TOTAL_POWER'], color='pink') + .background_gradient(subset=['PL_TARGET_DENSITY'], cmap='Greens') + .bar(color='lightblue', vmin=0.001, subset=['DIEAREA_mm^2'])) +``` + +## Plot experiments + +```python +import matplotlib.colors +from matplotlib import pyplot as plt + +cool = matplotlib.colormaps['cool'] +cool.set_bad(color='none') +ax = df.plot.scatter(x='DIEAREA_mm^2', y='PL_TARGET_DENSITY', c='TOTAL_POWER', + cmap=cool, s=50, sharex=False, plotnonfinite=True, alpha=1.0, edgecolor='black') +plt.savefig('{prefix}.png') +ax +``` + +```python +from matplotlib import pyplot as plt +from matplotlib import animation +from matplotlib import cm + +from tqdm import tqdm +from IPython.display import Image + +min_total_power = df['TOTAL_POWER'].min() +max_total_power = df['TOTAL_POWER'].max() +fig, ax = plt.subplots() +fig.colorbar(cm.ScalarMappable(matplotlib.colors.Normalize(min_total_power, max_total_power), cmap=cool), + label='TOTAL_POWER', + ax=ax) +ax.set_xlabel('DIEAREA_mm^2') +ax.set_ylabel('PL_TARGET_DENSITY') +plt.close(fig) # hide current figure + +def generate_frames(): + for n in range(50, last_trial_id, 50): + batch = df[0:n] + yield [ax.scatter( + batch['DIEAREA_mm^2'], batch['PL_TARGET_DENSITY'], c=batch['TOTAL_POWER'], + s=50, vmin=min_total_power, vmax=max_total_power, cmap=cool, plotnonfinite=True, edgecolor='black')] + +frames = list(generate_frames()) +anim = animation.ArtistAnimation(fig, frames) +anim.save('{prefix}.gif', writer=animation.PillowWriter(fps=10)) +Image('{prefix}.gif') +``` + +## Render chip layouts + +```python +from matplotlib import pyplot as plt +from matplotlib import animation +from matplotlib import cm +from tqdm import tqdm +from IPython.display import Image +from time import sleep +import matplotlib.colors +import io +import base64 +import PIL +import PIL.ImageOps +import PIL.ImageDraw +import numpy as np + + +min_total_power = df['TOTAL_POWER'].min() +max_total_power = df['TOTAL_POWER'].max() + +def images_with_power(): + for trial_id, trial in df.dropna().iterrows(): + book = books[f'subservient_out_{trial_id}'] + layout = book.scraps['layout'] + f = io.BytesIO(base64.b64decode(layout.display.data['image/png'])) + img = PIL.Image.open(f)#.convert('L') + total_power = trial['TOTAL_POWER'] + power = (total_power - min_total_power) / (max_total_power - min_total_power) + yield trial_id, img, power + +size = (500, 500) +fig, ax = plt.subplots(figsize=size) +cool = cm.get_cmap('cool') + +def generate_frames(): + for trial_id, img, power in tqdm(images_with_power()): + img = img.resize(size) + d = PIL.ImageDraw.Draw(img) + d.text((10, 10), f'SUBSERVIENT_{trial_id}', fill=(255, 255, 255, 255)) + yield img + +frames = list(generate_frames()) +frames[0].save('allthe{prefix}.gif', save_all=True, loop=0, append_images=frames[1:]) +Image('allthe{prefix}.gif') +``` diff --git a/modules/silicon_design/scripts/build/notebooks/xls/grid.md b/modules/silicon_design/scripts/build/notebooks/xls/grid.md new file mode 100644 index 00000000..d4787818 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/xls/grid.md @@ -0,0 +1,113 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.14.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# Parameter Tuning Sample + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to leverage [Vertex AI hyperparameter tuning](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) in order to find the right flow parameters value to optimize a given metric. + + + +## Define project parameters + + +```python tags=["parameters"] +import pathlib +import datetime + +worker_image = 'us-east4-docker.pkg.dev/catx-demo-radlab/radlab-silicon-tuning-containers/silicon-design-ubuntu-2004:202207270308' +project = 'catx-demo-radlab' +location = 'us-east4' +machine_type = 'n1-standard-32' +notebook = pathlib.Path('asap7.ipynb') +now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') +prefix = notebook.stem +staging_bucket = f'catx-demo-radlab-staging-{location}' +staging_dir = f'{prefix}-tuning-{now}' +staging_dir +``` + +## Stage the notebook for the experiment + +```python tags=[] +!gsutil mb -l {location} gs://{staging_bucket} +!gsutil cp {notebook} gs://{staging_bucket}/{staging_dir}/{notebook} +``` + +## Create Parameters and Metrics specs + +We want to find the best value for *target density* and *die area* in order optimize *total power* consumption. + +Those keys map to the [parameters](https://papermill.readthedocs.io/en/latest/usage-parameterize.html) and [metrics](https://github.com/GoogleCloudPlatform/cloudml-hypertune) advertised by the notebook. + +```python tags=[] +from google.cloud.aiplatform import hyperparameter_tuning as hpt + +parameter_spec = { + 'pipeline_stages': hpt.IntegerParameterSpec(min=1, max=16, scale='linear'), +} +metric_spec={'slack': 'maximize'} +``` + +## Create Custom Job spec + +```python tags=[] +from google.cloud import aiplatform +import pathlib + +worker_pool_specs = [{ + 'machine_spec': { + 'machine_type': machine_type, + }, + 'replica_count': 1, + 'container_spec': { + 'image_uri': worker_image, + 'args': ['/usr/local/bin/papermill-launcher', + f'gs://{staging_bucket}/{staging_dir}/{notebook}', + f'$AIP_MODEL_DIR/{prefix}_out.ipynb', + '--run_dirs=/tmp'] + } +}] +custom_job = aiplatform.CustomJob(display_name=f'{prefix}-custom-job', + worker_pool_specs=worker_pool_specs, + staging_bucket=staging_bucket, + base_output_dir=f'gs://{staging_bucket}/{staging_dir}') +``` + +## Run Hyperparameter tuning job + +```python tags=[] +from google.cloud import aiplatform +parameters_count = len(parameter_spec.keys()) +metrics_count = len(metric_spec.keys()) +max_trial_count = 16 +parallel_trial_count = 16 + +hpt_job = aiplatform.HyperparameterTuningJob( + display_name=f'{prefix}-tuning-job', + custom_job=custom_job, + metric_spec=metric_spec, + parameter_spec=parameter_spec, + max_trial_count=max_trial_count, + parallel_trial_count=parallel_trial_count, + max_failed_trial_count=max_trial_count, + location=location, + search_algorithm='grid') +hpt_job.run(sync=True) +``` diff --git a/modules/silicon_design/scripts/build/notebooks/xls/tuning.md b/modules/silicon_design/scripts/build/notebooks/xls/tuning.md new file mode 100644 index 00000000..2e9eebbd --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/xls/tuning.md @@ -0,0 +1,116 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.14.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# Parameter Tuning Sample + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to leverage [Vertex AI hyperparameter tuning](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) in order to find the right flow parameters value to optimize a given metric. + + + +## Define project parameters + + +```python tags=["parameters"] +import pathlib +import datetime + +worker_image = 'us-east4-docker.pkg.dev/catx-demo-radlab/radlab-silicon-tuning-containers/silicon-design-ubuntu-2004:202207270308' +staging_bucket = 'catx-demo-radlab-staging' +project = 'catx-demo-radlab' +location = 'us-east4' +machine_type = 'n1-standard-32' +notebook = pathlib.Path('asap7.ipynb') +now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') +prefix = notebook.stem +staging_dir = f'{prefix}-tuning-{now}' +staging_dir +``` + +## Stage the notebook for the experiment + +```python tags=[] +#!gsutil rm -r gs://{staging_bucket}/{staging_dir} +!gsutil cp {notebook} gs://{staging_bucket}/{staging_dir}/{notebook} +``` + +## Create Parameters and Metrics specs + +We want to find the best value for *target density* and *die area* in order optimize *total power* consumption. + +Those keys map to the [parameters](https://papermill.readthedocs.io/en/latest/usage-parameterize.html) and [metrics](https://github.com/GoogleCloudPlatform/cloudml-hypertune) advertised by the notebook. + +```python tags=[] +from google.cloud.aiplatform import hyperparameter_tuning as hpt + +parameter_spec = { + 'target_density': hpt.DoubleParameterSpec(min=0.1, max=1.0, scale='linear'), + 'die_width': hpt.IntegerParameterSpec(min=1, max=100, scale='linear'), + 'crc32_rounds': hpt.IntegerParameterSpec(min=1, max=32, scale='linear'), + 'pipeline_stages': hpt.IntegerParameterSpec(min=1, max=16, scale='linear'), +} + +metric_spec={'slack': 'maximize'} +``` + +## Create Custom Job spec + +```python tags=[] +from google.cloud import aiplatform +import pathlib + +worker_pool_specs = [{ + 'machine_spec': { + 'machine_type': machine_type, + }, + 'replica_count': 1, + 'container_spec': { + 'image_uri': worker_image, + 'args': ['/usr/local/bin/papermill-launcher', + f'gs://{staging_bucket}/{staging_dir}/{notebook}', + f'$AIP_MODEL_DIR/{prefix}_out.ipynb', + '--runs_dir=/tmp'] + } +}] +custom_job = aiplatform.CustomJob(display_name=f'{prefix}-custom-job', + worker_pool_specs=worker_pool_specs, + staging_bucket=staging_bucket, + base_output_dir=f'gs://{staging_bucket}/{staging_dir}') +``` + +## Run Hyperparameter tuning job + +```python tags=[] +from google.cloud import aiplatform +parameters_count = len(parameter_spec.keys()) +metrics_count = len(metric_spec.keys()) +max_trial_count = 100 * parameters_count * metrics_count +parallel_trial_count = int(max_trial_count / 20) + +hpt_job = aiplatform.HyperparameterTuningJob( + display_name=f'{prefix}-tuning-job', + custom_job=custom_job, + metric_spec=metric_spec, + parameter_spec=parameter_spec, + max_trial_count=max_trial_count, + parallel_trial_count=parallel_trial_count, + max_failed_trial_count=max_trial_count, + location=location) +hpt_job.run(sync=True) +``` diff --git a/modules/silicon_design/scripts/build/notebooks/xls/visualization.md b/modules/silicon_design/scripts/build/notebooks/xls/visualization.md new file mode 100644 index 00000000..08f92507 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/xls/visualization.md @@ -0,0 +1,200 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.14.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# Parameter Tuning Analysis + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to analyse data from [Vertex AI hyperparameter tuning](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) jobs. + + + +## Define project parameters + + +```python tags=["parameters"] +import pathlib + +location = 'us-east4' +#staging_bucket = 'catx-demo-radlab-staging' +staging_bucket = f'catx-demo-radlab-staging-{location}' + +notebook = pathlib.Path('asap7.ipynb') +prefix = notebook.stem +staging_dir = 'asap7-tuning-20220804141156' +``` + +## Fetch notebooks for all study trials + +```python +last_trial_id = 16 + +import pathlib +from google.cloud import storage +import tqdm + +local_dir = pathlib.Path(staging_dir) +local_dir.mkdir(exist_ok=True, parents=True) + +client = storage.Client() +bucket = client.bucket(staging_bucket) +for i in tqdm.tqdm(range(1, last_trial_id+1)): + src = bucket.blob(f'{staging_dir}/{i}/model/{prefix}_out.ipynb') + dst = local_dir / f'{prefix}_out_{i}.ipynb' + with dst.open('wb') as f: + src.download_to_file(f) +``` + +## Extract metrics from notebooks + +```python tags=[] +import scrapbook as sb +books = sb.read_notebooks(staging_dir) +``` + +```python + import pathlib +import math + +import pandas as pd +import tqdm +def metrics(): + for b in tqdm.tqdm(books): + trial_id = int(pathlib.Path(books[b].filename).stem.split('_')[-1]) + params = books[b].parameters + die_width_u = 90 #float(params.die_width) + target_density = 0.6 #float(params.target_density) + crc32_rounds = 8 #int(params.crc32_rounds) + pipeline_stages = int(params.pipeline_stages) + if ('ppa' in books[b].scraps) and not books[b].scraps['ppa'].data.empty: + metrics = books[b].scraps['metrics'].data + ppa = books[b].scraps['ppa'].data + yield trial_id, crc32_rounds, pipeline_stages, die_width_u, target_density, metrics['globalroute__timing__clock__slack'][0], metrics['finish__design__instance__utilization'][0] / 100.0, ppa['slack'][0], metrics['globalroute__timing__clock__slack'][0], metrics['finish__timing__setup__ws'][0], metrics['finish__timing__cp_delay'][0], ppa['power'][0] + else: + yield trial_id, crc32_rounds, pipeline_stages, die_width_u, target_density, math.nan, math.nan, math.nan, math.nan + +df = pd.DataFrame.from_records(metrics(), columns=['trial_id', 'crc32_rounds', 'pipeline_stages', 'die_width^2', 'target_density', 'finish__design__instance__area', 'finish__design__instance__utilization', 'critical_path_slack', 'globalroute__timing__clock__slack', 'finish__timing__setup__ws', 'finish__timing__cp_delay', 'power'], index='trial_id').sort_index() +df.to_csv(f'{prefix}.csv') +(df.sort_values(['pipeline_stages', 'finish__timing__setup__ws'], ascending=[True, True]) + .style + .format({'finish__design__instance__area': '{:.8f}', 'finish__design__instance__utilization': '{:.2%}', 'finish__timing__setup__ws': '{:.6f}'}) + .background_gradient(subset=['crc32_rounds'], cmap='Blues') + .background_gradient(subset=['pipeline_stages'], cmap='Oranges') + .bar(subset=['critical_path_slack'], color='pink') + .bar(subset=['globalroute__timing__clock__slack'], color='pink') + .bar(subset=['finish__timing__setup__ws'], color='pink') + .bar(subset=['finish__timing__cp_delay'], color='pink') + .bar(subset=['power'], color='pink') + .background_gradient(subset=['finish__design__instance__utilization'], cmap='Greens') + .bar(color='lightblue', vmin=0.001, subset=['finish__design__instance__area'])) +``` + +## Plot experiments + +```python +ax = pd.plotting.scatter_matrix(df, figsize=(30, 30)) +plt.savefig(f'{prefix}_matrix.png') +ax +``` + +```python +import matplotlib.colors +from matplotlib import pyplot as plt + +cool = matplotlib.colormaps['cool'] +cool.set_bad(color='none') +ax = df.plot.scatter(x='pipeline_stages', y='finish__timing__setup__ws', c='finish__timing__cp_delay', + cmap=cool, s=50, sharex=False, alpha=1.0, edgecolor='black') +plt.savefig(f'{prefix}.png') +ax +``` + +```python tags=[] +from matplotlib import pyplot as plt +from matplotlib import animation +from matplotlib import cm + +from tqdm import tqdm +from IPython.display import Image + +min_metric = df['slack'].min() +max_metric = df['slack'].max() +fig, ax = plt.subplots() +fig.colorbar(cm.ScalarMappable(matplotlib.colors.Normalize(min_metric, max_metric), cmap=cool), + label='slack', + ax=ax) +ax.set_xlabel('area') +ax.set_ylabel('pipelines') +plt.close(fig) # hide current figure + +def generate_frames(): + for n in range(20, last_trial_id, 20): + batch = df[0:n] + yield [ax.scatter( + batch['area'], batch['pipelines'], c=batch['slack'], + s=50, vmin=min_metric, vmax=max_metric, cmap=cool, edgecolor='black')] + +frames = list(generate_frames()) +anim = animation.ArtistAnimation(fig, frames) +anim.save(f'{prefix}.gif', writer=animation.PillowWriter(fps=10)) +Image(f'{prefix}.gif') +``` + +## Render chip layouts + +```python +from matplotlib import pyplot as plt +from matplotlib import animation +from matplotlib import cm +from tqdm import tqdm +from IPython.display import Image +from time import sleep +import matplotlib.colors +import io +import base64 +import PIL +import PIL.ImageOps +import PIL.ImageDraw +import numpy as np + +def trial_images(): + for trial_id, trial in df.dropna().sort_values(['trial_id'], ascending=[True]).iterrows(): + book = books[f'asap7_out_{trial_id}'] + layout = book.scraps['layout'] + f = io.BytesIO(base64.b64decode(layout.display.data['image/png'])) + img = PIL.Image.open(f)#.convert('L') + yield trial_id, img + +size = (500, 500) +fig, ax = plt.subplots(figsize=size) + +def generate_frames(): + for trial_id, img in tqdm(trial_images()): + img = img.resize(size) + d = PIL.ImageDraw.Draw(img) + d.text((10, 10), f'CRC32_ASAP7_{trial_id}', fill=(255, 255, 255, 255)) + yield img + +frames = list(generate_frames()) +frames[0].save(f'allthe{prefix}.gif', save_all=True, loop=0, append_images=frames[1:]) +Image(f'allthe{prefix}.gif') +``` + +```python +len(df.dropna()) +``` diff --git a/modules/silicon_design/scripts/build/notebooks/xls/xls-crc32-asap7.md b/modules/silicon_design/scripts/build/notebooks/xls/xls-crc32-asap7.md new file mode 100644 index 00000000..dadf8903 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/xls/xls-crc32-asap7.md @@ -0,0 +1,262 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.14.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +# OpenROAD Flow ASAP7 Sample + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to run a test design thru OpenROAD flow targetting the ASAP7 process node + + +## Define flow parameters + +```python tags=["parameters"] +import datetime + +die_width = 30 +core_padding = 2 +target_density = 0.90 +crc32_rounds = 8 +pipeline_stages = 4 +clock_period = 200 +runs_dir = 'runs' +``` + +## Timestamp run + +```python +now = datetime.datetime.now().strftime('%Y%m%d%H%M%S') +run_path = f'{runs_dir}/{now}' +run_path +``` + + +## Write and test DSLX module + +The CRC computation is written using the [DSLX](https://google.github.io/xls/dslx_reference/) HLS, a domain specific, dataflow-oriented functional language used to build hardware w/ a Rust-like syntax. + + +```bash colab={"base_uri": "https://localhost:8080/"} id="JKGxScUtoV4E" outputId="b9359a05-fa7f-4366-ecf8-40138acb11f1" magic_args="-c 'sed s/__CRC32_ROUNDS__/{crc32_rounds}/ > crc32.x; interpreter_main crc32.x'" +// Performs a table-less crc32 of the input data as in Hacker's Delight: +// https://github.com/hcs0/Hackers-Delight/blob/master/crc.c.txt (roughly flavor b) + +fn crc32_one_byte(byte: u8, polynomial: u32, crc: u32) -> u32 { + let crc = crc ^ (byte as u32); + // __CRC32_ROUNDS__ rounds of updates. + for (i, crc): (u32, u32) in range(u32:0, u32:__CRC32_ROUNDS__) { + let mask = -(crc & u32:1); + (crc >> u32:1) ^ (polynomial & mask) + }(crc) +} + +fn main(message: u8) -> u32 { + crc32_one_byte(message, u32:0xEDB88320, u32:-1) ^ u32:-1 +} +``` + + +## Generate IR and Verilog + +XLS can generate combinational or pipelined version of a given design. + + +```python colab={"base_uri": "https://localhost:8080/"} id="YMTh7WB6oxeW" outputId="a4e9d2f2-69e3-47e9-cad6-e1b89124553b" tags=[] +!ir_converter_main --top main crc32.x > crc32.ir +!opt_main crc32.ir > crc32_opt.ir +!codegen_main --generator=pipeline --delay_model="asap7" --module_name="crc32" --pipeline_stages={pipeline_stages} crc32_opt.ir > crc32.v +!cat crc32.v +``` + +## Configure OpenROAD Flow + +```python +%%writefile config.mk + +export PLATFORM = asap7 +export DESIGN_NAME = crc32 +export VERILOG_FILES = ${PWD}/crc32.v +export SDC_FILE = ${PWD}/constraint.sdc +export DIE_AREA = 0 0 $(DIE_WIDTH) $(DIE_WIDTH) +export CORE_AREA = $(CORE_PADDING) $(CORE_PADDING) $(CORE_WIDTH) $(CORE_WIDTH) + +export PLACE_DENSITY = $(TARGET_DENSITY) +``` + +```bash magic_args="-c \"sed s/__CLOCK_PERIOD__/{clock_period}/ | tee constraint.sdc\"" + +set clk_name clk +set clk_port_name clk +set clk_period __CLOCK_PERIOD__ +set clk_io_pct 0.1 + +set clk_port [get_ports $clk_port_name] + +create_clock -name $clk_name -period $clk_period $clk_port + +set non_clock_inputs [lsearch -inline -all -not -exact [all_inputs] $clk_port] + +set_input_delay [expr $clk_period * $clk_io_pct] -clock $clk_name $non_clock_inputs +set_output_delay [expr $clk_period * $clk_io_pct] -clock $clk_name [all_outputs] +``` + +## Run OpenROAD Flow + +[OpenROAD Flow](https://github.com/The-OpenROAD-Project/OpenROAD-flow-scripts) is a full RTL-to-GDS flow built entirely on open-source tools. The project aims for automated, no-human-in-the-loop digital circuit design with 24-hour turnaround time. + +```python jupyter={"outputs_hidden": true} tags=[] +import pathlib +import os + +work_dir = pathlib.Path(run_path) +work_dir.mkdir(parents=True, exist_ok=True) +work_dir = str(work_dir.resolve()) +pwd = os.getcwd() + +!make -C /OpenROAD-flow-scripts/flow \ + SHELL=/bin/bash \ + FLOW_HOME=/OpenROAD-flow-scripts/flow \ + PLATFORM_DIR=/OpenROAD-flow-scripts/flow/platforms/asap7 \ + DESIGN_CONFIG={pwd}/config.mk \ + DIE_WIDTH={die_width} \ + CORE_PADDING={core_padding} \ + CORE_WIDTH={float(die_width) - float(core_padding)} \ + TARGET_DENSITY={target_density} \ + WORK_HOME={work_dir} +``` + + +## Dump flow metrics + + +```python tags=[] +import pathlib + +flow_path = pathlib.Path(run_path).resolve() +!PLATFORM_DIR=/OpenROAD-flow-scripts/flow/platforms python /OpenROAD-flow-scripts/flow/util/genMetrics.py --flowPath {flow_path} --design crc32 --platform asap7 --output {flow_path}/metrics.json + +import json +import pandas as pd +from IPython.display import display +import scrapbook as sb + +pd.set_option('display.max_rows', None) +metrics = pathlib.Path(run_path) / 'metrics.json' +with metrics.open() as f: + data = json.load(f) + df = pd.DataFrame.from_records([data]) +sb.glue('metrics', df, 'pandas') +df.transpose().rename(columns={0: 'metrics'}) +``` + +## Display layout with KLayout + +```python +%%writefile /OpenROAD-flow-scripts/flow/util/gallery.json +[ + { + "name" : "final", + "layout_file": "6_final.def", + "min_hierarchy": 0, + "max_hierarchy": 1, + "x_resolution": 500, + "y_resolution": 500, + "hide_layers": false + }, + { + "name" : "final_no_power", + "layout_file": "6_final_no_power.def", + "min_hierarchy": 0, + "max_hierarchy": 1, + "x_resolution": 500, + "y_resolution": 500, + "hide_layers": false + } +] +``` + +```python +!make -C /OpenROAD-flow-scripts/flow/ \ + SHELL=/bin/bash \ + FLOW_HOME=/OpenROAD-flow-scripts/flow \ + PLATFORM_DIR=/OpenROAD-flow-scripts/flow/platforms/asap7 \ + DESIGN_CONFIG={pwd}/config.mk \ + WORK_HOME={work_dir} \ + gallery + +from IPython.display import Image + +gallery = pathlib.Path(run_path) / 'results/asap7/crc32/base/gallery_final_no_power.png' +sb.glue('layout', Image(gallery), 'display', display=True) +``` + +## Extract metrics + +Build a pandas dataframe with ppa. + +```python tags=[] +#papermill_description=ExtractingMetrics +import re +import scrapbook as sb +re_critical_path_delay = r'''critical path delay +-------------------------------------------------------------------------- +(\S+) +''' +re_critical_path_slack = r'''finish critical path slack +-------------------------------------------------------------------------- +(\S+) +''' +re_total_power = r'''^Total.*%$''' +re_design_area = r'''finish report_design_area +-------------------------------------------------------------------------- +Design area (\S+) u\^2 (\S+)% utilization.''' +def runs_ppa(): + for r in pathlib.Path(runs_dir).glob('*/logs/asap7/crc32/base/6_report.log'): + with r.open() as f: + report = f.read() + critical_path_delay = float(re.search(re_critical_path_delay, report).group(1)) + critical_path_slack = float(re.search(re_critical_path_slack, report).group(1)) + m = re.search(re_total_power, report, re.MULTILINE) + total_power = float(re.split('\s+', m.group())[-2]) + m = re.search(re_design_area, report, re.MULTILINE) + area = int(m.group(1)) + utilization = float(m.group(2)) / 100.0 + yield r.parts[1], total_power, critical_path_delay, critical_path_slack, area, utilization + +df = pd.DataFrame.from_records(runs_ppa(), columns=['run', 'power', 'delay', 'slack', 'area', 'utilization'], index='run').sort_index() +sb.glue('ppa', df, 'pandas') +(df.style + .format({'area': '{:.8f}', 'utilization': '{:.2%}', 'power': '{:.6f}', 'slack': '{:.6f}', 'delay': '{:.6f}'}) + .bar(subset=['power'], color='pink') + .bar(subset=['slack'], color='lime') + .background_gradient(subset=['utilization'], cmap='Greens') + .bar(subset=['area'], color='lightblue')) +``` + +Report metrics for hyper-parameters tuning. + +```python +#papermill_description=ReportingMetrics +import hypertune + +slack = df['slack'][0] +print('reporting metric:', 'slack', slack) +hpt = hypertune.HyperTune() +hpt.report_hyperparameter_tuning_metric( + hyperparameter_metric_tag='slack', + metric_value=slack, +) +``` diff --git a/modules/silicon_design/scripts/build/notebooks/xls/xls-crc32-sky130.md b/modules/silicon_design/scripts/build/notebooks/xls/xls-crc32-sky130.md new file mode 100644 index 00000000..312bf4e4 --- /dev/null +++ b/modules/silicon_design/scripts/build/notebooks/xls/xls-crc32-sky130.md @@ -0,0 +1,195 @@ +--- +jupyter: + jupytext: + text_representation: + extension: .md + format_name: markdown + format_version: '1.3' + jupytext_version: 1.13.8 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + + +# XLS Sample + +``` +Copyright 2022 Google LLC. +SPDX-License-Identifier: Apache-2.0 +``` + +This notebook shows how to run a [XLS](https://google.github.io/xls/)-based CRC checksum calculator design thru an end-to-end RTL to GDSII flow targetting the [SKY130](https://github.com/google/skywater-pdk/) process node. + + + +## Define flow parameters + + +```python tags=["parameters"] +die_width = 100 +target_density = 80 +run_path = 'runs/crc32' +``` + + +## Write and test DSLX module + +The CRC computation is written using the [DSLX](https://google.github.io/xls/dslx_reference/) HLS, a domain specific, dataflow-oriented functional language used to build hardware w/ a Rust-like syntax. + + +```bash colab={"base_uri": "https://localhost:8080/"} id="JKGxScUtoV4E" outputId="b9359a05-fa7f-4366-ecf8-40138acb11f1" magic_args="-c 'cat > crc32.x; interpreter_main crc32.x'" +// Performs a table-less crc32 of the input data as in Hacker's Delight: +// https://github.com/hcs0/Hackers-Delight/blob/master/crc.c.txt (roughly flavor b) + +fn crc32_one_byte(byte: u8, polynomial: u32, crc: u32) -> u32 { + let crc = crc ^ (byte as u32); + // 8 rounds of updates. + for (i, crc): (u32, u32) in range(u32:0, u32:8) { + let mask = -(crc & u32:1); + (crc >> u32:1) ^ (polynomial & mask) + }(crc) +} + +fn main(message: u8) -> u32 { + crc32_one_byte(message, u32:0xEDB88320, u32:-1) ^ u32:-1 +} + +#![test] +fn crc32_one_char() { + assert_eq(u32:0x83DCEFB7, main('1')) +} +``` + + +## Generate IR and Verilog + +XLS can generate combinational or pipelined version of a given design. + + +```python colab={"base_uri": "https://localhost:8080/"} id="YMTh7WB6oxeW" outputId="a4e9d2f2-69e3-47e9-cad6-e1b89124553b" +!ir_converter_main --top main crc32.x > crc32.ir +!opt_main crc32.ir > crc32_opt.ir +!codegen_main --generator=combinational crc32_opt.ir > crc32.v +!cat crc32.v +``` + +## Write flow configuration + +See [OpenLane Variables information](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/configuration/README.md) for the list of available variables. + +```python id="rBk7BdF0n_o5" +%%writefile config.tcl +set ::env(DESIGN_NAME) __crc32__main + +set script_dir [file dirname [file normalize [info script]]] +set ::env(VERILOG_FILES) "$script_dir/crc32.v" + +set ::env(CLOCK_TREE_SYNTH) 0 +set ::env(CLOCK_PORT) "" + +set ::env(FP_SIZING) "absolute" +set ::env(DIE_AREA) "0 0 $::env(DIE_WIDTH) $::env(DIE_WIDTH)" +set ::env(PL_TARGET_DENSITY) [expr {$::env(TARGET_DENSITY) / 100.0}] + +# TODO(proppy) find out why LVS fails +set ::env(RUN_LVS) 0 +``` + +## Run OpenLane flow + +[OpenLane](https://github.com/The-OpenROAD-Project/OpenLane) is an automated RTL to GDSII flow based on several components including [OpenROAD](https://github.com/The-OpenROAD-Project/OpenROAD), [Yosys](https://github.com/YosysHQ/yosys), Magic, Netgen, Fault, CVC, SPEF-Extractor, CU-GR, Klayout and a number of custom scripts for design exploration and optimization. + +```python colab={"base_uri": "https://localhost:8080/"} id="8gim7pEdozHv" outputId="3d4cccd8-bda2-4380-a1c3-5c9002560b7b" tags=[] +#papermill_description=RunningOpenLaneFlow +%env DIE_WIDTH={die_width} +%env TARGET_DENSITY={target_density} +!flow.tcl -design . -run_path {run_path} +``` + +## Display layout + +Use [GDSII Tool Kit](https://github.com/heitzmann/gdstk) and [CairoSVG](https://cairosvg.org/) to convert the resulting GDSII file to PNG. + +```python colab={"base_uri": "https://localhost:8080/", "height": 1000} id="1uSEdmRhtXdl" outputId="6830cf44-e85f-48fc-aa84-84f794c25dc8" +#papermill_description=RenderingGDS +import pathlib +import gdstk +import cairosvg + +import IPython.display +import scrapbook as sb + +gds_path = sorted(pathlib.Path(run_path).glob('*/results/final/gds/*.gds'))[-1] +library = gdstk.read_gds(gds_path) +top_cells = library.top_level() +svg_path = pathlib.Path(run_path) / 'xls.svg' +top_cells[0].write_svg(svg_path) +png_path = pathlib.Path(run_path) / 'xls.png' + +cairosvg.svg2png(url=str(svg_path), write_to=str(png_path)) +sb.glue('layout', IPython.display.Image(png_path), 'display', display=True) +``` + +## Dump flow report + +See [OpenLane Datapoint Definitions](https://github.com/The-OpenROAD-Project/OpenLane/blob/master/regression_results/datapoint_definitions.md) for the description of the report columns. + +```python tags=[] +#papermill_description=DumpingReport +import pandas as pd +import pathlib +import scrapbook as sb + +final_summary_report = sorted(pathlib.Path(run_path).glob('*/reports/final_summary_report.csv'))[-1] +df = pd.read_csv(final_summary_report) +pd.set_option('display.max_rows', None) +sb.glue('summary', df, 'pandas') +df.transpose() +``` + +## Extract power metrics + +Build a pandas dataframe with area, density and power consumption. + +```python tags=[] +#papermill_description=ExtractingMetrics +import scrapbook as sb + +def get_power(sta_power_report): + with sta_power_report.open() as f: + for l in f.readlines(): + if l.startswith('Total'): + return float(l.split(' ')[-2]) + +def area_density_ppa(): + for report in sorted(pathlib.Path(run_path).glob('*/reports')): + sta_power_report = report / 'routing/23-parasitics_sta.power.rpt' + final_summary_report = report / 'final_summary_report.csv' + if final_summary_report.exists() and sta_power_report.exists(): + df = pd.read_csv(final_summary_report) + power = get_power(sta_power_report) + yield (df['DIEAREA_mm^2'][0], df['PL_TARGET_DENSITY'][0], power) + +df = pd.DataFrame(area_density_ppa(), columns=('DIEAREA_mm^2', 'PL_TARGET_DENSITY', 'TOTAL_POWER')) +sb.glue('metrics', df, 'pandas') +(df.style.hide_index() + .format({'area': '{:.8f}', 'density': '{:.2%}', 'power': '{:.8f}'}) + .bar(subset=['TOTAL_POWER'], color='pink') + .background_gradient(subset=['PL_TARGET_DENSITY'], cmap='Greens') + .bar(color='lightblue', vmin=0.001, subset=['DIEAREA_mm^2'])) +``` + +```python +#papermill_description=ReportingMetrics +import hypertune + +total_power = df['TOTAL_POWER'][0] * 1e6 +print('reporting metric:', 'total_power', total_power) +hpt = hypertune.HyperTune() +hpt.report_hyperparameter_tuning_metric( + hyperparameter_metric_tag='total_power', + metric_value=total_power, +) +``` diff --git a/modules/silicon_design/scripts/usage/ai-notebook-desktop-script.sh b/modules/silicon_design/scripts/usage/ai-notebook-desktop-script.sh index 2726cc6b..59299edf 100755 --- a/modules/silicon_design/scripts/usage/ai-notebook-desktop-script.sh +++ b/modules/silicon_design/scripts/usage/ai-notebook-desktop-script.sh @@ -1,5 +1,5 @@ #!/bin/bash - +# # Copyright 2022 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/modules/silicon_design/variables.tf b/modules/silicon_design/variables.tf index 26c0bed2..900f991d 100644 --- a/modules/silicon_design/variables.tf +++ b/modules/silicon_design/variables.tf @@ -14,6 +14,11 @@ * limitations under the License. */ +variable "name" { + description = "Name prefix used for naming child radlab resources" + type = string +} + variable "billing_account_id" { description = "Billing Account associated to the GCP Resources" type = string @@ -70,7 +75,7 @@ variable "machine_type" { variable "network_name" { description = "Name of the network to be created." type = string - default = "ai-notebook" + default = null } variable "notebook_count" { @@ -96,6 +101,7 @@ variable "project_name" { type = string default = "radlab-silicon-design" } + variable "random_id" { description = "Adds a suffix of 4 random characters to the `project_id`" type = string @@ -123,7 +129,7 @@ variable "set_trustedimage_project_policy" { variable "subnet_name" { description = "Name of the subnet where to deploy the Notebooks." type = string - default = "subnet-ai-notebook" + default = null } variable "trusted_users" { @@ -137,3 +143,15 @@ variable "zone" { type = string default = "us-east4-c" } + +variable "image_name" { + description = "Basename for the compute and container image." + type = string + default = "silicon-design-ubuntu-2004" +} + +variable "image_tag" { + description = "Tag for the compute and container image." + type = string + default = "" +} diff --git a/modules/silicon_design/versions.tf b/modules/silicon_design/versions.tf index 5a19b143..3de89291 100644 --- a/modules/silicon_design/versions.tf +++ b/modules/silicon_design/versions.tf @@ -18,7 +18,7 @@ terraform { required_version = "~> 1.0" required_providers { - google = ">= 3.87.0" - google-beta = ">= 3.87.0" + google = ">= 4.22.0" + google-beta = ">= 4.22.0" } }