fix(m365): Correct issues when specifying input_storage_container_url

m365/README.adoc

@@ -97,16 +97,21 @@ Optional::
 Advanced::
 `create_app` (bool) [default=True]::: If true, the app will be created. If false, the app will be imported
 `prefix_override` (string) [default=None]::: Prefix for resource names. If null, one will be generated from app_name
-`input_storage_container_url` (string) [default=None]::: If not null, input container to read configs from (must give permissions to service account). Otherwise by default will create storage container. Expect an https url pointing to a container
-`output_storage_container_url` (string) [default=None]::: If not null, output container to put results in (must give permissions to service account or use SAS). Otherwise by default will create storage container. Expect an https url pointing to a container
+`input_storage_container_url` (string) [default=None]::: If not null, input container to read configs from (must assign blob reader role role to service account `sp_object_id` manually).
+Otherwise by default will create storage container. 
+Expect a container URL like: https://<account>.blob.core.windows.net/<container>
+Note that the container must have "adhoc" and "scheduled" directories. These are not created automatically in this case
+`output_storage_container_url` (string) [default=None]::: If not null, output container to put results in (must give permissions to service account `sp_object_id` or use SAS).
+Otherwise by default will create storage container.
+Expect a container URL like: https://<account>.blob.core.windows.net/<container>
 `output_storage_container_sas` (string) [default=None]::: If not null, shared access signature token (query string) to use when writing results to the output storage container. Set this when the container is in an external tenant (the owner of that container will provide the value).
 `tenants_dir_path` (string) [default=./tenants]::: Relative path to directory containing tenant configuration files in yaml
 `container_registry` (object) [default=None]::: Credentials for logging into registry with container image
 `container_image` (string) [default=ghcr.io/cisagov/scubaconnect-m365:latest]::: Docker image to use for running ScubaGear.
 `container_memory_gb` (number) [default=3]::: Amount of memory to allocate for ScubaGear container. Due to memory leaks in some dependencies, this may need to be increased if running on many tenants
 `secondary_app_info` (object) [default=None]::: Information for a secondary app. This can be used for one ScubaConnect instance to handle multiple environments (e.g., GCC and GCC High).
-    To use, manually create an app in the other environment and add the certificate created for the primary app to it.
-    Set `environment_to_use` to the environment the manual app is in, either "commericial" or "gcchigh"
+To use, manually create an app in the other environment and add the certificate created for the primary app to it.
+Set `environment_to_use` to the environment the manual app is in, either "commericial" or "gcchigh"
 
 [#onboard]
 === Onboarding a Tenant

m365/terraform/env/example/outputs.tf

@@ -10,7 +10,7 @@ output "output_storage_container_url" {
 
 output "input_storage_container_url" {
   description = "URL of the input storage account configs are read from"
-  value       = module.scuba_connect.output_storage_container_url
+  value       = module.scuba_connect.input_storage_container_url
 }
 
 output "sp_object_id" {

m365/terraform/env/example/variables.tf

@@ -102,13 +102,22 @@ variable "prefix_override" {
 variable "input_storage_container_url" {
   default     = null
   type        = string
-  description = "If not null, input container to read configs from (must give permissions to service account). Otherwise by default will create storage container. Expect an https url pointing to a container"
+  description = <<-EOT
+    If not null, input container to read configs from (must assign blob reader role role to service account `sp_object_id` manually).
+    Otherwise by default will create storage container. 
+    Expect a container URL like: https://<account>.blob.core.windows.net/<container>
+    Note that the container must have "adhoc" and "scheduled" directories. These are not created automatically in this case
+  EOT
 }
 
 variable "output_storage_container_url" {
   default     = null
   type        = string
-  description = "If not null, output container to put results in (must give permissions to service account or use SAS). Otherwise by default will create storage container. Expect an https url pointing to a container"
+  description = <<-EOT
+    If not null, output container to put results in (must give permissions to service account `sp_object_id` or use SAS).
+    Otherwise by default will create storage container.
+    Expect a container URL like: https://<account>.blob.core.windows.net/<container>
+  EOT
 }
 
 variable "output_storage_container_sas" {
@@ -151,11 +160,11 @@ variable "container_memory_gb" {
 }
 
 variable "secondary_app_info" {
-  description = <<EOF
+  description = <<-EOT
     Information for a secondary app. This can be used for one ScubaConnect instance to handle multiple environments (e.g., GCC and GCC High).
     To use, manually create an app in the other environment and add the certificate created for the primary app to it.
     Set `environment_to_use` to the environment the manual app is in, either "commericial" or "gcchigh"
-  EOF
+  EOT
   type = object({
     app_id             = string
     environment_to_use = string
@@ -165,4 +174,4 @@ variable "secondary_app_info" {
     condition     = var.secondary_app_info == null ? true : contains(["commercial", "gcchigh"], var.secondary_app_info.environment_to_use)
     error_message = "Valid values for create_mode are (Default, PointInTimeRestore, Replica)"
   }
-}
+}

m365/terraform/modules/container/storage.tf

@@ -61,7 +61,7 @@ resource "azurerm_storage_container" "input" {
 }
 
 resource "azurerm_storage_blob" "keep_files" {
-  for_each               = local.container_types
+  for_each               = toset([for l in local.container_types : l if var.input_storage_container_url == null])
   name                   = "${each.key}/.keep"
   storage_account_name   = azurerm_storage_account.storage[0].name
   storage_container_name = azurerm_storage_container.input[0].name
@@ -78,7 +78,7 @@ resource "azurerm_storage_blob" "keep_files" {
 
 # Blobs containing configuration for each tenant
 resource "azurerm_storage_blob" "tenants" {
-  for_each               = { for typeFile in setproduct(local.container_types, fileset(var.tenants_dir_path, "*")): "${typeFile[0]}/${typeFile[1]}" => typeFile[1] }
+  for_each               = { for typeFile in setproduct(local.container_types, fileset(var.tenants_dir_path, "*")) : "${typeFile[0]}/${typeFile[1]}" => typeFile[1] if var.input_storage_container_url == null }
   name                   = each.key
   storage_account_name   = azurerm_storage_account.storage[0].name
   storage_container_name = azurerm_storage_container.input[0].name
@@ -95,4 +95,4 @@ resource "azurerm_storage_blob" "tenants" {
 locals {
   input_storage_container_url  = var.input_storage_container_url == null ? "${azurerm_storage_account.storage[0].primary_blob_endpoint}${azurerm_storage_container.input[0].name}" : var.input_storage_container_url
   output_storage_container_url = var.output_storage_container_url == null ? "${azurerm_storage_account.storage[0].primary_blob_endpoint}${azurerm_storage_container.output[0].name}" : var.output_storage_container_url
-}
+}

m365/terraform/modules/container/variables.tf

@@ -25,13 +25,22 @@ variable "schedule_interval" {
 variable "input_storage_container_url" {
   default     = null
   type        = string
-  description = "If not null, input container to read configs from (must give permissions to service account). Otherwise by default will create storage container. Expect an https url pointing to a container"
+  description = <<-EOT
+    If not null, input container to read configs from (must assign blob reader role role to service account `sp_object_id` manually).
+    Otherwise by default will create storage container. 
+    Expect a container URL like: https://<account>.blob.core.windows.net/<container>
+    Note that the container must have "adhoc" and "scheduled" directories. These are not created automatically in this case
+  EOT
 }
 
 variable "output_storage_container_url" {
   default     = null
   type        = string
-  description = "If not null, output container to put results in (must give permissions to service account or use SAS). Otherwise by default will create storage container. Expect an https url pointing to a container"
+  description = <<-EOT
+    If not null, output container to put results in (must give permissions to service account `sp_object_id` or use SAS).
+    Otherwise by default will create storage container.
+    Expect a container URL like: https://<account>.blob.core.windows.net/<container>
+  EOT
 }
 
 variable "output_storage_container_sas" {
@@ -123,11 +132,11 @@ variable "cert_info" {
 }
 
 variable "secondary_app_info" {
-  description = <<EOF
+  description = <<-EOT
     Information for a secondary app. This can be used for one ScubaConnect instance to handle multiple environments (e.g., GCC and GCC High).
     To use, manually create an app in the other environment and add the certificate created for the primary app to it.
     Set `environment_to_use` to the environment the manual app is in, either "commericial" or "gcchigh"
-  EOF
+  EOT
   type = object({
     app_id             = string
     environment_to_use = string

m365/terraform/outputs.tf

@@ -10,7 +10,7 @@ output "output_storage_container_url" {
 
 output "input_storage_container_url" {
   description = "URL of the input storage account configs are read from"
-  value       = module.container.output_storage_container_url
+  value       = module.container.input_storage_container_url
 }
 
 output "sp_object_id" {

m365/terraform/variables.tf

@@ -102,13 +102,22 @@ variable "prefix_override" {
 variable "input_storage_container_url" {
   default     = null
   type        = string
-  description = "If not null, input container to read configs from (must give permissions to service account). Otherwise by default will create storage container. Expect an https url pointing to a container"
+  description = <<-EOT
+    If not null, input container to read configs from (must assign blob reader role role to service account `sp_object_id` manually).
+    Otherwise by default will create storage container. 
+    Expect a container URL like: https://<account>.blob.core.windows.net/<container>
+    Note that the container must have "adhoc" and "scheduled" directories. These are not created automatically in this case
+  EOT
 }
 
 variable "output_storage_container_url" {
   default     = null
   type        = string
-  description = "If not null, output container to put results in (must give permissions to service account or use SAS). Otherwise by default will create storage container. Expect an https url pointing to a container"
+  description = <<-EOT
+    If not null, output container to put results in (must give permissions to service account `sp_object_id` or use SAS).
+    Otherwise by default will create storage container.
+    Expect a container URL like: https://<account>.blob.core.windows.net/<container>
+  EOT
 }
 
 variable "output_storage_container_sas" {
@@ -151,11 +160,11 @@ variable "container_memory_gb" {
 }
 
 variable "secondary_app_info" {
-  description = <<EOF
+  description = <<-EOT
     Information for a secondary app. This can be used for one ScubaConnect instance to handle multiple environments (e.g., GCC and GCC High).
     To use, manually create an app in the other environment and add the certificate created for the primary app to it.
     Set `environment_to_use` to the environment the manual app is in, either "commericial" or "gcchigh"
-  EOF
+  EOT
   type = object({
     app_id             = string
     environment_to_use = string
@@ -165,4 +174,4 @@ variable "secondary_app_info" {
     condition     = var.secondary_app_info == null ? true : contains(["commercial", "gcchigh"], var.secondary_app_info.environment_to_use)
     error_message = "Valid values for create_mode are (Default, PointInTimeRestore, Replica)"
   }
-}
+}

utils/tf_vars_to_adoc.py

@@ -1,15 +1,18 @@
-import hcl2
+import hcl2  # written for version 8+
+from hcl2 import SerializationOptions
 import argparse
 import re
+import codecs
 
+hcl_opts = SerializationOptions(strip_string_quotes=True, preserve_heredocs=False)
 COMMENT_HEADER_REGEX = r'###\s*(.*?)\s*#*\s*variable\s*"(.*)"'
 
 parser = argparse.ArgumentParser("Converts a variables.tf file to an asciidoc description list. Treats comment blocks starting with ### as section headers")
 parser.add_argument("variables_tf")
 args = parser.parse_args()
 
 with open(args.variables_tf, 'r') as f:
-    d = hcl2.load(f)
+    d = hcl2.load(f, serialization_options=hcl_opts)
     f.seek(0)
     text = '\n'.join(f.readlines())
     category_matches = re.findall(COMMENT_HEADER_REGEX, text, flags=re.MULTILINE)
@@ -23,5 +26,5 @@
     if t.startswith("object"):
         t = "object"
     default = f"[default={v[name]['default']}]" if "default" in v[name] else ""
-    desc = v[name]["description"]
-    print(f"`{name}` ({t}) {default}::: {desc.strip()}")
+    desc = codecs.decode(v[name]["description"], 'unicode_escape')
+    print(f"`{name}` ({t}) {default}::: {desc}")