Skip to content

Adding job labels for FinOps #673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JulianUmbhau wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Adding job labels for FinOps #673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2c57e1a
adding labels to calls, check_labels function, default NULL for label…
JulianUmbhau Feb 23, 2026
ad400e8
changing from list to named vector
JulianUmbhau Mar 2, 2026
d621e08
Corrections based on PR comments. Returned to use list, cleaned up ch…
JulianUmbhau Mar 6, 2026
9ffeefa
Merge branch 'r-dbi:main' into job_labels
JulianUmbhau Mar 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 49 additions & 8 deletions R/bq-perform.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ bq_perform_extract <- function(
check_bool(print_header)
check_string(billing)

labels <- check_labels(getOption("bigrquery.labels"))
Copy link
Member

@hadley hadley Mar 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As a general principle, I don't think it's good idea to allow global options to change what a computation does, only how it looks. I think it's probably ok in this case because I think you can frame this as a sort of logging operation, but it should appear in function arguments too.

I would also wonder if it would instead make sense to make it a field in BigQueryConnection. Would that also solve the problem for you or are you usually calling these low-level functions directly?

Copy link
Author

@JulianUmbhau JulianUmbhau Mar 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand.
I do use the low-level functions directly currently, but it is also a good point with a field in BigQueryConnection.

Would it make sense to both:

  1. Add labels field to BigQueryConnection
  2. Add labels as arguments to each bq_perform_... function? - To set the labels explicitly

Copy link
Member

@hadley hadley Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think so.

Copy link
Member

@hadley hadley Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm also happy to do that myself, if you don't have the time.


url <- bq_path(billing, jobs = "")
body <- list(
configuration = list(
Expand All @@ -81,7 +83,8 @@ bq_perform_extract <- function(
destinationFormat = unbox(destination_format),
compression = unbox(compression),
printHeader = unbox(print_header)
)
),
labels = labels
)
)

Expand Down Expand Up @@ -140,6 +143,8 @@ bq_perform_upload <- function(
check_string(billing)
json_digits <- check_digits(json_digits)

labels <- check_labels(getOption("bigrquery.labels"))

load <- list(
sourceFormat = unbox(source_format),
destinationTable = tableReference(x),
Expand All @@ -154,11 +159,16 @@ bq_perform_upload <- function(
load$autodetect <- unbox(TRUE)
}

metadata <- list(configuration = list(load = load))
metadata <- list(
configuration = list(
load = load,
labels = labels
)
)
metadata <- bq_body(metadata, ...)
metadata <- list(
"type" = "application/json; charset=UTF-8",
"content" = jsonlite::toJSON(metadata, pretty = TRUE, digits = json_digits)
"content" = jsonlite::toJSON(metadata, auto_unbox = TRUE, pretty = TRUE, digits = json_digits)
)

if (source_format == "NEWLINE_DELIMITED_JSON") {
Expand Down Expand Up @@ -261,6 +271,8 @@ bq_perform_load <- function(
check_string(create_disposition)
check_string(write_disposition)

labels <- check_labels(getOption("bigrquery.labels"))

load <- list(
sourceUris = as.list(source_uris),
sourceFormat = unbox(source_format),
Expand All @@ -280,7 +292,12 @@ bq_perform_load <- function(
load$autodetect <- TRUE
}

body <- list(configuration = list(load = load))
body <- list(
configuration = list(
load = load,
labels = labels
)
)

url <- bq_path(billing, jobs = "")
res <- bq_post(
Expand Down Expand Up @@ -332,6 +349,8 @@ bq_perform_query <- function(
check_bool(use_legacy_sql)
check_string(priority)

labels <- check_labels(getOption("bigrquery.labels"))

query <- list(
query = unbox(query),
useLegacySql = unbox(use_legacy_sql),
Expand All @@ -357,7 +376,12 @@ bq_perform_query <- function(
}

url <- bq_path(billing, jobs = "")
body <- list(configuration = list(query = query))
body <- list(
configuration = list(
query = query,
labels = labels
)
)

res <- bq_post(
url,
Expand All @@ -383,9 +407,16 @@ bq_perform_query_dry_run <- function(
parameters = parameters,
use_legacy_sql = use_legacy_sql
)
labels <- check_labels(getOption("bigrquery.labels"))

url <- bq_path(billing, jobs = "")
body <- list(configuration = list(query = query, dryRun = unbox(TRUE)))
body <- list(
configuration = list(
query = query,
labels = labels,
dryRun = unbox(TRUE)
)
)

res <- bq_post(
url,
Expand All @@ -412,8 +443,16 @@ bq_perform_query_schema <- function(
use_legacy_sql = FALSE
)

labels <- check_labels(getOption("bigrquery.labels"))

url <- bq_path(billing, jobs = "")
body <- list(configuration = list(query = query, dryRun = unbox(TRUE)))
body <- list(
configuration = list(
query = query,
labels = labels,
dryRun = unbox(TRUE)
)
)

res <- bq_post(
url,
Expand Down Expand Up @@ -463,6 +502,7 @@ bq_perform_copy <- function(
) {
billing <- billing %||% dest$project
url <- bq_path(billing, jobs = "")
labels <- check_labels(getOption("bigrquery.labels"))

body <- list(
configuration = list(
Expand All @@ -471,7 +511,8 @@ bq_perform_copy <- function(
destinationTable = tableReference(dest),
createDisposition = unbox(create_disposition),
writeDisposition = unbox(write_disposition)
)
),
labels = labels
)
)

Expand Down
12 changes: 12 additions & 0 deletions R/utils.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,3 +85,15 @@ as_query <- function(x, error_arg = caller_arg(x), error_call = caller_env()) {
has_bigrquerystorage <- function() {
is_installed("bigrquerystorage")
}

check_labels <- function(labels) {
if (is.null(labels) || length(labels) == 0) {
return(NULL)
}

if (!is.list(labels) || any(names2(labels) == "")) {
cli::cli_abort("Labels must be a named list.")
}

labels
}
3 changes: 2 additions & 1 deletion R/zzz.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@
op <- options()
defaults <- list(
bigrquery.quiet = NA,
bigrquery.page.size = 1e4
bigrquery.page.size = 1e4,
bigrquery.labels = NULL
)
toset <- !(names(defaults) %in% names(op))
if (any(toset)) {
Expand Down
14 changes: 14 additions & 0 deletions tests/testthat/test-utils.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,17 @@ test_that("bq_check_namespace() works", {
error = TRUE
)
})

test_that("check_labels() accepts valid labels and NULL-like inputs", {
expect_null(check_labels(NULL))
expect_null(check_labels(list()))

expect_equal(check_labels(list(env = "prod")), list(env = "prod"))
expect_equal(check_labels(list(env = "prod", team = "data")), list(env = "prod", team = "data"))
expect_equal(check_labels(list(env = "")), list(env = ""))
})

test_that("check_labels() errors on invalid inputs", {
expect_error(check_labels(c(env = "prod")), "named list")
expect_error(check_labels(list("no-name")), "named list")
})
Loading