diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index f69fe11..d3d1d9a 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -32,6 +32,15 @@ jobs: r-version: '4.4.0' use-public-rspm: true + - uses: r-lib/actions/setup-tinytex@v2 + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} + + - name: Install PDF manual dependencies + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} + run: | + tlmgr update --self + tlmgr install makeindex + - uses: r-lib/actions/setup-r-dependencies@v2 with: extra-packages: any::rcmdcheck, any::lintr @@ -68,6 +77,22 @@ jobs: sed -i "/^Version:/c\Version: ${{ steps.get_version.outputs.VERSION }}" imotionsApi/DESCRIPTION shell: bash + - name: Generate PDF manual + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} + run: | + R CMD Rd2pdf . --output=../documentation/imotionsApi_${{ steps.get_version.outputs.VERSION }}.pdf --force --no-preview + shell: bash + working-directory: './imotionsApi' + + - name: Upload PDF manual for documentation branch + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} + uses: actions/upload-artifact@v4 + with: + name: release-documentation-${{ github.ref_name }} + path: ./documentation/imotionsApi_${{ steps.get_version.outputs.VERSION }}.pdf + if-no-files-found: error + retention-days: 1 + - uses: r-lib/actions/check-r-package@v2 with: error-on: '"error"' @@ -93,9 +118,78 @@ jobs: files: | LICENSE ./imotionsApi/check/*.tar.gz + ./documentation/imotionsApi_${{ steps.get_version.outputs.VERSION }}.pdf - name: Publish test results uses: EnricoMi/publish-unit-test-result-action/windows@v2 if: always() with: - files: './imotionsApi/check/imotionsApi.Rcheck/tests/test-results-tests.xml' \ No newline at end of file + files: './imotionsApi/check/imotionsApi.Rcheck/tests/test-results-tests.xml' + + update-tagged-branch: + needs: R-test-lint-build-release + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} + runs-on: windows-latest + concurrency: + group: tagged-branch-documentation + cancel-in-progress: false + + steps: + - name: Find branch tagged at this commit + id: target_branch + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + $headers = @{ + Accept = 'application/vnd.github+json' + Authorization = "Bearer $env:GH_TOKEN" + 'X-GitHub-Api-Version' = '2022-11-28' + } + $uri = "$env:GITHUB_API_URL/repos/$env:GITHUB_REPOSITORY/commits/$env:GITHUB_SHA/branches-where-head" + $response = Invoke-RestMethod -Uri $uri -Headers $headers + $defaultBranch = '${{ github.event.repository.default_branch }}' + $branches = @($response | Where-Object { $_.name -ne $defaultBranch } | ForEach-Object { $_.name }) + if ($branches.Count -eq 0) { + throw "No non-default branch has tagged commit $env:GITHUB_SHA as its head." + } + if ($branches.Count -gt 1) { + throw "Multiple non-default branches have tagged commit $env:GITHUB_SHA as their head: $($branches -join ', ')" + } + "branch=$($branches[0])" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append + + - uses: actions/checkout@v4 + with: + ref: ${{ steps.target_branch.outputs.branch }} + fetch-depth: 0 + + - name: Download PDF manual + uses: actions/download-artifact@v4 + with: + name: release-documentation-${{ github.ref_name }} + path: ./release-documentation + + - name: Prepare latest release documentation + shell: pwsh + run: | + $version = '${{ github.ref_name }}'.Substring(1) + Get-ChildItem -LiteralPath './documentation' -Filter 'imotionsApi_*.pdf' -File | Remove-Item -Force + Move-Item -LiteralPath "./release-documentation/imotionsApi_$version.pdf" -Destination './documentation/' + $description = Get-Content -Raw -LiteralPath './imotionsApi/DESCRIPTION' + $description = $description -replace '(?m)^Version:.*$', "Version: $version-0000000000" + Set-Content -NoNewline -LiteralPath './imotionsApi/DESCRIPTION' -Value $description + + - name: Commit release documentation to tagged branch + shell: pwsh + run: | + $branch = '${{ steps.target_branch.outputs.branch }}' + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add --all -- documentation imotionsApi/DESCRIPTION + git diff --cached --quiet + if ($LASTEXITCODE -eq 0) { + Write-Output 'Release documentation is already up to date.' + exit 0 + } + git commit -m "Update release documentation for ${{ github.ref_name }}" + git push origin "HEAD:$branch" diff --git a/documentation/imotionsApi_2.9.3.pdf b/documentation/imotionsApi_2.9.4.pdf similarity index 50% rename from documentation/imotionsApi_2.9.3.pdf rename to documentation/imotionsApi_2.9.4.pdf index c643f04..9525d08 100644 Binary files a/documentation/imotionsApi_2.9.3.pdf and b/documentation/imotionsApi_2.9.4.pdf differ diff --git a/imotionsApi/DESCRIPTION b/imotionsApi/DESCRIPTION index 288bd3e..f00f6cf 100644 --- a/imotionsApi/DESCRIPTION +++ b/imotionsApi/DESCRIPTION @@ -1,7 +1,7 @@ Package: imotionsApi Type: Package Title: iMotions R library -Version: 2.9.3-0000000000 +Version: 2.9.4-0000000000 Date: 2026-07-08 Authors@R: c( person("Amandine", "Grappe", role = "aut"), diff --git a/imotionsApi/NAMESPACE b/imotionsApi/NAMESPACE index c23e5b6..553dc10 100644 --- a/imotionsApi/NAMESPACE +++ b/imotionsApi/NAMESPACE @@ -4,8 +4,10 @@ S3method("[",imObject) S3method("[",imObjectList) S3method(getAoiDetailsUrl,imAOI) S3method(getAoiDetailsUrl,imStimulus) +S3method(getAoiMetricsUrl,imRespondent) S3method(getSensorsUrl,imRespondent) S3method(getSensorsUrl,imSegment) +S3method(getUploadAoiMetricsUrl,imRespondent) S3method(getUploadAoiMetricsUrl,imSegment) S3method(getUploadEventsUrl,imRespondent) S3method(getUploadMetricsUrl,imRespondent) @@ -59,7 +61,6 @@ importFrom(dplyr,mutate_at) importFrom(purrr,map_chr) importFrom(rlang,enquo) importFrom(tidyr,unnest) -importFrom(utils,download.file) importFrom(utils,tail) importFrom(utils,type.convert) importFrom(utils,unzip) diff --git a/imotionsApi/R/imotionsApi.R b/imotionsApi/R/imotionsApi.R index 2e0423c..eac828b 100644 --- a/imotionsApi/R/imotionsApi.R +++ b/imotionsApi/R/imotionsApi.R @@ -945,6 +945,7 @@ getSensors <- function(study, target, stimulus = NULL) { type <- tolower(gsub("^im|List", "", class(target)[1])) endpoint <- paste0(type, ": ", target$name) + if (!is.null(stimulus)) { endpoint <- paste0(endpoint, ", stimulus: ", stimulus$name) } @@ -963,12 +964,12 @@ getSensors <- function(study, target, stimulus = NULL) { if (study$connection$localIM == TRUE) { for (i in seq_along(sensors)) { suppressWarnings(signalsMetaData[[i]] <- rbindlist(sensors[[i]]$signalsMetaData, fill = TRUE)) - signals[[i]] <- sensors[[i]]$signals + signals[i] <- list(sensors[[i]]$signals) sensors[[i]][c("signals", "signalsMetaData")] <- NULL } } else { for (i in seq_along(sensors)) { - signals[[i]] <- sensors[[i]]$sampleDescription$signals + signals[i] <- list(sensors[[i]]$sampleDescription$signals) sensors[[i]][c("id", "respondent", "sampleDescription")] <- NULL sensors[[i]]$sensorSpecific <- list() } @@ -1398,11 +1399,11 @@ convertRecordingTsToIntervals <- function(recordingTs, intervals, keepTs = FALSE } # Converting timestamps to intervals fragments ranges - invisible(lapply(seq_len(nrow(intervals)), function(x) { + for (x in seq_len(nrow(intervals))) { isIn <- timestamps %inrange% intervals[x, c("fragments.start", "fragments.end")] - timestamps[isIn] <<- timestamps[isIn] - intervals[x, ]$fragments.start + + timestamps[isIn] <- timestamps[isIn] - intervals[x, ]$fragments.start + sum(intervals[0:(x - 1), ]$fragments.duration) - })) + } if (inherits(recordingTs, "data.frame")) { recordingTs$Timestamp <- timestamps @@ -1477,7 +1478,7 @@ getSensorData <- function(study, sensor, signalsName = NULL, intervals = NULL) { #' #' @importFrom arrow read_parquet set_cpu_count #' @importFrom rlang enquo -#' @importFrom utils download.file unzip +#' @importFrom utils unzip #' @return A data.table with all signals (or specified signals) from the sensor of interest. #' @keywords internal privateDownloadData <- function(study, sensor, signalsName = NULL) { @@ -1642,6 +1643,21 @@ getAoiRespondentData <- function(study, AOI, respondent) { } +#' Return the local cache path for AOI respondent metrics. +#' +#' @param study An imStudy object as returned from \code{\link{imStudy}}. +#' @param AOI An imAOI object as returned from \code{\link{getAois}}. +#' @param respondent An imRespondent object as returned from \code{\link{getRespondents}}. +#' +#' @keywords internal +privateGetAoiRespondentMetricsPath <- function(study, AOI, respondent) { + if (is.null(study$connection$localPath) || !length(AOI$fileId) || is.na(AOI$fileId)) return(NULL) + + return(file.path(study$connection$localPath, sub("^.*/aoiDefinition/", "", AOI$fileId), + paste0(respondent$id, "metrics.csv"))) +} + + #' Get the metrics for a specific AOI/respondent combination. #' #' @param study An imStudy object as returned from imStudy() @@ -1669,6 +1685,7 @@ getAoiRespondentMetrics <- function(study, AOI, respondent) { assertClass(respondent, "imRespondent", "`respondent` argument is not an imRespondent object") endpoint <- paste0("AOI: ", AOI$name, ", Respondent: ", respondent$name) + metricsFile <- NULL if (study$connection$localIM) { AOIDetails <- privateGetAoiDetails(study, AOI, respondent) @@ -1678,31 +1695,36 @@ getAoiRespondentMetrics <- function(study, AOI, respondent) { return(NULL) } - if (is.na(AOIDetails$resultId)) { - warning(paste0("No metrics found for ", endpoint)) - return(NULL) - } - - metrics <- fread(AOIDetails$resultId) + metricsFile <- AOIDetails$resultId } else { - # For online we load metrics written locally - if (is.null(study$connection$localPath)) { - warning("No localPath set when calling imConnection(), not possible to read metrics locally.") - return(NULL) + metricsFile <- privateGetAoiRespondentMetricsPath(study, AOI, respondent) + + # Retrieve the S3 location only when the metrics are not already cached locally. + if (is.null(metricsFile) || !file.exists(metricsFile)) { + metricsFile <- tryCatch({ + stats <- getJSON(study$connection, getAoiMetricsUrl(study, respondent, AOI), + message = paste("Retrieving AOI respondent metrics for", endpoint)) + + if (is.null(stats)) { + NULL + } else { + getFile(study$connection, stats$aoiRespondentStatsUrl, + message = paste("Downloading AOI respondent metrics for", endpoint), + localFilePath = metricsFile)$file_path + } + }, error = function(err) { + if (grepl("Resource not found", err$message, fixed = TRUE)) return(NULL) + stop(err) + }) } + } - tmp_dir <- study$connection$localPath - file_name <- file.path(tmp_dir, sub("^.*/aoiDefinition/", "", AOI$fileId), paste0(respondent$id, "metrics.csv")) - - if (file.exists(file_name)) { - metrics <- fread(file_name) - } else { - warning(paste0("No metrics found for ", endpoint)) - return(NULL) - } + if (is.null(metricsFile) || is.na(metricsFile)) { + warning(paste0("No metrics found for ", endpoint)) + return(NULL) } - metrics <- checkDataFormat(metrics) + metrics <- checkDataFormat(fread(metricsFile)) return(metrics) } @@ -1907,6 +1929,24 @@ uploadAoiMetrics <- function(study, AOI, target, metrics) { } +#' Get upload credentials and upload a file to the returned presigned URL. +#' +#' @param connection An imConnection object as returned from \code{\link{imConnection}}. +#' @param uploadUrl The URL used to request upload credentials. +#' @param postData The JSON body used to request upload credentials. +#' @param fileName The path of the file to upload. +#' @param credentialsMsg Message used if requesting credentials fails. +#' @param uploadMsg Message used if uploading the file fails. +#' +#' @return The upload credentials returned by the API. +#' @keywords internal +privateUploadToPresignedUrl <- function(connection, uploadUrl, postData, fileName, credentialsMsg, uploadMsg) { + fileInfos <- postJSON(connection, uploadUrl, postData, message = credentialsMsg) + putHttr(connection, fileInfos$presignedUrl, fileName, message = uploadMsg) + return(fileInfos) +} + + #' Generic privateUploadAoiMetrics function that takes as parameter a study object, a respondent/segment object, an AOI #' and some metrics to upload. #' @@ -1923,9 +1963,9 @@ privateUploadAoiMetrics <- function(study, obj, AOI, metrics) { } -#' Writes back metrics locally for a respondent of interest. +#' Write back or upload metrics for a respondent of interest. #' -#' S3 method to write back metrics locally for a specific respondent. +#' S3 method to write back metrics locally or upload them for a specific respondent. #' #' @inheritParams privateUploadAoiMetrics #' @@ -1942,19 +1982,28 @@ privateUploadAoiMetrics.imRespondent <- function(study, obj, AOI, metrics) { if (study$connection$localIM) { dataFileName <- paste0(tools::file_path_sans_ext(AOIDetails$fileId), "metrics.csv") } else { - if (is.null(study$connection$localPath)) { - warning("Please set a localPath when calling imConnection() to write back metrics locally.") - return() - } - - # For online we write back metrics locally - tmp_dir <- study$connection$localPath - dataFileName <- file.path(tmp_dir, sub("^.*/aoiDefinition/", "", AOI$fileId), paste0(obj$id, "metrics.csv")) - - dir.create(dirname(dataFileName), showWarnings = FALSE, recursive = TRUE) + dataFileName <- tempfile(fileext = ".csv") + on.exit(unlink(dataFileName), add = TRUE) } fwrite(x = metrics, file = dataFileName, col.names = TRUE, row.names = FALSE) + + if (!study$connection$localIM) { + uploadUrl <- getUploadAoiMetricsUrl(study, obj, AOI) + postData <- toJSON(list(name = AOI$name, fileName = dataFileName), null = "null") + endpoint <- paste0("respondent: ", obj$name, ", AOI: ", AOI$name) + uploadMessage <- paste("Uploading AOI metrics for", endpoint) + + fileInfos <- privateUploadToPresignedUrl(study$connection, uploadUrl, postData, dataFileName, + "Getting presignedUrl to upload AOI metrics", uploadMessage) + + # The uploaded file makes any locally cached metrics stale, even if confirmation fails. + cachedFile <- privateGetAoiRespondentMetricsPath(study, AOI, obj) + if (!is.null(cachedFile)) unlink(cachedFile) + + putHttr(study$connection, uploadUrl, reqBody = toJSON(fileInfos, null = "null"), + message = paste("Upload of AOI metrics for", endpoint, "confirmed")) + } } @@ -2197,7 +2246,7 @@ privateUpload <- function(params, study, data, target, sampleName, scriptName, m # Prepare the http request if (inherits(data, "imSignals")) { uploadUrl <- getUploadSensorDataUrl(study, target, stimulus) - postData <- privateCreatePostRequest(params, study, sampleName, tempFileName, overwrite) + postData <- privateCreatePostRequest(params, study, sampleName, tempFileName, overwrite, names(data)) endpoint_data <- "sensor data" } else if (inherits(data, "imEvents")) { uploadUrl <- getUploadEventsUrl(study, target) @@ -2220,16 +2269,15 @@ privateUpload <- function(params, study, data, target, sampleName, scriptName, m endpoint <- paste0(endpoint, ", stimulus: ", stimulus$name) } + uploadMessage <- paste("Uploading", endpoint_data, "for", endpoint) + if (study$connection$localIM) { - fileInfos <- postJSON(study$connection, uploadUrl, postData, - message = paste("Uploading", endpoint_data, "for", endpoint)) + fileInfos <- postJSON(study$connection, uploadUrl, postData, message = uploadMessage) } else { assertValid(exists("reportRunId", params), "Required `reportRunId` field in params for remote connection") uploadUrl <- str_replace(uploadUrl, "placeholder_reportId", params$reportRunId) - fileInfos <- postJSON(study$connection, uploadUrl, postData, message = "Getting presignedUrl to upload data") - - putHttr(study$connection, fileInfos$presignedUrl, tempFileName, - message = paste("Uploading", endpoint_data, "for", endpoint)) + fileInfos <- privateUploadToPresignedUrl(study$connection, uploadUrl, postData, tempFileName, + "Getting presignedUrl to upload data", uploadMessage) if (inherits(data, "imExport")) { confirmUrl <- uploadUrl @@ -2307,9 +2355,10 @@ privateSaveToFile <- function(params, study, data, sampleName, scriptName, metad #' Create headers specific to the data format (signals, events, exports). #' #' @inheritParams privateUpload +#' @param data_names Optional names of the data columns included in an online sensor upload. #' #' @keywords internal -privateCreatePostRequest <- function(params, study, sampleName, fileName, overwrite = TRUE) { +privateCreatePostRequest <- function(params, study, sampleName, fileName, overwrite = TRUE, data_names = NULL) { postRequest <- list(params$flowName, sampleName, fileName) if (study$connection$localIM) { @@ -2317,6 +2366,7 @@ privateCreatePostRequest <- function(params, study, sampleName, fileName, overwr names(postRequest) <- c("flowName", "sampleName", "fileName", "overwrite") } else { names(postRequest) <- c("instance", "name", "fileName") + if (!is.null(data_names)) postRequest$sampleDescription <- data_names } postRequest <- toJSON(postRequest, null = "null") @@ -2784,7 +2834,6 @@ getUploadMetricsUrl <- function(study, imObject) { UseMethod("getUploadMetricsUrl", object = imObject) } - #' getUploadMetricsUrl method to get the path/url to upload metrics to this respondent object. #' #' @inheritParams getUploadMetricsUrl @@ -2795,49 +2844,6 @@ getUploadMetricsUrl.imRespondent <- function(study, imObject) { file.path(getStudyBaseUrl(study), "rmetrics", study$id, "respondent", imObject$id, "data") } - -#' getUploadAoiMetadataUrl function that takes as parameter a study object. -#' -#' Return the path/url to upload AOI metadata for a specific study. -#' -#' @param study An imStudy object as returned from \code{\link{imStudy}}. -#' -#' @keywords internal -getUploadAoiMetadataUrl <- function(study) { - if (!study$connection$localIM) { - file.path(getStudyBaseUrl(study), "aoi", "sets", study$aoiSet$id, "metadata") - } -} - - -#' Generic getUploadAoiMetricsUrl function that takes as parameter a study object, a respondent/segment object and -#' an AOI -#' . -#' Return the path/url to upload AOI metrics to this respondent/segment object. -#' -#' @param study An imStudy object as returned from \code{\link{imStudy}}. -#' @param imObject An imRespondent or imSegment object of interest. -#' @param AOI An imAOI object as returned from \code{\link{getAois}}. -#' -#' @keywords internal -getUploadAoiMetricsUrl <- function(study, imObject, AOI) { - UseMethod("getUploadAoiMetricsUrl", object = imObject) -} - - -#' getUploadAoiMetricsUrl method to get the path/url to upload AOI metrics to this segment object. -#' -#' @inheritParams getUploadAoiMetricsUrl -#' -#' @keywords internal -#' @exportS3Method getUploadAoiMetricsUrl imSegment -getUploadAoiMetricsUrl.imSegment <- function(study, imObject, AOI) { - if (!study$connection$localIM) { - file.path(getAoisUrl(study), AOI$id, "segments", imObject$id, "stats") - } -} - - #' getAoiUrl function that takes as parameter a study object and optionally a respondent or a stimulus id. #' #' Return the path/url to the AOIs for this study/respondent/stimulus. @@ -2923,6 +2929,86 @@ getAoiDetailsUrl.imStimulus <- function(study, imObject, respondent = NULL) { } +#' Generic getAoiMetricsUrl function that takes as parameter a study, respondent object and AOI. +#' +#' Return the path/url to retrieve AOI metrics for the given object. +#' +#' @param study An imStudy object as returned from \code{\link{imStudy}}. +#' @param imObject An imRespondent object of interest. +#' @param AOI An imAOI object as returned from \code{\link{getAois}}. +#' +#' @keywords internal +getAoiMetricsUrl <- function(study, imObject, AOI) { + UseMethod("getAoiMetricsUrl", object = imObject) +} + + +#' getAoiMetricsUrl method to return the path/url to retrieve AOI metrics for a respondent. +#' +#' @inheritParams getAoiMetricsUrl +#' +#' @keywords internal +#' @exportS3Method getAoiMetricsUrl imRespondent +getAoiMetricsUrl.imRespondent <- function(study, imObject, AOI) { + if (!study$connection$localIM) { + file.path(getStudyBaseUrl(study), "aoi", "definitions", AOI$id, "respondents", imObject$id, "stats") + } +} + +#' getUploadAoiMetadataUrl function that takes as parameter a study object. +#' +#' Return the path/url to upload AOI metadata for a specific study. +#' +#' @param study An imStudy object as returned from \code{\link{imStudy}}. +#' +#' @keywords internal +getUploadAoiMetadataUrl <- function(study) { + if (!study$connection$localIM) { + file.path(getStudyBaseUrl(study), "aoi", "sets", study$aoiSet$id, "metadata") + } +} + +#' Generic getUploadAoiMetricsUrl function that takes as parameter a study object, a respondent/segment object and +#' an AOI. +#' +#' Return the path/url to upload AOI metrics to this respondent/segment object. +#' +#' @param study An imStudy object as returned from \code{\link{imStudy}}. +#' @param imObject An imRespondent or imSegment object of interest. +#' @param AOI An imAOI object as returned from \code{\link{getAois}}. +#' +#' @keywords internal +getUploadAoiMetricsUrl <- function(study, imObject, AOI) { + UseMethod("getUploadAoiMetricsUrl", object = imObject) +} + + +#' getUploadAoiMetricsUrl method to get the path/url to upload AOI metrics to this respondent object. +#' +#' @inheritParams getUploadAoiMetricsUrl +#' +#' @keywords internal +#' @exportS3Method getUploadAoiMetricsUrl imRespondent +getUploadAoiMetricsUrl.imRespondent <- function(study, imObject, AOI) { + if (!study$connection$localIM) { + file.path(getStudyBaseUrl(study), "aoi", "definitions", AOI$id, "respondents", imObject$id, "upload") + } +} + + +#' getUploadAoiMetricsUrl method to get the path/url to upload AOI metrics to this segment object. +#' +#' @inheritParams getUploadAoiMetricsUrl +#' +#' @keywords internal +#' @exportS3Method getUploadAoiMetricsUrl imSegment +getUploadAoiMetricsUrl.imSegment <- function(study, imObject, AOI) { + if (!study$connection$localIM) { + file.path(getAoisUrl(study), AOI$id, "segments", imObject$id, "stats") + } +} + + #' Return the path/url to the scenes of a specific respondent. #' #' @param study An imStudy object as returned from \code{\link{imStudy}}. @@ -2975,30 +3061,38 @@ getJSON <- function(connection, url, message = NULL, ...) { #' error message. #' #' @param fileName Optional - In case of a zip folder, the name of the file to look for. +#' @param localFilePath Optional - Exact local path where the downloaded file should be cached. #' #' @return A list with the temporary folder path and the downloaded file name. #' @keywords internal -getFile <- function(connection, url, message = NULL, fileName = NULL) { - response <- getHttr(connection, url, message) - - # Use temporary directory to download data - if (!is.null(connection$localPath)) { +getFile <- function(connection, url, message = NULL, fileName = NULL, localFilePath = NULL) { + # Use the requested file path, or fall back to the connection/temporary download directory. + if (!is.null(localFilePath)) { + tmp_dir <- dirname(localFilePath) + file_path <- localFilePath + } else if (!is.null(connection$localPath)) { tmp_dir <- connection$localPath file_path <- file.path(tmp_dir, sub("^.*/StudyUpload/", "", url)) - dir.create(dirname(file_path), showWarnings = FALSE, recursive = TRUE) } else { tmp_dir <- tempdir(check = TRUE) file_path <- file.path(tmp_dir, basename(url)) } + # Reuse files already downloaded to the selected path. if (!file.exists(file_path)) { - # We only download the file if it's not there yet - download.file(response$url, file_path, method = "auto", mode = "wb") + dir.create(dirname(file_path), showWarnings = FALSE, recursive = TRUE) + tmp_path <- paste0(file_path, ".part") + + getHttr(connection, url, message, writePath = tmp_path) + file.rename(tmp_path, file_path) } else { message("Retrieving local data for ", file_path) } - if (grepl("zip", response$headers$`content-type`)) { + # Check the retrieved/downloaded file for a zip signature directly + is_zip <- identical(readBin(file_path, "raw", n = 2), charToRaw("PK")) + + if (is_zip) { files_in_zip <- unzip(file_path, exdir = tmp_dir) assertValid(!is.null(fileName), "You need to provide a fileName for zip file extraction.") file_path <- str_subset(files_in_zip, fileName) @@ -3054,10 +3148,11 @@ jsonHeaders <- function() httr::add_headers("Content-Type" = "application/json") #' @param url The url/path where the file is located. #' @param message Optional - a short message indicating which steps are getting performed to get a more indicative #' error message. +#' @param writePath Optional - A local path where the response body should be streamed. #' #' @return The last response. #' @keywords internal -getHttr <- function(connection, url, message = NULL) { +getHttr <- function(connection, url, message = NULL, writePath = NULL) { if (connection$localIM) { # Locally there is no point to retry request if we get a 404 not found error terminate_on <- 404 @@ -3065,7 +3160,9 @@ getHttr <- function(connection, url, message = NULL) { terminate_on <- NULL } - response <- retryHttr(message, "GET", url, tokenHeaders(connection$token), terminate_on = terminate_on) + output <- if (is.null(writePath)) httr::write_memory() else httr::write_disk(writePath, overwrite = TRUE) + response <- retryHttr(message, "GET", url, tokenHeaders(connection$token), output, terminate_on = terminate_on) + return(response) } diff --git a/imotionsApi/man/getAoiMetricsUrl.Rd b/imotionsApi/man/getAoiMetricsUrl.Rd new file mode 100644 index 0000000..a70e133 --- /dev/null +++ b/imotionsApi/man/getAoiMetricsUrl.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/imotionsApi.R +\name{getAoiMetricsUrl} +\alias{getAoiMetricsUrl} +\title{Generic getAoiMetricsUrl function that takes as parameter a study, respondent object and AOI.} +\usage{ +getAoiMetricsUrl(study, imObject, AOI) +} +\arguments{ +\item{study}{An imStudy object as returned from \code{\link{imStudy}}.} + +\item{imObject}{An imRespondent object of interest.} + +\item{AOI}{An imAOI object as returned from \code{\link{getAois}}.} +} +\description{ +Return the path/url to retrieve AOI metrics for the given object. +} +\keyword{internal} diff --git a/imotionsApi/man/getAoiMetricsUrl.imRespondent.Rd b/imotionsApi/man/getAoiMetricsUrl.imRespondent.Rd new file mode 100644 index 0000000..5db3584 --- /dev/null +++ b/imotionsApi/man/getAoiMetricsUrl.imRespondent.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/imotionsApi.R +\name{getAoiMetricsUrl.imRespondent} +\alias{getAoiMetricsUrl.imRespondent} +\title{getAoiMetricsUrl method to return the path/url to retrieve AOI metrics for a respondent.} +\usage{ +\method{getAoiMetricsUrl}{imRespondent}(study, imObject, AOI) +} +\arguments{ +\item{study}{An imStudy object as returned from \code{\link{imStudy}}.} + +\item{imObject}{An imRespondent object of interest.} + +\item{AOI}{An imAOI object as returned from \code{\link{getAois}}.} +} +\description{ +getAoiMetricsUrl method to return the path/url to retrieve AOI metrics for a respondent. +} +\keyword{internal} diff --git a/imotionsApi/man/getFile.Rd b/imotionsApi/man/getFile.Rd index 8c6142b..9ad9eaa 100644 --- a/imotionsApi/man/getFile.Rd +++ b/imotionsApi/man/getFile.Rd @@ -5,7 +5,7 @@ \title{Retrieve a zip/csv file from the indicated path/url - optionally allow the user to add more information to know when an error occurred.} \usage{ -getFile(connection, url, message = NULL, fileName = NULL) +getFile(connection, url, message = NULL, fileName = NULL, localFilePath = NULL) } \arguments{ \item{connection}{An imConnection object as returned from \code{\link{imConnection}}.} @@ -16,6 +16,8 @@ getFile(connection, url, message = NULL, fileName = NULL) error message.} \item{fileName}{Optional - In case of a zip folder, the name of the file to look for.} + +\item{localFilePath}{Optional - Exact local path where the downloaded file should be cached.} } \value{ A list with the temporary folder path and the downloaded file name. diff --git a/imotionsApi/man/getHttr.Rd b/imotionsApi/man/getHttr.Rd index f2e188f..f6f6f78 100644 --- a/imotionsApi/man/getHttr.Rd +++ b/imotionsApi/man/getHttr.Rd @@ -4,7 +4,7 @@ \alias{getHttr} \title{Perform a GET HTTP request with authentication.} \usage{ -getHttr(connection, url, message = NULL) +getHttr(connection, url, message = NULL, writePath = NULL) } \arguments{ \item{connection}{An imConnection object as returned from \code{\link{imConnection}}.} @@ -13,6 +13,8 @@ getHttr(connection, url, message = NULL) \item{message}{Optional - a short message indicating which steps are getting performed to get a more indicative error message.} + +\item{writePath}{Optional - A local path where the response body should be streamed.} } \value{ The last response. diff --git a/imotionsApi/man/getUploadAoiMetricsUrl.Rd b/imotionsApi/man/getUploadAoiMetricsUrl.Rd index f1d0518..f9c1a96 100644 --- a/imotionsApi/man/getUploadAoiMetricsUrl.Rd +++ b/imotionsApi/man/getUploadAoiMetricsUrl.Rd @@ -3,9 +3,7 @@ \name{getUploadAoiMetricsUrl} \alias{getUploadAoiMetricsUrl} \title{Generic getUploadAoiMetricsUrl function that takes as parameter a study object, a respondent/segment object and -an AOI -. -Return the path/url to upload AOI metrics to this respondent/segment object.} +an AOI.} \usage{ getUploadAoiMetricsUrl(study, imObject, AOI) } @@ -17,9 +15,6 @@ getUploadAoiMetricsUrl(study, imObject, AOI) \item{AOI}{An imAOI object as returned from \code{\link{getAois}}.} } \description{ -Generic getUploadAoiMetricsUrl function that takes as parameter a study object, a respondent/segment object and -an AOI -. Return the path/url to upload AOI metrics to this respondent/segment object. } \keyword{internal} diff --git a/imotionsApi/man/getUploadAoiMetricsUrl.imRespondent.Rd b/imotionsApi/man/getUploadAoiMetricsUrl.imRespondent.Rd new file mode 100644 index 0000000..037c4bc --- /dev/null +++ b/imotionsApi/man/getUploadAoiMetricsUrl.imRespondent.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/imotionsApi.R +\name{getUploadAoiMetricsUrl.imRespondent} +\alias{getUploadAoiMetricsUrl.imRespondent} +\title{getUploadAoiMetricsUrl method to get the path/url to upload AOI metrics to this respondent object.} +\usage{ +\method{getUploadAoiMetricsUrl}{imRespondent}(study, imObject, AOI) +} +\arguments{ +\item{study}{An imStudy object as returned from \code{\link{imStudy}}.} + +\item{imObject}{An imRespondent or imSegment object of interest.} + +\item{AOI}{An imAOI object as returned from \code{\link{getAois}}.} +} +\description{ +getUploadAoiMetricsUrl method to get the path/url to upload AOI metrics to this respondent object. +} +\keyword{internal} diff --git a/imotionsApi/man/privateCreatePostRequest.Rd b/imotionsApi/man/privateCreatePostRequest.Rd index b2f5736..2cc600b 100644 --- a/imotionsApi/man/privateCreatePostRequest.Rd +++ b/imotionsApi/man/privateCreatePostRequest.Rd @@ -4,7 +4,14 @@ \alias{privateCreatePostRequest} \title{Create headers specific to the data format (signals, events, exports).} \usage{ -privateCreatePostRequest(params, study, sampleName, fileName, overwrite = TRUE) +privateCreatePostRequest( + params, + study, + sampleName, + fileName, + overwrite = TRUE, + data_names = NULL +) } \arguments{ \item{params}{The list of parameters provided to the script - specific parameters value will be stored as metadata.} @@ -16,6 +23,8 @@ this should correspond to the name of the file created (a path can also be provi \item{overwrite}{Optional - a boolean indicating if the new sensor/metrics/events/export should overwrite sensor/metrics/events/export generated for the same flowName. By default, they are overwritten.} + +\item{data_names}{Optional names of the data columns included in an online sensor upload.} } \description{ Create headers specific to the data format (signals, events, exports). diff --git a/imotionsApi/man/privateGetAoiRespondentMetricsPath.Rd b/imotionsApi/man/privateGetAoiRespondentMetricsPath.Rd new file mode 100644 index 0000000..b7a7c29 --- /dev/null +++ b/imotionsApi/man/privateGetAoiRespondentMetricsPath.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/imotionsApi.R +\name{privateGetAoiRespondentMetricsPath} +\alias{privateGetAoiRespondentMetricsPath} +\title{Return the local cache path for AOI respondent metrics.} +\usage{ +privateGetAoiRespondentMetricsPath(study, AOI, respondent) +} +\arguments{ +\item{study}{An imStudy object as returned from \code{\link{imStudy}}.} + +\item{AOI}{An imAOI object as returned from \code{\link{getAois}}.} + +\item{respondent}{An imRespondent object as returned from \code{\link{getRespondents}}.} +} +\description{ +Return the local cache path for AOI respondent metrics. +} +\keyword{internal} diff --git a/imotionsApi/man/privateUploadAoiMetrics.imRespondent.Rd b/imotionsApi/man/privateUploadAoiMetrics.imRespondent.Rd index 00b17bd..5faa006 100644 --- a/imotionsApi/man/privateUploadAoiMetrics.imRespondent.Rd +++ b/imotionsApi/man/privateUploadAoiMetrics.imRespondent.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/imotionsApi.R \name{privateUploadAoiMetrics.imRespondent} \alias{privateUploadAoiMetrics.imRespondent} -\title{Writes back metrics locally for a respondent of interest.} +\title{Write back or upload metrics for a respondent of interest.} \usage{ \method{privateUploadAoiMetrics}{imRespondent}(study, obj, AOI, metrics) } @@ -16,6 +16,6 @@ \item{metrics}{A data.table containing the metrics to upload.} } \description{ -S3 method to write back metrics locally for a specific respondent. +S3 method to write back metrics locally or upload them for a specific respondent. } \keyword{internal} diff --git a/imotionsApi/man/privateUploadToPresignedUrl.Rd b/imotionsApi/man/privateUploadToPresignedUrl.Rd new file mode 100644 index 0000000..03b0bcd --- /dev/null +++ b/imotionsApi/man/privateUploadToPresignedUrl.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/imotionsApi.R +\name{privateUploadToPresignedUrl} +\alias{privateUploadToPresignedUrl} +\title{Get upload credentials and upload a file to the returned presigned URL.} +\usage{ +privateUploadToPresignedUrl( + connection, + uploadUrl, + postData, + fileName, + credentialsMsg, + uploadMsg +) +} +\arguments{ +\item{connection}{An imConnection object as returned from \code{\link{imConnection}}.} + +\item{uploadUrl}{The URL used to request upload credentials.} + +\item{postData}{The JSON body used to request upload credentials.} + +\item{fileName}{The path of the file to upload.} + +\item{credentialsMsg}{Message used if requesting credentials fails.} + +\item{uploadMsg}{Message used if uploading the file fails.} +} +\value{ +The upload credentials returned by the API. +} +\description{ +Get upload credentials and upload a file to the returned presigned URL. +} +\keyword{internal} diff --git a/imotionsApi/tests/testthat/test-getAOIRespondentMetrics.R b/imotionsApi/tests/testthat/test-getAOIRespondentMetrics.R index 134f08b..e669d77 100644 --- a/imotionsApi/tests/testthat/test-getAOIRespondentMetrics.R +++ b/imotionsApi/tests/testthat/test-getAOIRespondentMetrics.R @@ -1,5 +1,5 @@ -# getAoiRespondentMetrics ============================================================================================= -context("getAoiRespondentMetrics()") +# privateGetAoiRespondentMetricsPath ================================================================================== +context("privateGetAoiRespondentMetricsPath()") library(mockery) @@ -12,31 +12,78 @@ study_cloud_local <- jsonlite::unserializeJSON(readLines("../data/imStudy_cloud_ AOI <- suppressWarnings(jsonlite::unserializeJSON(readLines("../data/imAOI.json"))) AOI_cloud <- suppressWarnings(jsonlite::unserializeJSON(readLines("../data/imAOI_cloud.json"))) AOIDetailsRespondentPath <- "../data/AOIDetailsRespondent.json" -AOIDetailsFile <- jsonlite::fromJSON(AOIDetailsRespondentPath) +AOIDetails <- jsonlite::fromJSON(AOIDetailsRespondentPath) metrics_output <- fread("../data/AOImetrics.csv") # Load respondent respondent <- getRespondents(study)[1, ] -mockedGetAoiRespondentMetrics <- function(study, AOI, respondent, AOIDetailsFile = NULL, expectedFilePath = NULL, - expectCallsDetails = 0, expectCallsFread = 0, fail = FALSE) { +expectedTmpPath <- paste0("myLocalPath/93fbaae0-8b6f-45b6-b5dd-9a5d4216d7fd/dd8a9342-f5c0-4a02-bf27-de68fc13f2bc/", + respondent$id, "metrics.csv") + +test_that("remote check - should return the local AOI respondent metrics path", { + expect_identical(privateGetAoiRespondentMetricsPath(study_cloud_local, AOI_cloud, respondent), expectedTmpPath) +}) + +test_that("remote check - should not return a local path when the AOI file ID is unavailable", { + AOI_cloud$fileId <- NULL + expect_null(privateGetAoiRespondentMetricsPath(study_cloud_local, AOI_cloud, respondent)) + + AOI_cloud$fileId <- NA_character_ + expect_null(privateGetAoiRespondentMetricsPath(study_cloud_local, AOI_cloud, respondent)) +}) + +# getAoiRespondentMetrics ============================================================================================= +context("getAoiRespondentMetrics()") + +mockGetAoiRespondentMetrics <- function(study, AOI, respondent, AOIDetailsFile = NULL, stats = NULL, + expectedFilePath = NULL, fileExists = FALSE, expectCallsDetails = 0, + expectCallsFileExists = 0, expectCallsFread = 0, expectCallsGetJSON = 0, + expectCallsGetFile = 0, fail_getFile = NULL) { + privateGetAoiDetails_Stub <- mock(AOIDetailsFile) - file.exists_Stub <- mock(!fail) + file.exists_Stub <- mock(fileExists) fread_Stub <- mock(metrics_output) + getJSON_Stub <- mock(stats) + getFile_Stub <- if (!is.null(fail_getFile)) mock(stop(fail_getFile)) else mock(list(file_path = expectedFilePath)) metrics <- mockr::with_mock(privateGetAoiDetails = privateGetAoiDetails_Stub, file.exists = file.exists_Stub, + getJSON = getJSON_Stub, + getFile = getFile_Stub, fread = fread_Stub, { getAoiRespondentMetrics(study, AOI, respondent) }) expect_called(privateGetAoiDetails_Stub, expectCallsDetails) + expect_called(file.exists_Stub, expectCallsFileExists) + expect_called(getJSON_Stub, expectCallsGetJSON) + expect_called(getFile_Stub, expectCallsGetFile) + expect_called(fread_Stub, expectCallsFread) if (expectCallsDetails > 0) { expect_args(privateGetAoiDetails_Stub, 1, study = study, imObject = AOI, respondent = respondent) } - expect_called(fread_Stub, expectCallsFread) + if (!study$connection$localIM) { + endpoint <- paste0("AOI: ", AOI$name, ", Respondent: ", respondent$name) + + if (expectCallsFileExists > 0) { + expect_args(file.exists_Stub, 1, expectedFilePath) + } + + if (expectCallsGetJSON > 0) { + expect_args(getJSON_Stub, 1, study$connection, getAoiMetricsUrl(study, respondent, AOI), + message = paste("Retrieving AOI respondent metrics for", endpoint)) + } + + if (expectCallsGetFile > 0) { + localFilePath <- if (!is.null(study$connection$localPath)) expectedFilePath else NULL + expect_args(getFile_Stub, 1, study$connection, stats$aoiRespondentStatsUrl, + message = paste("Downloading AOI respondent metrics for", endpoint), + localFilePath = localFilePath) + } + } if (expectCallsFread > 0) { expect_args(fread_Stub, 1, expectedFilePath) @@ -73,31 +120,30 @@ test_that("error - arguments are missing or not from the good class", { info = "respondent not being an imRespondent object should throw an error") }) -test_that("warning - AOI has not been defined for this respondent", { - AOIDetailsFile <- jsonlite::fromJSON("../data/no_scenes_annotations_aoidetails.json") - expect_warning(metrics <- mockedGetAoiRespondentMetrics(study, AOI, respondent, AOIDetailsFile, - expectCallsDetails = 1), +test_that("local warning - AOI has not been defined for this respondent", { + AOIDetails <- jsonlite::fromJSON("../data/no_scenes_annotations_aoidetails.json") + expect_warning(metrics <- mockGetAoiRespondentMetrics(study, AOI, respondent, AOIDetails, expectCallsDetails = 1), "AOI New Aoi was not found for respondent Wendy", info = "no AOI defined for this respondent should throw a warning") expect_null(metrics, info = "result should be null") }) -test_that("warning - no metrics have been found for this respondent", { - expect_warning(metrics <- mockedGetAoiRespondentMetrics(study, AOI, respondent, AOIDetailsFile, - expectCallsDetails = 1), +test_that("local warning - no metrics have been found for this respondent", { + expect_warning(metrics <- mockGetAoiRespondentMetrics(study, AOI, respondent, AOIDetails, expectCallsDetails = 1), "No metrics found for AOI: New Aoi, Respondent: Wendy", info = "no metrics found should throw an warning") expect_null(metrics, "result should be null") }) -# Modify AOIDetailsFile so it fit test data -AOIDetailsFile$resultId <- "../data/AOImetrics.csv" +# Modify AOIDetails so it fit test data +AOIDetails$resultId <- "../data/AOImetrics.csv" -test_that("local return - metrics for this AOI/respondent pair", { - metrics <- mockedGetAoiRespondentMetrics(study, AOI, respondent, AOIDetailsFile, expectCallsDetails = 1, - expectCallsFread = 1, expectedFilePath = "../data/AOImetrics.csv") +test_that("local check - should call privateGetAoiDetails and fread for a specific respondent", { + metrics <- mockGetAoiRespondentMetrics(study, AOI, respondent, AOIDetails, + expectedFilePath = "../data/AOImetrics.csv", expectCallsDetails = 1, + expectCallsFread = 1) # Check dimensions and class of metrics expect_equal(nrow(metrics), 1, infos = "metrics should always only have one row") @@ -105,31 +151,85 @@ test_that("local return - metrics for this AOI/respondent pair", { expect_s3_class(metrics, "imAOIMetrics") }) -test_that("remote warning - in case no local path is set, should send a warning and return NULL", { - expect_warning(metrics <- mockedGetAoiRespondentMetrics(study_cloud, AOI_cloud, respondent), - "No localPath set when calling imConnection(), not possible to read metrics locally.", - fixed = TRUE, info = "no local path, should throw an warning") +test_that("remote warning - no AOI respondent stats have been returned", { + expect_warning(metrics <- mockGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, fileExists = FALSE, + expectedFilePath = expectedTmpPath, expectCallsFileExists = 1, + expectCallsGetJSON = 1), + "No metrics found for AOI: El Manuel Area, Respondent: Wendy", + info = "no metrics found should throw an warning") expect_null(metrics, "result should be null") }) -test_that("remote warning - in case no metrics path found", { - expect_warning(metrics <- mockedGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, fail = TRUE), - "No metrics found for AOI: El Manuel Area, Respondent: Wendy", - info = "no metrics found should throw an warning") +test_that("remote warning - no metrics have been uploaded for this respondent", { + expect_warning(metrics <- mockGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, + stats = stop("Resource not found"), + expectedFilePath = expectedTmpPath, + expectCallsFileExists = 1, expectCallsGetJSON = 1), + "No metrics found for AOI: El Manuel Area, Respondent: Wendy", fixed = TRUE) - expect_null(metrics, "result should be null") + expect_null(metrics) }) -test_that("remote return - metrics for this AOI/respondent pair", { - expectedDirectory <- "myLocalPath/93fbaae0-8b6f-45b6-b5dd-9a5d4216d7fd/dd8a9342-f5c0-4a02-bf27-de68fc13f2bc" - expectedFilepath <- paste0(expectedDirectory, "/", respondent$id, "metrics.csv") +test_that("remote error - getJSON errors other than resource not found should be rethrown", { + expect_error(mockGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, stats = stop("Server error"), + expectedFilePath = expectedTmpPath, expectCallsFileExists = 1, + expectCallsGetJSON = 1), + "Server error", fixed = TRUE) +}) + +stats <- list(aoiRespondentStatsUrl = "https://s3.test/respondent-aoi-metrics.csv") + +test_that("remote warning - missing AOI respondent metrics file should return no metrics", { + expect_warning(metrics <- mockGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, stats = stats, + expectedFilePath = expectedTmpPath, expectCallsFileExists = 1, + expectCallsGetJSON = 1, expectCallsGetFile = 1, + fail_getFile = "Resource not found"), + "No metrics found for AOI: El Manuel Area, Respondent: Wendy", fixed = TRUE) + + expect_null(metrics) +}) + +test_that("remote error - getFile errors other than resource not found should be rethrown", { + expect_error(mockGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, stats = stats, + expectedFilePath = expectedTmpPath, expectCallsFileExists = 1, + expectCallsGetJSON = 1, expectCallsGetFile = 1, + fail_getFile = "Server error"), + "Server error", fixed = TRUE) +}) + +test_that("remote check - should call getJSON, getFile and fread when no local path is set", { + stats <- list(aoiRespondentStatsUrl = "https://s3.test/respondent-aoi-metrics.csv") - metrics <- mockedGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, expectCallsFread = 1, - expectedFilePath = expectedFilepath) + metrics <- mockGetAoiRespondentMetrics(study_cloud, AOI_cloud, respondent, stats = stats, + expectedFilePath = "../data/AOImetrics.csv", expectCallsFread = 1, + expectCallsGetJSON = 1, expectCallsGetFile = 1) # Check dimensions and class of metrics expect_equal(nrow(metrics), 1, infos = "metrics should always only have one row") expect_equal(ncol(metrics), 37, infos = "no column should be lost") expect_s3_class(metrics, "imAOIMetrics") }) + + + +test_that("remote check - should only call fread when metrics are cached locally", { + metrics <- mockGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, expectedFilePath = expectedTmpPath, + fileExists = TRUE, expectCallsFileExists = 1, expectCallsFread = 1) + + expect_equal(nrow(metrics), 1, infos = "metrics should always only have one row") + expect_equal(ncol(metrics), 37, infos = "no column should be lost") + expect_s3_class(metrics, "imAOIMetrics") +}) + +test_that("remote check - should call getJSON, getFile and fread when metrics are not cached locally", { + stats <- list(aoiRespondentStatsUrl = "https://s3.test/respondent-aoi-metrics.csv") + + metrics <- mockGetAoiRespondentMetrics(study_cloud_local, AOI_cloud, respondent, stats = stats, + expectedFilePath = expectedTmpPath, expectCallsFileExists = 1, + expectCallsFread = 1, expectCallsGetJSON = 1, expectCallsGetFile = 1) + + expect_equal(nrow(metrics), 1, infos = "metrics should always only have one row") + expect_equal(ncol(metrics), 37, infos = "no column should be lost") + expect_s3_class(metrics, "imAOIMetrics") +}) diff --git a/imotionsApi/tests/testthat/test-getSensors.R b/imotionsApi/tests/testthat/test-getSensors.R index e8ee894..8d1ea09 100644 --- a/imotionsApi/tests/testthat/test-getSensors.R +++ b/imotionsApi/tests/testthat/test-getSensors.R @@ -95,6 +95,19 @@ test_that("local return - imSensorList object", { expect_s3_class(sensors, c("data.table", "data.frame"), exact = TRUE) }) +test_that("local return - should preserve a NULL signals entry for the last sensor", { + sensors_with_null <- jsonlite::fromJSON(sensorsRespondentPath, simplifyDataFrame = FALSE) + sensors_with_null[[4]]["signals"] <- list(NULL) + + sensors <- mockr::with_mock(getJSON = mock(sensors_with_null), { + getSensors(study, respondent) + }) + + expect_equal(nrow(sensors), 4) + expect_length(sensors$signals, 4) + expect_null(sensors$signals[[4]]) +}) + test_that("remote return - imSensorList object", { respondent <- getRespondents(study_cloud)[1, ] expectedEndpoint <- "Retrieving sensors for respondent: bab55356-43fc-4c25-a39d-a1d513965614" diff --git a/imotionsApi/tests/testthat/test-getUrls.R b/imotionsApi/tests/testthat/test-getUrls.R index 386a42a..3e84131 100644 --- a/imotionsApi/tests/testthat/test-getUrls.R +++ b/imotionsApi/tests/testthat/test-getUrls.R @@ -57,6 +57,16 @@ test_that("local - all getUrl function should work as expected", { expect_identical(getUploadSensorDataUrl(study, respondent, stimulus), paste0(sensorStimulusUrl, "/data")) expect_identical(getUploadSensorDataUrl(study, segment, stimulus), paste0(sensorsSegmentUrl, "/data")) + #getUploadEventsUrl + eventUrl <- paste0(baseUrl, "/revents/af8c2165-4389-4cc3-8b1e-b2d3a4bd8be1") + expect_identical(getUploadEventsUrl(study, respondent), + paste0(eventUrl, "/respondent/09bd22e6-29b6-4a8a-8cc1-4780a5163e63/data")) + + #getUploadMetricsUrl + metricsUrl <- paste0(baseUrl, "/rmetrics/af8c2165-4389-4cc3-8b1e-b2d3a4bd8be1") + expect_identical(getUploadMetricsUrl(study, respondent), + paste0(metricsUrl, "/respondent/09bd22e6-29b6-4a8a-8cc1-4780a5163e63/data")) + #getAoisUrl AOIUrl <- paste0(baseUrl, "/aois/af8c2165-4389-4cc3-8b1e-b2d3a4bd8be1") expect_identical(getAoisUrl(study), AOIUrl) @@ -68,26 +78,16 @@ test_that("local - all getUrl function should work as expected", { "Please provide either stimulusId or respondentId, not both.", info = "too many params not handled properly") - #getUploadEventsUrl - eventUrl <- paste0(baseUrl, "/revents/af8c2165-4389-4cc3-8b1e-b2d3a4bd8be1") - expect_identical(getUploadEventsUrl(study, respondent), - paste0(eventUrl, "/respondent/09bd22e6-29b6-4a8a-8cc1-4780a5163e63/data")) - - #getUploadMetricsUrl - metricsUrl <- paste0(baseUrl, "/rmetrics/af8c2165-4389-4cc3-8b1e-b2d3a4bd8be1") - expect_identical(getUploadMetricsUrl(study, respondent), - paste0(metricsUrl, "/respondent/09bd22e6-29b6-4a8a-8cc1-4780a5163e63/data")) + #getAoiDetailsUrl (AOI) + expect_identical(getAoiDetailsUrl(study, AOI), paste0(AOIUrl, "/stimuli/", AOI$stimulusId, "/", AOI$id)) + expect_identical(getAoiDetailsUrl(study, AOI, respondent), + paste0(AOIUrl, "/stimuli/", AOI$stimulusId, "/respondent/", respondentId, "/", AOI$id)) - #getAoiDetailsForStimulusUrl + #getAoiDetailsUrl (Stimulus) expect_identical(getAoiDetailsUrl(study, stimulus), paste0(AOIUrl, "/stimuli/", stimulusId, "/*")) expect_identical(getAoiDetailsUrl(study, stimulus, respondent), paste0(AOIUrl, "/stimuli/", stimulusId, "/respondent/", respondentId, "/*")) - #getAoiDetailsUrl - expect_identical(getAoiDetailsUrl(study, AOI), paste0(AOIUrl, "/stimuli/", AOI$stimulusId, "/", AOI$id)) - expect_identical(getAoiDetailsUrl(study, AOI, respondent), - paste0(AOIUrl, "/stimuli/", AOI$stimulusId, "/respondent/", respondentId, "/", AOI$id)) - #getRespondentScenesUrl sceneUrl <- paste0(baseUrl, "/scenes/af8c2165-4389-4cc3-8b1e-b2d3a4bd8be1/", "respondent/09bd22e6-29b6-4a8a-8cc1-4780a5163e63") @@ -99,6 +99,11 @@ test_that("local - all getUrl function should work as expected", { "respondent/09bd22e6-29b6-4a8a-8cc1-4780a5163e63") expect_identical(getRespondentAnnotationsUrl(study, respondent), annotationsUrl) + + # List of functions that have no support locally + #getAoiMetricsUrl + #getUploadAoiMetadataUrl + #getUploadAoiMetricsUrl }) # Also checking remote study path @@ -159,14 +164,27 @@ test_that("remote - all getUrl function should work as expected", { "Please provide either stimulusId or respondentId, not both.", info = "too many params not handled properly") - #getUploadAoiMetricsUrl - uploadAOIUrl <- paste0(AOIUrl, "/09a234fc-00ad-4257-b7e1-da5754986c9d/segments/1010/stats") - expect_identical(getUploadAoiMetricsUrl(study_cloud, segment, AOI), uploadAOIUrl) + #getAoiMetricsUrl + aoiMetricsUrl <- paste0(baseUrl, "/aoi/definitions/09a234fc-00ad-4257-b7e1-da5754986c9d/respondents/", + "09bd22e6-29b6-4a8a-8cc1-4780a5163e63/stats") + + expect_identical(getAoiMetricsUrl(study_cloud, respondent, AOI), aoiMetricsUrl) #getUploadAoiMetadataUrl uploadMetadataUrl <- paste0(baseUrl, "/aoi/sets/fd7891cf-3f51-4fdd-bf48-0ee2f75d53b9/metadata") expect_identical(getUploadAoiMetadataUrl(study_cloud), uploadMetadataUrl) + #getUploadAoiMetricsUrl + respondentMetricAoiUrl <- paste0(baseUrl, "/aoi/definitions/09a234fc-00ad-4257-b7e1-da5754986c9d/respondents/", + "09bd22e6-29b6-4a8a-8cc1-4780a5163e63/upload") + + expect_identical(getUploadAoiMetricsUrl(study_cloud, respondent, AOI), respondentMetricAoiUrl) + + segmentMetricAoiUrl <- paste0(AOIUrl, "/09a234fc-00ad-4257-b7e1-da5754986c9d/segments/1010/stats") + expect_identical(getUploadAoiMetricsUrl(study_cloud, segment, AOI), segmentMetricAoiUrl) + + # List of functions that have no support in online + #getAoiDetailsUrl #getUploadEventsUrl #getUploadMetricsUrl #getRespondentScenesUrl diff --git a/imotionsApi/tests/testthat/test-httrRequests.R b/imotionsApi/tests/testthat/test-httrRequests.R index d5992d4..c0f5ae7 100644 --- a/imotionsApi/tests/testthat/test-httrRequests.R +++ b/imotionsApi/tests/testthat/test-httrRequests.R @@ -1,304 +1,380 @@ -context("Testing `httrRequests` functions") -# stopOnHttpError ===================================================================================================== -context("stopOnHttpError()") - -library(mockery) - -mockResponse <- list(status_code = 200) - -mockedStopOnHttpError <- function(response) { - status_code_Stub <- mock(response) - - mockr::with_mock(status_code = status_code_Stub, { - stopOnHttpError(mockResponse, "Test API") - }) -} - -test_that("error - token is not authorized", { - expect_error(mockedStopOnHttpError(response = 401), "Test API - Token not authorized to access requested resource", - info = "error 401 is not handled properly") -}) - -test_that("error - resource is not found", { - expect_error(mockedStopOnHttpError(response = 404), "Test API - Resource not found", - info = "error 404 is not handled properly") -}) - -test_that("error - something unexpected happened", { - expect_error(mockedStopOnHttpError(response = 135), "Test API - unexpected response status (135)", - fixed = TRUE, info = "unexpected error is not handled properly") -}) - -test_that("remote/local check - should work correctly", { - # Local connection - error <- mockedStopOnHttpError(response = 200) - expect_null(error, info = "should not produce an error") - - # Remote connection - error <- mockedStopOnHttpError(response = 204) - expect_null(error, info = "should not produce an error") -}) - -# retryHttr =========================================================================================================== -context("retryHttr()") - -connection <- jsonlite::unserializeJSON(readLines("../data/imConnection.json")) -connection_cloud <- jsonlite::unserializeJSON(readLines("../data/imConnection_cloud.json")) -terminate_on <- 404 -config <- tokenHeaders(connection$token) - - -mockedRetryHttr <- function(message, url, mockResponse) { - if (mockResponse == "error") { - RETRY_Stub <- mock(stop("Error")) - } else { - RETRY_Stub <- mock(mockResponse) - } - - response <- mockr::with_mock(RETRY = RETRY_Stub, { - retryHttr(message, "GET", url, config, terminate_on) - }) - - expect_args(RETRY_Stub, 1, "GET", url, config, terminate_on, 3, 2, 60, 1, FALSE) - return(response) -} - -test_that("return - a valid response in case of a valid request", { - mockResponse$status_code <- 200 - class(mockResponse) <- "response" - response <- mockedRetryHttr("Test Message", "testurl", mockResponse) - expect_identical(response, mockResponse, "response should not have been modified") - -}) - -test_that("error - non existing url should throw an error", { - mockResponse <- "error" - - expect_error(mockedRetryHttr("Test Message", "testurl", mockResponse), - "Test Message - Cannot access: testurl", - info = "should handle error from the RETRY function") -}) - -test_that("error - invalid response", { - mockResponse$status_code <- 404 - class(mockResponse) <- "response" - - expect_error(mockedRetryHttr("Test Message", "testurl", mockResponse), - "Test Message - url: testurl - Resource not found", - info = "should handle error from the stopOnHttpError function") -}) - -# getHttr ============================================================================================================= -context("getHttr()") - -mockedGetHttr <- function(connection, url, mockResponse, terminate_on) { - config <- tokenHeaders(connection$token) - retryHttr_Stub <- mock(mockResponse) - - response <- mockr::with_mock(retryHttr = retryHttr_Stub, { - getHttr(connection, url, "Test API") - }) - - expect_args(retryHttr_Stub, 1, "Test API", "GET", url, config, terminate_on) - return(response) -} - -test_that("remote/local check - should work correctly", { - # Local connection - mockResponse$status_code <- 200 - class(mockResponse) <- "response" - response <- mockedGetHttr(connection, "url", mockResponse, terminate_on) - expect_identical(response, mockResponse, "response should not have been modified") - - # Remote connection - mockResponse$status_code <- 204 - class(mockResponse) <- "response" - response <- mockedGetHttr(connection_cloud, "url", mockResponse, terminate_on = NULL) - expect_identical(response, mockResponse, "response should not have been modified") -}) - -# getJSON ============================================================================================================= -context("getJSON()") - -mockedGetJSON <- function(connection, url, mockResponse) { - getHttr_Stub <- mock(mockResponse) - - data <- mockr::with_mock(getHttr = getHttr_Stub, { - getJSON(connection, url, "Test API") - }) - - expect_args(getHttr_Stub, 1, connection, url, "Test API") - return(data) -} - -test_that("return - JSON file if a good request has been sent", { - mockData <- '{"userId":1,"id":1,"title":"delectus aut autem"}' - mockResponse <- list(status_code = 200, content = charToRaw(mockData)) - class(mockResponse) <- "response" - - expectedReturn <- list(userId = 1, id = 1, title = "delectus aut autem") - - #in case of good request - send back a JSON file - data <- mockedGetJSON(connection, "url", mockResponse) - expect_equal(data, expectedReturn, info = "wrong object returned") -}) - -# getFile ============================================================================================================= -context("getFile()") - -# Get the sensors through the cloud -sensors_cloud <- suppressWarnings(jsonlite::unserializeJSON(readLines("../data/imSensorList_cloud.json"))) - -mockedGetFile <- function(connection, url, mockResponse, fileName) { - getHttr_Stub <- mock(mockResponse) - - fileInfos <- mockr::with_mock(getHttr = getHttr_Stub, { - getFile(connection, url, "Test API", fileName) - }) - - expect_args(getHttr_Stub, 1, connection, url, "Test API") - return(fileInfos) -} - -test_that("remote return - correct paths to the file", { - # Case with eyetracking data (zip file) - eyetracking_fileName <- sensors_cloud[2, ]$fileName - url <- "file://../data/example_data_cloud.zip" - mockResponse <- list(headers = list("content-type" = "application/zip"), url = url) - class(mockResponse) <- "response" - - fileInfos <- mockedGetFile(connection_cloud, url, mockResponse, eyetracking_fileName) - - expected_path <- paste0("ProgramData/iMotions/Lab_NG/Data/RRRock The R/Signals/", - "20e73a6c-f2ae-4146-90f7-1430bbc9857b/ET_Eyetracker.csv") - - expect_identical(fileInfos$file_path, file.path(gsub("\\\\", "/", fileInfos$tmp_dir), expected_path), - "wrong file found") - - expect_true(file.exists(fileInfos$file_path), info = "file should exists") - unlink(c(fileInfos$file_path, file.path(fileInfos$tmp_dir, "ProgramData")), recursive = TRUE) - expect_false(file.exists(fileInfos$file_path), info = "file should have been deleted") - - # Case with slideEvents data (csv file) - events_fileName <- sensors_cloud[2, ]$fileName - url <- "file://../data/Native_SlideEvents_cloud.csv" - mockResponse <- list(headers = list("content-type" = "application/octet-stream"), url = url) - class(mockResponse) <- "response" - - fileInfos <- mockedGetFile(connection_cloud, url, mockResponse, events_fileName) - - expected_path <- file.path(fileInfos$tmp_dir, "Native_SlideEvents_cloud.csv") - expect_identical(fileInfos$file_path, expected_path, "wrong file found") - - expect_true(file.exists(fileInfos$file_path), info = "file should exists") - unlink(c(fileInfos$file_path, file.path(fileInfos$tmp_dir, "ProgramData")), recursive = TRUE) - expect_false(file.exists(fileInfos$file_path), info = "file should have been deleted") -}) - -# postHttr ============================================================================================================ -context("postHttr()") - -body <- '{"flowName":"flowName","sampleName":"TestSensor","fileName":"../data/testFile.csv"}' -class(body) <- "json" - -mockedPostHttr <- function(connection, url, mockResponse, body) { - config <- tokenHeaders(connection$token) - retryHttr_Stub <- mock(mockResponse) - - response <- mockr::with_mock(retryHttr = retryHttr_Stub, { - postHttr(connection, url, body, "Test API") - }) - - expect_args(retryHttr_Stub, 1, "Test API", "POST", url, body, config, jsonHeaders(), "json") - return(response) -} - -test_that("remote/local check - should work correctly", { - # Local connection - mockResponse$status_code <- 200 - class(mockResponse) <- "response" - response <- mockedPostHttr(connection, "url", mockResponse, body) - expect_identical(response, mockResponse, "response should not have been modified") - - # Remote connection - mockResponse$status_code <- 204 - class(mockResponse) <- "response" - response <- mockedPostHttr(connection_cloud, "url", mockResponse, body) - expect_identical(response, mockResponse, "response should not have been modified") -}) - -# postJSON ============================================================================================================ -context("postJSON()") - -mockedPostJSON <- function(connection, url, body, mockResponse = NULL) { - postHttr_Stub <- mock(mockResponse) - - fileInfos <- mockr::with_mock(postHttr = postHttr_Stub, { - postJSON(connection, url, body, "Test Upload API") - }) - - expect_args(postHttr_Stub, 1, connection, url, body, "Test Upload API") - return(fileInfos) -} - -test_that("local check - should upload a JSON file if a good request has been sent", { - # prepare mock results - mockData <- '{"filePath":"C:\\\\Users\\\\exampleUrls"}' - mockResponse <- list(status_code = 200, content = charToRaw(mockData)) - class(mockResponse) <- "response" - - # in case of good request - send back the url - expectedReturn <- list(filePath = "C:\\Users\\exampleUrls") - fileInfos <- mockedPostJSON(connection, "url", body, mockResponse) - expect_identical(fileInfos, expectedReturn, info = "wrong object returned") -}) - - -test_that("remote check - should get back correct information if asking for credential", { - # prepare mock results - expectedReturn <- fromJSON("../data/uploadCredential_cloud.json") - mockData <- as.character(toJSON(expectedReturn)) - mockResponse <- list(status_code = 200, content = charToRaw(mockData)) - class(mockResponse) <- "response" - - # in case of good request - send back the whole table - fileInfos <- mockedPostJSON(connection_cloud, "url", body, mockResponse) - expect_identical(fileInfos, expectedReturn, info = "wrong object returned") -}) - -# putHttr ============================================================================================================= -context("putHttr()") - -mockedPutHttr <- function(connection, url, mockResponse, fileName = NULL, reqBody = NULL, expectedBody = NULL, - expectedConfig = NULL) { - - retryHttr_Stub <- mock(mockResponse) - - mockr::with_mock(retryHttr = retryHttr_Stub, { - putHttr(connection, url, fileName, reqBody, "Test Put API") - }) - - expect_args(retryHttr_Stub, 1, "Test Put API", "PUT", url, body = expectedBody, config = expectedConfig) -} - -test_that("remote check - should work correctly", { - # With a file pushed - mockResponse$status_code <- 200 - class(mockResponse) <- "response" - config <- csvHeaders() - fileName <- "../data/testFile.csv" - expectedBody <- httr::upload_file(fileName) - mockedPutHttr(connection_cloud, "url", mockResponse, fileName, expectedBody = expectedBody, expectedConfig = config) - - # With no file pushed - mockResponse$status_code <- 204 - class(mockResponse) <- "response" - config <- tokenHeaders(connection_cloud$token) - mockedPutHttr(connection_cloud, "url", mockResponse, expectedConfig = config) - - # With a json object pushed - config <- c(tokenHeaders(connection_cloud$token), jsonHeaders()) - expectedBody <- "test" - mockedPutHttr(connection_cloud, "url", mockResponse, reqBody = "test", expectedBody = expectedBody, - expectedConfig = config) -}) +context("Testing `httrRequests` functions") +# stopOnHttpError ===================================================================================================== +context("stopOnHttpError()") + +library(mockery) + +mockResponse <- list(status_code = 200) + +mockedStopOnHttpError <- function(response) { + status_code_Stub <- mock(response) + + mockr::with_mock(status_code = status_code_Stub, { + stopOnHttpError(mockResponse, "Test API") + }) +} + +test_that("error - token is not authorized", { + expect_error(mockedStopOnHttpError(response = 401), "Test API - Token not authorized to access requested resource", + info = "error 401 is not handled properly") +}) + +test_that("error - resource is not found", { + expect_error(mockedStopOnHttpError(response = 404), "Test API - Resource not found", + info = "error 404 is not handled properly") +}) + +test_that("error - something unexpected happened", { + expect_error(mockedStopOnHttpError(response = 135), "Test API - unexpected response status (135)", + fixed = TRUE, info = "unexpected error is not handled properly") +}) + +test_that("remote/local check - should work correctly", { + # Local connection + error <- mockedStopOnHttpError(response = 200) + expect_null(error, info = "should not produce an error") + + # Remote connection + error <- mockedStopOnHttpError(response = 204) + expect_null(error, info = "should not produce an error") +}) + +# retryHttr =========================================================================================================== +context("retryHttr()") + +connection <- jsonlite::unserializeJSON(readLines("../data/imConnection.json")) +connection_cloud <- jsonlite::unserializeJSON(readLines("../data/imConnection_cloud.json")) +terminate_on <- 404 +config <- tokenHeaders(connection$token) + + +mockedRetryHttr <- function(message, url, mockResponse) { + if (mockResponse == "error") { + RETRY_Stub <- mock(stop("Error")) + } else { + RETRY_Stub <- mock(mockResponse) + } + + response <- mockr::with_mock(RETRY = RETRY_Stub, { + retryHttr(message, "GET", url, config, terminate_on) + }) + + expect_args(RETRY_Stub, 1, "GET", url, config, terminate_on, 3, 2, 60, 1, FALSE) + return(response) +} + +test_that("return - a valid response in case of a valid request", { + mockResponse$status_code <- 200 + class(mockResponse) <- "response" + response <- mockedRetryHttr("Test Message", "testurl", mockResponse) + expect_identical(response, mockResponse, "response should not have been modified") + +}) + +test_that("error - non existing url should throw an error", { + mockResponse <- "error" + + expect_error(mockedRetryHttr("Test Message", "testurl", mockResponse), + "Test Message - Cannot access: testurl", + info = "should handle error from the RETRY function") +}) + +test_that("error - invalid response", { + mockResponse$status_code <- 404 + class(mockResponse) <- "response" + + expect_error(mockedRetryHttr("Test Message", "testurl", mockResponse), + "Test Message - url: testurl - Resource not found", + info = "should handle error from the stopOnHttpError function") +}) + +# getHttr ============================================================================================================= +context("getHttr()") + +mockedGetHttr <- function(connection, url, mockResponse, terminate_on, writePath = NULL) { + config <- tokenHeaders(connection$token) + output <- if (is.null(writePath)) httr::write_memory() else httr::write_disk(writePath, overwrite = TRUE) + retryHttr_Stub <- mock(mockResponse) + + response <- mockr::with_mock(retryHttr = retryHttr_Stub, { + getHttr(connection, url, "Test API", writePath = writePath) + }) + + expect_args(retryHttr_Stub, 1, "Test API", "GET", url, config, output, terminate_on) + + return(response) +} + +test_that("remote/local check - should work correctly", { + # Local connection + mockResponse$status_code <- 200 + class(mockResponse) <- "response" + response <- mockedGetHttr(connection, "url", mockResponse, terminate_on) + expect_identical(response, mockResponse, "response should not have been modified") + + # Remote connection + mockResponse$status_code <- 204 + class(mockResponse) <- "response" + response <- mockedGetHttr(connection_cloud, "url", mockResponse, terminate_on = NULL) + expect_identical(response, mockResponse, "response should not have been modified") +}) + +test_that("remote check - should stream a GET response to the requested file", { + mockResponse$status_code <- 200 + class(mockResponse) <- "response" + + response <- mockedGetHttr(connection_cloud, "url", mockResponse, terminate_on = NULL, writePath = "download.part") + expect_identical(response, mockResponse, "response should not have been modified") +}) + +# getJSON ============================================================================================================= +context("getJSON()") + +mockedGetJSON <- function(connection, url, mockResponse) { + getHttr_Stub <- mock(mockResponse) + + data <- mockr::with_mock(getHttr = getHttr_Stub, { + getJSON(connection, url, "Test API") + }) + + expect_args(getHttr_Stub, 1, connection, url, "Test API") + return(data) +} + +test_that("return - JSON file if a good request has been sent", { + mockData <- '{"userId":1,"id":1,"title":"delectus aut autem"}' + mockResponse <- list(status_code = 200, content = charToRaw(mockData)) + class(mockResponse) <- "response" + + expectedReturn <- list(userId = 1, id = 1, title = "delectus aut autem") + + #in case of good request - send back a JSON file + data <- mockedGetJSON(connection, "url", mockResponse) + expect_equal(data, expectedReturn, info = "wrong object returned") +}) + +# getFile ============================================================================================================= +context("getFile()") + +# Get the sensors through the cloud +sensors_cloud <- suppressWarnings(jsonlite::unserializeJSON(readLines("../data/imSensorList_cloud.json"))) + +mockGetFile <- function(connection, url, mockResponse, fileName = NULL, localFilePath = NULL) { + getHttr_Stub <- function(connection, url, message, writePath = NULL) { + # Mimic write_disk() by copying the fixture to the requested path. + file.copy(sub("^file://", "", mockResponse$url), writePath, overwrite = TRUE) + return(mockResponse) + } + + fileInfos <- mockr::with_mock(getHttr = getHttr_Stub, { + getFile(connection, url, "Test API", fileName, localFilePath) + }) + + return(fileInfos) +} + +test_that("remote check - should call getHttr and return the correct file paths", { + # Case with eyetracking data (zip file) + eyetracking_fileName <- sensors_cloud[2, ]$fileName + url <- "file://../data/example_data_cloud.zip" + mockResponse <- list(headers = list("content-type" = "application/zip"), url = url) + class(mockResponse) <- "response" + + fileInfos <- mockGetFile(connection_cloud, url, mockResponse, eyetracking_fileName) + + expected_path <- paste0("ProgramData/iMotions/Lab_NG/Data/RRRock The R/Signals/", + "20e73a6c-f2ae-4146-90f7-1430bbc9857b/ET_Eyetracker.csv") + + expect_identical(fileInfos$file_path, file.path(gsub("\\\\", "/", fileInfos$tmp_dir), expected_path), + "wrong file found") + + expect_true(file.exists(fileInfos$file_path), info = "file should exists") + unlink(c(fileInfos$file_path, file.path(fileInfos$tmp_dir, "ProgramData")), recursive = TRUE) + expect_false(file.exists(fileInfos$file_path), info = "file should have been deleted") + + # Case with slideEvents data (csv file) + events_fileName <- sensors_cloud[2, ]$fileName + url <- "file://../data/Native_SlideEvents_cloud.csv" + mockResponse <- list(headers = list("content-type" = "application/octet-stream"), url = url) + class(mockResponse) <- "response" + + fileInfos <- mockGetFile(connection_cloud, url, mockResponse, events_fileName) + + expected_path <- file.path(fileInfos$tmp_dir, "Native_SlideEvents_cloud.csv") + expect_identical(fileInfos$file_path, expected_path, "wrong file found") + + expect_true(file.exists(fileInfos$file_path), info = "file should exists") + unlink(c(fileInfos$file_path, file.path(fileInfos$tmp_dir, "ProgramData")), recursive = TRUE) + expect_false(file.exists(fileInfos$file_path), info = "file should have been deleted") +}) + +test_that("remote check - should call getHttr and download a file to an explicit local path", { + url <- "https://s3.test/respondent-aoi-metrics.csv" + expected_path <- tempfile("aoi-metrics-cache-", fileext = ".csv") + mockResponse <- list(headers = list("content-type" = "text/csv"), url = "file://../data/AOImetrics.csv") + class(mockResponse) <- "response" + + fileInfos <- mockGetFile(connection_cloud, url, mockResponse, localFilePath = expected_path) + + expect_identical(fileInfos$file_path, expected_path, "wrong file found") + + expect_true(file.exists(expected_path)) + unlink(expected_path) + expect_false(file.exists(expected_path)) +}) + +test_that("remote check - should call getHttr and reuse a file from an explicit local path", { + url <- "https://s3.test/respondent-aoi-metrics.csv" + expected_path <- "../data/AOImetrics.csv" + getHttr_Stub <- mock() + + fileInfos <- mockr::with_mock(getHttr = getHttr_Stub, { + getFile(connection_cloud, url, "Test API", localFilePath = expected_path) + }) + + expect_called(getHttr_Stub, 0) + expect_identical(fileInfos$file_path, expected_path) + expect_true(file.exists(expected_path)) +}) + +test_that("remote check - should reuse a cached ZIP without a file extension", { + connection_cloud$localPath <- "../data" + file.copy("../data/example_data_cloud.zip", "../data/example_data_cloud", overwrite = TRUE) + getHttr_Stub <- mock() + + fileInfos <- mockr::with_mock(getHttr = getHttr_Stub, { + getFile(connection_cloud, "api/content/StudyUpload/example_data_cloud", "Test API", + sensors_cloud[2, ]$fileName) + }) + + expect_called(getHttr_Stub, 0) + expect_true(file.exists(fileInfos$file_path)) + unlink(c("../data/example_data_cloud", "../data/ProgramData"), recursive = TRUE) +}) + +test_that("remote error - should not promote a partial download", { + expected_path <- file.path(tempdir(), "aoi-metrics-cache.csv") + on.exit(unlink(paste0(expected_path, ".part")), add = TRUE) + + expect_error( + mockr::with_mock(getHttr = function(connection, url, message, writePath = NULL) { + writeBin(charToRaw("partial"), writePath) + stop("Network error") + }, { + getFile(connection_cloud, "https://s3.test/respondent-aoi-metrics.csv", "Test API", + localFilePath = expected_path) + }), + "Network error", + fixed = TRUE + ) + + expect_false(file.exists(expected_path)) +}) + +# postHttr ============================================================================================================ +context("postHttr()") + +body <- '{"flowName":"flowName","sampleName":"TestSensor","fileName":"../data/testFile.csv"}' +class(body) <- "json" + +mockedPostHttr <- function(connection, url, mockResponse, body) { + config <- tokenHeaders(connection$token) + retryHttr_Stub <- mock(mockResponse) + + response <- mockr::with_mock(retryHttr = retryHttr_Stub, { + postHttr(connection, url, body, "Test API") + }) + + expect_args(retryHttr_Stub, 1, "Test API", "POST", url, body, config, jsonHeaders(), "json") + return(response) +} + +test_that("remote/local check - should work correctly", { + # Local connection + mockResponse$status_code <- 200 + class(mockResponse) <- "response" + response <- mockedPostHttr(connection, "url", mockResponse, body) + expect_identical(response, mockResponse, "response should not have been modified") + + # Remote connection + mockResponse$status_code <- 204 + class(mockResponse) <- "response" + response <- mockedPostHttr(connection_cloud, "url", mockResponse, body) + expect_identical(response, mockResponse, "response should not have been modified") +}) + +# postJSON ============================================================================================================ +context("postJSON()") + +mockedPostJSON <- function(connection, url, body, mockResponse = NULL) { + postHttr_Stub <- mock(mockResponse) + + fileInfos <- mockr::with_mock(postHttr = postHttr_Stub, { + postJSON(connection, url, body, "Test Upload API") + }) + + expect_args(postHttr_Stub, 1, connection, url, body, "Test Upload API") + return(fileInfos) +} + +test_that("local check - should upload a JSON file if a good request has been sent", { + # prepare mock results + mockData <- '{"filePath":"C:\\\\Users\\\\exampleUrls"}' + mockResponse <- list(status_code = 200, content = charToRaw(mockData)) + class(mockResponse) <- "response" + + # in case of good request - send back the url + expectedReturn <- list(filePath = "C:\\Users\\exampleUrls") + fileInfos <- mockedPostJSON(connection, "url", body, mockResponse) + expect_identical(fileInfos, expectedReturn, info = "wrong object returned") +}) + + +test_that("remote check - should get back correct information if asking for credential", { + # prepare mock results + expectedReturn <- fromJSON("../data/uploadCredential_cloud.json") + mockData <- as.character(toJSON(expectedReturn)) + mockResponse <- list(status_code = 200, content = charToRaw(mockData)) + class(mockResponse) <- "response" + + # in case of good request - send back the whole table + fileInfos <- mockedPostJSON(connection_cloud, "url", body, mockResponse) + expect_identical(fileInfos, expectedReturn, info = "wrong object returned") +}) + +# putHttr ============================================================================================================= +context("putHttr()") + +mockedPutHttr <- function(connection, url, mockResponse, fileName = NULL, reqBody = NULL, expectedBody = NULL, + expectedConfig = NULL) { + + retryHttr_Stub <- mock(mockResponse) + + mockr::with_mock(retryHttr = retryHttr_Stub, { + putHttr(connection, url, fileName, reqBody, "Test Put API") + }) + + expect_args(retryHttr_Stub, 1, "Test Put API", "PUT", url, body = expectedBody, config = expectedConfig) +} + +test_that("remote check - should work correctly", { + # With a file pushed + mockResponse$status_code <- 200 + class(mockResponse) <- "response" + config <- csvHeaders() + fileName <- "../data/testFile.csv" + expectedBody <- httr::upload_file(fileName) + mockedPutHttr(connection_cloud, "url", mockResponse, fileName, expectedBody = expectedBody, expectedConfig = config) + + # With no file pushed + mockResponse$status_code <- 204 + class(mockResponse) <- "response" + config <- tokenHeaders(connection_cloud$token) + mockedPutHttr(connection_cloud, "url", mockResponse, expectedConfig = config) + + # With a json object pushed + config <- c(tokenHeaders(connection_cloud$token), jsonHeaders()) + expectedBody <- "test" + mockedPutHttr(connection_cloud, "url", mockResponse, reqBody = "test", expectedBody = expectedBody, + expectedConfig = config) +}) diff --git a/imotionsApi/tests/testthat/test-uploadAOIMetrics.R b/imotionsApi/tests/testthat/test-uploadAOIMetrics.R index 274eac8..cba57eb 100644 --- a/imotionsApi/tests/testthat/test-uploadAOIMetrics.R +++ b/imotionsApi/tests/testthat/test-uploadAOIMetrics.R @@ -1,5 +1,5 @@ -# privateUploadAoiMetrics ========================================================================================== -context("privateUploadAoiMetrics()") +# privateUploadToPresignedUrl ========================================================================================= +context("privateUploadToPresignedUrl()") library(mockery) @@ -8,7 +8,30 @@ study <- jsonlite::unserializeJSON(readLines("../data/imStudy.json")) study_cloud <- jsonlite::unserializeJSON(readLines("../data/imStudy_cloud.json")) study_cloud_local <- jsonlite::unserializeJSON(readLines("../data/imStudy_cloud_local.json")) +test_that("remote check - presigned uploads request should work as expected and upload metrics", { + uploadUrl <- "myurl" + fileName <- "metrics.csv" + postData <- '{"name":"AOI","fileName":"metrics.csv"}' + fileInfos <- list(presignedUrl = "presignedUrl") + postJSON_Stub <- mock(fileInfos) + putHttr_Stub <- mock() + + result <- mockr::with_mock(postJSON = postJSON_Stub, + putHttr = putHttr_Stub, { + privateUploadToPresignedUrl(study_cloud$connection, uploadUrl, postData, fileName, + "Getting upload credentials", "Uploading") + }) + + expect_identical(result, fileInfos) + expect_args(postJSON_Stub, 1, study_cloud$connection, uploadUrl, postData, message = "Getting upload credentials") + expect_args(putHttr_Stub, 1, study_cloud$connection, fileInfos$presignedUrl, fileName, message = "Uploading") +}) + +# privateUploadAoiMetrics ========================================================================================== +context("privateUploadAoiMetrics()") + respondent <- getRespondent(study, "09bd22e6-29b6-4a8a-8cc1-4780a5163e63") +respondent_cloud <- getRespondent(study_cloud, "6c74637b-6250-4d13-bbbc-c999d9c8a74b") AOI <- suppressWarnings(jsonlite::unserializeJSON(readLines("../data/imAOI.json"))) AOI_cloud <- suppressWarnings(jsonlite::unserializeJSON(readLines("../data/imAOI_cloud.json"))) @@ -18,44 +41,62 @@ AOIDetailsFile <- jsonlite::fromJSON("../data/AOIDetailsRespondent.json") metrics <- data.frame("metric1" = 2, "metric2" = 234, "metric3" = 1234) mockPrivateUploadAoiMetrics <- function(study, obj, AOI, metrics, AOIDetailsFile = NULL, expectedFilepath = NULL, - expectedDirectory = NULL, expectedEndpoint = NULL, expectedBody = NULL, - expectCallDetails = 0, expectCallsfwrite = 0, expectCallsDir = 0, - expectCallsPut = 0) { + expectedEndpoint = NULL, expectedBody = NULL, expectCallsfwrite = 0, + expectCallsPut = 0, expectCallsPresigned = 0, uploadStub = NULL, + confirmationStub = NULL, expectFileRemoved = FALSE) { privateGetAoiDetails_Stub <- mock(AOIDetailsFile) fwrite_Stub <- mock() - dir.create_Stub <- mock() + fwriteMock <- if (expectFileRemoved) function(...) data.table::fwrite(...) else fwrite_Stub getUploadAoiMetricsUrl_Stub <- mock("myurl") - putHttr_Stub <- mock() + fileInfos <- list(assetId = "assetId", presignedUrl = "presignedUrl") + privateUploadToPresignedUrl_Stub <- if (is.null(uploadStub)) mock(fileInfos) else mock(stop(uploadStub)) + putHttr_Stub <- if (is.null(confirmationStub)) mock() else mock(stop(confirmationStub)) + + if (expectFileRemoved) { + on.exit({ + uploadedFile <- mock_args(privateUploadToPresignedUrl_Stub)[[1]][[4]] + expect_false(file.exists(uploadedFile), info = "temporary metrics file should have been deleted") + }, add = TRUE) + } mockr::with_mock(privateGetAoiDetails = privateGetAoiDetails_Stub, - fwrite = fwrite_Stub, - dir.create = dir.create_Stub, + fwrite = fwriteMock, getUploadAoiMetricsUrl = getUploadAoiMetricsUrl_Stub, + privateUploadToPresignedUrl = privateUploadToPresignedUrl_Stub, putHttr = putHttr_Stub, { privateUploadAoiMetrics(study, obj, AOI, metrics) }) - expect_called(privateGetAoiDetails_Stub, expectCallDetails) + expect_called(privateGetAoiDetails_Stub, as.integer(inherits(obj, "imRespondent"))) expect_called(fwrite_Stub, expectCallsfwrite) - expect_called(dir.create_Stub, expectCallsDir) expect_called(getUploadAoiMetricsUrl_Stub, expectCallsPut) + expect_called(privateUploadToPresignedUrl_Stub, expectCallsPresigned) expect_called(putHttr_Stub, expectCallsPut) - if (expectCallDetails > 0) { - expect_args(privateGetAoiDetails_Stub, 1, study = study, imObject = AOI, respondent = obj) + if (!study$connection$localIM) { + expect_args(getUploadAoiMetricsUrl_Stub, 1, study, obj, AOI) } - if (expectCallsfwrite > 0) { - expect_args(fwrite_Stub, 1, x = metrics, file = expectedFilepath, col.names = TRUE, row.names = FALSE) - } + if (inherits(obj, "imRespondent")) { + expect_args(privateGetAoiDetails_Stub, 1, study = study, imObject = AOI, respondent = obj) - if (expectCallsDir > 0) { - expect_args(dir.create_Stub, 1, expectedDirectory, showWarnings = FALSE, recursive = TRUE) - } + if (expectCallsPresigned > 0) { + expectedFilepath <- mock_args(privateUploadToPresignedUrl_Stub)[[1]][[4]] + postData <- toJSON(list(name = AOI$name, fileName = expectedFilepath), null = "null") + endpoint <- paste0("respondent: ", obj$name, ", AOI: ", AOI$name) - if (expectCallsPut > 0) { - expect_args(getUploadAoiMetricsUrl_Stub, 1, study, obj, AOI) + expect_args(privateUploadToPresignedUrl_Stub, 1, study$connection, "myurl", postData, expectedFilepath, + "Getting presignedUrl to upload AOI metrics", paste("Uploading AOI metrics for", endpoint)) + + expect_args(putHttr_Stub, 1, study$connection, "myurl", reqBody = toJSON(fileInfos, null = "null"), + message = paste("Upload of AOI metrics for", endpoint, "confirmed")) + } + + if (expectCallsfwrite > 0) { + expect_args(fwrite_Stub, 1, x = metrics, file = expectedFilepath, col.names = TRUE, row.names = FALSE) + } + } else if (inherits(obj, "imSegment") && expectCallsPut > 0) { expect_args(putHttr_Stub, 1, study$connection, "myurl", expectedBody, expectedEndpoint) } } @@ -63,7 +104,7 @@ mockPrivateUploadAoiMetrics <- function(study, obj, AOI, metrics, AOIDetailsFile test_that("warning - AOI is not found for a specific respondent", { AOIDetailsFile <- jsonlite::fromJSON("../data/no_scenes_annotations_aoidetails.json") - expect_warning(mockPrivateUploadAoiMetrics(study, respondent, AOI, metrics, AOIDetailsFile, expectCallDetails = 1), + expect_warning(mockPrivateUploadAoiMetrics(study, respondent, AOI, metrics, AOIDetailsFile), "AOI New Aoi was not found for respondent Wendy", info = "no AOI defined for this respondent should throw a warning") }) @@ -72,33 +113,56 @@ test_that("local check - should call privateGetAoiDetails and fwrite for a speci expectedFilepath <- paste0(tools::file_path_sans_ext(AOIDetailsFile$fileId), "metrics.csv") mockPrivateUploadAoiMetrics(study, respondent, AOI, metrics, AOIDetailsFile, expectedFilepath, - expectCallDetails = 1, expectCallsfwrite = 1) + expectCallsfwrite = 1) }) -test_that("remote warning - should throw a warning and exit if no local path was set for a specific respondent", { - expect_warning(mockPrivateUploadAoiMetrics(study_cloud, respondent, AOI_cloud, metrics, AOIDetailsFile, - expectCallDetails = 1), fixed = TRUE, - "Please set a localPath when calling imConnection() to write back metrics locally.", - info = "no local path should throw a warning") +test_that("remote check - should call privateUploadToPresignedUrl and putHttr for a specific respondent", { + mockPrivateUploadAoiMetrics(study_cloud, respondent_cloud, AOI_cloud, metrics, AOIDetailsFile = AOI_cloud, + expectCallsPut = 1, expectCallsPresigned = 1, expectFileRemoved = TRUE) +}) +test_that("remote error - should remove the temporary metrics file when upload fails", { + expect_error(mockPrivateUploadAoiMetrics(study_cloud, respondent_cloud, AOI_cloud, metrics, + AOIDetailsFile = AOI_cloud, uploadStub = "Upload failed", + expectFileRemoved = TRUE), + "Upload failed", fixed = TRUE) }) -segment <- getSegments(study_cloud) +expectedTmpPath <- paste0("myLocalPath/93fbaae0-8b6f-45b6-b5dd-9a5d4216d7fd/", + "dd8a9342-f5c0-4a02-bf27-de68fc13f2bc/", respondent_cloud$id, "metrics.csv") -test_that("remote check - should call privateGetAoiDetails, dir.create and fwrite for a specific respondent", { - expectedDirectory <- "myLocalPath/93fbaae0-8b6f-45b6-b5dd-9a5d4216d7fd/dd8a9342-f5c0-4a02-bf27-de68fc13f2bc" - expectedFilepath <- paste0(expectedDirectory, "/", respondent$id, "metrics.csv") +test_that("remote check - should remove cached metrics after uploading for a specific respondent", { + dir.create(dirname(expectedTmpPath), recursive = TRUE) + file.create(expectedTmpPath) - mockPrivateUploadAoiMetrics(study_cloud_local, respondent, AOI_cloud, metrics, AOIDetailsFile, expectedFilepath, - expectedDirectory, expectCallDetails = 1, expectCallsfwrite = 1, expectCallsDir = 1) + mockPrivateUploadAoiMetrics(study_cloud_local, respondent_cloud, AOI_cloud, metrics, AOIDetailsFile = AOI_cloud, + expectCallsPut = 1, expectCallsPresigned = 1, expectFileRemoved = TRUE) + + expect_false(file.exists(expectedTmpPath), info = "cached metrics file should have been deleted") + unlink(study_cloud_local$connection$localPath, recursive = TRUE) }) +test_that("remote error - should invalidate cached metrics before upload confirmation", { + dir.create(dirname(expectedTmpPath), recursive = TRUE) + file.create(expectedTmpPath) + + expect_error(mockPrivateUploadAoiMetrics(study_cloud_local, respondent_cloud, AOI_cloud, metrics, + AOIDetailsFile = AOI_cloud, confirmationStub = "Confirmation failed", + expectFileRemoved = TRUE), + "Confirmation failed", fixed = TRUE) + + expect_false(file.exists(expectedTmpPath)) + unlink(study_cloud_local$connection$localPath, recursive = TRUE) +}) + +segment <- getSegments(study_cloud) + test_that("remote check - should call getUploadAoiMetricsUrl and putHttr for a specific segment", { expectedEndpoint <- "Updating AOI metrics for segment: All Respondents, AOI: El Manuel Area" expectedBody <- toJSON(fromJSON("../data/AOImetrics_cloud.json")) - mockPrivateUploadAoiMetrics(study_cloud, segment, AOI_cloud, metrics, expectedUrl, - expectedEndpoint = expectedEndpoint, expectedBody = expectedBody, expectCallsPut = 1) + mockPrivateUploadAoiMetrics(study_cloud, segment, AOI_cloud, metrics, expectedEndpoint = expectedEndpoint, + expectedBody = expectedBody, expectCallsPut = 1) }) # uploadAoiMetrics ========================================================================================== diff --git a/imotionsApi/tests/testthat/test-uploadSensorData.R b/imotionsApi/tests/testthat/test-uploadSensorData.R index 100daf8..00425b9 100644 --- a/imotionsApi/tests/testthat/test-uploadSensorData.R +++ b/imotionsApi/tests/testthat/test-uploadSensorData.R @@ -435,7 +435,9 @@ test_that("remote check - upload signals to a given respondent/segment", { # upload to a respondent data <- checkDataFormat(data) params$reportRunId <- "1234" - expectedBody <- '{"instance":"flowName","name":"TestSensor","fileName":"../data/testFile.csv"}' + expectedBody <- paste0('{"instance":"flowName","name":"TestSensor","fileName":"../data/testFile.csv",', + '"sampleDescription":["Timestamp","Thresholded value"]}') + expectedEndpoint <- "Getting presignedUrl to upload data" res <- mockedPrivateUpload(params, study_cloud, data, respondent, expectedBody, expectedEndpoint, sampleName,