diff --git a/doc/release-notes/11695-change-api-get-storage-driver.md b/doc/release-notes/11695-change-api-get-storage-driver.md new file mode 100644 index 00000000000..aa993232f4d --- /dev/null +++ b/doc/release-notes/11695-change-api-get-storage-driver.md @@ -0,0 +1,12 @@ +## Get Dataset/Dataverse Storage Driver API + +### Changed Json response - breaking change! + +The API for getting the Storage Driver info has been changed/extended. +/api/datasets/{identifier}/storageDriver +/api/admin/dataverse/{dataverse-alias}/storageDriver +changed "message" to "name" and added "type" and "label" + +Also added query param for /api/admin/dataverse/{dataverse-alias}/storageDriver?getEffective=true to recurse the chain of parents to find the effective storageDriver + +See also [the guides](https://dataverse-guide--11664.org.readthedocs.build/en/11664/api/native-api.html#configure-a-dataset-to-store-all-new-files-in-a-specific-file-store), #11695, and #11664. diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index a37819c90e1..0fa5bcf69f1 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -60,6 +60,10 @@ The current driver can be seen using:: curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver +Or to recurse the chain of parents to find the effective storageDriver:: + + curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver?getEffective=true + (Note that for ``dataverse.files.store1.label=MyLabel``, ``store1`` will be returned.) and can be reset to the default store with:: diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index b793981e761..84ac2ba962c 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -13,6 +13,7 @@ v6.8 - For POST /api/files/{id}/metadata passing an empty string ("description":"") or array ("categories":[]) will no longer be ignored. Empty fields will now clear out the values in the file's metadata. To ignore the fields simply do not include them in the JSON string. - For PUT /api/datasets/{id}/editMetadata the query parameter "sourceInternalVersionNumber" has been removed and replaced with "sourceLastUpdateTime" to verify that the data being edited hasn't been modified and isn't stale. - For GET /api/dataverses/$dataverse-alias/links the Json response has changed breaking the backward compatibility of the API. +- For GET /api/admin/dataverse/{dataverse-alias}/storageDriver and /api/datasets/{identifier}/storageDriver the driver name is no longer returned in data.message. This value is now returned in data.name. - For PUT /api/dataverses/$dataverse-alias/inputLevels custom input levels that had been previously set will no longer be deleted. To delete input levels send an empty list (deletes all), then send the new/modified list. - For GET /api/externalTools and /api/externalTools/{id} the responses are now formatted as JSON (previously the toolParameters and allowedApiCalls were a JSON object and array (respectively) that were serialized as JSON strings) and any configured "requirements" are included. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index d55f582ecae..12989a1b70c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -20,6 +20,7 @@ import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.util.cache.CacheFactoryBean; +import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; import edu.harvard.iq.dataverse.validation.EMailValidator; import edu.harvard.iq.dataverse.EjbDataverseEngine; @@ -2180,7 +2181,8 @@ public Response addRoleAssignementsToChildren(@Context ContainerRequestContext c @GET @AuthRequired @Path("/dataverse/{alias}/storageDriver") - public Response getStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse { + public Response getStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias, + @QueryParam("getEffective") Boolean getEffective) throws WrappedResponse { Dataverse dataverse = dataverseSvc.findByAlias(alias); if (dataverse == null) { return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + "."); @@ -2193,8 +2195,12 @@ public Response getStorageDriver(@Context ContainerRequestContext crc, @PathPara } catch (WrappedResponse wr) { return wr.getResponse(); } - //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver - return ok(dataverse.getStorageDriverId()); + + if (getEffective != null && getEffective) { + return ok(JsonPrinter.jsonStorageDriver(dataverse.getEffectiveStorageDriverId(), null)); + } else { + return ok(JsonPrinter.jsonStorageDriver(dataverse.getStorageDriverId(), null)); + } } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 3334f2ec8bb..1ada5f78b01 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -3690,7 +3690,7 @@ public Response getFileStore(@Context ContainerRequestContext crc, @PathParam("i return error(Response.Status.NOT_FOUND, "No such dataset"); } - return response(req -> ok(dataset.getEffectiveStorageDriverId()), getRequestUser(crc)); + return ok(JsonPrinter.jsonStorageDriver(dataset.getEffectiveStorageDriverId(), dataset)); } @PUT diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index bbc834e0cc4..1a2697180e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -40,6 +40,7 @@ import edu.harvard.iq.dataverse.workflow.Workflow; import edu.harvard.iq.dataverse.workflow.step.WorkflowStepData; +import java.io.IOException; import java.util.*; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; @@ -1653,6 +1654,23 @@ public static JsonArrayBuilder jsonTemplateInstructions(Map temp return jsonArrayBuilder; } + public static JsonObjectBuilder jsonStorageDriver(String storageDriverId, Dataset dataset) { + JsonObjectBuilder jsonObjectBuilder = new NullSafeJsonBuilder(); + jsonObjectBuilder.add("name", storageDriverId); + jsonObjectBuilder.add("type", DataAccess.getDriverType(storageDriverId)); + jsonObjectBuilder.add("label", DataAccess.getStorageDriverLabelFor(storageDriverId)); + if (dataset != null) { + jsonObjectBuilder.add("directUpload", DataAccess.uploadToDatasetAllowed(dataset, storageDriverId)); + try { + jsonObjectBuilder.add("directDownload", DataAccess.getStorageIO(dataset).downloadRedirectEnabled()); + } catch (IOException ex) { + logger.fine("Failed to get Storage IO for dataset " + ex.getMessage()); + } + } + + return jsonObjectBuilder; + } + public static JsonArrayBuilder json(List notifications, AuthenticatedUser authenticatedUser, boolean inAppNotificationFormat) { JsonArrayBuilder notificationsArray = Json.createArrayBuilder(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 5dac4b1d05c..16552869a80 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -6118,6 +6118,60 @@ public void testGetGlobusUploadParameters() { GlobusOverlayAccessIOTest.tearDown(); } + @Test + public void testSetGetDatasetStorageDriver() { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + String username = UtilIT.getUsernameFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + Response createDataverse = UtilIT.createRandomDataverse(apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverse); + Response createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + Integer datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + + Response storageDrivers = UtilIT.listStorageDrivers(apiToken); + storageDrivers.prettyPrint(); + JsonObject data = JsonUtil.getJsonObject(storageDrivers.getBody().asString()); + String first = data.getJsonObject("data").keySet().iterator().next(); + String name = data.getJsonObject("data").getString(first); + + Response setDriver = UtilIT.setDatasetStorageDriver(datasetId, first, apiToken); + setDriver.prettyPrint(); + assertEquals(200, setDriver.getStatusCode()); + Response getDriver = UtilIT.getDatasetStorageDriver(datasetId, apiToken); + getDriver.prettyPrint(); + assertEquals(200, getDriver.getStatusCode()); + getDriver.then().assertThat() + .body("data.name", CoreMatchers.equalTo(name)) + .body("data.type", CoreMatchers.notNullValue()) + .body("data.label", CoreMatchers.notNullValue()) + .body("data.directUpload", CoreMatchers.notNullValue()) + .body("data.directDownload", CoreMatchers.notNullValue()) + .statusCode(OK.getStatusCode()); + + // Test dataset under root with default storage driver + Response getStorageDriverResponse = UtilIT.getStorageDriver("root", apiToken, Boolean.TRUE); + getStorageDriverResponse.prettyPrint(); + data = JsonUtil.getJsonObject(getStorageDriverResponse.getBody().asString()); + name = data.getJsonObject("data").getString("name"); + String type = data.getJsonObject("data").getString("type"); + String label = data.getJsonObject("data").getString("label"); + createDataset = UtilIT.createRandomDatasetViaNativeApi("root", apiToken); + datasetId = UtilIT.getDatasetIdFromResponse(createDataset); + getDriver = UtilIT.getDatasetStorageDriver(datasetId, apiToken); + getDriver.prettyPrint(); + assertEquals(200, getDriver.getStatusCode()); + getDriver.then().assertThat() + .body("data.name", CoreMatchers.equalTo(name)) + .body("data.type", CoreMatchers.equalTo(type)) + .body("data.label", CoreMatchers.equalTo(label)) + .body("data.directUpload", CoreMatchers.notNullValue()) + .body("data.directDownload", CoreMatchers.notNullValue()) + .statusCode(OK.getStatusCode()); + } + @Test public void testGetCanDownloadAtLeastOneFile() { Response createUserResponse = UtilIT.createRandomUser(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 155166a386d..45f9f8a1ce0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.util.json.JsonParseException; import edu.harvard.iq.dataverse.util.json.JsonParser; import edu.harvard.iq.dataverse.util.json.JsonUtil; @@ -2450,6 +2451,26 @@ public void testCreateAndGetTemplates() { getTemplateResponse.then().assertThat().statusCode(UNAUTHORIZED.getStatusCode()); } + @Test + public void testGetStorageDriver() { + Response updatedStorageDriver = UtilIT.getStorageDriver("root", getSuperuserToken(), Boolean.TRUE); + updatedStorageDriver.prettyPrint(); + updatedStorageDriver.then().assertThat() + .body("data.name", CoreMatchers.notNullValue()) + .body("data.type", CoreMatchers.notNullValue()) + .body("data.label", CoreMatchers.notNullValue()) + .body("data.directUpload", CoreMatchers.nullValue()) + .body("data.directDownload", CoreMatchers.nullValue()) + .statusCode(200); + + // Root without default is undefined + updatedStorageDriver = UtilIT.getStorageDriver("root", getSuperuserToken(), null); + updatedStorageDriver.prettyPrint(); + updatedStorageDriver.then().assertThat() + .body("data.name", CoreMatchers.equalTo(DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER)) + .statusCode(200); + } + private String getSuperuserToken() { Response createResponse = UtilIT.createRandomUser(); String adminApiToken = UtilIT.getApiTokenFromResponse(createResponse); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java index 137c2ef4c7b..24625c87ce2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java @@ -1,5 +1,6 @@ package edu.harvard.iq.dataverse.api; +import org.hamcrest.CoreMatchers; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.ResponseInputStream; @@ -142,7 +143,7 @@ public void testNonDirectUpload() { Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); originalStorageDriver.prettyPrint(); originalStorageDriver.then().assertThat() - .body("data.message", equalTo("undefined")) + .body("data.name", equalTo("undefined")) .statusCode(200); Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); @@ -153,6 +154,9 @@ public void testNonDirectUpload() { Response updatedStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); updatedStorageDriver.prettyPrint(); updatedStorageDriver.then().assertThat() + .body("data.type", CoreMatchers.notNullValue()) + .body("data.label", CoreMatchers.notNullValue()) + .body("data.directUpload", CoreMatchers.nullValue()) .statusCode(200); Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); @@ -273,7 +277,7 @@ public void testDirectUpload() { Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); originalStorageDriver.prettyPrint(); originalStorageDriver.then().assertThat() - .body("data.message", equalTo("undefined")) + .body("data.name", equalTo("undefined")) .statusCode(200); Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); @@ -491,7 +495,7 @@ public void testDirectUploadDetectStataFile() { Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); originalStorageDriver.prettyPrint(); originalStorageDriver.then().assertThat() - .body("data.message", equalTo("undefined")) + .body("data.name", equalTo("undefined")) .statusCode(200); Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); @@ -689,7 +693,7 @@ public void testDirectUploadWithFileCountLimit() throws JsonParseException { Response originalStorageDriver = UtilIT.getStorageDriver(dataverseAlias, superuserApiToken); originalStorageDriver.prettyPrint(); originalStorageDriver.then().assertThat() - .body("data.message", equalTo("undefined")) + .body("data.name", equalTo("undefined")) .statusCode(200); Response setStorageDriverToS3 = UtilIT.setStorageDriver(dataverseAlias, driverLabel, superuserApiToken); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index e5d3f908349..9c0cc213df0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2850,9 +2850,13 @@ static Response listStorageDrivers(String apiToken) { } static Response getStorageDriver(String dvAlias, String apiToken) { + return getStorageDriver(dvAlias, apiToken, null); + } + static Response getStorageDriver(String dvAlias, String apiToken, Boolean getEffective) { + String params = getEffective != null ? "?getEffective=" + getEffective : ""; return given() .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("/api/admin/dataverse/" + dvAlias + "/storageDriver"); + .get("/api/admin/dataverse/" + dvAlias + "/storageDriver" + params); } static Response setStorageDriver(String dvAlias, String label, String apiToken) { @@ -4460,6 +4464,12 @@ static Response setDatasetStorageDriver(Integer datasetId, String driverLabel, S .put("/api/datasets/" + datasetId + "/storageDriver"); } + static Response getDatasetStorageDriver(Integer datasetId, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/datasets/" + datasetId + "/storageDriver"); + } + /** GET on /api/admin/savedsearches/list */ static Response getSavedSearchList() { return given().get("/api/admin/savedsearches/list");