diff --git a/conf/keycloak/docker-compose-dev.yml b/conf/keycloak/docker-compose-dev.yml index 7356161ec47..7e57cd7d83c 100644 --- a/conf/keycloak/docker-compose-dev.yml +++ b/conf/keycloak/docker-compose-dev.yml @@ -53,16 +53,16 @@ services: -Ddataverse.files.localstack1.download-redirect=true -Ddataverse.files.localstack1.access-key=default -Ddataverse.files.localstack1.secret-key=default - -Ddataverse.files.minio1.type=s3 - -Ddataverse.files.minio1.label=MinIO - -Ddataverse.files.minio1.custom-endpoint-url=http://minio:9000 - -Ddataverse.files.minio1.custom-endpoint-region=us-east-1 - -Ddataverse.files.minio1.bucket-name=mybucket - -Ddataverse.files.minio1.path-style-access=true - -Ddataverse.files.minio1.upload-redirect=false - -Ddataverse.files.minio1.download-redirect=false - -Ddataverse.files.minio1.access-key=4cc355_k3y - -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k3y + -Ddataverse.files.localstack_noredirect.type=s3 + -Ddataverse.files.localstack_noredirect.label=LocalStackNoRedirect + -Ddataverse.files.localstack_noredirect.custom-endpoint-url=http://localstack:4566 + -Ddataverse.files.localstack_noredirect.custom-endpoint-region=us-east-2 + -Ddataverse.files.localstack_noredirect.bucket-name=mybucket-noredirect + -Ddataverse.files.localstack_noredirect.path-style-access=true + -Ddataverse.files.localstack_noredirect.upload-redirect=false + -Ddataverse.files.localstack_noredirect.download-redirect=false + -Ddataverse.files.localstack_noredirect.access-key=default + -Ddataverse.files.localstack_noredirect.secret-key=default -Ddataverse.pid.providers=fake -Ddataverse.pid.default-provider=fake -Ddataverse.pid.fake.type=FAKE @@ -260,23 +260,6 @@ services: tmpfs: - /localstack:mode=770,size=128M,uid=1000,gid=1000 - dev_minio: - container_name: "dev_minio" - hostname: "minio" - image: minio/minio - restart: on-failure - ports: - - "9000:9000" - - "9001:9001" - networks: - - dataverse - volumes: - - ./docker-dev-volumes/minio_storage:/data - environment: - MINIO_ROOT_USER: 4cc355_k3y - MINIO_ROOT_PASSWORD: s3cr3t_4cc355_k3y - command: server /data - previewers-provider: container_name: previewers-provider hostname: previewers-provider diff --git a/conf/localstack/buckets.sh b/conf/localstack/buckets.sh index fe940d9890d..bd901c19634 100755 --- a/conf/localstack/buckets.sh +++ b/conf/localstack/buckets.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash # https://stackoverflow.com/questions/53619901/auto-create-s3-buckets-on-localstack awslocal s3 mb s3://mybucket +awslocal s3 mb s3://mybucket-noredirect diff --git a/doc/sphinx-guides/source/installation/big-data-support.rst b/doc/sphinx-guides/source/installation/big-data-support.rst index 45b94f71a9f..411bd6b54d8 100644 --- a/doc/sphinx-guides/source/installation/big-data-support.rst +++ b/doc/sphinx-guides/source/installation/big-data-support.rst @@ -68,7 +68,6 @@ If the bucket allows the wildcard ``*`` but the Dataverse application only allow Detailed information for the most common S3 admin tools around CORS: - `AWS `_ -- `Minio mc `_ - `s3cmd `_ Get Current CORS Policy on Bucket @@ -80,9 +79,6 @@ If you'd like to check the CORS configuration on your bucket before making chang .. group-tab:: AWS CLI :code:`aws s3api get-bucket-cors --bucket ` - .. group-tab:: Minio Client (mc) - :code:`mc cors get /` - Set CORS Policy on Bucket +++++++++++++++++++++++++ @@ -107,9 +103,6 @@ Both JSON and XML format are explained in detail in `AWS Docs ` as follows: - .. literalinclude:: /_static/installation/cors/cors.xml :name: xml-cors :language: xml @@ -124,7 +117,7 @@ Both JSON and XML format are explained in detail in `AWS Docs `_ - Set ``dataverse.files..path-style-access=true``, as Minio works path-based. Works pretty smooth, easy to setup. - **Can be used for quick testing, too:** just use the example values above. Uses the public (read: unsecure and - possibly slow) https://play.minio.io:9000 service. - `StorJ Object Store `_ StorJ is a distributed object store that can be configured with an S3 gateway. Per the S3 Storage instructions above, you'll first set up the StorJ S3 store by defining the id, type, and label. After following the general installation, set the following configuration to use a StorJ object store: ``dataverse.files..chunked-encoding=false``. For step-by-step instructions see https://docs.storj.io/dcs/how-tos/dataverse-integration-guide/ diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index b24bf0ed6f6..c176597c990 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -44,16 +44,16 @@ services: -Ddataverse.files.localstack1.download-redirect=true -Ddataverse.files.localstack1.access-key=default -Ddataverse.files.localstack1.secret-key=default - -Ddataverse.files.minio1.type=s3 - -Ddataverse.files.minio1.label=MinIO - -Ddataverse.files.minio1.custom-endpoint-url=http://minio:9000 - -Ddataverse.files.minio1.custom-endpoint-region=us-east-1 - -Ddataverse.files.minio1.bucket-name=mybucket - -Ddataverse.files.minio1.path-style-access=true - -Ddataverse.files.minio1.upload-redirect=false - -Ddataverse.files.minio1.download-redirect=false - -Ddataverse.files.minio1.access-key=4cc355_k3y - -Ddataverse.files.minio1.secret-key=s3cr3t_4cc355_k3y + -Ddataverse.files.localstack_noredirect.type=s3 + -Ddataverse.files.localstack_noredirect.label=LocalStackNoRedirect + -Ddataverse.files.localstack_noredirect.custom-endpoint-url=http://localstack:4566 + -Ddataverse.files.localstack_noredirect.custom-endpoint-region=us-east-2 + -Ddataverse.files.localstack_noredirect.bucket-name=mybucket-noredirect + -Ddataverse.files.localstack_noredirect.path-style-access=true + -Ddataverse.files.localstack_noredirect.upload-redirect=false + -Ddataverse.files.localstack_noredirect.download-redirect=false + -Ddataverse.files.localstack_noredirect.access-key=default + -Ddataverse.files.localstack_noredirect.secret-key=default -Ddataverse.pid.providers=fake -Ddataverse.pid.default-provider=fake -Ddataverse.pid.fake.type=FAKE @@ -252,23 +252,6 @@ services: tmpfs: - /localstack:mode=770,size=128M,uid=1000,gid=1000 - dev_minio: - container_name: "dev_minio" - hostname: "minio" - image: minio/minio - restart: on-failure - ports: - - "9000:9000" - - "9001:9001" - networks: - - dataverse - volumes: - - ./docker-dev-volumes/minio_storage:/data - environment: - MINIO_ROOT_USER: 4cc355_k3y - MINIO_ROOT_PASSWORD: s3cr3t_4cc355_k3y - command: server /data - previewers-provider: container_name: previewers-provider hostname: previewers-provider diff --git a/scripts/dev/dev-start-frd.sh b/scripts/dev/dev-start-frd.sh index d113f677bad..d76bed85770 100755 --- a/scripts/dev/dev-start-frd.sh +++ b/scripts/dev/dev-start-frd.sh @@ -28,7 +28,6 @@ mkdir -p docker-dev-volumes/app/secrets mkdir -p docker-dev-volumes/postgresql/data mkdir -p docker-dev-volumes/solr/data mkdir -p docker-dev-volumes/solr/conf -mkdir -p docker-dev-volumes/minio_storage # Only disable DDL generation if database is already initialized # (on first run, we need create-tables to bootstrap the schema) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 6d3fe205639..74c22f4ce3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -1331,8 +1331,8 @@ public void removeTempTag() throws IOException { if (e.getCause() instanceof S3Exception) { S3Exception s3e = (S3Exception) e.getCause(); if (s3e.statusCode() == 501) { - // In this case, it's likely that tags are not implemented at all (e.g. by - // Minio) so no tag was set either and it's just something to be aware of + // In this case, it's likely that tags are not implemented at all + // so no tag was set either and it's just something to be aware of logger.warning("Temp tag not deleted: Object tags not supported by storage: " + driverId); } else { // In this case, the assumption is that adding tags has worked, so not removing diff --git a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java index 48a64490796..bdc9049e519 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/S3AccessIT.java @@ -46,7 +46,7 @@ import org.junit.jupiter.api.Test; /** - * This test requires LocalStack and Minio to be running. Developers can use our + * This test requires LocalStack to be running. Developers can use our * docker-compose file, which has all the necessary configuration. */ public class S3AccessIT { @@ -54,8 +54,8 @@ public class S3AccessIT { private static final Logger logger = Logger.getLogger(S3AccessIT.class.getCanonicalName()); static final String BUCKET_NAME = "mybucket"; + static final String BUCKET_NAME_NOREDIRECT = "mybucket-noredirect"; static S3Client s3localstack = null; - static S3Client s3minio = null; @BeforeAll public static void setUp() { @@ -71,45 +71,34 @@ public static void setUp() { .region(Region.US_EAST_2) .build(); - String accessKeyMinio = "4cc355_k3y"; - String secretKeyMinio = "s3cr3t_4cc355_k3y"; - s3minio = S3Client.builder() - .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyMinio, secretKeyMinio))) - .endpointOverride(URI.create("http://localhost:9000")) - .region(Region.US_EAST_1) - .forcePathStyle(true) - .build(); - - // create bucket if it doesn't exist - try { - s3localstack.headBucket(HeadBucketRequest.builder().bucket(BUCKET_NAME).build()); - } catch (NoSuchBucketException ex) { - s3localstack.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); - } + ensureBucketExists(BUCKET_NAME); + ensureBucketExists(BUCKET_NAME_NOREDIRECT); + } + private static void ensureBucketExists(String bucketName) { try { - s3minio.headBucket(HeadBucketRequest.builder().bucket(BUCKET_NAME).build()); - } catch (NoSuchBucketException ex) { - try { - CreateBucketResponse createBucketResponse = s3minio.createBucket(CreateBucketRequest.builder().bucket(BUCKET_NAME).build()); - if (createBucketResponse.sdkHttpResponse().isSuccessful()) { - System.out.println("Bucket created successfully"); - } else { - System.err.println("Failed to create bucket: " + createBucketResponse.sdkHttpResponse().statusCode()); - } - } catch (S3Exception e) { - System.err.println("Error creating bucket: " + e.getMessage()); + s3localstack.headBucket(HeadBucketRequest.builder().bucket(bucketName).build()); + } catch (S3Exception ex) { + String errorCode = ex.awsErrorDetails() == null ? null : ex.awsErrorDetails().errorCode(); + if (ex.statusCode() == 404 || "NoSuchBucket".equals(errorCode)) { + s3localstack.createBucket(CreateBucketRequest.builder().bucket(bucketName).build()); + } else { + throw ex; } } } /** - * We're using MinIO for testing non-direct upload. + * We're using LocalStack (with redirects disabled) for testing non-direct + * upload. Using localstack_noredirect ensures the non-redirect + * (proxy-through-Dataverse) code path is actually exercised. If localstack1 + * (redirect-enabled) were used, RestAssured would silently follow the 303 + * redirect and the proxy path would never be tested. */ @Test public void testNonDirectUpload() { - String driverId = "minio1"; - String driverLabel = "MinIO"; + String driverId = "localstack_noredirect"; + String driverLabel = "LocalStackNoRedirect"; Response createSuperuser = UtilIT.createRandomUser(); createSuperuser.then().assertThat().statusCode(200); @@ -124,7 +113,7 @@ public void testNonDirectUpload() { "status": "OK", "data": { "LocalStack": "localstack1", - "MinIO": "minio1", + "LocalStackNoRedirect": "localstack_noredirect", "Local": "local", "Filesystem": "file1" } @@ -186,13 +175,13 @@ public void testNonDirectUpload() { String storageIdentifier = JsonPath.from(addFileResponse.body().asString()).getString("data.files[0].dataFile.storageIdentifier"); String keyInDataverse = storageIdentifier.split(":")[2]; - Assertions.assertEquals(driverId + "://" + BUCKET_NAME + ":" + keyInDataverse, storageIdentifier); + Assertions.assertEquals(driverId + "://" + BUCKET_NAME_NOREDIRECT + ":" + keyInDataverse, storageIdentifier); String keyInS3 = datasetStorageIdentifier + "/" + keyInDataverse; String s3Object = null; try { - ResponseInputStream s3ObjectResponse = s3minio.getObject(GetObjectRequest.builder() - .bucket(BUCKET_NAME) + ResponseInputStream s3ObjectResponse = s3localstack.getObject(GetObjectRequest.builder() + .bucket(BUCKET_NAME_NOREDIRECT) .key(keyInS3) .build()); // Read the content of the object into a string @@ -207,8 +196,11 @@ public void testNonDirectUpload() { fail("Failed to read S3 object content: " + ex.getMessage()); } + // Use downloadFileNoRedirect to verify Dataverse serves the content directly + // (status 200). If the driver were misconfigured with download-redirect=true, + // this would return 303 instead, causing the test to fail explicitly. System.out.println("non-direct download..."); - Response downloadFile = UtilIT.downloadFile(Integer.valueOf(fileId), apiToken); + Response downloadFile = UtilIT.downloadFileNoRedirect(Integer.valueOf(fileId), apiToken); downloadFile.then().assertThat().statusCode(200); String contentsOfDownloadedFile = downloadFile.getBody().asString(); @@ -220,8 +212,8 @@ public void testNonDirectUpload() { S3Exception expectedException = null; try { - ResponseInputStream s3ObjectResponse = s3minio.getObject(GetObjectRequest.builder() - .bucket(BUCKET_NAME) + ResponseInputStream s3ObjectResponse = s3localstack.getObject(GetObjectRequest.builder() + .bucket(BUCKET_NAME_NOREDIRECT) .key(keyInS3) .build()); // Read the content of the object into a string @@ -258,7 +250,7 @@ public void testDirectUpload() { "status": "OK", "data": { "LocalStack": "localstack1", - "MinIO": "minio1", + "LocalStackNoRedirect": "localstack_noredirect", "Local": "local", "Filesystem": "file1" } @@ -336,18 +328,6 @@ public void testDirectUpload() { InputStream inputStream = new ByteArrayInputStream(contentsOfFile.getBytes(StandardCharsets.UTF_8)); Response uploadFileDirect = UtilIT.uploadFileDirect(localhostUrl, inputStream); uploadFileDirect.prettyPrint(); - /* - Direct upload to MinIO is failing with errors like this: - - SignatureDoesNotMatch - The request signature we calculated does not match the signature you provided. Check your key and signing method. - 10.5072/FK2/KGFCEJ/18b8c06688c-21b8320a3ee5 - mybucket - /mybucket/10.5072/FK2/KGFCEJ/18b8c06688c-21b8320a3ee5 - 1793915CCC5BC95C - dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8 - - */ uploadFileDirect.then().assertThat().statusCode(200); // TODO: Use MD5 or whatever Dataverse is configured for and @@ -441,7 +421,7 @@ public void testDirectUpload() { S3Exception expectedException = null; try { - ResponseInputStream s3ObjectResponse = s3minio.getObject(GetObjectRequest.builder() + ResponseInputStream s3ObjectResponse = s3localstack.getObject(GetObjectRequest.builder() .bucket(BUCKET_NAME) .key(keyInS3) .build()); @@ -476,7 +456,7 @@ public void testDirectUploadDetectStataFile() { "status": "OK", "data": { "LocalStack": "localstack1", - "MinIO": "minio1", + "LocalStackNoRedirect": "localstack_noredirect", "Local": "local", "Filesystem": "file1" } @@ -550,18 +530,6 @@ public void testDirectUploadDetectStataFile() { } Response uploadFileDirect = UtilIT.uploadFileDirect(localhostUrl, inputStream); uploadFileDirect.prettyPrint(); - /* - Direct upload to MinIO is failing with errors like this: - - SignatureDoesNotMatch - The request signature we calculated does not match the signature you provided. Check your key and signing method. - 10.5072/FK2/KGFCEJ/18b8c06688c-21b8320a3ee5 - mybucket - /mybucket/10.5072/FK2/KGFCEJ/18b8c06688c-21b8320a3ee5 - 1793915CCC5BC95C - dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8 - - */ uploadFileDirect.then().assertThat().statusCode(200); // TODO: Use MD5 or whatever Dataverse is configured for and @@ -663,7 +631,7 @@ public void testDirectUploadWithFileCountLimit() throws JsonParseException { "status": "OK", "data": { "LocalStack": "localstack1", - "MinIO": "minio1", + "LocalStackNoRedirect": "localstack_noredirect", "Local": "local", "Filesystem": "file1" }