diff --git a/.github/workflows/pic-sure-services-test.yml b/.github/workflows/pic-sure-services-test.yml
new file mode 100644
index 000000000..1160494ba
--- /dev/null
+++ b/.github/workflows/pic-sure-services-test.yml
@@ -0,0 +1,22 @@
+name: pic-sure-services tests
+on:
+ pull_request:
+ paths: ['services/pic-sure-services/**']
+ push:
+ branches: [main]
+ paths: ['services/pic-sure-services/**']
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
+ - name: Set up JDK 25
+ uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 #v5.2.0
+ with:
+ java-version: '25'
+ distribution: 'temurin'
+ cache: maven
+ - name: Test with Maven
+ run: mvn -B -pl services/pic-sure-services/info-service,services/pic-sure-services/uploader -am verify
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
diff --git a/pom.xml b/pom.xml
index d113ab150..ce55a69f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -17,6 +17,8 @@
services/pic-sure-hpds-query-service
services/pic-sure-visualization-service
+ services/pic-sure-services/info-service
+ services/pic-sure-services/uploader
diff --git a/services/pic-sure-services/info-service/.sdkmanrc b/services/pic-sure-services/info-service/.sdkmanrc
new file mode 100644
index 000000000..d2344323e
--- /dev/null
+++ b/services/pic-sure-services/info-service/.sdkmanrc
@@ -0,0 +1 @@
+java=25.0.3-tem
diff --git a/services/pic-sure-services/info-service/Dockerfile b/services/pic-sure-services/info-service/Dockerfile
index 02fbd5683..90118c769 100644
--- a/services/pic-sure-services/info-service/Dockerfile
+++ b/services/pic-sure-services/info-service/Dockerfile
@@ -1,17 +1,9 @@
-FROM maven:3-amazoncorretto-21 as build
+FROM amazoncorretto:25
-COPY pom.xml .
-
-RUN mvn -B dependency:go-offline
-
-COPY src src
-
-RUN mvn -B package -DskipTests
-
-FROM amazoncorretto:21.0.1-alpine3.18
-
-# Copy jar and access token from maven build
-COPY --from=build target/*.jar /service.jar
+# Runtime-only image (reactor convention): build the jar first with
+# mvn -pl services/pic-sure-services/info-service -am package
+# from the monorepo root, then build this image from the module directory.
+COPY target/infoservice-*.jar /service.jar
# Time zone
ENV TZ="US/Eastern"
diff --git a/services/pic-sure-services/info-service/pom.xml b/services/pic-sure-services/info-service/pom.xml
index 576160bf9..884bc1c0c 100644
--- a/services/pic-sure-services/info-service/pom.xml
+++ b/services/pic-sure-services/info-service/pom.xml
@@ -3,37 +3,19 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
4.0.0
- org.springframework.boot
- spring-boot-starter-parent
- 3.1.5
-
+ edu.harvard.hms.dbmi.avillach
+ pic-sure-api
+ ${revision}
+ ../../../pom.xml
edu.harvard.dbmi.avillach
infoservice
- 0.0.1-SNAPSHOT
Info Service
Info Service API
-
-
- jitpack.io
- https://jitpack.io
-
-
-
- 21
-
- com.github.hms-dbmi
- pic-sure
- e421ae9caff5ed02e9cd27812edfe24ec455281c
-
-
-
- org.slf4j
- *
-
-
+ edu.harvard.hms.dbmi.avillach
+ pic-sure-api-model
org.springframework.boot
diff --git a/services/pic-sure-services/info-service/src/main/java/edu/harvard/dbmi/avillach/infoservice/InfoServiceApplication.java b/services/pic-sure-services/info-service/src/main/java/edu/harvard/dbmi/avillach/infoservice/InfoServiceApplication.java
index 85791fde4..cde4e5f78 100644
--- a/services/pic-sure-services/info-service/src/main/java/edu/harvard/dbmi/avillach/infoservice/InfoServiceApplication.java
+++ b/services/pic-sure-services/info-service/src/main/java/edu/harvard/dbmi/avillach/infoservice/InfoServiceApplication.java
@@ -6,8 +6,8 @@
@SpringBootApplication
public class InfoServiceApplication {
- public static void main(String[] args) {
- SpringApplication.run(InfoServiceApplication.class, args);
- }
+ public static void main(String[] args) {
+ SpringApplication.run(InfoServiceApplication.class, args);
+ }
}
diff --git a/services/pic-sure-services/info-service/src/test/java/edu/harvard/dbmi/avillach/infoservice/InfoControllerTest.java b/services/pic-sure-services/info-service/src/test/java/edu/harvard/dbmi/avillach/infoservice/InfoControllerTest.java
index 2f6c8a0fc..cda2511f2 100644
--- a/services/pic-sure-services/info-service/src/test/java/edu/harvard/dbmi/avillach/infoservice/InfoControllerTest.java
+++ b/services/pic-sure-services/info-service/src/test/java/edu/harvard/dbmi/avillach/infoservice/InfoControllerTest.java
@@ -32,4 +32,4 @@ void shouldDoHealthCheck() {
Assertions.assertEquals(expected.getId(), actual.getBody().getId());
Assertions.assertEquals(expected.getQueryFormats(), actual.getBody().getQueryFormats());
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/info-service/src/test/java/edu/harvard/dbmi/avillach/infoservice/InfoServiceApplicationTests.java b/services/pic-sure-services/info-service/src/test/java/edu/harvard/dbmi/avillach/infoservice/InfoServiceApplicationTests.java
index 1f2370813..005ea62dc 100644
--- a/services/pic-sure-services/info-service/src/test/java/edu/harvard/dbmi/avillach/infoservice/InfoServiceApplicationTests.java
+++ b/services/pic-sure-services/info-service/src/test/java/edu/harvard/dbmi/avillach/infoservice/InfoServiceApplicationTests.java
@@ -6,8 +6,7 @@
@SpringBootTest
class InfoServiceApplicationTests {
- @Test
- void contextLoads() {
- }
+ @Test
+ void contextLoads() {}
}
diff --git a/services/pic-sure-services/uploader/.sdkmanrc b/services/pic-sure-services/uploader/.sdkmanrc
new file mode 100644
index 000000000..d2344323e
--- /dev/null
+++ b/services/pic-sure-services/uploader/.sdkmanrc
@@ -0,0 +1 @@
+java=25.0.3-tem
diff --git a/services/pic-sure-services/uploader/Dockerfile b/services/pic-sure-services/uploader/Dockerfile
index 2e8c8ae7a..3c15162f9 100644
--- a/services/pic-sure-services/uploader/Dockerfile
+++ b/services/pic-sure-services/uploader/Dockerfile
@@ -1,17 +1,9 @@
-FROM maven:3-amazoncorretto-21 as build
+FROM amazoncorretto:25
-COPY pom.xml .
-
-RUN mvn -B dependency:go-offline
-
-COPY src src
-
-RUN mvn -B package -DskipTests
-
-FROM amazoncorretto:21.0.1-alpine3.18
-
-# Copy jar and access token from maven build
-COPY --from=build target/uploader-*.jar /dataupload.jar
+# Runtime-only image (reactor convention): build the jar first with
+# mvn -pl services/pic-sure-services/uploader -am package
+# from the monorepo root, then build this image from the module directory.
+COPY target/uploader-*.jar /dataupload.jar
# Time zone
ENV TZ="US/Eastern"
diff --git a/services/pic-sure-services/uploader/pom.xml b/services/pic-sure-services/uploader/pom.xml
index 836ceb616..2089c83cf 100644
--- a/services/pic-sure-services/uploader/pom.xml
+++ b/services/pic-sure-services/uploader/pom.xml
@@ -3,29 +3,32 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
4.0.0
- org.springframework.boot
- spring-boot-starter-parent
- 3.4.2
-
+ edu.harvard.hms.dbmi.avillach
+ pic-sure-api
+ ${revision}
+ ../../../pom.xml
edu.harvard.dbmi.avillach
uploader
- 0.0.1-SNAPSHOT
dataupload
Data Upload Client
- 21
2.30.24
-
-
- jitpack.io
- https://jitpack.io
-
-
+
+
+
+
+ software.amazon.awssdk
+ bom
+ ${aws.version}
+ pom
+ import
+
+
+
-
-
org.springframework.boot
spring-boot-starter-jdbc
@@ -74,16 +77,12 @@
- com.github.hms-dbmi
- pic-sure
- fdfd61388980b8f3dad74e4c40e715d9350ccee4
-
-
-
- org.slf4j
- *
-
-
+ edu.harvard.hms.dbmi.avillach
+ pic-sure-api-model
+
+
+ org.apache.httpcomponents.client5
+ httpclient5
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/DatauploadApplication.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/DatauploadApplication.java
index 63f5e7c5d..40d8e061b 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/DatauploadApplication.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/DatauploadApplication.java
@@ -6,8 +6,8 @@
@SpringBootApplication
public class DatauploadApplication {
- public static void main(String[] args) {
- SpringApplication.run(DatauploadApplication.class, args);
- }
+ public static void main(String[] args) {
+ SpringApplication.run(DatauploadApplication.class, args);
+ }
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSClientBuilder.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSClientBuilder.java
index 455fc4a01..ce74a0b1b 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSClientBuilder.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSClientBuilder.java
@@ -33,11 +33,8 @@ public class AWSClientBuilder {
@Autowired
public AWSClientBuilder(
- Map sites,
- StsClientProvider stsClientProvider,
- S3ClientBuilder s3ClientBuilder,
- @Autowired(required = false) SdkHttpClient sdkHttpClient,
- @Value("${s3.retain_role:false}") boolean retainRole
+ Map sites, StsClientProvider stsClientProvider, S3ClientBuilder s3ClientBuilder,
+ @Autowired(required = false) SdkHttpClient sdkHttpClient, @Value("${s3.retain_role:false}") boolean retainRole
) {
this.sites = sites;
this.stsClientProvider = stsClientProvider;
@@ -56,24 +53,18 @@ public Optional buildClientForSite(String siteName) {
if (retainRole) {
log.info("s3.retain_role set to true. Will retain current role rather than assuming one for site");
InstanceProfileCredentialsProvider credentialsProvider = InstanceProfileCredentialsProvider.create();
- S3Client client = s3ClientBuilder
- .credentialsProvider(credentialsProvider)
- .build();
+ S3Client client = s3ClientBuilder.credentialsProvider(credentialsProvider).build();
return Optional.of(client);
}
log.info("Found site, making assume role request");
SiteAWSInfo site = sites.get(siteName);
- AssumeRoleRequest roleRequest = AssumeRoleRequest.builder()
- .roleArn(site.roleARN())
- .roleSessionName("test_session" + System.nanoTime())
- .externalId(site.externalId())
- .durationSeconds(60*60) // 1 hour
+ AssumeRoleRequest roleRequest = AssumeRoleRequest.builder().roleArn(site.roleARN())
+ .roleSessionName("test_session" + System.nanoTime()).externalId(site.externalId()).durationSeconds(60 * 60) // 1 hour
.build();
- Optional assumeRoleResponse = stsClientProvider.createClient()
- .map(c -> c.assumeRole(roleRequest))
- .map(AssumeRoleResponse::credentials);
- if (assumeRoleResponse.isEmpty() ) {
+ Optional assumeRoleResponse =
+ stsClientProvider.createClient().map(c -> c.assumeRole(roleRequest)).map(AssumeRoleResponse::credentials);
+ if (assumeRoleResponse.isEmpty()) {
log.error("Error assuming role {} , no credentials returned", site.roleARN());
return Optional.empty();
}
@@ -82,12 +73,9 @@ public Optional buildClientForSite(String siteName) {
log.info("Building S3 client for site {}", site.siteName());
// Use the credentials from the role to create the S3 client
Credentials credentials = assumeRoleResponse.get();
- AwsSessionCredentials sessionCredentials = AwsSessionCredentials.builder()
- .accessKeyId(credentials.accessKeyId())
- .secretAccessKey(credentials.secretAccessKey())
- .sessionToken(credentials.sessionToken())
- .expirationTime(credentials.expiration())
- .build();
+ AwsSessionCredentials sessionCredentials =
+ AwsSessionCredentials.builder().accessKeyId(credentials.accessKeyId()).secretAccessKey(credentials.secretAccessKey())
+ .sessionToken(credentials.sessionToken()).expirationTime(credentials.expiration()).build();
StaticCredentialsProvider provider = StaticCredentialsProvider.create(sessionCredentials);
return Optional.of(buildFromProvider(provider));
}
@@ -97,10 +85,7 @@ private S3Client buildFromProvider(StaticCredentialsProvider provider) {
return s3ClientBuilder.credentialsProvider(provider).build();
}
log.info("Http proxy detected and added to S3 client");
- return s3ClientBuilder
- .credentialsProvider(provider)
- .httpClient(sdkHttpClient)
- .build();
+ return s3ClientBuilder.credentialsProvider(provider).httpClient(sdkHttpClient).build();
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfiguration.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfiguration.java
index 0836dd90f..143ee6d86 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfiguration.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfiguration.java
@@ -55,20 +55,15 @@ public class AWSConfiguration {
@Bean
@ConditionalOnProperty(name = "production", havingValue = "true")
Map roleARNs() {
- boolean badConfig = Stream.of(roleArns, externalIDs, kmsKeyIds, buckets)
- .filter(l -> l.size() != institutions.size())
- .peek(l -> LOG.error("Mismatched aws credentials {}", l))
- .findAny()
- .isPresent();
+ boolean badConfig = Stream.of(roleArns, externalIDs, kmsKeyIds, buckets).filter(l -> l.size() != institutions.size())
+ .peek(l -> LOG.error("Mismatched aws credentials {}", l)).findAny().isPresent();
if (badConfig) {
context.close();
return Map.of();
}
HashMap roles = new HashMap<>();
for (int i = 0; i < institutions.size(); i++) {
- SiteAWSInfo info = new SiteAWSInfo(
- institutions.get(i), roleArns.get(i), externalIDs.get(i), buckets.get(i), kmsKeyIds.get(i)
- );
+ SiteAWSInfo info = new SiteAWSInfo(institutions.get(i), roleArns.get(i), externalIDs.get(i), buckets.get(i), kmsKeyIds.get(i));
roles.put(info.siteName(), info);
}
return roles;
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSCredentialsService.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSCredentialsService.java
index 7e82f0170..3163b301a 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSCredentialsService.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSCredentialsService.java
@@ -28,10 +28,8 @@ public class AWSCredentialsService {
@Autowired
public AWSCredentialsService(
- @Value("${aws.authentication.method:}") String authMethod,
- @Value("${aws.s3.access_key_secret:}") String secret,
- @Value("${aws.s3.access_key_id:}") String key,
- @Value("${aws.s3.session_token:}") String token,
+ @Value("${aws.authentication.method:}") String authMethod, @Value("${aws.s3.access_key_secret:}") String secret,
+ @Value("${aws.s3.access_key_id:}") String key, @Value("${aws.s3.session_token:}") String token,
ConfigurableApplicationContext context
) {
this.authMethod = authMethod;
@@ -42,7 +40,7 @@ public AWSCredentialsService(
}
public AwsCredentials constructCredentials() {
- //noinspection SwitchStatementWithTooFewBranches
+ // noinspection SwitchStatementWithTooFewBranches
return switch (authMethod) {
case "instance-profile" -> createInstanceProfileBasedCredentials();
default -> createUserBasedCredentials();
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java
index 4356681f3..a62574e97 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/aws/S3StateVerifier.java
@@ -46,23 +46,16 @@ private void verifyS3Status() {
private void asyncVerify(SiteAWSInfo institution) {
LOG.info("Checking S3 connection to {} ...", institution.siteName());
- createTempFileWithText(institution)
- .flatMap(p -> uploadFileFromPath(p, institution))
- .orElseThrow();
+ createTempFileWithText(institution).flatMap(p -> uploadFileFromPath(p, institution)).orElseThrow();
LOG.info("S3 connection to {} verified.", institution.siteName());
}
private Optional uploadFileFromPath(Path p, SiteAWSInfo info) {
LOG.info("Verifying upload capabilities");
RequestBody body = RequestBody.fromFile(p.toFile());
- PutObjectRequest request = PutObjectRequest.builder()
- .bucket(info.bucket())
- .serverSideEncryption(ServerSideEncryption.AWS_KMS)
- .ssekmsKeyId(info.kmsKeyID())
- .key(p.getFileName().toString())
- .build();
- return clientBuilder.buildClientForSite(info.siteName())
- .map(client -> client.putObject(request, body))
+ PutObjectRequest request = PutObjectRequest.builder().bucket(info.bucket()).serverSideEncryption(ServerSideEncryption.AWS_KMS)
+ .ssekmsKeyId(info.kmsKeyID()).key(p.getFileName().toString()).build();
+ return clientBuilder.buildClientForSite(info.siteName()).map(client -> client.putObject(request, body))
.map(resp -> p.getFileName().toString());
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java
index 1f9f1562c..c5180fdad 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClient.java
@@ -6,20 +6,14 @@
import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.ResultType;
import edu.harvard.dbmi.avillach.domain.GeneralQueryRequest;
import edu.harvard.dbmi.avillach.domain.QueryRequest;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.entity.StringEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.util.Optional;
+import org.springframework.web.client.RestClient;
+import org.springframework.web.client.RestClientException;
@Service
public class HPDSClient {
@@ -28,11 +22,12 @@ public class HPDSClient {
private static final String HPDS_URI = "http://hpds:8080/PIC-SURE/";
private static final ObjectMapper mapper = new ObjectMapper();
- @Autowired
- private HttpClient client;
+ private final RestClient restClient;
@Autowired
- private HttpClientContext context;
+ public HPDSClient(RestClient restClient) {
+ this.restClient = restClient;
+ }
public boolean writeTestData(Query query) {
return writeData(query, "test_upload");
@@ -60,11 +55,7 @@ public boolean initializeQuery(Query query) {
return false;
}
- Optional maybePost = createPost(HPDS_URI + "query/sync", body);
-
- return maybePost
- .map(this::sendAndVerifyRequest)
- .orElse(false);
+ return sendAndVerifyRequest(HPDS_URI + "query/sync", body);
}
private boolean writeData(Query query, String mode) {
@@ -73,31 +64,15 @@ private boolean writeData(Query query, String mode) {
return false;
}
- Optional maybePost = createPost(HPDS_URI + "write/" + mode, body);
-
- return maybePost
- .map(this::sendAndVerifyRequest)
- .orElse(false);
- }
-
- private Optional createPost(String uri, String body) {
- HttpPost request = new HttpPost(URI.create(uri));
- try {
- request.setEntity(new StringEntity(body));
- } catch (UnsupportedEncodingException e) {
- LOG.error("Error making request body", e);
- return Optional.empty();
- }
- request.setHeader("Content-Type", "application/json");
-
- return Optional.of(request);
+ return sendAndVerifyRequest(HPDS_URI + "write/" + mode, body);
}
- private boolean sendAndVerifyRequest(HttpPost request) {
+ private boolean sendAndVerifyRequest(String uri, String body) {
try {
- HttpResponse response = client.execute(request, context);
- return response.getStatusLine().getStatusCode() == 200;
- } catch (IOException e) {
+ ResponseEntity response =
+ restClient.post().uri(uri).contentType(MediaType.APPLICATION_JSON).body(body).retrieve().toBodilessEntity();
+ return response.getStatusCode().value() == 200;
+ } catch (RestClientException e) {
LOG.error("Error making request", e);
return false;
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java
index 5d1edeeec..39d574e88 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifier.java
@@ -47,10 +47,7 @@ public boolean verifyConnection() {
log.info("File found! Connection to HPDS verified!");
return testData.delete();
}
- log.info(
- "File {} not found. HPDS is running, but the shared directory is probably misconfigured",
- testData.getPath()
- );
+ log.info("File {} not found. HPDS is running, but the shared directory is probably misconfigured", testData.getPath());
return false;
}
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HttpClientConfig.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HttpClientConfig.java
index e1f16f0c5..41ba8e6e4 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HttpClientConfig.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/HttpClientConfig.java
@@ -1,19 +1,16 @@
package edu.harvard.dbmi.avillach.dataupload.hpds;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.util.StringUtils;
+import org.springframework.web.client.RestClient;
import software.amazon.awssdk.http.SdkHttpClient;
import software.amazon.awssdk.http.apache.ApacheHttpClient;
import software.amazon.awssdk.http.apache.ProxyConfiguration;
@@ -29,47 +26,27 @@ public class HttpClientConfig {
private String proxyPassword;
@Bean
- public HttpClient getHttpClient() {
+ public RestClient restClient(RestClient.Builder builder) {
+ CloseableHttpClient httpClient;
if (!StringUtils.hasLength(proxyUser)) {
- return HttpClients.createDefault();
+ httpClient = HttpClients.createDefault();
+ } else {
+ LOG.info("Found proxy user {}, will configure proxy from system properties", proxyUser);
+ httpClient =
+ HttpClients.custom().setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create().setMaxConnTotal(100).build())
+ .useSystemProperties().build();
}
- LOG.info("Found proxy user {}, will configure proxy", proxyUser);
- PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager();
- manager.setMaxTotal(100);
- return HttpClients
- .custom()
- .setConnectionManager(new PoolingHttpClientConnectionManager())
- .useSystemProperties()
- .build();
+ return builder.requestFactory(new HttpComponentsClientHttpRequestFactory(httpClient)).build();
}
-
@Bean
public SdkHttpClient getSdkClient() {
if (!StringUtils.hasLength(proxyUser)) {
return null;
}
LOG.info("Found proxy user {}, will configure sdk proxy", proxyUser);
- ProxyConfiguration proxy = ProxyConfiguration.builder()
- .useSystemPropertyValues(true)
- .username(proxyUser)
- .password(proxyPassword)
- .build();
- return ApacheHttpClient.builder()
- .proxyConfiguration(proxy)
- .build();
- }
-
- @Bean
- public HttpClientContext getClientConfig() {
- if (StringUtils.hasLength(proxyUser) && StringUtils.hasLength(proxyPassword)) {
- HttpClientContext httpClientContext = HttpClientContext.create();
- CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
- credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxyUser, proxyPassword));
- httpClientContext.setCredentialsProvider(credentialsProvider);
-
- return httpClientContext;
- }
- return HttpClientContext.create();
+ ProxyConfiguration proxy =
+ ProxyConfiguration.builder().useSystemPropertyValues(true).username(proxyUser).password(proxyPassword).build();
+ return ApacheHttpClient.builder().proxyConfiguration(proxy).build();
}
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Filter.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Filter.java
index 525fefd92..9bee608ed 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Filter.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Filter.java
@@ -1,101 +1,101 @@
package edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange;
public interface Filter {
- public boolean apply(T value);
-
- public static class DoubleFilter implements Filter {
-
- Double min, max;
-
- public Double getMin() {
- return min;
- }
-
- public void setMin(Double min) {
- this.min = min;
- }
-
- public Double getMax() {
- return max;
- }
-
- public void setMax(Double max) {
- this.max = max;
- }
-
- public DoubleFilter() {
-
- }
-
- public DoubleFilter(Double min, Double max) {
- this.min = min;
- this.max = max;
- }
-
- public boolean apply(Double value) {
- return value >= min && value <= max;
- }
-
- public String toString() {
- String strVal = "";
- if(min != null) {
- strVal = "Greater than " + min;
- if(max != null) {
- strVal += " and ";
- }
- }
- if(max != null) {
- strVal += "Less than " + max;
- }
- return strVal;
- }
- }
- public static class FloatFilter implements Filter {
-
- Float min, max;
-
- public Float getMin() {
- return min;
- }
-
- public void setMin(Float min) {
- this.min = min;
- }
-
- public Float getMax() {
- return max;
- }
-
- public void setMax(Float max) {
- this.max = max;
- }
-
- public FloatFilter() {
-
- }
-
- public FloatFilter(Float min, Float max) {
- this.min = min;
- this.max = max;
- }
-
- public boolean apply(Float value) {
- return value >= min && value <= max;
- }
-
- public String toString() {
- String strVal = "";
- if(min != null) {
- strVal = "Greater than " + min;
- if(max != null) {
- strVal += " and ";
- }
- }
- if(max != null) {
- strVal += "Less than " + max;
- }
- return strVal;
- }
- }
+ public boolean apply(T value);
+
+ public static class DoubleFilter implements Filter {
+
+ Double min, max;
+
+ public Double getMin() {
+ return min;
+ }
+
+ public void setMin(Double min) {
+ this.min = min;
+ }
+
+ public Double getMax() {
+ return max;
+ }
+
+ public void setMax(Double max) {
+ this.max = max;
+ }
+
+ public DoubleFilter() {
+
+ }
+
+ public DoubleFilter(Double min, Double max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ public boolean apply(Double value) {
+ return value >= min && value <= max;
+ }
+
+ public String toString() {
+ String strVal = "";
+ if (min != null) {
+ strVal = "Greater than " + min;
+ if (max != null) {
+ strVal += " and ";
+ }
+ }
+ if (max != null) {
+ strVal += "Less than " + max;
+ }
+ return strVal;
+ }
+ }
+ public static class FloatFilter implements Filter {
+
+ Float min, max;
+
+ public Float getMin() {
+ return min;
+ }
+
+ public void setMin(Float min) {
+ this.min = min;
+ }
+
+ public Float getMax() {
+ return max;
+ }
+
+ public void setMax(Float max) {
+ this.max = max;
+ }
+
+ public FloatFilter() {
+
+ }
+
+ public FloatFilter(Float min, Float max) {
+ this.min = min;
+ this.max = max;
+ }
+
+ public boolean apply(Float value) {
+ return value >= min && value <= max;
+ }
+
+ public String toString() {
+ String strVal = "";
+ if (min != null) {
+ strVal = "Greater than " + min;
+ if (max != null) {
+ strVal += " and ";
+ }
+ }
+ if (max != null) {
+ strVal += "Less than " + max;
+ }
+ return strVal;
+ }
+ }
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java
index bb8410698..4d995f9d1 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/Query.java
@@ -7,159 +7,163 @@
public class Query {
- private static final Logger log = LoggerFactory.getLogger(Query.class);
-
- public Query() {
-
- }
-
- public Query(Query query) {
- this.expectedResultType = query.expectedResultType;
- this.crossCountFields = new ArrayList(query.crossCountFields);
- this.fields = new ArrayList(query.fields);
- this.requiredFields = new ArrayList(query.requiredFields);
- this.anyRecordOf = new ArrayList(query.anyRecordOf);
- this.numericFilters = new TreeMap(query.numericFilters);
- this.categoryFilters = new TreeMap(query.categoryFilters);
- this.variantInfoFilters = new ArrayList();
- if (query.variantInfoFilters != null) {
- query.variantInfoFilters.forEach((filter) -> {
- this.variantInfoFilters.add(new VariantInfoFilter(filter));
- });
- }
- this.id = query.id;
- this.picSureId = query.picSureId;
- }
-
- private ResultType expectedResultType = ResultType.COUNT;
- private List crossCountFields = new ArrayList<>();
- private List fields = new ArrayList<>();
- private List requiredFields = new ArrayList<>();
- private List anyRecordOf = new ArrayList<>();
- private List> anyRecordOfMulti = new ArrayList<>();
- private Map numericFilters = new HashMap<>();
- private Map categoryFilters = new HashMap<>();
- private List variantInfoFilters = new ArrayList<>();
- private String id;
-
- private String picSureId;
-
-
- public ResultType getExpectedResultType() {
- return expectedResultType;
- }
-
- public List getCrossCountFields() {
- return crossCountFields;
- }
-
- public List getFields() {
- return fields;
- }
-
- public List getRequiredFields() {
- return requiredFields;
- }
-
- public List getAnyRecordOf() {
- return anyRecordOf;
- }
- public List> getAnyRecordOfMulti() {
- return anyRecordOfMulti;
- }
- public List> getAllAnyRecordOf() {
- List> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti);
- anyRecordOfMultiCopy.add(anyRecordOf);
- return anyRecordOfMultiCopy;
- }
-
- public Map getNumericFilters() {
- return numericFilters;
- }
-
- public Map getCategoryFilters() {
- return categoryFilters;
- }
-
- public List getVariantInfoFilters() {
- return variantInfoFilters;
- }
-
- public String getId() {
- return id;
- }
-
- public void setExpectedResultType(ResultType expectedResultType) {
- this.expectedResultType = expectedResultType;
- }
-
- public void setCrossCountFields(Collection crossCountFields) {
- this.crossCountFields = crossCountFields != null ? new ArrayList<>(crossCountFields) : new ArrayList<>();
- }
-
- public void setFields(Collection fields) {
- this.fields = fields != null ? new ArrayList<>(fields) : new ArrayList<>();
- }
-
- public void setRequiredFields(Collection requiredFields) {
- this.requiredFields = requiredFields!= null ? new ArrayList<>(requiredFields) : new ArrayList<>();
- }
-
- public void setAnyRecordOf(Collection anyRecordOf) {
- this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>();
- }
- public void setAnyRecordOfMulti(Collection> anyRecordOfMulti) {
- this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>();
- }
-
- public void setNumericFilters(Map numericFilters) {
- this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>();
- }
-
- public void setCategoryFilters(Map categoryFilters) {
- this.categoryFilters = categoryFilters != null ? new HashMap<>(categoryFilters) : new HashMap<>();
- }
-
- public void setVariantInfoFilters(Collection variantInfoFilters) {
- this.variantInfoFilters = variantInfoFilters != null ? new ArrayList<>(variantInfoFilters) : new ArrayList<>();
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public String getPicSureId() {
- return picSureId;
- }
-
- public void setPicSureId(String picSureId) {
- this.picSureId = picSureId;
- }
-
- public static class VariantInfoFilter {
- public VariantInfoFilter() {
-
- }
-
- public VariantInfoFilter(VariantInfoFilter filter) {
- this.numericVariantInfoFilters = new TreeMap<>(filter.numericVariantInfoFilters);
- this.categoryVariantInfoFilters = new TreeMap<>(filter.categoryVariantInfoFilters);
- }
-
- public Map numericVariantInfoFilters;
- public Map categoryVariantInfoFilters;
- }
-
- @Override
- public boolean equals(Object object) {
- if (this == object) return true;
- if (object == null || getClass() != object.getClass()) return false;
- Query query = (Query) object;
- return getExpectedResultType() == query.getExpectedResultType() && Objects.equals(getId(), query.getId()) && Objects.equals(getPicSureId(), query.getPicSureId());
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(getExpectedResultType(), getId(), getPicSureId());
- }
+ private static final Logger log = LoggerFactory.getLogger(Query.class);
+
+ public Query() {
+
+ }
+
+ public Query(Query query) {
+ this.expectedResultType = query.expectedResultType;
+ this.crossCountFields = new ArrayList(query.crossCountFields);
+ this.fields = new ArrayList(query.fields);
+ this.requiredFields = new ArrayList(query.requiredFields);
+ this.anyRecordOf = new ArrayList(query.anyRecordOf);
+ this.numericFilters = new TreeMap(query.numericFilters);
+ this.categoryFilters = new TreeMap(query.categoryFilters);
+ this.variantInfoFilters = new ArrayList();
+ if (query.variantInfoFilters != null) {
+ query.variantInfoFilters.forEach((filter) -> {
+ this.variantInfoFilters.add(new VariantInfoFilter(filter));
+ });
+ }
+ this.id = query.id;
+ this.picSureId = query.picSureId;
+ }
+
+ private ResultType expectedResultType = ResultType.COUNT;
+ private List crossCountFields = new ArrayList<>();
+ private List fields = new ArrayList<>();
+ private List requiredFields = new ArrayList<>();
+ private List anyRecordOf = new ArrayList<>();
+ private List> anyRecordOfMulti = new ArrayList<>();
+ private Map numericFilters = new HashMap<>();
+ private Map categoryFilters = new HashMap<>();
+ private List variantInfoFilters = new ArrayList<>();
+ private String id;
+
+ private String picSureId;
+
+
+ public ResultType getExpectedResultType() {
+ return expectedResultType;
+ }
+
+ public List getCrossCountFields() {
+ return crossCountFields;
+ }
+
+ public List getFields() {
+ return fields;
+ }
+
+ public List getRequiredFields() {
+ return requiredFields;
+ }
+
+ public List getAnyRecordOf() {
+ return anyRecordOf;
+ }
+
+ public List> getAnyRecordOfMulti() {
+ return anyRecordOfMulti;
+ }
+
+ public List> getAllAnyRecordOf() {
+ List> anyRecordOfMultiCopy = new ArrayList<>(anyRecordOfMulti);
+ anyRecordOfMultiCopy.add(anyRecordOf);
+ return anyRecordOfMultiCopy;
+ }
+
+ public Map getNumericFilters() {
+ return numericFilters;
+ }
+
+ public Map getCategoryFilters() {
+ return categoryFilters;
+ }
+
+ public List getVariantInfoFilters() {
+ return variantInfoFilters;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setExpectedResultType(ResultType expectedResultType) {
+ this.expectedResultType = expectedResultType;
+ }
+
+ public void setCrossCountFields(Collection crossCountFields) {
+ this.crossCountFields = crossCountFields != null ? new ArrayList<>(crossCountFields) : new ArrayList<>();
+ }
+
+ public void setFields(Collection fields) {
+ this.fields = fields != null ? new ArrayList<>(fields) : new ArrayList<>();
+ }
+
+ public void setRequiredFields(Collection requiredFields) {
+ this.requiredFields = requiredFields != null ? new ArrayList<>(requiredFields) : new ArrayList<>();
+ }
+
+ public void setAnyRecordOf(Collection anyRecordOf) {
+ this.anyRecordOf = anyRecordOf != null ? new ArrayList<>(anyRecordOf) : new ArrayList<>();
+ }
+
+ public void setAnyRecordOfMulti(Collection> anyRecordOfMulti) {
+ this.anyRecordOfMulti = anyRecordOfMulti != null ? new ArrayList<>(anyRecordOfMulti) : new ArrayList<>();
+ }
+
+ public void setNumericFilters(Map numericFilters) {
+ this.numericFilters = numericFilters != null ? new HashMap<>(numericFilters) : new HashMap<>();
+ }
+
+ public void setCategoryFilters(Map categoryFilters) {
+ this.categoryFilters = categoryFilters != null ? new HashMap<>(categoryFilters) : new HashMap<>();
+ }
+
+ public void setVariantInfoFilters(Collection variantInfoFilters) {
+ this.variantInfoFilters = variantInfoFilters != null ? new ArrayList<>(variantInfoFilters) : new ArrayList<>();
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getPicSureId() {
+ return picSureId;
+ }
+
+ public void setPicSureId(String picSureId) {
+ this.picSureId = picSureId;
+ }
+
+ public static class VariantInfoFilter {
+ public VariantInfoFilter() {
+
+ }
+
+ public VariantInfoFilter(VariantInfoFilter filter) {
+ this.numericVariantInfoFilters = new TreeMap<>(filter.numericVariantInfoFilters);
+ this.categoryVariantInfoFilters = new TreeMap<>(filter.categoryVariantInfoFilters);
+ }
+
+ public Map numericVariantInfoFilters;
+ public Map categoryVariantInfoFilters;
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (this == object) return true;
+ if (object == null || getClass() != object.getClass()) return false;
+ Query query = (Query) object;
+ return getExpectedResultType() == query.getExpectedResultType() && Objects.equals(getId(), query.getId())
+ && Objects.equals(getPicSureId(), query.getPicSureId());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getExpectedResultType(), getId(), getPicSureId());
+ }
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/ResultType.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/ResultType.java
index 5443627e0..964e8095c 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/ResultType.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/hpds/hpdsartifactsdonotchange/ResultType.java
@@ -1,103 +1,85 @@
package edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange;
public enum ResultType {
- /**
- * Just a patient count
- */
- COUNT,
- /**
- * Return a CSV with an observation for each concept for each
- * patient included in the query. If there are multiple facts
- * for a specific patient, you will get the one that happens
- * to resolve from a binary search of the facts for the patient
- * id.
- */
- DATAFRAME,
+ /**
+ * Just a patient count
+ */
+ COUNT,
+ /**
+ * Return a CSV with an observation for each concept for each patient included in the query. If there are multiple facts for a specific
+ * patient, you will get the one that happens to resolve from a binary search of the facts for the patient id.
+ */
+ DATAFRAME,
- /**
- * Create a dataframe, but do not allow conventional access to it.
- * Instead, the dataframe will be accessed in the backend only,
- * where it is sent to a S3 bucket by a GIC admin.
- *
- */
- SECRET_ADMIN_DATAFRAME,
- /**
- * Return one patient count for each concept path included in
- * the crossCountFields
- */
- CROSS_COUNT,
- /**
- * Return multiple patient count for each concept and its given variables
- * included in the categoryFilters field
- */
- CATEGORICAL_CROSS_COUNT,
- /**
- * Return one patient count for each concept path included in
- * the numericFilters field
- */
- CONTINUOUS_CROSS_COUNT,
- /**
- * Return all variant info column metadata
- */
- INFO_COLUMN_LISTING,
- /**
- * Return the number of total observations for included patients and
- * included fields.
- */
- OBSERVATION_COUNT,
- /**
- * Return the number of observations for included patients and
- * included fields, broken up across the included cross count fields.
- */
- OBSERVATION_CROSS_COUNT,
- /**
- * This was developed for UDN, but is completely useless and should
- * be deleted.
- */
- DATAFRAME_MERGED,
- /**
- * Not completely implemented and currently dead code. Someone with
- * statistics experience needs to develop a p-value based filter for
- * the subset of patients.
- */
- VARIANTS_OF_INTEREST,
- /**
- * The count is the size of the intersection of VariantSpecs that is
- * the result of applying all INFO filters in the query.
- *
- * This is used by clients to limit queries to reasonable numbers of
- * variants.
- */
- VARIANT_COUNT_FOR_QUERY,
- /**
- * This returns the list of string representations of VariantSpecs
- * involved in a query.
- */
- VARIANT_LIST_FOR_QUERY,
- /**
- * This returns quasi-VCF lines for the variants expressed in the
- * query.
- */
- VCF_EXCERPT,
- /**
- * This returns quasi-VCF lines for the variants expressed in the
- * query without patient data.
- */
- AGGREGATE_VCF_EXCERPT,
- /**
- * This returns data to feed a timeline,
- * TODO: add more details later.
- */
- TIMELINE_DATA,
- /**
- * This exports data in the same format as the HPDS csv loader, which
- * is suitable to time series analysis and/or loading into another
- * instance of HPDS.
- */
- DATAFRAME_TIMESERIES,
+ /**
+ * Create a dataframe, but do not allow conventional access to it. Instead, the dataframe will be accessed in the backend only, where it
+ * is sent to a S3 bucket by a GIC admin.
+ *
+ */
+ SECRET_ADMIN_DATAFRAME,
+ /**
+ * Return one patient count for each concept path included in the crossCountFields
+ */
+ CROSS_COUNT,
+ /**
+ * Return multiple patient count for each concept and its given variables included in the categoryFilters field
+ */
+ CATEGORICAL_CROSS_COUNT,
+ /**
+ * Return one patient count for each concept path included in the numericFilters field
+ */
+ CONTINUOUS_CROSS_COUNT,
+ /**
+ * Return all variant info column metadata
+ */
+ INFO_COLUMN_LISTING,
+ /**
+ * Return the number of total observations for included patients and included fields.
+ */
+ OBSERVATION_COUNT,
+ /**
+ * Return the number of observations for included patients and included fields, broken up across the included cross count fields.
+ */
+ OBSERVATION_CROSS_COUNT,
+ /**
+ * This was developed for UDN, but is completely useless and should be deleted.
+ */
+ DATAFRAME_MERGED,
+ /**
+ * Not completely implemented and currently dead code. Someone with statistics experience needs to develop a p-value based filter for
+ * the subset of patients.
+ */
+ VARIANTS_OF_INTEREST,
+ /**
+ * The count is the size of the intersection of VariantSpecs that is the result of applying all INFO filters in the query.
+ *
+ * This is used by clients to limit queries to reasonable numbers of variants.
+ */
+ VARIANT_COUNT_FOR_QUERY,
+ /**
+ * This returns the list of string representations of VariantSpecs involved in a query.
+ */
+ VARIANT_LIST_FOR_QUERY,
+ /**
+ * This returns quasi-VCF lines for the variants expressed in the query.
+ */
+ VCF_EXCERPT,
+ /**
+ * This returns quasi-VCF lines for the variants expressed in the query without patient data.
+ */
+ AGGREGATE_VCF_EXCERPT,
+ /**
+ * This returns data to feed a timeline, TODO: add more details later.
+ */
+ TIMELINE_DATA,
+ /**
+ * This exports data in the same format as the HPDS csv loader, which is suitable to time series analysis and/or loading into another
+ * instance of HPDS.
+ */
+ DATAFRAME_TIMESERIES,
- /**
- * Patients associated with this query
- */
- PATIENTS
+ /**
+ * Patients associated with this query
+ */
+ PATIENTS
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/site/SiteListing.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/site/SiteListing.java
index b3425171f..3719617e4 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/site/SiteListing.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/site/SiteListing.java
@@ -3,9 +3,8 @@
import java.util.List;
/**
- * A Site is an organization participating in GIC. Don't say institute!
- * These names should be all lower case abbreviations for the site:
- * bch, cchmc, pitt, washu, chop, uthsc, etc
+ * A Site is an organization participating in GIC. Don't say institute! These names should be all lower case abbreviations for the site:
+ * bch, cchmc, pitt, washu, chop, uthsc, etc
* @param sites A list of all sites participating in data uploading
* @param homeSite The site where this application is installed
* @param homeSite The short display for the home site
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatuses.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatuses.java
index 47b100d0c..0e7ca2d6c 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatuses.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatuses.java
@@ -5,7 +5,7 @@
import java.time.LocalDate;
public record DataUploadStatuses(
- UploadStatus genomic, UploadStatus phenotypic, UploadStatus patient, UploadStatus query,
- String queryId, @Nullable LocalDate approved, String site
+ UploadStatus genomic, UploadStatus phenotypic, UploadStatus patient, UploadStatus query, String queryId, @Nullable LocalDate approved,
+ String site
) {
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatusesMapper.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatusesMapper.java
index 25a9e1cdb..c42709fb0 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatusesMapper.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/DataUploadStatusesMapper.java
@@ -19,16 +19,12 @@ public DataUploadStatuses mapRow(ResultSet rs, int rowNum) throws SQLException {
String query = fromDashlessString(rs.getString("QUERY")).toString();
Date approved = rs.getDate("APPROVED");
String site = rs.getString("SITE");
- return new DataUploadStatuses(
- genomic, pheno, patient, queryStatus, query, approved == null ? null : approved.toLocalDate(), site
- );
+ return new DataUploadStatuses(genomic, pheno, patient, queryStatus, query, approved == null ? null : approved.toLocalDate(), site);
}
private UUID fromDashlessString(String uuid) {
- String dashed = uuid.replaceFirst(
- "(\\p{XDigit}{8})(\\p{XDigit}{4})(\\p{XDigit}{4})(\\p{XDigit}{4})(\\p{XDigit}+)",
- "$1-$2-$3-$4-$5"
- );
+ String dashed =
+ uuid.replaceFirst("(\\p{XDigit}{8})(\\p{XDigit}{4})(\\p{XDigit}{4})(\\p{XDigit}{4})(\\p{XDigit}+)", "$1-$2-$3-$4-$5");
return UUID.fromString(dashed);
}
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusController.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusController.java
index 6f207e4e4..e8fa924ec 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusController.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusController.java
@@ -26,19 +26,15 @@ public ResponseEntity getServerStatus() {
@GetMapping("/status/{queryId}")
public ResponseEntity getUploadStatus(@PathVariable("queryId") String queryId) {
- return statusService.getStatus(queryId)
- .map(ResponseEntity::ok)
- .orElse(ResponseEntity.notFound().build());
+ return statusService.getStatus(queryId).map(ResponseEntity::ok).orElse(ResponseEntity.notFound().build());
}
@GetMapping("/status/{queryId}/approve")
public ResponseEntity getUploadStatus(
- @PathVariable("queryId") String queryId,
- @RequestParam("date") String approvalDate
+ @PathVariable("queryId") String queryId, @RequestParam("date") String approvalDate
) {
- return statusService.approve(queryId, parseDate(approvalDate))
- .map(ResponseEntity::ok)
- .orElse(ResponseEntity.internalServerError().build());
+ return statusService.approve(queryId, parseDate(approvalDate)).map(ResponseEntity::ok)
+ .orElse(ResponseEntity.internalServerError().build());
}
private LocalDate parseDate(@Nullable String date) {
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepository.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepository.java
index 5e8af93a6..a689594ee 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepository.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepository.java
@@ -18,76 +18,74 @@ public class StatusRepository {
public Optional getQueryStatus(String queryId) {
String sql = """
- SELECT
- GENOMIC_STATUS, PHENOTYPIC_STATUS, PATIENT_STATUS, QUERY_JSON_STATUS,
- hex(QUERY) as QUERY, APPROVED, SITE
- FROM
- query_status
- WHERE
- QUERY = unhex(?)
- """;
- return template.query(sql, statusesMapper, queryId.replace("-", ""))
- .stream()
- .findFirst();
+ SELECT
+ GENOMIC_STATUS, PHENOTYPIC_STATUS, PATIENT_STATUS, QUERY_JSON_STATUS,
+ hex(QUERY) as QUERY, APPROVED, SITE
+ FROM
+ query_status
+ WHERE
+ QUERY = unhex(?)
+ """;
+ return template.query(sql, statusesMapper, queryId.replace("-", "")).stream().findFirst();
}
public void setGenomicStatus(String queryId, UploadStatus status) {
String sql = """
- INSERT INTO query_status
- (query, genomic_status)
- VALUES (unhex(?), ?)
- ON DUPLICATE KEY UPDATE genomic_status=?
- """;
+ INSERT INTO query_status
+ (query, genomic_status)
+ VALUES (unhex(?), ?)
+ ON DUPLICATE KEY UPDATE genomic_status=?
+ """;
template.update(sql, queryId.replace("-", ""), status.toString(), status.toString());
}
public void setPhenotypicStatus(String queryId, UploadStatus status) {
String sql = """
- INSERT INTO query_status
- (query, phenotypic_status)
- VALUES (unhex(?), ?)
- ON DUPLICATE KEY UPDATE phenotypic_status=?
- """;
+ INSERT INTO query_status
+ (query, phenotypic_status)
+ VALUES (unhex(?), ?)
+ ON DUPLICATE KEY UPDATE phenotypic_status=?
+ """;
template.update(sql, queryId.replace("-", ""), status.toString(), status.toString());
}
public void setApproved(String queryId, LocalDate approvalDate) {
String sql = """
- INSERT INTO query_status
- (QUERY, APPROVED)
- VALUES (unhex(?), ?)
- ON DUPLICATE KEY UPDATE APPROVED=?
- """;
+ INSERT INTO query_status
+ (QUERY, APPROVED)
+ VALUES (unhex(?), ?)
+ ON DUPLICATE KEY UPDATE APPROVED=?
+ """;
template.update(sql, queryId.replace("-", ""), approvalDate, approvalDate);
}
public void setSite(String picSureId, String site) {
String sql = """
- INSERT INTO query_status
- (QUERY, SITE)
- VALUES (unhex(?), ?)
- ON DUPLICATE KEY UPDATE SITE=?
- """;
+ INSERT INTO query_status
+ (QUERY, SITE)
+ VALUES (unhex(?), ?)
+ ON DUPLICATE KEY UPDATE SITE=?
+ """;
template.update(sql, picSureId.replace("-", ""), site, site);
}
public void setPatientStatus(String queryId, UploadStatus status) {
String sql = """
- INSERT INTO query_status
- (query, patient_status)
- VALUES (unhex(?), ?)
- ON DUPLICATE KEY UPDATE patient_status=?
- """;
+ INSERT INTO query_status
+ (query, patient_status)
+ VALUES (unhex(?), ?)
+ ON DUPLICATE KEY UPDATE patient_status=?
+ """;
template.update(sql, queryId.replace("-", ""), status.toString(), status.toString());
}
public void setQueryUploadStatus(String queryId, UploadStatus status) {
String sql = """
- INSERT INTO query_status
- (query, query_json_status)
- VALUES (unhex(?), ?)
- ON DUPLICATE KEY UPDATE query_json_status=?
- """;
+ INSERT INTO query_status
+ (query, query_json_status)
+ VALUES (unhex(?), ?)
+ ON DUPLICATE KEY UPDATE query_json_status=?
+ """;
template.update(sql, queryId.replace("-", ""), status.toString(), status.toString());
}
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/UploadStatus.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/UploadStatus.java
index ab73ccc81..abed3e14d 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/UploadStatus.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/status/UploadStatus.java
@@ -6,9 +6,6 @@ public enum UploadStatus {
Uploading, Querying, Uploaded, Error, Unsent, Unknown, Queued;
public static UploadStatus fromString(String status) {
- return Arrays.stream(UploadStatus.values())
- .filter(v -> v.name().equalsIgnoreCase(status.trim()))
- .findAny()
- .orElse(Unknown);
+ return Arrays.stream(UploadStatus.values()).filter(v -> v.name().equalsIgnoreCase(status.trim())).findAny().orElse(Unknown);
}
}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataType.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataType.java
index 6122b3c9d..3f10b6d3d 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataType.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataType.java
@@ -10,6 +10,7 @@
public enum DataType {
Genomic("genomic_data.tsv"), Phenotypic("phenotypic_data.csv"), Patient("patients.txt");
+
public final String fileName;
DataType(String fileName) {
@@ -31,4 +32,4 @@ public Function getHPDSUpload(HPDSClient client) {
case Patient -> client::writePatientData;
};
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadService.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadService.java
index 58c58bdcd..8f022a4f1 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadService.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadService.java
@@ -70,8 +70,8 @@ public DataUploadStatuses asyncUpload(Query query, String site, DataType dataTyp
dataType.getStatusSetter(statusService).accept(query, UploadStatus.Queued);
Thread.ofVirtual().start(() -> uploadData(query, dataType, site));
statusService.setSite(query, site);
- return statusService.getStatus(query.getPicSureId())
- .orElse(null); // this should never happen. the status object is created during the set calls above
+ return statusService.getStatus(query.getPicSureId()).orElse(null); // this should never happen. the status object is created during
+ // the set calls above
}
protected void uploadData(Query query, DataType dataType, String site) {
@@ -104,7 +104,7 @@ protected void uploadData(Query query, DataType dataType, String site) {
uploadLock.release();
return;
}
-
+
LOG.info("File location verified. Uploading for {} to AWS", query.getPicSureId());
success = uploadFileFromPath(data, roleARNs.get(site), query.getPicSureId());
deleteFile(data);
@@ -120,9 +120,8 @@ protected void uploadData(Query query, DataType dataType, String site) {
}
private void uploadQueryJson(Query query, SiteAWSInfo site) {
- UploadStatus queryUploadStatus = statusService.getStatus(query.getPicSureId())
- .map(DataUploadStatuses::query)
- .orElse(UploadStatus.Unsent);
+ UploadStatus queryUploadStatus =
+ statusService.getStatus(query.getPicSureId()).map(DataUploadStatuses::query).orElse(UploadStatus.Unsent);
if (queryUploadStatus == UploadStatus.Uploaded || queryUploadStatus == UploadStatus.Uploading) {
return;
}
@@ -166,12 +165,9 @@ private boolean uploadFileFromPath(Path p, SiteAWSInfo site, String dir) {
S3Client s3 = maybeClient.get();
LOG.info("Starting multipart upload for file {} to site {} in dir {}", p, site, dir);
- CreateMultipartUploadRequest createRequest = CreateMultipartUploadRequest.builder()
- .bucket(site.bucket())
- .serverSideEncryption(ServerSideEncryption.AWS_KMS)
- .ssekmsKeyId(site.kmsKeyID())
- .key(Path.of(dir, home + "_" + p.getFileName().toString()).toString())
- .build();
+ CreateMultipartUploadRequest createRequest =
+ CreateMultipartUploadRequest.builder().bucket(site.bucket()).serverSideEncryption(ServerSideEncryption.AWS_KMS)
+ .ssekmsKeyId(site.kmsKeyID()).key(Path.of(dir, home + "_" + p.getFileName().toString()).toString()).build();
String uploadId;
try {
uploadId = s3.createMultipartUpload(createRequest).uploadId();
@@ -189,15 +185,10 @@ private boolean uploadFileFromPath(Path p, SiteAWSInfo site, String dir) {
LOG.info("Upload complete! Uploaded {} parts", completedParts.size());
LOG.info("Notifying S3 of completed upload...");
- CompletedMultipartUpload completedUpload = CompletedMultipartUpload.builder()
- .parts(completedParts)
- .build();
+ CompletedMultipartUpload completedUpload = CompletedMultipartUpload.builder().parts(completedParts).build();
- CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder()
- .bucket(site.bucket())
- .key(Path.of(dir, home + "_" + p.getFileName().toString()).toString())
- .uploadId(uploadId)
- .multipartUpload(completedUpload)
+ CompleteMultipartUploadRequest completeRequest = CompleteMultipartUploadRequest.builder().bucket(site.bucket())
+ .key(Path.of(dir, home + "_" + p.getFileName().toString()).toString()).uploadId(uploadId).multipartUpload(completedUpload)
.build();
try {
@@ -225,21 +216,14 @@ private List uploadAllParts(Path p, SiteAWSInfo site, String dir,
LOG.info("Uploading file {} part {}", p.getFileName(), part);
buffer.flip();
- UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
- .bucket(site.bucket())
- .key(Path.of(dir, home + "_" + p.getFileName().toString()).toString())
- .uploadId(uploadId)
- .partNumber(part)
- .contentLength((long) bytesRead)
- .build();
+ UploadPartRequest uploadPartRequest =
+ UploadPartRequest.builder().bucket(site.bucket()).key(Path.of(dir, home + "_" + p.getFileName().toString()).toString())
+ .uploadId(uploadId).partNumber(part).contentLength((long) bytesRead).build();
UploadPartResponse response = s3.uploadPart(uploadPartRequest, RequestBody.fromByteBuffer(buffer));
- completedParts.add(CompletedPart.builder()
- .partNumber(part)
- .eTag(response.eTag())
- .build());
+ completedParts.add(CompletedPart.builder().partNumber(part).eTag(response.eTag()).build());
buffer.clear();
position += bytesRead;
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/lambda/CumulusUploadService.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/lambda/CumulusUploadService.java
index 857ee6e0e..02469efdf 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/lambda/CumulusUploadService.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/lambda/CumulusUploadService.java
@@ -42,6 +42,7 @@ public boolean asyncUpload(Query query) {
Thread.ofVirtual().start(() -> upload(query));
return true;
}
+
private void upload(Query query) {
log.info("Fetching upload URL");
Optional uploadURL = urlFetcher.getPreSignedUploadURL(query.getPicSureId(), "patients.txt");
@@ -84,9 +85,8 @@ private boolean uploadFileToPresignedUrl(String presignedUrlString, Path filePat
connection.setRequestMethod("PUT");
OutputStream out = connection.getOutputStream();
- try (RandomAccessFile file = new RandomAccessFile(filePath.toString(), "r");
- FileChannel inChannel = file.getChannel()) {
- ByteBuffer buffer = ByteBuffer.allocate(8192); //Buffer size is 8k
+ try (RandomAccessFile file = new RandomAccessFile(filePath.toString(), "r"); FileChannel inChannel = file.getChannel()) {
+ ByteBuffer buffer = ByteBuffer.allocate(8192); // Buffer size is 8k
while (inChannel.read(buffer) > 0) {
buffer.flip();
diff --git a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/lambda/POSTUrlFetcher.java b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/lambda/POSTUrlFetcher.java
index d578275da..d0bbb4985 100644
--- a/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/lambda/POSTUrlFetcher.java
+++ b/services/pic-sure-services/uploader/src/main/java/edu/harvard/dbmi/avillach/dataupload/upload/lambda/POSTUrlFetcher.java
@@ -31,9 +31,7 @@ public class POSTUrlFetcher {
@Autowired
public POSTUrlFetcher(
- AWSCredentialsService credentialsService,
- @Value("${aws.region}") String region,
- @Value("${cumulus.lambda:}") String labdaARN,
+ AWSCredentialsService credentialsService, @Value("${aws.region}") String region, @Value("${cumulus.lambda:}") String labdaARN,
@Value("${cumulus.bucket:}") String bucketName
) {
this.credentialsService = credentialsService;
@@ -47,20 +45,12 @@ public Optional getPreSignedUploadURL(String uploadUUID, String fileName
StaticCredentialsProvider credentialsProvider = StaticCredentialsProvider.create(awsCredentials);
- try (
- LambdaClient lambdaClient = LambdaClient.builder()
- .region(region)
- .credentialsProvider(credentialsProvider)
- .build()
- ) {
+ try (LambdaClient lambdaClient = LambdaClient.builder().region(region).credentialsProvider(credentialsProvider).build()) {
String payload = new Payload(uploadUUID, fileName, bucketName).asJson();
log.info("Created upload request payload of {}", payload);
- InvokeRequest invokeRequest = InvokeRequest.builder()
- .functionName(labdaARN)
- .payload(SdkBytes.fromUtf8String(payload))
- .build();
+ InvokeRequest invokeRequest = InvokeRequest.builder().functionName(labdaARN).payload(SdkBytes.fromUtf8String(payload)).build();
log.info("Invoking lambda");
InvokeResponse invokeResponse = lambdaClient.invoke(invokeRequest);
@@ -94,4 +84,4 @@ public String asJson() {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/DatauploadApplicationTests.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/DatauploadApplicationTests.java
index 77c4b69a2..af8c49f1a 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/DatauploadApplicationTests.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/DatauploadApplicationTests.java
@@ -6,8 +6,7 @@
@SpringBootTest
class DatauploadApplicationTests {
- @Test
- void contextLoads() {
- }
+ @Test
+ void contextLoads() {}
}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSClientBuilderTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSClientBuilderTest.java
index 9cfe40da9..015d72971 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSClientBuilderTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSClientBuilderTest.java
@@ -49,17 +49,15 @@ void shouldCreateCredentialsWithoutAssumingRole() {
Mockito.when(sites.containsKey("bch")).thenReturn(true);
Mockito.when(s3ClientBuilder.credentialsProvider(Mockito.any(InstanceProfileCredentialsProvider.class)))
.thenReturn(s3ClientBuilder);
- Mockito.when(s3ClientBuilder.build())
- .thenReturn(s3Client);
- ReflectionTestUtils.setField(subject, "retainRole", true);
+ Mockito.when(s3ClientBuilder.build()).thenReturn(s3Client);
+ ReflectionTestUtils.setField(subject, "retainRole", true);
Optional actual = subject.buildClientForSite("bch");
Assertions.assertEquals(Optional.of(s3Client), actual);
}
@Test
void shouldNotBuildClientIfSiteDNE() {
- Mockito.when(sites.get("Narnia"))
- .thenReturn(null);
+ Mockito.when(sites.get("Narnia")).thenReturn(null);
Optional actual = subject.buildClientForSite("Narnia");
Optional expected = Optional.empty();
@@ -70,20 +68,14 @@ void shouldNotBuildClientIfSiteDNE() {
@Test
void shouldNotBuildClientIfRoleRequestFails() {
SiteAWSInfo siteAWSInfo = new SiteAWSInfo("bch", "aws:arn:420", "external", "bucket", "aws:kms:420");
- Mockito.when(sites.get("bch"))
- .thenReturn(siteAWSInfo);
+ Mockito.when(sites.get("bch")).thenReturn(siteAWSInfo);
Mockito.when(sites.containsKey("bch")).thenReturn(true);
- ArgumentMatcher requestMatcher =
- (r) -> r.roleArn().equals("aws:arn:420")
- && r.roleSessionName().startsWith("test_session")
- && r.externalId().equals("external")
- && r.durationSeconds().equals(3600);
+ ArgumentMatcher requestMatcher = (r) -> r.roleArn().equals("aws:arn:420")
+ && r.roleSessionName().startsWith("test_session") && r.externalId().equals("external") && r.durationSeconds().equals(3600);
AssumeRoleResponse response = Mockito.mock(AssumeRoleResponse.class);
- Mockito.when(stsClient.assumeRole(Mockito.argThat(requestMatcher)))
- .thenReturn(response);
- Mockito.when(stsClientProvider.createClient())
- .thenReturn(Optional.of(stsClient));
+ Mockito.when(stsClient.assumeRole(Mockito.argThat(requestMatcher))).thenReturn(response);
+ Mockito.when(stsClientProvider.createClient()).thenReturn(Optional.of(stsClient));
Optional actual = subject.buildClientForSite("bch");
Optional expected = Optional.empty();
@@ -93,10 +85,9 @@ void shouldNotBuildClientIfRoleRequestFails() {
@Test
void shouldBuildClient() {
- ReflectionTestUtils.setField(subject, "retainRole", false);
+ ReflectionTestUtils.setField(subject, "retainRole", false);
SiteAWSInfo siteAWSInfo = new SiteAWSInfo("bch", "aws:arn:420", "external", "bucket", "aws:kms:420");
- Mockito.when(sites.get("bch"))
- .thenReturn(siteAWSInfo);
+ Mockito.when(sites.get("bch")).thenReturn(siteAWSInfo);
Mockito.when(sites.containsKey("bch")).thenReturn(true);
Credentials credentials = Mockito.mock(Credentials.class);
@@ -105,35 +96,24 @@ void shouldBuildClient() {
Mockito.when(credentials.sessionToken()).thenReturn("session");
Mockito.when(credentials.expiration()).thenReturn(Instant.MAX);
AssumeRoleResponse assumeRoleResponse = Mockito.mock(AssumeRoleResponse.class);
- Mockito.when(assumeRoleResponse.credentials())
- .thenReturn(credentials);
- AwsSessionCredentials sessionCredentials = AwsSessionCredentials.builder()
- .accessKeyId(credentials.accessKeyId())
- .secretAccessKey(credentials.secretAccessKey())
- .sessionToken(credentials.sessionToken())
- .expirationTime(credentials.expiration())
- .build();
- ArgumentMatcher requestMatcher =
- (r) -> r.roleArn().equals("aws:arn:420")
- && r.roleSessionName().startsWith("test_session")
- && r.externalId().equals("external")
- && r.durationSeconds().equals(3600);
- Mockito.when(stsClient.assumeRole(Mockito.argThat(requestMatcher)))
- .thenReturn(assumeRoleResponse);
- Mockito.when(stsClientProvider.createClient())
- .thenReturn(Optional.of(stsClient));
+ Mockito.when(assumeRoleResponse.credentials()).thenReturn(credentials);
+ AwsSessionCredentials sessionCredentials =
+ AwsSessionCredentials.builder().accessKeyId(credentials.accessKeyId()).secretAccessKey(credentials.secretAccessKey())
+ .sessionToken(credentials.sessionToken()).expirationTime(credentials.expiration()).build();
+ ArgumentMatcher requestMatcher = (r) -> r.roleArn().equals("aws:arn:420")
+ && r.roleSessionName().startsWith("test_session") && r.externalId().equals("external") && r.durationSeconds().equals(3600);
+ Mockito.when(stsClient.assumeRole(Mockito.argThat(requestMatcher))).thenReturn(assumeRoleResponse);
+ Mockito.when(stsClientProvider.createClient()).thenReturn(Optional.of(stsClient));
StaticCredentialsProvider provider = StaticCredentialsProvider.create(sessionCredentials);
ArgumentMatcher credsMatcher = (AwsCredentialsProvider p) -> p.toString().equals(provider.toString());
S3Client s3Client = Mockito.mock(S3Client.class);
- Mockito.when(s3ClientBuilder.credentialsProvider(Mockito.argThat(credsMatcher)))
- .thenReturn(s3ClientBuilder);
- Mockito.when(s3ClientBuilder.build())
- .thenReturn(s3Client);
+ Mockito.when(s3ClientBuilder.credentialsProvider(Mockito.argThat(credsMatcher))).thenReturn(s3ClientBuilder);
+ Mockito.when(s3ClientBuilder.build()).thenReturn(s3Client);
Optional actual = subject.buildClientForSite("bch");
Optional expected = Optional.of(s3Client);
Assertions.assertEquals(expected, actual);
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfigurationTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfigurationTest.java
index a288825f8..40b1e9c3f 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfigurationTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSConfigurationTest.java
@@ -42,10 +42,8 @@ void shouldCreateRoles() {
ReflectionTestUtils.setField(subject, "kmsKeyIds", List.of("k1", "k2"));
Map actual = subject.roleARNs();
- Map expected = Map.of(
- "i1", new SiteAWSInfo("i1", ":)", "frodo", "b1", "k1"),
- "i2", new SiteAWSInfo("i2", ">:|", "gimli", "b2", "k2")
- );
+ Map expected =
+ Map.of("i1", new SiteAWSInfo("i1", ":)", "frodo", "b1", "k1"), "i2", new SiteAWSInfo("i2", ">:|", "gimli", "b2", "k2"));
Assertions.assertEquals(expected, actual);
}
@@ -62,4 +60,4 @@ void shouldNotCreateRoles() {
Mockito.verify(context, Mockito.times(1)).close();
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSCredentialsServiceTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSCredentialsServiceTest.java
index f12ec3f77..81d799aac 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSCredentialsServiceTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/aws/AWSCredentialsServiceTest.java
@@ -44,4 +44,4 @@ void shouldNotCreateCredentials() {
Mockito.verify(context, Mockito.times(1)).close();
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java
index 204393192..4b0ae6c4b 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSClientTest.java
@@ -1,83 +1,49 @@
package edu.harvard.dbmi.avillach.dataupload.hpds;
import edu.harvard.dbmi.avillach.dataupload.hpds.hpdsartifactsdonotchange.Query;
-import org.apache.http.HttpResponse;
-import org.apache.http.StatusLine;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.protocol.HttpClientContext;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.test.web.client.MockRestServiceServer;
+import org.springframework.web.client.RestClient;
-import java.io.IOException;
+import static org.springframework.test.web.client.match.MockRestRequestMatchers.method;
+import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo;
+import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus;
+import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess;
-@SpringBootTest
class HPDSClientTest {
- @Mock
- HttpClient client;
+ private static final String HPDS_URI = "http://hpds:8080/PIC-SURE/";
- @Mock
- HttpClientContext context;
+ private MockRestServiceServer server;
+ private HPDSClient subject;
- @Mock
- HttpResponse response;
-
- @Mock
- StatusLine line;
-
- @InjectMocks
- HPDSClient subject;
-
- @Test
- void shouldInitializeQuery() throws IOException, InterruptedException {
- Query query = new Query();
- query.setPicSureId("my id");
-
- Mockito.when(response.getStatusLine())
- .thenReturn(line);
- Mockito.when(line.getStatusCode())
- .thenReturn(200);
- Mockito.when(client.execute(Mockito.any(), Mockito.eq(context)))
- .thenReturn(response);
-
- boolean actual = subject.initializeQuery(query);
-
- Assertions.assertTrue(actual);
+ @BeforeEach
+ void setUp() {
+ RestClient.Builder builder = RestClient.builder();
+ server = MockRestServiceServer.bindTo(builder).build();
+ subject = new HPDSClient(builder.build());
}
@Test
- void shouldNotInitializeQuery() throws IOException {
+ void shouldInitializeQuery() {
Query query = new Query();
query.setPicSureId("my id");
-
- Mockito.when(response.getStatusLine())
- .thenReturn(line);
- Mockito.when(line.getStatusCode())
- .thenReturn(404);
- Mockito.when(client.execute(Mockito.any(), Mockito.eq(context)))
- .thenReturn(response);
+ server.expect(requestTo(HPDS_URI + "query/sync")).andExpect(method(HttpMethod.POST)).andRespond(withSuccess());
boolean actual = subject.initializeQuery(query);
- Assertions.assertFalse(actual);
+ Assertions.assertTrue(actual);
}
@Test
- void shouldHandleException() throws IOException, InterruptedException {
+ void shouldNotInitializeQuery() {
Query query = new Query();
query.setPicSureId("my id");
-
- Mockito.when(response.getStatusLine())
- .thenReturn(line);
- Mockito.when(line.getStatusCode())
- .thenReturn(404);
- Mockito.when(client.execute(Mockito.any(), Mockito.eq(context)))
- .thenReturn(response);
+ server.expect(requestTo(HPDS_URI + "query/sync")).andExpect(method(HttpMethod.POST)).andRespond(withStatus(HttpStatus.NOT_FOUND));
boolean actual = subject.initializeQuery(query);
@@ -85,16 +51,10 @@ void shouldHandleException() throws IOException, InterruptedException {
}
@Test
- void shouldWritePhenotypicData() throws IOException, InterruptedException {
+ void shouldWritePhenotypicData() {
Query query = new Query();
query.setPicSureId("my id");
-
- Mockito.when(response.getStatusLine())
- .thenReturn(line);
- Mockito.when(line.getStatusCode())
- .thenReturn(200);
- Mockito.when(client.execute(Mockito.any(), Mockito.eq(context)))
- .thenReturn(response);
+ server.expect(requestTo(HPDS_URI + "write/phenotypic")).andExpect(method(HttpMethod.POST)).andRespond(withSuccess());
boolean actual = subject.writePhenotypicData(query);
@@ -102,16 +62,11 @@ void shouldWritePhenotypicData() throws IOException, InterruptedException {
}
@Test
- void shouldNotWriteGenomicData() throws IOException, InterruptedException {
+ void shouldNotWriteGenomicData() {
Query query = new Query();
query.setPicSureId("my id");
-
- Mockito.when(response.getStatusLine())
- .thenReturn(line);
- Mockito.when(line.getStatusCode())
- .thenReturn(500);
- Mockito.when(client.execute(Mockito.any(), Mockito.eq(context)))
- .thenReturn(response);
+ server.expect(requestTo(HPDS_URI + "write/genomic")).andExpect(method(HttpMethod.POST))
+ .andRespond(withStatus(HttpStatus.INTERNAL_SERVER_ERROR));
boolean actual = subject.writeGenomicData(query);
@@ -119,19 +74,13 @@ void shouldNotWriteGenomicData() throws IOException, InterruptedException {
}
@Test
- void shouldWriteTestData() throws IOException {
+ void shouldWriteTestData() {
Query query = new Query();
query.setPicSureId("my id");
-
- Mockito.when(response.getStatusLine())
- .thenReturn(line);
- Mockito.when(line.getStatusCode())
- .thenReturn(200);
- Mockito.when(client.execute(Mockito.any(), Mockito.eq(context)))
- .thenReturn(response);
+ server.expect(requestTo(HPDS_URI + "write/test_upload")).andExpect(method(HttpMethod.POST)).andRespond(withSuccess());
boolean actual = subject.writeTestData(query);
Assertions.assertTrue(actual);
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java
index 0e20ec476..45aca06a5 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/hpds/HPDSConnectionVerifierTest.java
@@ -32,32 +32,26 @@ class HPDSConnectionVerifierTest {
@Test
void shouldFailWhenHPDS400s(@TempDir Path sharingRoot) {
- Mockito.when(client.writeTestData(query))
- .thenReturn(false);
- Mockito.when(generator.generate())
- .thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
+ Mockito.when(client.writeTestData(query)).thenReturn(false);
+ Mockito.when(generator.generate()).thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator);
boolean result = subject.verifyConnection();
Assertions.assertFalse(result);
- Mockito.verify(client, Mockito.times(1))
- .writeTestData(query);
+ Mockito.verify(client, Mockito.times(1)).writeTestData(query);
}
@Test
void shouldFailWhenHPDSDoesNotWrite(@TempDir Path sharingRoot) {
- Mockito.when(client.writeTestData(query))
- .thenReturn(true);
- Mockito.when(generator.generate())
- .thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
+ Mockito.when(client.writeTestData(query)).thenReturn(true);
+ Mockito.when(generator.generate()).thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator);
boolean result = subject.verifyConnection();
Assertions.assertFalse(result);
- Mockito.verify(client, Mockito.times(1))
- .writeTestData(query);
+ Mockito.verify(client, Mockito.times(1)).writeTestData(query);
}
@Test
@@ -65,33 +59,24 @@ void shouldFailWhenHPDSMakesADirectory(@TempDir Path sharingRoot) throws IOExcep
Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e"));
Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e", "test_data.txt"));
- Mockito.when(client.writeTestData(query))
- .thenReturn(true);
- Mockito.when(generator.generate())
- .thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
+ Mockito.when(client.writeTestData(query)).thenReturn(true);
+ Mockito.when(generator.generate()).thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator);
Assertions.assertFalse(subject.verifyConnection());
- Mockito.verify(client, Mockito.times(1))
- .writeTestData(query);
+ Mockito.verify(client, Mockito.times(1)).writeTestData(query);
}
@Test
void shouldPass(@TempDir Path sharingRoot) throws IOException {
Files.createDirectory(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e"));
- Files.writeString(
- Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e", "test_data.txt"),
- "Howdy :)"
- );
-
- Mockito.when(client.writeTestData(query))
- .thenReturn(true);
- Mockito.when(generator.generate())
- .thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
+ Files.writeString(Path.of(sharingRoot.toString(), "9f1fc383-611b-4c6a-af37-a33c07feea5e", "test_data.txt"), "Howdy :)");
+
+ Mockito.when(client.writeTestData(query)).thenReturn(true);
+ Mockito.when(generator.generate()).thenReturn(UUID.fromString("9f1fc383-611b-4c6a-af37-a33c07feea5e"));
HPDSConnectionVerifier subject = new HPDSConnectionVerifier(client, sharingRoot, generator);
Assertions.assertTrue(subject.verifyConnection());
- Mockito.verify(client, Mockito.times(1))
- .writeTestData(query);
+ Mockito.verify(client, Mockito.times(1)).writeTestData(query);
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/site/SiteConfigurationTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/site/SiteConfigurationTest.java
index 186bff560..997f1ee9a 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/site/SiteConfigurationTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/site/SiteConfigurationTest.java
@@ -42,4 +42,4 @@ void shouldNotGetSiteInfo() {
Mockito.verify(context, Mockito.times(1)).close();
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/site/SiteInfoControllerTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/site/SiteInfoControllerTest.java
index 457ce846b..fc47d4f22 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/site/SiteInfoControllerTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/site/SiteInfoControllerTest.java
@@ -23,4 +23,4 @@ void shouldListSites() {
Assertions.assertEquals(listing, actual.getBody());
Assertions.assertTrue(actual.getStatusCode().is2xxSuccessful());
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepositoryTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepositoryTest.java
index 2e3267a41..bbd787773 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepositoryTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusRepositoryTest.java
@@ -22,8 +22,7 @@
class StatusRepositoryTest {
@Container
- static final MySQLContainer databaseContainer =
- new MySQLContainer<>("mysql:8.0").withReuse(true);
+ static final MySQLContainer databaseContainer = new MySQLContainer<>("mysql:8.0").withReuse(true);
@DynamicPropertySource
static void mySQLProperties(DynamicPropertyRegistry registry) {
@@ -41,8 +40,7 @@ void shouldGetGenomicStatus() {
Query query = new Query();
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
- Optional actual = subject.getQueryStatus(query.getPicSureId())
- .map(DataUploadStatuses::genomic);
+ Optional actual = subject.getQueryStatus(query.getPicSureId()).map(DataUploadStatuses::genomic);
Optional expected = Optional.of(UploadStatus.Uploaded);
Assertions.assertEquals(expected, actual);
@@ -53,8 +51,7 @@ void shouldGetPhenotypicStatus() {
Query query = new Query();
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
- Optional actual = subject.getQueryStatus(query.getPicSureId())
- .map(DataUploadStatuses::phenotypic);
+ Optional actual = subject.getQueryStatus(query.getPicSureId()).map(DataUploadStatuses::phenotypic);
Optional expected = Optional.of(UploadStatus.Error);
Assertions.assertEquals(expected, actual);
@@ -66,8 +63,7 @@ void shouldSetGenomicStatus() {
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
subject.setGenomicStatus(query.getPicSureId(), UploadStatus.Uploading);
- Optional actual = subject.getQueryStatus(query.getPicSureId())
- .map(DataUploadStatuses::genomic);
+ Optional actual = subject.getQueryStatus(query.getPicSureId()).map(DataUploadStatuses::genomic);
Optional expected = Optional.of(UploadStatus.Uploading);
Assertions.assertEquals(expected, actual);
@@ -79,8 +75,7 @@ void shouldSetPhenotypicStatus() {
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
subject.setPhenotypicStatus(query.getPicSureId(), UploadStatus.Unsent);
- Optional actual = subject.getQueryStatus(query.getPicSureId())
- .map(DataUploadStatuses::phenotypic);
+ Optional actual = subject.getQueryStatus(query.getPicSureId()).map(DataUploadStatuses::phenotypic);
Optional expected = Optional.of(UploadStatus.Unsent);
Assertions.assertEquals(expected, actual);
@@ -93,8 +88,7 @@ void shouldSetStatusesOfNewQuery() {
subject.setGenomicStatus(query.getPicSureId(), UploadStatus.Uploading);
- Optional actual = subject.getQueryStatus(query.getPicSureId())
- .map(DataUploadStatuses::phenotypic);
+ Optional actual = subject.getQueryStatus(query.getPicSureId()).map(DataUploadStatuses::phenotypic);
Optional expected = Optional.of(UploadStatus.Unsent);
Assertions.assertEquals(expected, actual);
}
@@ -129,8 +123,8 @@ void shouldGetQueryId() {
DataUploadStatuses actual = subject.getQueryStatus(query.getPicSureId()).orElseThrow();
DataUploadStatuses expected = new DataUploadStatuses(
- UploadStatus.Uploaded, UploadStatus.Error, UploadStatus.Unsent, UploadStatus.Uploading,
- "33613336-3934-3761-2d38-3233312d3131", LocalDate.of(2022, 2, 22), "bch"
+ UploadStatus.Uploaded, UploadStatus.Error, UploadStatus.Unsent, UploadStatus.Uploading, "33613336-3934-3761-2d38-3233312d3131",
+ LocalDate.of(2022, 2, 22), "bch"
);
Assertions.assertEquals(expected, actual);
@@ -142,8 +136,7 @@ void shouldSetSite() {
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
subject.setSite(query.getPicSureId(), "Narnia");
- Optional actual = subject.getQueryStatus(query.getPicSureId())
- .map(DataUploadStatuses::site);
+ Optional actual = subject.getQueryStatus(query.getPicSureId()).map(DataUploadStatuses::site);
Optional expected = Optional.of("Narnia");
Assertions.assertEquals(expected, actual);
@@ -155,8 +148,7 @@ void shouldSetPatient() {
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
subject.setPatientStatus(query.getPicSureId(), UploadStatus.Uploaded);
- Optional actual = subject.getQueryStatus(query.getPicSureId())
- .map(DataUploadStatuses::patient);
+ Optional actual = subject.getQueryStatus(query.getPicSureId()).map(DataUploadStatuses::patient);
Optional expected = Optional.of(UploadStatus.Uploaded);
Assertions.assertEquals(expected, actual);
@@ -168,10 +160,9 @@ void shouldSetQueryStatus() {
query.setPicSureId(UUID.fromString("33613336-3934-3761-2d38-3233312d3131").toString());
subject.setQueryUploadStatus(query.getPicSureId(), UploadStatus.Error);
- Optional actual = subject.getQueryStatus(query.getPicSureId())
- .map(DataUploadStatuses::query);
+ Optional actual = subject.getQueryStatus(query.getPicSureId()).map(DataUploadStatuses::query);
Optional expected = Optional.of(UploadStatus.Error);
Assertions.assertEquals(expected, actual);
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusServiceTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusServiceTest.java
index f1743ebc1..bff8a7172 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusServiceTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/status/StatusServiceTest.java
@@ -76,11 +76,9 @@ void shouldGetQueryStatus() {
Query q = new Query();
q.setPicSureId(":)");
DataUploadStatuses statuses = new DataUploadStatuses(
- UploadStatus.Error, UploadStatus.Error, UploadStatus.Unsent, UploadStatus.Unsent,
- ":)", LocalDate.now(), "bch"
+ UploadStatus.Error, UploadStatus.Error, UploadStatus.Unsent, UploadStatus.Unsent, ":)", LocalDate.now(), "bch"
);
- Mockito.when(repository.getQueryStatus(":)"))
- .thenReturn(Optional.of(statuses));
+ Mockito.when(repository.getQueryStatus(":)")).thenReturn(Optional.of(statuses));
Optional actual = subject.getStatus(q.getPicSureId());
Optional expected = Optional.of(statuses);
@@ -107,4 +105,4 @@ void shouldSetQueryStatus() {
Mockito.verify(repository, Mockito.times(1)).setQueryUploadStatus(":)", UploadStatus.Uploading);
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadControllerTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadControllerTest.java
index 91df87358..3b91aeb84 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadControllerTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadControllerTest.java
@@ -41,23 +41,19 @@ void shouldUpload() {
Query query = new Query();
query.setPicSureId("my id");
DataUploadStatuses before = new DataUploadStatuses(
- UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent,
- query.getPicSureId(), LocalDate.EPOCH, "bch"
+ UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.EPOCH, "bch"
);
DataUploadStatuses after = new DataUploadStatuses(
- UploadStatus.Uploading, UploadStatus.Uploading, UploadStatus.Unsent, UploadStatus.Unsent,
- query.getPicSureId(), LocalDate.EPOCH, "bch"
+ UploadStatus.Uploading, UploadStatus.Uploading, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.EPOCH,
+ "bch"
);
- Mockito.when(statusService.getStatus(query.getPicSureId()))
- .thenReturn(Optional.of(before));
- Mockito.when(uploadService.asyncUpload(query, "bch", DataType.Genomic))
- .thenReturn(after);
+ Mockito.when(statusService.getStatus(query.getPicSureId())).thenReturn(Optional.of(before));
+ Mockito.when(uploadService.asyncUpload(query, "bch", DataType.Genomic)).thenReturn(after);
ResponseEntity actual = subject.startUpload(query, "bch", DataType.Genomic);
ResponseEntity expected = ResponseEntity.ok(after);
- Mockito.verify(uploadService, Mockito.times(1))
- .asyncUpload(query, "bch", DataType.Genomic);
+ Mockito.verify(uploadService, Mockito.times(1)).asyncUpload(query, "bch", DataType.Genomic);
Assertions.assertEquals(expected, actual);
}
@@ -66,8 +62,7 @@ void shouldSayNotFound() {
Query query = new Query();
query.setPicSureId("my id");
- Mockito.when(statusService.getStatus(query.getPicSureId()))
- .thenReturn(Optional.empty());
+ Mockito.when(statusService.getStatus(query.getPicSureId())).thenReturn(Optional.empty());
ResponseEntity actual = subject.startUpload(query, "bch", DataType.Genomic);
@@ -88,10 +83,10 @@ void shouldSayNotFoundForSite() {
void shouldBlockUnapproved() {
Query query = new Query();
query.setPicSureId("my id");
- DataUploadStatuses nullApprovalDate =
- new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), null, "bch");
- Mockito.when(statusService.getStatus(query.getPicSureId()))
- .thenReturn(Optional.of(nullApprovalDate));
+ DataUploadStatuses nullApprovalDate = new DataUploadStatuses(
+ UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), null, "bch"
+ );
+ Mockito.when(statusService.getStatus(query.getPicSureId())).thenReturn(Optional.of(nullApprovalDate));
ResponseEntity actual = subject.startUpload(query, "bch", DataType.Genomic);
@@ -102,14 +97,14 @@ void shouldBlockUnapproved() {
void shouldBlockApprovedInFuture() {
Query query = new Query();
query.setPicSureId("my id");
- DataUploadStatuses nullApprovalDate =
- new DataUploadStatuses(UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.MAX, "bch");
- Mockito.when(statusService.getStatus(query.getPicSureId()))
- .thenReturn(Optional.of(nullApprovalDate));
+ DataUploadStatuses nullApprovalDate = new DataUploadStatuses(
+ UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, query.getPicSureId(), LocalDate.MAX, "bch"
+ );
+ Mockito.when(statusService.getStatus(query.getPicSureId())).thenReturn(Optional.of(nullApprovalDate));
ResponseEntity actual = subject.startUpload(query, "bch", DataType.Genomic);
Assertions.assertEquals(HttpStatus.UNAUTHORIZED, actual.getStatusCode());
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadServiceTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadServiceTest.java
index 3b835b104..a34df7c8d 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadServiceTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/DataUploadServiceTest.java
@@ -102,8 +102,7 @@ void shouldNotUploadDataIfAWSUpset(@TempDir Path tempDir) throws IOException, In
Files.writeString(Path.of(tempDir.toString(), q.getPicSureId(), DataType.Phenotypic.fileName), ":)");
ReflectionTestUtils.setField(subject, "roleARNs", roleARNs);
DataUploadStatuses statuses = new DataUploadStatuses(
- UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Uploaded,
- q.getPicSureId(), LocalDate.MIN, "bch"
+ UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Uploaded, q.getPicSureId(), LocalDate.MIN, "bch"
);
Mockito.when(statusService.getStatus(q.getPicSureId())).thenReturn(Optional.of(statuses));
@@ -111,18 +110,14 @@ void shouldNotUploadDataIfAWSUpset(@TempDir Path tempDir) throws IOException, In
Mockito.when(hpds.writePhenotypicData(q)).thenReturn(true);
Mockito.when(s3.buildClientForSite("bch")).thenReturn(Optional.of(s3Client));
Mockito.when(s3Client.createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class)))
- .thenThrow(AwsServiceException.builder().build());
+ .thenThrow(AwsServiceException.builder().build());
subject.uploadData(q, DataType.Phenotypic, "bch");
- Mockito.verify(statusService, Mockito.times(1)).
- setPhenotypicStatus(q, UploadStatus.Querying);
- Mockito.verify(statusService, Mockito.times(1)).
- setPhenotypicStatus(q, UploadStatus.Uploading);
- Mockito.verify(s3Client, Mockito.times(1))
- .createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class));
- Mockito.verify(statusService, Mockito.times(1)).
- setPhenotypicStatus(q, UploadStatus.Error);
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Querying);
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Uploading);
+ Mockito.verify(s3Client, Mockito.times(1)).createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class));
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Error);
Mockito.verify(uploadLock, Mockito.times(1)).acquire();
Mockito.verify(uploadLock, Mockito.times(1)).release();
}
@@ -139,31 +134,24 @@ void shouldNotUploadDataIfAWSUploadFails(@TempDir Path tempDir) throws IOExcepti
CreateMultipartUploadResponse createResp = Mockito.mock(CreateMultipartUploadResponse.class);
Mockito.when(createResp.uploadId()).thenReturn("frank");
DataUploadStatuses statuses = new DataUploadStatuses(
- UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Uploaded,
- q.getPicSureId(), LocalDate.MIN, "bch"
+ UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Uploaded, q.getPicSureId(), LocalDate.MIN, "bch"
);
Mockito.when(statusService.getStatus(q.getPicSureId())).thenReturn(Optional.of(statuses));
Mockito.when(sharingRoot.toString()).thenReturn(tempDir.toString());
Mockito.when(hpds.writePhenotypicData(q)).thenReturn(true);
Mockito.when(s3.buildClientForSite("bch")).thenReturn(Optional.of(s3Client));
- Mockito.when(s3Client.createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class)))
- .thenReturn(createResp);
+ Mockito.when(s3Client.createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class))).thenReturn(createResp);
Mockito.when(s3Client.uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class)))
.thenThrow(AwsServiceException.builder().build());
subject.uploadData(q, DataType.Phenotypic, "bch");
- Mockito.verify(statusService, Mockito.times(1)).
- setPhenotypicStatus(q, UploadStatus.Querying);
- Mockito.verify(statusService, Mockito.times(1)).
- setPhenotypicStatus(q, UploadStatus.Uploading);
- Mockito.verify(s3Client, Mockito.times(1))
- .createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class));
- Mockito.verify(s3Client, Mockito.times(1))
- .uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class));
- Mockito.verify(statusService, Mockito.times(1)).
- setPhenotypicStatus(q, UploadStatus.Error);
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Querying);
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Uploading);
+ Mockito.verify(s3Client, Mockito.times(1)).createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class));
+ Mockito.verify(s3Client, Mockito.times(1)).uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class));
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Error);
Mockito.verify(uploadLock, Mockito.times(1)).acquire();
Mockito.verify(uploadLock, Mockito.times(1)).release();
}
@@ -184,36 +172,27 @@ void shouldNotUploadDataWhenCompleteFails(@TempDir Path tempDir) throws IOExcept
UploadPartResponse uploadResp = Mockito.mock(UploadPartResponse.class);
Mockito.when(uploadResp.eTag()).thenReturn("gus");
DataUploadStatuses statuses = new DataUploadStatuses(
- UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Uploaded,
- q.getPicSureId(), LocalDate.MIN, "bch"
+ UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Uploaded, q.getPicSureId(), LocalDate.MIN, "bch"
);
Mockito.when(statusService.getStatus(q.getPicSureId())).thenReturn(Optional.of(statuses));
Mockito.when(sharingRoot.toString()).thenReturn(tempDir.toString());
Mockito.when(hpds.writePhenotypicData(q)).thenReturn(true);
Mockito.when(s3.buildClientForSite("bch")).thenReturn(Optional.of(s3Client));
- Mockito.when(s3Client.uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class)))
- .thenReturn(uploadResp);
- Mockito.when(s3Client.createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class)))
- .thenReturn(createResp);
+ Mockito.when(s3Client.uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class))).thenReturn(uploadResp);
+ Mockito.when(s3Client.createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class))).thenReturn(createResp);
Mockito.when(s3Client.completeMultipartUpload(Mockito.any(CompleteMultipartUploadRequest.class)))
- .thenThrow(AwsServiceException.builder().build());
+ .thenThrow(AwsServiceException.builder().build());
subject.uploadData(q, DataType.Phenotypic, "bch");
- Mockito.verify(statusService, Mockito.times(1))
- .setPhenotypicStatus(q, UploadStatus.Querying);
- Mockito.verify(statusService, Mockito.times(1))
- .setPhenotypicStatus(q, UploadStatus.Uploading);
- Mockito.verify(s3Client, Mockito.times(1))
- .createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class));
- Mockito.verify(s3Client, Mockito.times(1))
- .completeMultipartUpload(Mockito.any(CompleteMultipartUploadRequest.class));
- Mockito.verify(s3Client, Mockito.times(1))
- .uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class));
- Mockito.verify(statusService, Mockito.times(1))
- .setPhenotypicStatus(q, UploadStatus.Error);
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Querying);
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Uploading);
+ Mockito.verify(s3Client, Mockito.times(1)).createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class));
+ Mockito.verify(s3Client, Mockito.times(1)).completeMultipartUpload(Mockito.any(CompleteMultipartUploadRequest.class));
+ Mockito.verify(s3Client, Mockito.times(1)).uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class));
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Error);
Assertions.assertFalse(Files.exists(fileToUpload));
Mockito.verify(uploadLock, Mockito.times(1)).acquire();
Mockito.verify(uploadLock, Mockito.times(1)).release();
@@ -236,39 +215,29 @@ void shouldUploadData(@TempDir Path tempDir) throws IOException, InterruptedExce
Mockito.when(uploadResp.eTag()).thenReturn("gus");
DataUploadStatuses statuses = new DataUploadStatuses(
- UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent,
- q.getPicSureId(), LocalDate.MIN, "bch"
+ UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, UploadStatus.Unsent, q.getPicSureId(), LocalDate.MIN, "bch"
);
Mockito.when(statusService.getStatus(q.getPicSureId())).thenReturn(Optional.of(statuses));
Mockito.when(sharingRoot.toString()).thenReturn(tempDir.toString());
Mockito.when(hpds.writePhenotypicData(q)).thenReturn(true);
Mockito.when(s3.buildClientForSite("bch")).thenReturn(Optional.of(s3Client));
- Mockito.when(s3Client.uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class)))
- .thenReturn(uploadResp);
- Mockito.when(s3Client.createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class)))
- .thenReturn(createResp);
+ Mockito.when(s3Client.uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class))).thenReturn(uploadResp);
+ Mockito.when(s3Client.createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class))).thenReturn(createResp);
subject.uploadData(q, DataType.Phenotypic, "bch");
- Mockito.verify(statusService, Mockito.times(1))
- .setPhenotypicStatus(q, UploadStatus.Querying);
- Mockito.verify(statusService, Mockito.times(1))
- .setPhenotypicStatus(q, UploadStatus.Uploading);
- Mockito.verify(statusService, Mockito.times(1))
- .setQueryUploadStatus(q, UploadStatus.Uploading);
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Querying);
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Uploading);
+ Mockito.verify(statusService, Mockito.times(1)).setQueryUploadStatus(q, UploadStatus.Uploading);
Mockito.verify(s3Client, Mockito.times(2)) // query json and pheno data
.createMultipartUpload(Mockito.any(CreateMultipartUploadRequest.class));
- Mockito.verify(s3Client, Mockito.times(2))
- .completeMultipartUpload(Mockito.any(CompleteMultipartUploadRequest.class));
- Mockito.verify(s3Client, Mockito.times(2))
- .uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class));
- Mockito.verify(statusService, Mockito.times(1))
- .setPhenotypicStatus(q, UploadStatus.Uploaded);
- Mockito.verify(statusService, Mockito.times(1))
- .setQueryUploadStatus(q, UploadStatus.Uploaded);
+ Mockito.verify(s3Client, Mockito.times(2)).completeMultipartUpload(Mockito.any(CompleteMultipartUploadRequest.class));
+ Mockito.verify(s3Client, Mockito.times(2)).uploadPart(Mockito.any(UploadPartRequest.class), Mockito.any(RequestBody.class));
+ Mockito.verify(statusService, Mockito.times(1)).setPhenotypicStatus(q, UploadStatus.Uploaded);
+ Mockito.verify(statusService, Mockito.times(1)).setQueryUploadStatus(q, UploadStatus.Uploaded);
Assertions.assertFalse(Files.exists(fileToUpload));
Mockito.verify(uploadLock, Mockito.times(1)).acquire();
Mockito.verify(uploadLock, Mockito.times(1)).release();
}
-}
\ No newline at end of file
+}
diff --git a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/UploadConfigTest.java b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/UploadConfigTest.java
index b61ece056..f1fb3ad28 100644
--- a/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/UploadConfigTest.java
+++ b/services/pic-sure-services/uploader/src/test/java/edu/harvard/dbmi/avillach/dataupload/upload/UploadConfigTest.java
@@ -15,4 +15,4 @@ void shouldGetLock() {
Assertions.assertEquals(1, lock.availablePermits());
}
-}
\ No newline at end of file
+}