Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .github/workflows/pic-sure-services-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: pic-sure-services tests
on:
pull_request:
paths: ['services/pic-sure-services/**']
push:
branches: [main]
paths: ['services/pic-sure-services/**']
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #v6.0.2
- name: Set up JDK 25
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 #v5.2.0
with:
java-version: '25'
distribution: 'temurin'
cache: maven
- name: Test with Maven
run: mvn -B -pl services/pic-sure-services/info-service,services/pic-sure-services/uploader -am verify
env:
GITHUB_TOKEN: ${{ github.token }}
2 changes: 2 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
<module>services/pic-sure-hpds-query-service</module>
<!-- Imported services adopted onto the reactor (carved from pic_sure_api_mono_repo) -->
<module>services/pic-sure-visualization-service</module>
<module>services/pic-sure-services/info-service</module>
<module>services/pic-sure-services/uploader</module>
<!-- pic-sure-legacy is deliberately NOT aggregated: it is quarantined
(Java 11/javax, own parent pom) and builds independently. -->
</modules>
Expand Down
1 change: 1 addition & 0 deletions services/pic-sure-services/info-service/.sdkmanrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
java=25.0.3-tem
18 changes: 5 additions & 13 deletions services/pic-sure-services/info-service/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,17 +1,9 @@
FROM maven:3-amazoncorretto-21 as build
FROM amazoncorretto:25

COPY pom.xml .

RUN mvn -B dependency:go-offline

COPY src src

RUN mvn -B package -DskipTests

FROM amazoncorretto:21.0.1-alpine3.18

# Copy jar and access token from maven build
COPY --from=build target/*.jar /service.jar
# Runtime-only image (reactor convention): build the jar first with
# mvn -pl services/pic-sure-services/info-service -am package
# from the monorepo root, then build this image from the module directory.
COPY target/infoservice-*.jar /service.jar

# Time zone
ENV TZ="US/Eastern"
Expand Down
30 changes: 6 additions & 24 deletions services/pic-sure-services/info-service/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,37 +3,19 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.1.5</version>
<relativePath/> <!-- lookup parent from repository -->
<groupId>edu.harvard.hms.dbmi.avillach</groupId>
<artifactId>pic-sure-api</artifactId>
<version>${revision}</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<groupId>edu.harvard.dbmi.avillach</groupId>
<artifactId>infoservice</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>Info Service</name>
<description>Info Service API</description>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
<properties>
<java.version>21</java.version>
</properties>
<dependencies>
<dependency>
<groupId>com.github.hms-dbmi</groupId>
<artifactId>pic-sure</artifactId>
<version>e421ae9caff5ed02e9cd27812edfe24ec455281c</version>
<exclusions>
<!--Exclude all transitive dependencies. I just want the ResourceInfo class-->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
<groupId>edu.harvard.hms.dbmi.avillach</groupId>
<artifactId>pic-sure-api-model</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
@SpringBootApplication
public class InfoServiceApplication {

public static void main(String[] args) {
SpringApplication.run(InfoServiceApplication.class, args);
}
public static void main(String[] args) {
SpringApplication.run(InfoServiceApplication.class, args);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,4 @@ void shouldDoHealthCheck() {
Assertions.assertEquals(expected.getId(), actual.getBody().getId());
Assertions.assertEquals(expected.getQueryFormats(), actual.getBody().getQueryFormats());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
@SpringBootTest
class InfoServiceApplicationTests {

@Test
void contextLoads() {
}
@Test
void contextLoads() {}

}
1 change: 1 addition & 0 deletions services/pic-sure-services/uploader/.sdkmanrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
java=25.0.3-tem
18 changes: 5 additions & 13 deletions services/pic-sure-services/uploader/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,17 +1,9 @@
FROM maven:3-amazoncorretto-21 as build
FROM amazoncorretto:25

COPY pom.xml .

RUN mvn -B dependency:go-offline

COPY src src

RUN mvn -B package -DskipTests

FROM amazoncorretto:21.0.1-alpine3.18

# Copy jar and access token from maven build
COPY --from=build target/uploader-*.jar /dataupload.jar
# Runtime-only image (reactor convention): build the jar first with
# mvn -pl services/pic-sure-services/uploader -am package
# from the monorepo root, then build this image from the module directory.
COPY target/uploader-*.jar /dataupload.jar

# Time zone
ENV TZ="US/Eastern"
Expand Down
47 changes: 23 additions & 24 deletions services/pic-sure-services/uploader/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,32 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.4.2</version>
<relativePath/> <!-- lookup parent from repository -->
<groupId>edu.harvard.hms.dbmi.avillach</groupId>
<artifactId>pic-sure-api</artifactId>
<version>${revision}</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<groupId>edu.harvard.dbmi.avillach</groupId>
<artifactId>uploader</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>dataupload</name>
<description>Data Upload Client</description>
<properties>
<java.version>21</java.version>
<aws.version>2.30.24</aws.version>
</properties>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
<dependencyManagement>
<dependencies>
<!-- Converges every software.amazon.awssdk artifact (the s3-encryption-client
drags an older s3) onto ${aws.version} — dependencyConvergence enforcement. -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${aws.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!--Checkmarx winges about a vulnerability in snakeyaml, but the CVE is for 1.30, not 1.33. Ignoring-->
<!--suppress VulnerableLibrariesLocal -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
Expand Down Expand Up @@ -74,16 +77,12 @@
</dependency>

<dependency>
<groupId>com.github.hms-dbmi</groupId>
<artifactId>pic-sure</artifactId>
<version>fdfd61388980b8f3dad74e4c40e715d9350ccee4</version>
<exclusions>
<!--Exclude all transitive dependencies. I just want the ResourceInfo class-->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
<groupId>edu.harvard.hms.dbmi.avillach</groupId>
<artifactId>pic-sure-api-model</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
</dependency>

<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
@SpringBootApplication
public class DatauploadApplication {

public static void main(String[] args) {
SpringApplication.run(DatauploadApplication.class, args);
}
public static void main(String[] args) {
SpringApplication.run(DatauploadApplication.class, args);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,8 @@ public class AWSClientBuilder {

@Autowired
public AWSClientBuilder(
Map<String, SiteAWSInfo> sites,
StsClientProvider stsClientProvider,
S3ClientBuilder s3ClientBuilder,
@Autowired(required = false) SdkHttpClient sdkHttpClient,
@Value("${s3.retain_role:false}") boolean retainRole
Map<String, SiteAWSInfo> sites, StsClientProvider stsClientProvider, S3ClientBuilder s3ClientBuilder,
@Autowired(required = false) SdkHttpClient sdkHttpClient, @Value("${s3.retain_role:false}") boolean retainRole
) {
this.sites = sites;
this.stsClientProvider = stsClientProvider;
Expand All @@ -56,24 +53,18 @@ public Optional<S3Client> buildClientForSite(String siteName) {
if (retainRole) {
log.info("s3.retain_role set to true. Will retain current role rather than assuming one for site");
InstanceProfileCredentialsProvider credentialsProvider = InstanceProfileCredentialsProvider.create();
S3Client client = s3ClientBuilder
.credentialsProvider(credentialsProvider)
.build();
S3Client client = s3ClientBuilder.credentialsProvider(credentialsProvider).build();
return Optional.of(client);
}

log.info("Found site, making assume role request");
SiteAWSInfo site = sites.get(siteName);
AssumeRoleRequest roleRequest = AssumeRoleRequest.builder()
.roleArn(site.roleARN())
.roleSessionName("test_session" + System.nanoTime())
.externalId(site.externalId())
.durationSeconds(60*60) // 1 hour
AssumeRoleRequest roleRequest = AssumeRoleRequest.builder().roleArn(site.roleARN())
.roleSessionName("test_session" + System.nanoTime()).externalId(site.externalId()).durationSeconds(60 * 60) // 1 hour
.build();
Optional<Credentials> assumeRoleResponse = stsClientProvider.createClient()
.map(c -> c.assumeRole(roleRequest))
.map(AssumeRoleResponse::credentials);
if (assumeRoleResponse.isEmpty() ) {
Optional<Credentials> assumeRoleResponse =
stsClientProvider.createClient().map(c -> c.assumeRole(roleRequest)).map(AssumeRoleResponse::credentials);
if (assumeRoleResponse.isEmpty()) {
log.error("Error assuming role {} , no credentials returned", site.roleARN());
return Optional.empty();
}
Expand All @@ -82,12 +73,9 @@ public Optional<S3Client> buildClientForSite(String siteName) {
log.info("Building S3 client for site {}", site.siteName());
// Use the credentials from the role to create the S3 client
Credentials credentials = assumeRoleResponse.get();
AwsSessionCredentials sessionCredentials = AwsSessionCredentials.builder()
.accessKeyId(credentials.accessKeyId())
.secretAccessKey(credentials.secretAccessKey())
.sessionToken(credentials.sessionToken())
.expirationTime(credentials.expiration())
.build();
AwsSessionCredentials sessionCredentials =
AwsSessionCredentials.builder().accessKeyId(credentials.accessKeyId()).secretAccessKey(credentials.secretAccessKey())
.sessionToken(credentials.sessionToken()).expirationTime(credentials.expiration()).build();
StaticCredentialsProvider provider = StaticCredentialsProvider.create(sessionCredentials);
return Optional.of(buildFromProvider(provider));
}
Expand All @@ -97,10 +85,7 @@ private S3Client buildFromProvider(StaticCredentialsProvider provider) {
return s3ClientBuilder.credentialsProvider(provider).build();
}
log.info("Http proxy detected and added to S3 client");
return s3ClientBuilder
.credentialsProvider(provider)
.httpClient(sdkHttpClient)
.build();
return s3ClientBuilder.credentialsProvider(provider).httpClient(sdkHttpClient).build();

}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,20 +55,15 @@ public class AWSConfiguration {
@Bean
@ConditionalOnProperty(name = "production", havingValue = "true")
Map<String, SiteAWSInfo> roleARNs() {
boolean badConfig = Stream.of(roleArns, externalIDs, kmsKeyIds, buckets)
.filter(l -> l.size() != institutions.size())
.peek(l -> LOG.error("Mismatched aws credentials {}", l))
.findAny()
.isPresent();
boolean badConfig = Stream.of(roleArns, externalIDs, kmsKeyIds, buckets).filter(l -> l.size() != institutions.size())
.peek(l -> LOG.error("Mismatched aws credentials {}", l)).findAny().isPresent();
if (badConfig) {
context.close();
return Map.of();
}
HashMap<String, SiteAWSInfo> roles = new HashMap<>();
for (int i = 0; i < institutions.size(); i++) {
SiteAWSInfo info = new SiteAWSInfo(
institutions.get(i), roleArns.get(i), externalIDs.get(i), buckets.get(i), kmsKeyIds.get(i)
);
SiteAWSInfo info = new SiteAWSInfo(institutions.get(i), roleArns.get(i), externalIDs.get(i), buckets.get(i), kmsKeyIds.get(i));
roles.put(info.siteName(), info);
}
return roles;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@ public class AWSCredentialsService {

@Autowired
public AWSCredentialsService(
@Value("${aws.authentication.method:}") String authMethod,
@Value("${aws.s3.access_key_secret:}") String secret,
@Value("${aws.s3.access_key_id:}") String key,
@Value("${aws.s3.session_token:}") String token,
@Value("${aws.authentication.method:}") String authMethod, @Value("${aws.s3.access_key_secret:}") String secret,
@Value("${aws.s3.access_key_id:}") String key, @Value("${aws.s3.session_token:}") String token,
ConfigurableApplicationContext context
) {
this.authMethod = authMethod;
Expand All @@ -42,7 +40,7 @@ public AWSCredentialsService(
}

public AwsCredentials constructCredentials() {
//noinspection SwitchStatementWithTooFewBranches
// noinspection SwitchStatementWithTooFewBranches
return switch (authMethod) {
case "instance-profile" -> createInstanceProfileBasedCredentials();
default -> createUserBasedCredentials();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,23 +46,16 @@ private void verifyS3Status() {

private void asyncVerify(SiteAWSInfo institution) {
LOG.info("Checking S3 connection to {} ...", institution.siteName());
createTempFileWithText(institution)
.flatMap(p -> uploadFileFromPath(p, institution))
.orElseThrow();
createTempFileWithText(institution).flatMap(p -> uploadFileFromPath(p, institution)).orElseThrow();
LOG.info("S3 connection to {} verified.", institution.siteName());
}

private Optional<String> uploadFileFromPath(Path p, SiteAWSInfo info) {
LOG.info("Verifying upload capabilities");
RequestBody body = RequestBody.fromFile(p.toFile());
PutObjectRequest request = PutObjectRequest.builder()
.bucket(info.bucket())
.serverSideEncryption(ServerSideEncryption.AWS_KMS)
.ssekmsKeyId(info.kmsKeyID())
.key(p.getFileName().toString())
.build();
return clientBuilder.buildClientForSite(info.siteName())
.map(client -> client.putObject(request, body))
PutObjectRequest request = PutObjectRequest.builder().bucket(info.bucket()).serverSideEncryption(ServerSideEncryption.AWS_KMS)
.ssekmsKeyId(info.kmsKeyID()).key(p.getFileName().toString()).build();
return clientBuilder.buildClientForSite(info.siteName()).map(client -> client.putObject(request, body))
.map(resp -> p.getFileName().toString());
}

Expand Down
Loading
Loading