diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index ca1b6e20a1..b6a21467c0 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -9,6 +9,7 @@ exports org.cloudfoundry.multiapps.controller.persistence.model; exports org.cloudfoundry.multiapps.controller.persistence.model.adapter; exports org.cloudfoundry.multiapps.controller.persistence.model.filters; + exports org.cloudfoundry.multiapps.controller.persistence.monitoring; exports org.cloudfoundry.multiapps.controller.persistence.query; exports org.cloudfoundry.multiapps.controller.persistence.query.criteria; exports org.cloudfoundry.multiapps.controller.persistence.query.impl; @@ -73,4 +74,5 @@ requires software.amazon.awssdk.retries.api; requires static java.compiler; requires static org.immutables.value; + requires io.netty.handler; } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 65fa73fbf9..7ee1d015df 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -67,6 +67,10 @@ public final class Messages { // INFO log messages: public static final String DELETING_FILES_WITHOUT_CONTENT_WITH_IDS_0 = "Deleting files without content with ids: {0}"; + public static final String TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for JClouds object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for GCP object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for Azure object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_AWS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for AWS object store upload: {0} in millis"; // DEBUG log messages: public static final String STORED_FILE_0 = "Stored file: \"{0}\""; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntry.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntry.java index a74c152602..325f2572b6 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntry.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntry.java @@ -1,6 +1,7 @@ package org.cloudfoundry.multiapps.controller.persistence.model; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import org.cloudfoundry.multiapps.common.Nullable; @@ -11,6 +12,10 @@ public interface AsyncUploadJobEntry { String STALE_JOB_DETAILS_FORMAT = "Stale job details - id: {0}, state: {1}, updatedAt: {2}, addedAt: {3}, startedAt: {4}, bytesRead: {5}, url: {6}, space: {7}, namespace: {8}, user: {9}, instance: {10}"; + String ASYNC_UPLOAD_JOB_SUMMARY_FORMAT = "id: {0}, state: {1}, fileId: {2}, mtaId: {3}, schemaVersion: {4}, instanceIndex: {5}, bytesRead: {6}, addedAt: {7}, startedAt: {8}, finishedAt: {9}, updatedAt: {10}, queueWaitTime: {11}, uploadDuration: {12}, totalTime: {13}, error: {14}"; + + String NOT_AVAILABLE = "N/A"; + enum State { INITIAL, RUNNING, FINISHED, ERROR } @@ -61,4 +66,42 @@ default String buildStaleDetailsLogMessage() { return MessageFormat.format(STALE_JOB_DETAILS_FORMAT, getId(), getState(), getUpdatedAt(), getAddedAt(), getStartedAt(), getBytesRead(), getUrl(), getSpaceGuid(), getNamespace(), getUser(), getInstanceIndex()); } + + default String buildLogSummary() { + return MessageFormat.format(ASYNC_UPLOAD_JOB_SUMMARY_FORMAT, getId(), getState(), getFileId(), getMtaId(), getSchemaVersion(), + getInstanceIndex(), getBytesRead(), getAddedAt(), getStartedAt(), getFinishedAt(), getUpdatedAt(), + formatDuration(getQueueWaitTime()), formatDuration(getUploadDuration()), formatDuration(getTotalTime()), + getError()); + } + + @Nullable + default Duration getQueueWaitTime() { + if (getAddedAt() == null || getStartedAt() == null) { + return null; + } + return Duration.between(getAddedAt(), getStartedAt()); + } + + @Nullable + default Duration getUploadDuration() { + if (getStartedAt() == null || getFinishedAt() == null) { + return null; + } + return Duration.between(getStartedAt(), getFinishedAt()); + } + + @Nullable + default Duration getTotalTime() { + if (getAddedAt() == null || getFinishedAt() == null) { + return null; + } + return Duration.between(getAddedAt(), getFinishedAt()); + } + + private static String formatDuration(Duration duration) { + if (duration == null) { + return NOT_AVAILABLE; + } + return duration.toMillis() + " ms"; + } } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java new file mode 100644 index 0000000000..d6ac887b8b --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java @@ -0,0 +1,99 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import jakarta.inject.Named; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +@Named +public class UploadDurationTracker { + + private final UploadPathStatistics appBinaryStatistics = new UploadPathStatistics(); + + private final UploadPathStatistics objectStoreStatistics = new UploadPathStatistics(); + + public void recordAppBinaryUpload(long durationMillis, boolean timedOut) { + appBinaryStatistics.record(durationMillis, timedOut); + } + + public void recordObjectStoreUpload(long durationMillis, boolean timedOut) { + objectStoreStatistics.record(durationMillis, timedOut); + } + + public void recordAppBinaryUploadRejection() { + appBinaryStatistics.recordRejection(); + } + + public UploadPathStatistics getAppBinaryStatistics() { + return this.appBinaryStatistics; + } + + public UploadPathStatistics getObjectStoreStatistics() { + return this.objectStoreStatistics; + } + + public static final class UploadPathStatistics { + + private final LongAdder total = new LongAdder(); + + private final LongAdder timeouts = new LongAdder(); + + private final LongAdder sumDuration = new LongAdder(); + + private final LongAdder rejections = new LongAdder(); + + private final AtomicLong rejectionsInWindow = new AtomicLong(0); + + private final AtomicLong maxDuration = new AtomicLong(0); + + private final AtomicLong timeoutsInWindow = new AtomicLong(0); + + public void record(long durationMillis, boolean timedOut) { + long duration = Math.max(0, durationMillis); + total.increment(); + + if (timedOut) { + timeouts.increment(); + timeoutsInWindow.incrementAndGet(); + } + + sumDuration.add(duration); + maxDuration.accumulateAndGet(duration, Math::max); + } + + public void recordRejection() { + rejections.increment(); + rejectionsInWindow.incrementAndGet(); + } + + public long totalCount() { + return total.sum(); + } + + public long timeoutCount() { + return timeouts.sum(); + } + + public long maxDurationMs() { + return maxDuration.getAndSet(0); + } + + public long sumDurationMs() { + return sumDuration.sum(); + } + + public long timeoutsInWindow() { + return timeoutsInWindow.getAndSet(0); + } + + public long rejectionCount() { + return rejections.sum(); + } + + public long rejectionsInWindow() { + return rejectionsInWindow.getAndSet(0); + } + + } + +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java new file mode 100644 index 0000000000..f229a53aee --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java @@ -0,0 +1,31 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import com.google.cloud.storage.StorageException; +import io.netty.handler.timeout.ReadTimeoutException; + +import java.net.SocketTimeoutException; +import java.util.concurrent.TimeoutException; + +public class UploadTimeoutMatcher { + + private UploadTimeoutMatcher() { + + } + + public static boolean isUploadTimeoutException(Throwable throwable) { + if (throwable == null) { + return false; + } + + Throwable cause = throwable.getCause(); + while (cause != null) { + if (cause instanceof SocketTimeoutException || cause instanceof TimeoutException || cause instanceof ReadTimeoutException || ( + cause instanceof StorageException + && ((StorageException) cause).getCode() == 504)) { + return true; + } + cause = cause.getCause(); + } + return false; + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java index a075038c51..53e813a877 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.net.URI; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -16,8 +17,11 @@ import java.util.concurrent.Executors; import java.util.function.BiPredicate; import java.util.function.Consumer; + import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -57,10 +61,12 @@ public class AwsS3ObjectStoreFileStorage extends ObjectStoreFileStorage { private final S3Client s3Client; private final String bucketName; + private final UploadDurationTracker uploadDurationTracker; - public AwsS3ObjectStoreFileStorage(Map credentials) { + public AwsS3ObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.s3Client = createS3Client(credentials); + this.uploadDurationTracker = uploadDurationTracker; } protected S3Client createS3Client(Map credentials) { @@ -103,6 +109,7 @@ protected ClientOverrideConfiguration buildClientOverrideConfig() { @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + LocalDateTime startTime = LocalDateTime.now(); long fileSize = fileEntry.getSize() .longValue(); PutObjectRequest request = PutObjectRequest.builder() @@ -115,8 +122,12 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage try { s3Client.putObject(request, RequestBody.fromInputStream(new BufferedInputStream(content), fileSize)); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_AWS_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } catch (Exception e) { LOGGER.error(MessageFormat.format(Messages.S3_UPLOAD_FAILED_FILE_0_SIZE_1, fileEntry.getName(), fileSize, e)); + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(MessageFormat.format(Messages.UPLOAD_OF_FILE_WITH_NAMESPACE_FAILED, fileEntry.getName(), fileEntry.getNamespace()), e); } @@ -183,14 +194,15 @@ public void deleteFilesBySpaceIds(List spaceIds) { @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - int deletedFiles = deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); + int deletedFiles = deleteByFilterAndCount( + (key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2, deletedFiles, space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { int deletedFiles = deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterByModificationTime(metadata, key, - modificationTime)); + modificationTime)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_MODIFIED_BEFORE_1, deletedFiles, modificationTime)); return deletedFiles; } @@ -319,6 +331,11 @@ public void destroy() { s3Client.close(); } + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + private static final class CredentialKeys { static final String ACCESS_KEY_ID = "access_key_id"; static final String SECRET_ACCESS_KEY = "secret_access_key"; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java index 721e7234b8..6bc2799a2d 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -5,6 +5,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.List; import java.util.Map; @@ -29,6 +30,8 @@ import com.azure.storage.blob.options.BlobParallelUploadOptions; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -46,15 +49,18 @@ public class AzureObjectStoreFileStorage extends ObjectStoreFileStorage { private static final int MAX_CONCURRENCY = 5; private final HttpClient httpClient; private final BlobContainerClient containerClient; + private final UploadDurationTracker uploadDurationTracker; - public AzureObjectStoreFileStorage(Map credentials) { + public AzureObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.httpClient = new JdkHttpClientBuilder().build(); this.containerClient = createContainerClient(credentials); + this.uploadDurationTracker = uploadDurationTracker; } @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); + LocalDateTime startTime = LocalDateTime.now(); try { ParallelTransferOptions pto = new ParallelTransferOptions().setMaxSingleUploadSizeLong(MAX_SINGLE_UPLOAD_SIZE) .setMaxConcurrency(MAX_CONCURRENCY) @@ -67,10 +73,14 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage blobClient.uploadWithResponse(blobParallelUploadOptions, ObjectStoreConstants.AZURE_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, null); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileEntry.getSize() - .longValue())); + .longValue())); } catch (BlobStorageException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(e); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -218,6 +228,11 @@ private int removeBlobsByFilter(Predicate filter) { return deletedBlobsResult; } + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + protected Set getEntryNames(Predicate filter) { BlobListDetails blobListDetails = new BlobListDetails().setRetrieveMetadata(true); ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(blobListDetails); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 2a0757eb5f..c4d73c8711 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -13,6 +13,8 @@ import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -25,6 +27,7 @@ import java.io.InputStream; import java.nio.channels.Channels; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Base64; @@ -41,10 +44,12 @@ public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; + private final UploadDurationTracker uploadDurationTracker; - public GcpObjectStoreFileStorage(Map credentials) { + public GcpObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.storage = createObjectStoreStorage(credentials); + this.uploadDurationTracker = uploadDurationTracker; } protected Storage createObjectStoreStorage(Map credentials) { @@ -92,11 +97,16 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage } private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageException { + LocalDateTime startTime = LocalDateTime.now(); try { storage.createFrom(blobInfo, content); } catch (IOException | StorageException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(e); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -283,4 +293,10 @@ private static final class CredentialKeys { static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; static final String BUCKET = "bucket"; } + + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 287aa5a344..e7bea41567 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.HashSet; import java.util.List; @@ -10,9 +11,12 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; + import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.jclouds.blobstore.BlobStore; @@ -37,10 +41,12 @@ public class JCloudsObjectStoreFileStorage extends ObjectStoreFileStorage { private final BlobStore blobStore; private final String container; + private final UploadDurationTracker uploadDurationTracker; - public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { + public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container, UploadDurationTracker uploadDurationTracker) { this.blobStore = blobStore; this.container = container; + this.uploadDurationTracker = uploadDurationTracker; } @Override @@ -55,13 +61,18 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentLength(fileSize) .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); + LocalDateTime startTime = LocalDateTime.now(); try { putBlobWithRetries(blob, MAX_RETRIES_COUNT); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); } catch (ContainerNotFoundException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(MessageFormat.format(Messages.FILE_UPLOAD_FAILED, fileEntry.getName(), fileEntry.getNamespace())); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -95,7 +106,7 @@ public void deleteFilesBySpaceIds(List spaceIds) { @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { int deletedFiles = removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getUserMetadata(), space, - namespace)); + namespace)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2, deletedFiles, space, namespace)); } @@ -260,4 +271,10 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } + + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntryTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntryTest.java new file mode 100644 index 0000000000..a90bc99402 --- /dev/null +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntryTest.java @@ -0,0 +1,110 @@ +package org.cloudfoundry.multiapps.controller.persistence.model; + +import java.time.Duration; +import java.time.LocalDateTime; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class AsyncUploadJobEntryTest { + + private static final String JOB_ID = "job-id"; + private static final String FILE_ID = "file-id"; + private static final String SPACE_GUID = "space-guid"; + private static final String USER = "user"; + private static final String URL = "https://user:secret@example.com/my.mtar"; + private static final LocalDateTime ADDED_AT = LocalDateTime.of(2026, 8, 3, 10, 0, 0); + private static final LocalDateTime STARTED_AT = ADDED_AT.plusSeconds(30); + private static final LocalDateTime FINISHED_AT = STARTED_AT.plusSeconds(90); + + @Test + void testTimingsForFinishedJob() { + AsyncUploadJobEntry job = baseBuilder().state(AsyncUploadJobEntry.State.FINISHED) + .addedAt(ADDED_AT) + .startedAt(STARTED_AT) + .finishedAt(FINISHED_AT) + .build(); + + Assertions.assertEquals(Duration.ofSeconds(30), job.getQueueWaitTime()); + Assertions.assertEquals(Duration.ofSeconds(90), job.getUploadDuration()); + Assertions.assertEquals(Duration.ofSeconds(120), job.getTotalTime()); + } + + @Test + void testTimingsAreNullWhenTimestampsMissing() { + AsyncUploadJobEntry runningJob = baseBuilder().state(AsyncUploadJobEntry.State.RUNNING) + .addedAt(ADDED_AT) + .startedAt(STARTED_AT) + .build(); + + Assertions.assertEquals(Duration.ofSeconds(30), runningJob.getQueueWaitTime()); + Assertions.assertNull(runningJob.getUploadDuration()); + Assertions.assertNull(runningJob.getTotalTime()); + + AsyncUploadJobEntry initialJob = baseBuilder().state(AsyncUploadJobEntry.State.INITIAL) + .addedAt(ADDED_AT) + .build(); + + Assertions.assertNull(initialJob.getQueueWaitTime()); + Assertions.assertNull(initialJob.getUploadDuration()); + Assertions.assertNull(initialJob.getTotalTime()); + } + + @Test + void testLogSafeSummaryContainsTimingsAndFileId() { + AsyncUploadJobEntry job = baseBuilder().state(AsyncUploadJobEntry.State.FINISHED) + .mtaId("my-mta") + .bytesRead(2048L) + .addedAt(ADDED_AT) + .startedAt(STARTED_AT) + .finishedAt(FINISHED_AT) + .build(); + + String summary = job.buildLogSummary(); + + Assertions.assertTrue(summary.contains(JOB_ID), summary); + Assertions.assertTrue(summary.contains(FILE_ID), summary); + Assertions.assertTrue(summary.contains("my-mta"), summary); + Assertions.assertTrue(summary.contains("queueWaitTime: 30000 ms"), summary); + Assertions.assertTrue(summary.contains("uploadDuration: 90000 ms"), summary); + Assertions.assertTrue(summary.contains("totalTime: 120000 ms"), summary); + } + + @Test + void testLogSafeSummaryHidesSensitiveData() { + AsyncUploadJobEntry job = baseBuilder().state(AsyncUploadJobEntry.State.RUNNING) + .addedAt(ADDED_AT) + .startedAt(STARTED_AT) + .build(); + + String summary = job.buildLogSummary(); + + Assertions.assertFalse(summary.contains(URL), summary); + Assertions.assertFalse(summary.contains("secret"), summary); + Assertions.assertFalse(summary.contains(USER), summary); + Assertions.assertFalse(summary.contains(SPACE_GUID), summary); + } + + @Test + void testLogSafeSummaryRendersMissingTimingsAsNotAvailable() { + AsyncUploadJobEntry job = baseBuilder().state(AsyncUploadJobEntry.State.INITIAL) + .addedAt(ADDED_AT) + .build(); + + String summary = job.buildLogSummary(); + + Assertions.assertTrue(summary.contains("queueWaitTime: " + AsyncUploadJobEntry.NOT_AVAILABLE), summary); + Assertions.assertTrue(summary.contains("uploadDuration: " + AsyncUploadJobEntry.NOT_AVAILABLE), summary); + Assertions.assertTrue(summary.contains("totalTime: " + AsyncUploadJobEntry.NOT_AVAILABLE), summary); + } + + private ImmutableAsyncUploadJobEntry.Builder baseBuilder() { + return ImmutableAsyncUploadJobEntry.builder() + .id(JOB_ID) + .fileId(FILE_ID) + .user(USER) + .url(URL) + .spaceGuid(SPACE_GUID) + .instanceIndex(0); + } +} diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java index b5b5a08011..e41c95d561 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java @@ -2,6 +2,7 @@ import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -62,16 +63,19 @@ class AwsS3ObjectStoreFileStorageTest { @Mock private FileContentProcessor fileContentProcessor; + @Mock + private UploadDurationTracker uploadDurationTracker; + private AwsS3ObjectStoreFileStorage fileStorage; private final InputStream inputStream = new ByteArrayInputStream(new byte[] {}); private static final String TEST_SPACE_ID = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_SPACE_ID_2 = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_ID = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_ID_2 = UUID.randomUUID() - .toString(); + .toString(); private static final String NAMESPACE = "namespace"; private static final String NAMESPACE_2 = "namespace_2"; @@ -80,7 +84,7 @@ void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME)) { + fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME), uploadDurationTracker) { @Override protected S3Client createS3Client(Map credentials) { @@ -136,7 +140,7 @@ void testGetFileEntriesWithoutContentWithoutMatches() { } @Test - void testExistsInObjectStoreWhenFileExists() { + void testExistsInObjectStoreWhenFileExists() { when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(HeadObjectResponse.builder() .build()); FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); @@ -232,7 +236,7 @@ void testDeleteFilesBySpaceAndNamespaceWithOneMatch() { @Test void testDeleteFilesModifiedBefore() { LocalDateTime oldModified = FILE_TIMESTAMP - .minusMinutes(15); + .minusMinutes(15); setupListObjectsWithKeys(TEST_ID, TEST_ID_2); setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, oldModified); @@ -250,7 +254,7 @@ void testDeleteFilesModifiedBeforeWithNoOldFiles() { setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, FILE_TIMESTAMP); LocalDateTime cutoff = FILE_TIMESTAMP - .minusDays(1); + .minusDays(1); int deletedCount = fileStorage.deleteFilesModifiedBefore(cutoff); assertEquals(0, deletedCount); @@ -532,7 +536,7 @@ void testBuildClientOverrideConfigSetsApiCallAttemptTimeout() { assertTrue(config.apiCallAttemptTimeout() .isPresent()); assertEquals(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, config.apiCallAttemptTimeout() - .get()); + .get()); } @Test @@ -544,8 +548,8 @@ void testBuildClientOverrideConfigSetsRetryStrategy() { assertInstanceOf(StandardRetryStrategy.class, config.retryStrategy() .get()); assertEquals(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG, config.retryStrategy() - .get() - .maxAttempts()); + .get() + .maxAttempts()); } @Test @@ -555,7 +559,7 @@ void testCreateS3ClientWithValidCredentials() { "bucket", "test-bucket", "host", "s3.amazonaws.com", "region", "eu-central-1"); - AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials); + AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials, uploadDurationTracker); assertNotNull(storage); storage.destroy(); diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java index 437981fa8a..adb3361faa 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java @@ -20,6 +20,7 @@ import com.azure.storage.blob.specialized.BlobInputStream; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -51,6 +52,9 @@ class AzureObjectStoreFileStorageTest { @Mock private BlobInputStream blobInputStream; + @Mock + private UploadDurationTracker uploadDurationTracker; + private AzureObjectStoreFileStorage fileStorage; private InputStream inputStream = new ByteArrayInputStream(new byte[] {}); private final String TEST_SPACE_ID = UUID.randomUUID() @@ -70,7 +74,7 @@ void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - fileStorage = new AzureObjectStoreFileStorage(Map.of()) { + fileStorage = new AzureObjectStoreFileStorage(Map.of(), uploadDurationTracker) { @Override protected BlobContainerClient createContainerClient(Map credentials) { diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 14264ffa72..6fb0f3cf44 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -8,6 +8,7 @@ import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,13 +37,14 @@ class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { private Storage storage; private Storage mockedStorage; private GcpObjectStoreFileStorage mockedGcpFileStorage; + private final UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); @Override @BeforeEach public void setUp() { storage = LocalStorageHelper.getOptions() .getService(); - fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER), uploadDurationTracker) { @Override protected Storage createObjectStoreStorage(Map credentials) { @@ -54,7 +56,7 @@ protected Storage createObjectStoreStorage(Map credentials) { namespace = UUID.randomUUID() .toString(); mockedStorage = mock(Storage.class); - mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER), uploadDurationTracker) { @Override protected Storage createObjectStoreStorage(Map credentials) { return mockedStorage; diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index 896cf90ee1..77de41d591 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -16,10 +16,12 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; + import jakarta.xml.bind.DatatypeConverter; import org.cloudfoundry.multiapps.common.util.DigestHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; @@ -28,6 +30,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; + import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -57,10 +60,12 @@ class JCloudsObjectStoreFileStorageTest { private BlobStoreContext blobStoreContext; + private UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); + @BeforeEach protected void setUp() { createBlobStoreContext(); - fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { + fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER, uploadDurationTracker) { @Override protected long getRetryWaitTime() { return 1; @@ -233,7 +238,7 @@ void testConnection() { @Test void testTestConnectionWhenContainerExists() { BlobStore mockBlobStore = mock(BlobStore.class); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.containerExists(CONTAINER)).thenReturn(true); @@ -244,7 +249,7 @@ void testTestConnectionWhenContainerExists() { @Test void testTestConnectionWhenContainerDoesNotExist() { BlobStore mockBlobStore = mock(BlobStore.class); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.containerExists(CONTAINER)).thenReturn(false); @@ -404,7 +409,7 @@ protected void assertFileExists(boolean expectedFileExist, FileEntry actualFile) void existsInObjectStoreWithMockedBlobStoreVerifiesBlobExistsCalled() { BlobStore mockBlobStore = mock(BlobStore.class); FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.blobExists(CONTAINER, fileEntry.getId())).thenReturn(true); @@ -418,7 +423,7 @@ void existsInObjectStoreWithMockedBlobStoreVerifiesBlobExistsCalled() { void existsInObjectStoreWithMockedBlobStoreFileNotFound() { BlobStore mockBlobStore = mock(BlobStore.class); FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.blobExists(CONTAINER, fileEntry.getId())).thenReturn(false); @@ -433,7 +438,8 @@ void existsInObjectStoreWithMockedBlobStoreVerifiesCorrectContainer() { BlobStore mockBlobStore = mock(BlobStore.class); String testContainer = "test-container"; FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, testContainer); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, testContainer, + uploadDurationTracker); when(mockBlobStore.blobExists(testContainer, fileEntry.getId())).thenReturn(true); diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java index 6549708ff4..071dece894 100755 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/Messages.java @@ -820,6 +820,9 @@ public class Messages { public static final String IGNORING_NOT_FOUND_OPTIONAL_SERVICE = "Service {0} not found but is optional"; public static final String IGNORING_NOT_FOUND_INACTIVE_SERVICE = "Service {0} not found but is inactive"; + public static final String ASYNC_UPLOAD_JOB_FOR_OPERATION_0_IS_1 = "Async upload job for operation \"{0}\" - {1}"; + public static final String COULD_NOT_LOG_ASYNC_UPLOAD_JOBS_FOR_OPERATION_0 = "Could not log async upload jobs for operation \"{0}\""; + // Not log messages public static final String SERVICE_TYPE = "{0}/{1}"; public static final String PARSE_NULL_STRING_ERROR = "Cannot parse null string"; diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/listeners/StartProcessListener.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/listeners/StartProcessListener.java index fbca97c731..58dc7c2c22 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/listeners/StartProcessListener.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/listeners/StartProcessListener.java @@ -19,8 +19,10 @@ import org.cloudfoundry.multiapps.controller.api.model.ProcessType; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.LoggingUtil; +import org.cloudfoundry.multiapps.controller.persistence.model.AsyncUploadJobEntry; import org.cloudfoundry.multiapps.controller.persistence.model.HistoricOperationEvent; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableHistoricOperationEvent; +import org.cloudfoundry.multiapps.controller.persistence.services.AsyncUploadJobService; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.persistence.services.HistoricOperationEventService; @@ -56,6 +58,7 @@ public class StartProcessListener extends AbstractProcessExecutionListener { private final ProcessTypeToOperationMetadataMapper operationMetadataMapper; private final DynatracePublisher dynatracePublisher; private final FileService fileService; + private final AsyncUploadJobService asyncUploadJobService; @Inject public StartProcessListener(ProgressMessageService progressMessageService, StepLogger.Factory stepLoggerFactory, @@ -63,7 +66,8 @@ public StartProcessListener(ProgressMessageService progressMessageService, StepL HistoricOperationEventService historicOperationEventService, FlowableFacade flowableFacade, ApplicationConfiguration configuration, ProcessTypeParser processTypeParser, OperationService operationService, ProcessTypeToOperationMetadataMapper operationMetadataMapper, - DynatracePublisher dynatracePublisher, FileService fileService) { + DynatracePublisher dynatracePublisher, FileService fileService, + AsyncUploadJobService asyncUploadJobService) { super(progressMessageService, stepLoggerFactory, processLoggerProvider, @@ -76,6 +80,7 @@ public StartProcessListener(ProgressMessageService progressMessageService, StepL this.operationMetadataMapper = operationMetadataMapper; this.dynatracePublisher = dynatracePublisher; this.fileService = fileService; + this.asyncUploadJobService = asyncUploadJobService; } @Override @@ -92,6 +97,7 @@ protected void notifyInternal(DelegateExecution execution) { } updateOperationFiles(execution, correlationId); + logAsyncUploadJobs(execution, correlationId); getHistoricOperationEventService().add(ImmutableHistoricOperationEvent.of(correlationId, HistoricOperationEvent.EventType.STARTED)); logProcessEnvironment(); logProcessVariables(execution, processType, correlationId); @@ -158,6 +164,24 @@ private void updateOperationFiles(DelegateExecution execution, String correlatio } } + private void logAsyncUploadJobs(DelegateExecution execution, String correlationId) { + List operationFileIds = OperationFileIdsUtil.getOperationFileIds(execution); + if (operationFileIds.isEmpty()) { + return; + } + try { + List asyncUploadJobs = asyncUploadJobService.createQuery() + .withFileIds(operationFileIds) + .list(); + for (AsyncUploadJobEntry asyncUploadJob : asyncUploadJobs) { + LOGGER.info(MessageFormat.format(Messages.ASYNC_UPLOAD_JOB_FOR_OPERATION_0_IS_1, correlationId, + asyncUploadJob.buildLogSummary())); + } + } catch (Exception e) { + LOGGER.warn(MessageFormat.format(Messages.COULD_NOT_LOG_ASYNC_UPLOAD_JOBS_FOR_OPERATION_0, correlationId), e); + } + } + private void publishDynatraceEvent(DelegateExecution execution, ProcessType processType, String correlationId) { DynatraceProcessEvent startEvent = ImmutableDynatraceProcessEvent.builder() .processId(correlationId) diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java index f68e1bd55f..17d5054779 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java @@ -23,6 +23,8 @@ import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.services.ProcessLoggerPersister; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.context.ApplicationToUploadContext; @@ -44,13 +46,16 @@ public class UploadAppAsyncExecution implements AsyncExecution { private final ProcessLoggerPersister processLoggerPersister; private final ApplicationConfiguration applicationConfiguration; private final ExecutorService appUploaderThreadPool; + private final UploadDurationTracker uploadDurationTracker; public UploadAppAsyncExecution(ApplicationZipBuilder applicationZipBuilder, ProcessLoggerPersister processLoggerPersister, - ApplicationConfiguration applicationConfiguration, ExecutorService appUploaderThreadPool) { + ApplicationConfiguration applicationConfiguration, ExecutorService appUploaderThreadPool, + UploadDurationTracker uploadDurationTracker) { this.applicationZipBuilder = applicationZipBuilder; this.processLoggerPersister = processLoggerPersister; this.applicationConfiguration = applicationConfiguration; this.appUploaderThreadPool = appUploaderThreadPool; + this.uploadDurationTracker = uploadDurationTracker; } @Override @@ -70,6 +75,7 @@ public AsyncExecutionState execute(ProcessContext context) { try { runningUpload = appUploaderThreadPool.submit(() -> doUpload(context, applicationToProcess, applicationToUploadContext)); } catch (RejectedExecutionException rejectedExecutionException) { + uploadDurationTracker.recordAppBinaryUploadRejection(); LOGGER.warn(rejectedExecutionException.getMessage(), rejectedExecutionException); context.getStepLogger() .warn(Messages.UPLOAD_OF_APPLICATION_0_WAS_NOT_ACCEPTED_BY_INSTANCE_1, applicationToProcess.getName(), @@ -80,13 +86,12 @@ public AsyncExecutionState execute(ProcessContext context) { try { cloudPackage = runningUpload.get(); } catch (InterruptedException | ExecutionException e) { + uploadDurationTracker.recordAppBinaryUpload(getElapsedTimeInMillis(context), UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new SLException(e, e.getMessage()); } - LocalDateTime startTime = context.getVariable(Variables.UPLOAD_START_TIME); - long timeElapsedForUpload = Duration.between(startTime, LocalDateTime.now()) - .toMillis(); + uploadDurationTracker.recordAppBinaryUpload(getElapsedTimeInMillis(context), false); context.getStepLogger() - .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_UPLOAD_0_IN_MILLIS, timeElapsedForUpload); + .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(context)); return processCloudPackage(context, client, cloudPackage); } @@ -201,6 +206,12 @@ private void attemptToUpdateApplicationDigest(CloudControllerClient client, Clou newApplicationDigest); } + private long getElapsedTimeInMillis(ProcessContext context) { + LocalDateTime startTime = context.getVariable(Variables.UPLOAD_START_TIME); + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + @Override public String getPollingErrorMessage(ProcessContext context) { CloudApplicationExtended appToProcess = context.getVariable(Variables.APP_TO_PROCESS); diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java index 0f19e1fa0e..33339f03c0 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java @@ -21,6 +21,7 @@ import org.cloudfoundry.multiapps.controller.core.helpers.ApplicationFileDigestDetector; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.security.serialization.DynamicSecureSerialization; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.security.util.SecureLoggingUtil; @@ -53,6 +54,8 @@ public class UploadAppStep extends TimeoutAsyncFlowableStep { protected CloudPackagesGetter cloudPackagesGetter; @Inject private ExecutorService appUploaderThreadPool; + @Inject + private UploadDurationTracker uploadDurationTracker; @Override public StepPhase executeAsyncStep(ProcessContext context) throws FileStorageException { @@ -185,7 +188,8 @@ private void removeApplicationDigestIfSet(ProcessContext context, Map getAsyncStepExecutions(ProcessContext context) { - return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, appUploaderThreadPool), + return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, appUploaderThreadPool, + uploadDurationTracker), new PollUploadAppStatusExecution()); } diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/listeners/StartProcessListenerTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/listeners/StartProcessListenerTest.java index bd4c656ab0..06b308e0ca 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/listeners/StartProcessListenerTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/listeners/StartProcessListenerTest.java @@ -18,6 +18,8 @@ import org.cloudfoundry.multiapps.controller.api.model.ProcessType; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.query.OperationQuery; +import org.cloudfoundry.multiapps.controller.persistence.query.AsyncUploadJobsQuery; +import org.cloudfoundry.multiapps.controller.persistence.services.AsyncUploadJobService; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.persistence.services.HistoricOperationEventService; @@ -78,6 +80,10 @@ class StartProcessListenerTest { private HistoricOperationEventService historicOperationEventService; @Mock private FileService fileService; + @Mock(answer = Answers.RETURNS_SELF) + private AsyncUploadJobService asyncUploadJobService; + @Mock(answer = Answers.RETURNS_SELF) + private AsyncUploadJobsQuery asyncUploadJobsQuery; @Spy private ProcessTypeToOperationMetadataMapper operationMetadataMapper; @Mock @@ -114,7 +120,8 @@ void setUp() throws Exception { operationService, operationMetadataMapper, dynatracePublisher, - fileService); + fileService, + asyncUploadJobService); } @ParameterizedTest @@ -139,6 +146,10 @@ private void prepare() { Mockito.doReturn(null) .when(operationQuery) .singleResult(); + Mockito.when(asyncUploadJobService.createQuery()) + .thenReturn(asyncUploadJobsQuery); + Mockito.when(asyncUploadJobsQuery.list()) + .thenReturn(Collections.emptyList()); } private void prepareContext() { diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java index 452fb59c15..b1a5d77a56 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java @@ -26,6 +26,7 @@ import org.cloudfoundry.multiapps.controller.client.lib.domain.ImmutableCloudApplicationExtended; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveIterator; @@ -86,6 +87,7 @@ class UploadAppAsyncExecutionTest extends AsyncStepOperationTest .build(); private final MtaArchiveElements mtaArchiveElements = new MtaArchiveElements(); private final ExecutorService appUploaderThreadPool = mock(ExecutorService.class); + private final UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); @TempDir Path tempDir; @@ -220,7 +222,7 @@ protected List getAsyncStepExecutions(ProcessContext context) { return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, - appUploaderThreadPool) { + appUploaderThreadPool, uploadDurationTracker) { }); } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java index c892373baa..85cdcb3c31 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java @@ -1,9 +1,11 @@ package org.cloudfoundry.multiapps.controller.web.configuration; +import jakarta.inject.Inject; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.resilience.NoRetryErrorClassifier; import org.cloudfoundry.multiapps.controller.persistence.services.resilience.RetryableErrorClassifier; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.configuration.factory.ObjectStoreSelectorFactory; import org.springframework.context.annotation.Bean; @@ -16,10 +18,14 @@ public class FileStorageConfiguration { private static final String OBJECT_STORE_SERVICE_NAME = "deploy-service-os"; + @Inject + private UploadDurationTracker uploadDurationTracker; + @Bean public ObjectStoreSelectorFactory objectStoreSelector(EnvironmentServicesFinder vcapServiceFinder, ApplicationConfiguration applicationConfiguration) { - return new ObjectStoreSelectorFactory(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration); + return new ObjectStoreSelectorFactory(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration, + uploadDurationTracker); } @Bean diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java index cb52bb629a..a8aca8efc2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java @@ -3,6 +3,7 @@ import java.util.Map; import org.cloudfoundry.multiapps.controller.web.monitoring.Metrics; +import org.cloudfoundry.multiapps.controller.web.monitoring.UploadDurationMetrics; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.jmx.export.MBeanExporter; @@ -12,10 +13,12 @@ public class JmxConfiguration { private static final String METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=MetricsMBean"; + private static final String UPLOAD_METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=UploadMetricsMBean"; + @Bean - public MBeanExporter jmxExporter(Metrics metrics) { + public MBeanExporter jmxExporter(Metrics metrics, UploadDurationMetrics uploadDurationMetrics) { MBeanExporter mBeanExporter = new MBeanExporter(); - mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics)); + mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics, UPLOAD_METRICS_BEAN, uploadDurationMetrics)); return mBeanExporter; } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java index 117ed389ad..ceeb850bf2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java @@ -7,11 +7,13 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; + import io.pivotal.cfenv.core.CfService; import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; @@ -39,12 +41,15 @@ public class ObjectStoreSelectorFactory { private final EnvironmentServicesFinder environmentServicesFinder; private final ApplicationConfiguration applicationConfiguration; private final SelectedObjectStore selectedObjectStore; + private final UploadDurationTracker uploadDurationTracker; public ObjectStoreSelectorFactory(String serviceName, EnvironmentServicesFinder environmentServicesFinder, - ApplicationConfiguration applicationConfiguration) { + ApplicationConfiguration applicationConfiguration, + UploadDurationTracker uploadDurationTracker) { this.serviceName = serviceName; this.environmentServicesFinder = environmentServicesFinder; this.applicationConfiguration = applicationConfiguration; + this.uploadDurationTracker = uploadDurationTracker; this.selectedObjectStore = doSelect(); } @@ -155,15 +160,15 @@ private SelectedObjectStore selectFor(ObjectStoreServiceInfo objectStoreServiceI } protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } protected AzureObjectStoreFileStorage createAzureFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } protected AwsS3ObjectStoreFileStorage createAwsS3FileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -203,7 +208,7 @@ private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextB protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { return new JCloudsObjectStoreFileStorage(context.getBlobStore(), (String) objectStoreServiceInfo.getCredentials() - .get(Constants.BUCKET)); + .get(Constants.BUCKET), uploadDurationTracker); } private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java new file mode 100644 index 0000000000..3207495a23 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java @@ -0,0 +1,28 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; + +@Named +public class AppUploaderThreadPoolInformation { + + private final ThreadPoolExecutor appUploaderPool; + + @Inject + public AppUploaderThreadPoolInformation( + @Named("appUploaderThreadPool") ExecutorService appUploaderThreadPool) { + this.appUploaderPool = (ThreadPoolExecutor) appUploaderThreadPool; + } + + public int getActiveThreads() { + return appUploaderPool.getActiveCount(); + } + + public int getMaxThreads() { + return appUploaderPool.getMaximumPoolSize(); + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java new file mode 100644 index 0000000000..36e111505b --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java @@ -0,0 +1,103 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import jakarta.inject.Inject; +import jakarta.inject.Named; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; + +@Named +public class UploadDurationMetrics implements UploadMetricsMBean { + + private final UploadDurationTracker tracker; + + private final AppUploaderThreadPoolInformation appUploaderInfo; + + @Inject + public UploadDurationMetrics(UploadDurationTracker tracker, + AppUploaderThreadPoolInformation appUploaderInfo) { + this.tracker = tracker; + this.appUploaderInfo = appUploaderInfo; + } + + @Override + public long getAppBinaryUploadTotalCount() { + return tracker.getAppBinaryStatistics() + .totalCount(); + } + + @Override + public long getAppBinaryUploadTimeoutCount() { + return tracker.getAppBinaryStatistics() + .timeoutCount(); + } + + @Override + public long getAppBinaryUploadMaxDurationMs() { + return tracker.getAppBinaryStatistics() + .maxDurationMs(); + } + + @Override + public long getAppBinaryUploadSumDurationMs() { + return tracker.getAppBinaryStatistics() + .sumDurationMs(); + } + + @Override + public int getAppUploaderActiveThreads() { + return appUploaderInfo.getActiveThreads(); + } + + @Override + public int getAppUploaderMaxThreads() { + return appUploaderInfo.getMaxThreads(); + } + + @Override + public long getAppUploaderRejectionCount() { + return tracker.getAppBinaryStatistics() + .rejectionCount(); + } + + @Override + public long getAppUploaderRejectionCountInWindow() { + return tracker.getAppBinaryStatistics() + .rejectionsInWindow(); + } + + @Override + public long getObjectStoreUploadTotalCount() { + return tracker.getObjectStoreStatistics() + .totalCount(); + } + + @Override + public long getObjectStoreUploadTimeoutCount() { + return tracker.getObjectStoreStatistics() + .timeoutCount(); + } + + @Override + public long getObjectStoreUploadMaxDurationMs() { + return tracker.getObjectStoreStatistics() + .maxDurationMs(); + } + + @Override + public long getObjectStoreUploadSumDurationMs() { + return tracker.getObjectStoreStatistics() + .sumDurationMs(); + } + + @Override + public long getAppBinaryUploadTimeoutsInWindow() { + return tracker.getAppBinaryStatistics() + .timeoutsInWindow(); + } + + @Override + public long getObjectStoreUploadTimeoutsInWindow() { + return tracker.getObjectStoreStatistics() + .timeoutsInWindow(); + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java new file mode 100644 index 0000000000..396c352db1 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java @@ -0,0 +1,33 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +public interface UploadMetricsMBean { + + long getAppBinaryUploadTotalCount(); + + long getAppBinaryUploadTimeoutCount(); + + long getAppBinaryUploadMaxDurationMs(); + + long getAppBinaryUploadSumDurationMs(); + + int getAppUploaderActiveThreads(); + + int getAppUploaderMaxThreads(); + + long getAppUploaderRejectionCount(); + + long getAppUploaderRejectionCountInWindow(); + + long getObjectStoreUploadTotalCount(); + + long getObjectStoreUploadTimeoutCount(); + + long getObjectStoreUploadMaxDurationMs(); + + long getObjectStoreUploadSumDurationMs(); + + long getAppBinaryUploadTimeoutsInWindow(); + + long getObjectStoreUploadTimeoutsInWindow(); + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java index b28570c678..911f956f89 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java @@ -4,6 +4,7 @@ import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; @@ -64,6 +65,8 @@ class ObjectStoreSelectorFactoryTest { private AwsS3ObjectStoreFileStorage awsS3ObjectStoreFileStorage; @Mock private AzureObjectStoreFileStorage azureObjectStoreFileStorage; + @Mock + private UploadDurationTracker uploadDurationTracker; private ObjectStoreServiceInfo capturedGcpServiceInfo; private ObjectStoreServiceInfo capturedAzureServiceInfo; @@ -103,7 +106,7 @@ void testObjectStoreCreationWhenEnvIsValidForAws() { assertInstanceOf(AwsS3ObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(AwsTransientErrorClassifier.class, selector.classifier()); verify(awsS3ObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -116,7 +119,7 @@ void testObjectStoreCreationWhenEnvIsValidForAzure() { assertInstanceOf(AzureObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(AzureTransientErrorClassifier.class, selector.classifier()); verify(azureObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -129,12 +132,12 @@ void testObjectStoreCreationWhenEnvIsValidForGcp() { assertInstanceOf(GcpObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(GcpTransientErrorClassifier.class, selector.classifier()); verify(gcpObjectStoreFileStorage) - .testConnection(); + .testConnection(); } static Stream testObjectStoreCreationFallsBackToFirstReachableProviderWhenEnvIsInvalid() { return Stream.of( - // @formatter:off + // @formatter:off // (0) Unknown provider name: Arguments.of("WRONG_PROVIDER"), // (1) Null env value: @@ -169,7 +172,7 @@ void testObjectStoreCreationFallsBackToNextProviderWhenFirstFails() { assertInstanceOf(JCloudsObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(JCloudsTransientErrorClassifier.class, selector.classifier()); verify(jCloudsObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -289,7 +292,7 @@ private class ObjectStoreSelectorFactoryMock extends ObjectStoreSelectorFactory ObjectStoreSelectorFactoryMock(String serviceName, EnvironmentServicesFinder environmentServicesFinder, ApplicationConfiguration applicationConfiguration) { - super(serviceName, environmentServicesFinder, applicationConfiguration); + super(serviceName, environmentServicesFinder, applicationConfiguration, uploadDurationTracker); } @Override