From f7777ca13c82f438d4e9aa6ca666b3ba37b5905a Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Fri, 31 Jul 2026 09:27:47 +0300 Subject: [PATCH 1/4] Adding upload statistics and alerts for both app binary uploads towards the CF API and OS uploads LMCROSSITXSADEPLOY-3054 --- .../src/main/java/module-info.java | 1 + .../controller/persistence/Messages.java | 3 + .../monitoring/UploadDurationTracker.java | 99 +++++++++++++++++ .../monitoring/UploadTimeoutMatcher.java | 31 ++++++ .../services/AzureObjectStoreFileStorage.java | 19 +++- .../services/GcpObjectStoreFileStorage.java | 18 ++- .../JCloudsObjectStoreFileStorage.java | 21 +++- .../AzureObjectStoreFileStorageTest.java | 6 +- .../GcpObjectStoreFileStorageTest.java | 6 +- .../JCloudsObjectStoreFileStorageTest.java | 18 ++- .../steps/UploadAppAsyncExecution.java | 21 +++- .../process/steps/UploadAppStep.java | 6 +- .../steps/UploadAppAsyncExecutionTest.java | 4 +- .../FileStorageConfiguration.java | 8 +- .../web/configuration/JmxConfiguration.java | 7 +- .../factory/ObjectStoreSelectorFactory.java | 15 ++- .../AppUploaderThreadPoolInformation.java | 28 +++++ .../web/monitoring/UploadDurationMetrics.java | 103 ++++++++++++++++++ .../web/monitoring/UploadMetricsMBean.java | 33 ++++++ .../ObjectStoreSelectorFactoryTest.java | 15 ++- 20 files changed, 427 insertions(+), 35 deletions(-) create mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java create mode 100644 multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java create mode 100644 multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java create mode 100644 multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java create mode 100644 multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index ca1b6e20a1..b3d692cb0b 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -9,6 +9,7 @@ exports org.cloudfoundry.multiapps.controller.persistence.model; exports org.cloudfoundry.multiapps.controller.persistence.model.adapter; exports org.cloudfoundry.multiapps.controller.persistence.model.filters; + exports org.cloudfoundry.multiapps.controller.persistence.monitoring; exports org.cloudfoundry.multiapps.controller.persistence.query; exports org.cloudfoundry.multiapps.controller.persistence.query.criteria; exports org.cloudfoundry.multiapps.controller.persistence.query.impl; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 65fa73fbf9..22707e64c3 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -67,6 +67,9 @@ public final class Messages { // INFO log messages: public static final String DELETING_FILES_WITHOUT_CONTENT_WITH_IDS_0 = "Deleting files without content with ids: {0}"; + public static final String TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for JClouds object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for GCP object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for Azure object store upload: {0} in millis"; // DEBUG log messages: public static final String STORED_FILE_0 = "Stored file: \"{0}\""; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java new file mode 100644 index 0000000000..d6ac887b8b --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadDurationTracker.java @@ -0,0 +1,99 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import jakarta.inject.Named; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +@Named +public class UploadDurationTracker { + + private final UploadPathStatistics appBinaryStatistics = new UploadPathStatistics(); + + private final UploadPathStatistics objectStoreStatistics = new UploadPathStatistics(); + + public void recordAppBinaryUpload(long durationMillis, boolean timedOut) { + appBinaryStatistics.record(durationMillis, timedOut); + } + + public void recordObjectStoreUpload(long durationMillis, boolean timedOut) { + objectStoreStatistics.record(durationMillis, timedOut); + } + + public void recordAppBinaryUploadRejection() { + appBinaryStatistics.recordRejection(); + } + + public UploadPathStatistics getAppBinaryStatistics() { + return this.appBinaryStatistics; + } + + public UploadPathStatistics getObjectStoreStatistics() { + return this.objectStoreStatistics; + } + + public static final class UploadPathStatistics { + + private final LongAdder total = new LongAdder(); + + private final LongAdder timeouts = new LongAdder(); + + private final LongAdder sumDuration = new LongAdder(); + + private final LongAdder rejections = new LongAdder(); + + private final AtomicLong rejectionsInWindow = new AtomicLong(0); + + private final AtomicLong maxDuration = new AtomicLong(0); + + private final AtomicLong timeoutsInWindow = new AtomicLong(0); + + public void record(long durationMillis, boolean timedOut) { + long duration = Math.max(0, durationMillis); + total.increment(); + + if (timedOut) { + timeouts.increment(); + timeoutsInWindow.incrementAndGet(); + } + + sumDuration.add(duration); + maxDuration.accumulateAndGet(duration, Math::max); + } + + public void recordRejection() { + rejections.increment(); + rejectionsInWindow.incrementAndGet(); + } + + public long totalCount() { + return total.sum(); + } + + public long timeoutCount() { + return timeouts.sum(); + } + + public long maxDurationMs() { + return maxDuration.getAndSet(0); + } + + public long sumDurationMs() { + return sumDuration.sum(); + } + + public long timeoutsInWindow() { + return timeoutsInWindow.getAndSet(0); + } + + public long rejectionCount() { + return rejections.sum(); + } + + public long rejectionsInWindow() { + return rejectionsInWindow.getAndSet(0); + } + + } + +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java new file mode 100644 index 0000000000..f229a53aee --- /dev/null +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/monitoring/UploadTimeoutMatcher.java @@ -0,0 +1,31 @@ +package org.cloudfoundry.multiapps.controller.persistence.monitoring; + +import com.google.cloud.storage.StorageException; +import io.netty.handler.timeout.ReadTimeoutException; + +import java.net.SocketTimeoutException; +import java.util.concurrent.TimeoutException; + +public class UploadTimeoutMatcher { + + private UploadTimeoutMatcher() { + + } + + public static boolean isUploadTimeoutException(Throwable throwable) { + if (throwable == null) { + return false; + } + + Throwable cause = throwable.getCause(); + while (cause != null) { + if (cause instanceof SocketTimeoutException || cause instanceof TimeoutException || cause instanceof ReadTimeoutException || ( + cause instanceof StorageException + && ((StorageException) cause).getCode() == 504)) { + return true; + } + cause = cause.getCause(); + } + return false; + } +} diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java index 721e7234b8..6bc2799a2d 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorage.java @@ -5,6 +5,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.List; import java.util.Map; @@ -29,6 +30,8 @@ import com.azure.storage.blob.options.BlobParallelUploadOptions; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -46,15 +49,18 @@ public class AzureObjectStoreFileStorage extends ObjectStoreFileStorage { private static final int MAX_CONCURRENCY = 5; private final HttpClient httpClient; private final BlobContainerClient containerClient; + private final UploadDurationTracker uploadDurationTracker; - public AzureObjectStoreFileStorage(Map credentials) { + public AzureObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.httpClient = new JdkHttpClientBuilder().build(); this.containerClient = createContainerClient(credentials); + this.uploadDurationTracker = uploadDurationTracker; } @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { BlobClient blobClient = containerClient.getBlobClient(fileEntry.getId()); + LocalDateTime startTime = LocalDateTime.now(); try { ParallelTransferOptions pto = new ParallelTransferOptions().setMaxSingleUploadSizeLong(MAX_SINGLE_UPLOAD_SIZE) .setMaxConcurrency(MAX_CONCURRENCY) @@ -67,10 +73,14 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage blobClient.uploadWithResponse(blobParallelUploadOptions, ObjectStoreConstants.AZURE_OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, null); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileEntry.getSize() - .longValue())); + .longValue())); } catch (BlobStorageException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(e); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -218,6 +228,11 @@ private int removeBlobsByFilter(Predicate filter) { return deletedBlobsResult; } + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + protected Set getEntryNames(Predicate filter) { BlobListDetails blobListDetails = new BlobListDetails().setRetrieveMetadata(true); ListBlobsOptions listBlobsOptions = new ListBlobsOptions().setDetails(blobListDetails); diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java index 2a0757eb5f..c4d73c8711 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorage.java @@ -13,6 +13,8 @@ import com.google.cloud.storage.StorageRetryStrategy; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; @@ -25,6 +27,7 @@ import java.io.InputStream; import java.nio.channels.Channels; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Base64; @@ -41,10 +44,12 @@ public class GcpObjectStoreFileStorage implements FileStorage { private final String bucketName; private final Storage storage; + private final UploadDurationTracker uploadDurationTracker; - public GcpObjectStoreFileStorage(Map credentials) { + public GcpObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.storage = createObjectStoreStorage(credentials); + this.uploadDurationTracker = uploadDurationTracker; } protected Storage createObjectStoreStorage(Map credentials) { @@ -92,11 +97,16 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage } private void putBlob(BlobInfo blobInfo, InputStream content) throws FileStorageException { + LocalDateTime startTime = LocalDateTime.now(); try { storage.createFrom(blobInfo, content); } catch (IOException | StorageException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(e); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -283,4 +293,10 @@ private static final class CredentialKeys { static final String BASE_64_ENCODED_PRIVATE_KEY_DATA = "base64EncodedPrivateKeyData"; static final String BUCKET = "bucket"; } + + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java index 287aa5a344..e7bea41567 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorage.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.io.InputStream; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.HashSet; import java.util.List; @@ -10,9 +11,12 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; + import org.cloudfoundry.multiapps.common.util.MiscUtil; import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.jclouds.blobstore.BlobStore; @@ -37,10 +41,12 @@ public class JCloudsObjectStoreFileStorage extends ObjectStoreFileStorage { private final BlobStore blobStore; private final String container; + private final UploadDurationTracker uploadDurationTracker; - public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container) { + public JCloudsObjectStoreFileStorage(BlobStore blobStore, String container, UploadDurationTracker uploadDurationTracker) { this.blobStore = blobStore; this.container = container; + this.uploadDurationTracker = uploadDurationTracker; } @Override @@ -55,13 +61,18 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage .contentLength(fileSize) .userMetadata(ObjectStoreMapper.createFileEntryMetadata(fileEntry)) .build(); + LocalDateTime startTime = LocalDateTime.now(); try { putBlobWithRetries(blob, MAX_RETRIES_COUNT); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); } catch (ContainerNotFoundException e) { + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(MessageFormat.format(Messages.FILE_UPLOAD_FAILED, fileEntry.getName(), fileEntry.getNamespace())); } + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } @Override @@ -95,7 +106,7 @@ public void deleteFilesBySpaceIds(List spaceIds) { @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { int deletedFiles = removeBlobsByFilter(blob -> ObjectStoreFilter.filterBySpaceAndNamespace(blob.getUserMetadata(), space, - namespace)); + namespace)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2, deletedFiles, space, namespace)); } @@ -260,4 +271,10 @@ private Set getAllEntries(ListContainerOptions options) { } return entries; } + + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + } diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java index 437981fa8a..adb3361faa 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AzureObjectStoreFileStorageTest.java @@ -20,6 +20,7 @@ import com.azure.storage.blob.specialized.BlobInputStream; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -51,6 +52,9 @@ class AzureObjectStoreFileStorageTest { @Mock private BlobInputStream blobInputStream; + @Mock + private UploadDurationTracker uploadDurationTracker; + private AzureObjectStoreFileStorage fileStorage; private InputStream inputStream = new ByteArrayInputStream(new byte[] {}); private final String TEST_SPACE_ID = UUID.randomUUID() @@ -70,7 +74,7 @@ void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - fileStorage = new AzureObjectStoreFileStorage(Map.of()) { + fileStorage = new AzureObjectStoreFileStorage(Map.of(), uploadDurationTracker) { @Override protected BlobContainerClient createContainerClient(Map credentials) { diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java index 14264ffa72..6fb0f3cf44 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/GcpObjectStoreFileStorageTest.java @@ -8,6 +8,7 @@ import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,13 +37,14 @@ class GcpObjectStoreFileStorageTest extends JCloudsObjectStoreFileStorageTest { private Storage storage; private Storage mockedStorage; private GcpObjectStoreFileStorage mockedGcpFileStorage; + private final UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); @Override @BeforeEach public void setUp() { storage = LocalStorageHelper.getOptions() .getService(); - fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + fileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER), uploadDurationTracker) { @Override protected Storage createObjectStoreStorage(Map credentials) { @@ -54,7 +56,7 @@ protected Storage createObjectStoreStorage(Map credentials) { namespace = UUID.randomUUID() .toString(); mockedStorage = mock(Storage.class); - mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER)) { + mockedGcpFileStorage = new GcpObjectStoreFileStorage(Map.of("bucket", CONTAINER), uploadDurationTracker) { @Override protected Storage createObjectStoreStorage(Map credentials) { return mockedStorage; diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java index 896cf90ee1..77de41d591 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/JCloudsObjectStoreFileStorageTest.java @@ -16,10 +16,12 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; + import jakarta.xml.bind.DatatypeConverter; import org.cloudfoundry.multiapps.common.util.DigestHelper; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; @@ -28,6 +30,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.http.MediaType; + import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -57,10 +60,12 @@ class JCloudsObjectStoreFileStorageTest { private BlobStoreContext blobStoreContext; + private UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); + @BeforeEach protected void setUp() { createBlobStoreContext(); - fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER) { + fileStorage = new JCloudsObjectStoreFileStorage(blobStoreContext.getBlobStore(), CONTAINER, uploadDurationTracker) { @Override protected long getRetryWaitTime() { return 1; @@ -233,7 +238,7 @@ void testConnection() { @Test void testTestConnectionWhenContainerExists() { BlobStore mockBlobStore = mock(BlobStore.class); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.containerExists(CONTAINER)).thenReturn(true); @@ -244,7 +249,7 @@ void testTestConnectionWhenContainerExists() { @Test void testTestConnectionWhenContainerDoesNotExist() { BlobStore mockBlobStore = mock(BlobStore.class); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.containerExists(CONTAINER)).thenReturn(false); @@ -404,7 +409,7 @@ protected void assertFileExists(boolean expectedFileExist, FileEntry actualFile) void existsInObjectStoreWithMockedBlobStoreVerifiesBlobExistsCalled() { BlobStore mockBlobStore = mock(BlobStore.class); FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.blobExists(CONTAINER, fileEntry.getId())).thenReturn(true); @@ -418,7 +423,7 @@ void existsInObjectStoreWithMockedBlobStoreVerifiesBlobExistsCalled() { void existsInObjectStoreWithMockedBlobStoreFileNotFound() { BlobStore mockBlobStore = mock(BlobStore.class); FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, CONTAINER, uploadDurationTracker); when(mockBlobStore.blobExists(CONTAINER, fileEntry.getId())).thenReturn(false); @@ -433,7 +438,8 @@ void existsInObjectStoreWithMockedBlobStoreVerifiesCorrectContainer() { BlobStore mockBlobStore = mock(BlobStore.class); String testContainer = "test-container"; FileEntry fileEntry = createFileEntry(); - JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, testContainer); + JCloudsObjectStoreFileStorage testFileStorage = new JCloudsObjectStoreFileStorage(mockBlobStore, testContainer, + uploadDurationTracker); when(mockBlobStore.blobExists(testContainer, fileEntry.getId())).thenReturn(true); diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java index f68e1bd55f..17d5054779 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecution.java @@ -23,6 +23,8 @@ import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.FileUtils; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.services.ProcessLoggerPersister; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.context.ApplicationToUploadContext; @@ -44,13 +46,16 @@ public class UploadAppAsyncExecution implements AsyncExecution { private final ProcessLoggerPersister processLoggerPersister; private final ApplicationConfiguration applicationConfiguration; private final ExecutorService appUploaderThreadPool; + private final UploadDurationTracker uploadDurationTracker; public UploadAppAsyncExecution(ApplicationZipBuilder applicationZipBuilder, ProcessLoggerPersister processLoggerPersister, - ApplicationConfiguration applicationConfiguration, ExecutorService appUploaderThreadPool) { + ApplicationConfiguration applicationConfiguration, ExecutorService appUploaderThreadPool, + UploadDurationTracker uploadDurationTracker) { this.applicationZipBuilder = applicationZipBuilder; this.processLoggerPersister = processLoggerPersister; this.applicationConfiguration = applicationConfiguration; this.appUploaderThreadPool = appUploaderThreadPool; + this.uploadDurationTracker = uploadDurationTracker; } @Override @@ -70,6 +75,7 @@ public AsyncExecutionState execute(ProcessContext context) { try { runningUpload = appUploaderThreadPool.submit(() -> doUpload(context, applicationToProcess, applicationToUploadContext)); } catch (RejectedExecutionException rejectedExecutionException) { + uploadDurationTracker.recordAppBinaryUploadRejection(); LOGGER.warn(rejectedExecutionException.getMessage(), rejectedExecutionException); context.getStepLogger() .warn(Messages.UPLOAD_OF_APPLICATION_0_WAS_NOT_ACCEPTED_BY_INSTANCE_1, applicationToProcess.getName(), @@ -80,13 +86,12 @@ public AsyncExecutionState execute(ProcessContext context) { try { cloudPackage = runningUpload.get(); } catch (InterruptedException | ExecutionException e) { + uploadDurationTracker.recordAppBinaryUpload(getElapsedTimeInMillis(context), UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new SLException(e, e.getMessage()); } - LocalDateTime startTime = context.getVariable(Variables.UPLOAD_START_TIME); - long timeElapsedForUpload = Duration.between(startTime, LocalDateTime.now()) - .toMillis(); + uploadDurationTracker.recordAppBinaryUpload(getElapsedTimeInMillis(context), false); context.getStepLogger() - .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_UPLOAD_0_IN_MILLIS, timeElapsedForUpload); + .infoWithoutProgressMessage(Messages.TIME_ELAPSED_FOR_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(context)); return processCloudPackage(context, client, cloudPackage); } @@ -201,6 +206,12 @@ private void attemptToUpdateApplicationDigest(CloudControllerClient client, Clou newApplicationDigest); } + private long getElapsedTimeInMillis(ProcessContext context) { + LocalDateTime startTime = context.getVariable(Variables.UPLOAD_START_TIME); + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + @Override public String getPollingErrorMessage(ProcessContext context) { CloudApplicationExtended appToProcess = context.getVariable(Variables.APP_TO_PROCESS); diff --git a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java index 0f19e1fa0e..33339f03c0 100644 --- a/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java +++ b/multiapps-controller-process/src/main/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppStep.java @@ -21,6 +21,7 @@ import org.cloudfoundry.multiapps.controller.core.helpers.ApplicationFileDigestDetector; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.security.serialization.DynamicSecureSerialization; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorageException; import org.cloudfoundry.multiapps.controller.process.Messages; import org.cloudfoundry.multiapps.controller.process.security.util.SecureLoggingUtil; @@ -53,6 +54,8 @@ public class UploadAppStep extends TimeoutAsyncFlowableStep { protected CloudPackagesGetter cloudPackagesGetter; @Inject private ExecutorService appUploaderThreadPool; + @Inject + private UploadDurationTracker uploadDurationTracker; @Override public StepPhase executeAsyncStep(ProcessContext context) throws FileStorageException { @@ -185,7 +188,8 @@ private void removeApplicationDigestIfSet(ProcessContext context, Map getAsyncStepExecutions(ProcessContext context) { - return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, appUploaderThreadPool), + return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, appUploaderThreadPool, + uploadDurationTracker), new PollUploadAppStatusExecution()); } diff --git a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java index 452fb59c15..b1a5d77a56 100644 --- a/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java +++ b/multiapps-controller-process/src/test/java/org/cloudfoundry/multiapps/controller/process/steps/UploadAppAsyncExecutionTest.java @@ -26,6 +26,7 @@ import org.cloudfoundry.multiapps.controller.client.lib.domain.ImmutableCloudApplicationExtended; import org.cloudfoundry.multiapps.controller.core.helpers.MtaArchiveElements; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.FileContentConsumer; import org.cloudfoundry.multiapps.controller.persistence.services.FileService; import org.cloudfoundry.multiapps.controller.process.util.ApplicationArchiveIterator; @@ -86,6 +87,7 @@ class UploadAppAsyncExecutionTest extends AsyncStepOperationTest .build(); private final MtaArchiveElements mtaArchiveElements = new MtaArchiveElements(); private final ExecutorService appUploaderThreadPool = mock(ExecutorService.class); + private final UploadDurationTracker uploadDurationTracker = mock(UploadDurationTracker.class); @TempDir Path tempDir; @@ -220,7 +222,7 @@ protected List getAsyncStepExecutions(ProcessContext context) { return List.of(new UploadAppAsyncExecution(applicationZipBuilder, getProcessLogsPersister(), configuration, - appUploaderThreadPool) { + appUploaderThreadPool, uploadDurationTracker) { }); } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java index c892373baa..85cdcb3c31 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/FileStorageConfiguration.java @@ -1,9 +1,11 @@ package org.cloudfoundry.multiapps.controller.web.configuration; +import jakarta.inject.Inject; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.resilience.NoRetryErrorClassifier; import org.cloudfoundry.multiapps.controller.persistence.services.resilience.RetryableErrorClassifier; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.util.EnvironmentServicesFinder; import org.cloudfoundry.multiapps.controller.web.configuration.factory.ObjectStoreSelectorFactory; import org.springframework.context.annotation.Bean; @@ -16,10 +18,14 @@ public class FileStorageConfiguration { private static final String OBJECT_STORE_SERVICE_NAME = "deploy-service-os"; + @Inject + private UploadDurationTracker uploadDurationTracker; + @Bean public ObjectStoreSelectorFactory objectStoreSelector(EnvironmentServicesFinder vcapServiceFinder, ApplicationConfiguration applicationConfiguration) { - return new ObjectStoreSelectorFactory(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration); + return new ObjectStoreSelectorFactory(OBJECT_STORE_SERVICE_NAME, vcapServiceFinder, applicationConfiguration, + uploadDurationTracker); } @Bean diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java index cb52bb629a..a8aca8efc2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/JmxConfiguration.java @@ -3,6 +3,7 @@ import java.util.Map; import org.cloudfoundry.multiapps.controller.web.monitoring.Metrics; +import org.cloudfoundry.multiapps.controller.web.monitoring.UploadDurationMetrics; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.jmx.export.MBeanExporter; @@ -12,10 +13,12 @@ public class JmxConfiguration { private static final String METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=MetricsMBean"; + private static final String UPLOAD_METRICS_BEAN = "org.cloudfoundry.multiapps.controller.web.monitoring:type=Metrics,name=UploadMetricsMBean"; + @Bean - public MBeanExporter jmxExporter(Metrics metrics) { + public MBeanExporter jmxExporter(Metrics metrics, UploadDurationMetrics uploadDurationMetrics) { MBeanExporter mBeanExporter = new MBeanExporter(); - mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics)); + mBeanExporter.setBeans(Map.of(METRICS_BEAN, metrics, UPLOAD_METRICS_BEAN, uploadDurationMetrics)); return mBeanExporter; } } diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java index 117ed389ad..d4a4d27284 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java @@ -7,11 +7,13 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; + import io.pivotal.cfenv.core.CfService; import org.apache.commons.lang3.StringUtils; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.core.util.UriUtil; import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; @@ -39,13 +41,16 @@ public class ObjectStoreSelectorFactory { private final EnvironmentServicesFinder environmentServicesFinder; private final ApplicationConfiguration applicationConfiguration; private final SelectedObjectStore selectedObjectStore; + private final UploadDurationTracker uploadDurationTracker; public ObjectStoreSelectorFactory(String serviceName, EnvironmentServicesFinder environmentServicesFinder, - ApplicationConfiguration applicationConfiguration) { + ApplicationConfiguration applicationConfiguration, + UploadDurationTracker uploadDurationTracker) { this.serviceName = serviceName; this.environmentServicesFinder = environmentServicesFinder; this.applicationConfiguration = applicationConfiguration; this.selectedObjectStore = doSelect(); + this.uploadDurationTracker = uploadDurationTracker; } public FileStorage fileStorage() { @@ -155,15 +160,15 @@ private SelectedObjectStore selectFor(ObjectStoreServiceInfo objectStoreServiceI } protected GcpObjectStoreFileStorage createGcpFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new GcpObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } protected AzureObjectStoreFileStorage createAzureFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new AzureObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } protected AwsS3ObjectStoreFileStorage createAwsS3FileStorage(ObjectStoreServiceInfo objectStoreServiceInfo) { - return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials()); + return new AwsS3ObjectStoreFileStorage(objectStoreServiceInfo.getCredentials(), uploadDurationTracker); } private BlobStoreContext getBlobStoreContext(ObjectStoreServiceInfo serviceInfo) { @@ -203,7 +208,7 @@ private void resolveContextEndpoint(ObjectStoreServiceInfo serviceInfo, ContextB protected JCloudsObjectStoreFileStorage createFileStorage(ObjectStoreServiceInfo objectStoreServiceInfo, BlobStoreContext context) { return new JCloudsObjectStoreFileStorage(context.getBlobStore(), (String) objectStoreServiceInfo.getCredentials() - .get(Constants.BUCKET)); + .get(Constants.BUCKET), uploadDurationTracker); } private Optional createObjectStoreBasedOnProvider(String objectStoreProviderName, diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java new file mode 100644 index 0000000000..3207495a23 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/AppUploaderThreadPoolInformation.java @@ -0,0 +1,28 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import jakarta.inject.Inject; +import jakarta.inject.Named; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; + +@Named +public class AppUploaderThreadPoolInformation { + + private final ThreadPoolExecutor appUploaderPool; + + @Inject + public AppUploaderThreadPoolInformation( + @Named("appUploaderThreadPool") ExecutorService appUploaderThreadPool) { + this.appUploaderPool = (ThreadPoolExecutor) appUploaderThreadPool; + } + + public int getActiveThreads() { + return appUploaderPool.getActiveCount(); + } + + public int getMaxThreads() { + return appUploaderPool.getMaximumPoolSize(); + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java new file mode 100644 index 0000000000..36e111505b --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadDurationMetrics.java @@ -0,0 +1,103 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +import jakarta.inject.Inject; +import jakarta.inject.Named; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; + +@Named +public class UploadDurationMetrics implements UploadMetricsMBean { + + private final UploadDurationTracker tracker; + + private final AppUploaderThreadPoolInformation appUploaderInfo; + + @Inject + public UploadDurationMetrics(UploadDurationTracker tracker, + AppUploaderThreadPoolInformation appUploaderInfo) { + this.tracker = tracker; + this.appUploaderInfo = appUploaderInfo; + } + + @Override + public long getAppBinaryUploadTotalCount() { + return tracker.getAppBinaryStatistics() + .totalCount(); + } + + @Override + public long getAppBinaryUploadTimeoutCount() { + return tracker.getAppBinaryStatistics() + .timeoutCount(); + } + + @Override + public long getAppBinaryUploadMaxDurationMs() { + return tracker.getAppBinaryStatistics() + .maxDurationMs(); + } + + @Override + public long getAppBinaryUploadSumDurationMs() { + return tracker.getAppBinaryStatistics() + .sumDurationMs(); + } + + @Override + public int getAppUploaderActiveThreads() { + return appUploaderInfo.getActiveThreads(); + } + + @Override + public int getAppUploaderMaxThreads() { + return appUploaderInfo.getMaxThreads(); + } + + @Override + public long getAppUploaderRejectionCount() { + return tracker.getAppBinaryStatistics() + .rejectionCount(); + } + + @Override + public long getAppUploaderRejectionCountInWindow() { + return tracker.getAppBinaryStatistics() + .rejectionsInWindow(); + } + + @Override + public long getObjectStoreUploadTotalCount() { + return tracker.getObjectStoreStatistics() + .totalCount(); + } + + @Override + public long getObjectStoreUploadTimeoutCount() { + return tracker.getObjectStoreStatistics() + .timeoutCount(); + } + + @Override + public long getObjectStoreUploadMaxDurationMs() { + return tracker.getObjectStoreStatistics() + .maxDurationMs(); + } + + @Override + public long getObjectStoreUploadSumDurationMs() { + return tracker.getObjectStoreStatistics() + .sumDurationMs(); + } + + @Override + public long getAppBinaryUploadTimeoutsInWindow() { + return tracker.getAppBinaryStatistics() + .timeoutsInWindow(); + } + + @Override + public long getObjectStoreUploadTimeoutsInWindow() { + return tracker.getObjectStoreStatistics() + .timeoutsInWindow(); + } + +} diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java new file mode 100644 index 0000000000..396c352db1 --- /dev/null +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/monitoring/UploadMetricsMBean.java @@ -0,0 +1,33 @@ +package org.cloudfoundry.multiapps.controller.web.monitoring; + +public interface UploadMetricsMBean { + + long getAppBinaryUploadTotalCount(); + + long getAppBinaryUploadTimeoutCount(); + + long getAppBinaryUploadMaxDurationMs(); + + long getAppBinaryUploadSumDurationMs(); + + int getAppUploaderActiveThreads(); + + int getAppUploaderMaxThreads(); + + long getAppUploaderRejectionCount(); + + long getAppUploaderRejectionCountInWindow(); + + long getObjectStoreUploadTotalCount(); + + long getObjectStoreUploadTimeoutCount(); + + long getObjectStoreUploadMaxDurationMs(); + + long getObjectStoreUploadSumDurationMs(); + + long getAppBinaryUploadTimeoutsInWindow(); + + long getObjectStoreUploadTimeoutsInWindow(); + +} diff --git a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java index b28570c678..911f956f89 100644 --- a/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java +++ b/multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/configuration/bean/factory/ObjectStoreSelectorFactoryTest.java @@ -4,6 +4,7 @@ import io.pivotal.cfenv.core.CfService; import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration; import org.cloudfoundry.multiapps.controller.persistence.services.AwsS3ObjectStoreFileStorage; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.services.AzureObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.GcpObjectStoreFileStorage; import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage; @@ -64,6 +65,8 @@ class ObjectStoreSelectorFactoryTest { private AwsS3ObjectStoreFileStorage awsS3ObjectStoreFileStorage; @Mock private AzureObjectStoreFileStorage azureObjectStoreFileStorage; + @Mock + private UploadDurationTracker uploadDurationTracker; private ObjectStoreServiceInfo capturedGcpServiceInfo; private ObjectStoreServiceInfo capturedAzureServiceInfo; @@ -103,7 +106,7 @@ void testObjectStoreCreationWhenEnvIsValidForAws() { assertInstanceOf(AwsS3ObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(AwsTransientErrorClassifier.class, selector.classifier()); verify(awsS3ObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -116,7 +119,7 @@ void testObjectStoreCreationWhenEnvIsValidForAzure() { assertInstanceOf(AzureObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(AzureTransientErrorClassifier.class, selector.classifier()); verify(azureObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -129,12 +132,12 @@ void testObjectStoreCreationWhenEnvIsValidForGcp() { assertInstanceOf(GcpObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(GcpTransientErrorClassifier.class, selector.classifier()); verify(gcpObjectStoreFileStorage) - .testConnection(); + .testConnection(); } static Stream testObjectStoreCreationFallsBackToFirstReachableProviderWhenEnvIsInvalid() { return Stream.of( - // @formatter:off + // @formatter:off // (0) Unknown provider name: Arguments.of("WRONG_PROVIDER"), // (1) Null env value: @@ -169,7 +172,7 @@ void testObjectStoreCreationFallsBackToNextProviderWhenFirstFails() { assertInstanceOf(JCloudsObjectStoreFileStorage.class, selector.fileStorage()); assertInstanceOf(JCloudsTransientErrorClassifier.class, selector.classifier()); verify(jCloudsObjectStoreFileStorage) - .testConnection(); + .testConnection(); } @Test @@ -289,7 +292,7 @@ private class ObjectStoreSelectorFactoryMock extends ObjectStoreSelectorFactory ObjectStoreSelectorFactoryMock(String serviceName, EnvironmentServicesFinder environmentServicesFinder, ApplicationConfiguration applicationConfiguration) { - super(serviceName, environmentServicesFinder, applicationConfiguration); + super(serviceName, environmentServicesFinder, applicationConfiguration, uploadDurationTracker); } @Override From 20645c9931f6cb015799942deff780f61b623537 Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Fri, 31 Jul 2026 12:23:09 +0300 Subject: [PATCH 2/4] Fix merge conflicts and add AWS OS upload tracking LMCROSSITXSADEPLOY-3054 --- .../src/main/java/module-info.java | 1 + .../controller/persistence/Messages.java | 1 + .../services/AwsS3ObjectStoreFileStorage.java | 24 ++++++++++++++-- .../AwsS3ObjectStoreFileStorageTest.java | 28 +++++++++++-------- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index b3d692cb0b..b6a21467c0 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -74,4 +74,5 @@ requires software.amazon.awssdk.retries.api; requires static java.compiler; requires static org.immutables.value; + requires io.netty.handler; } diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java index 22707e64c3..7ee1d015df 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/Messages.java @@ -70,6 +70,7 @@ public final class Messages { public static final String TIME_ELAPSED_FOR_JCLOUDS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for JClouds object store upload: {0} in millis"; public static final String TIME_ELAPSED_FOR_GCP_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for GCP object store upload: {0} in millis"; public static final String TIME_ELAPSED_FOR_AZURE_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for Azure object store upload: {0} in millis"; + public static final String TIME_ELAPSED_FOR_AWS_OS_UPLOAD_0_IN_MILLIS = "Time elapsed for AWS object store upload: {0} in millis"; // DEBUG log messages: public static final String STORED_FILE_0 = "Stored file: \"{0}\""; diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java index a075038c51..0647856341 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.net.URI; import java.text.MessageFormat; +import java.time.Duration; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashSet; @@ -16,13 +17,17 @@ import java.util.concurrent.Executors; import java.util.function.BiPredicate; import java.util.function.Consumer; + import org.cloudfoundry.multiapps.controller.persistence.Messages; import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadTimeoutMatcher; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreFilter; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.cglib.core.Local; import org.springframework.http.MediaType; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -57,10 +62,12 @@ public class AwsS3ObjectStoreFileStorage extends ObjectStoreFileStorage { private final S3Client s3Client; private final String bucketName; + private final UploadDurationTracker uploadDurationTracker; - public AwsS3ObjectStoreFileStorage(Map credentials) { + public AwsS3ObjectStoreFileStorage(Map credentials, UploadDurationTracker uploadDurationTracker) { this.bucketName = (String) credentials.get(CredentialKeys.BUCKET); this.s3Client = createS3Client(credentials); + this.uploadDurationTracker = uploadDurationTracker; } protected S3Client createS3Client(Map credentials) { @@ -103,6 +110,7 @@ protected ClientOverrideConfiguration buildClientOverrideConfig() { @Override public void addFile(FileEntry fileEntry, InputStream content) throws FileStorageException { + LocalDateTime startTime = LocalDateTime.now(); long fileSize = fileEntry.getSize() .longValue(); PutObjectRequest request = PutObjectRequest.builder() @@ -115,8 +123,12 @@ public void addFile(FileEntry fileEntry, InputStream content) throws FileStorage try { s3Client.putObject(request, RequestBody.fromInputStream(new BufferedInputStream(content), fileSize)); LOGGER.debug(MessageFormat.format(Messages.STORED_FILE_0_WITH_SIZE_1, fileEntry.getId(), fileSize)); + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), false); + LOGGER.info(MessageFormat.format(Messages.TIME_ELAPSED_FOR_AWS_OS_UPLOAD_0_IN_MILLIS, getElapsedTimeInMillis(startTime))); } catch (Exception e) { LOGGER.error(MessageFormat.format(Messages.S3_UPLOAD_FAILED_FILE_0_SIZE_1, fileEntry.getName(), fileSize, e)); + uploadDurationTracker.recordObjectStoreUpload(getElapsedTimeInMillis(startTime), + UploadTimeoutMatcher.isUploadTimeoutException(e)); throw new FileStorageException(MessageFormat.format(Messages.UPLOAD_OF_FILE_WITH_NAMESPACE_FAILED, fileEntry.getName(), fileEntry.getNamespace()), e); } @@ -183,14 +195,15 @@ public void deleteFilesBySpaceIds(List spaceIds) { @Override public void deleteFilesBySpaceAndNamespace(String space, String namespace) { - int deletedFiles = deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); + int deletedFiles = deleteByFilterAndCount( + (key, metadata) -> ObjectStoreFilter.filterBySpaceAndNamespace(metadata, space, namespace)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_WITH_SPACE_1_AND_NAMESPACE_2, deletedFiles, space, namespace)); } @Override public int deleteFilesModifiedBefore(LocalDateTime modificationTime) { int deletedFiles = deleteByFilterAndCount((key, metadata) -> ObjectStoreFilter.filterByModificationTime(metadata, key, - modificationTime)); + modificationTime)); LOGGER.debug(MessageFormat.format(Messages.DELETED_0_FILES_MODIFIED_BEFORE_1, deletedFiles, modificationTime)); return deletedFiles; } @@ -319,6 +332,11 @@ public void destroy() { s3Client.close(); } + private long getElapsedTimeInMillis(LocalDateTime startTime) { + return Duration.between(startTime, LocalDateTime.now()) + .toMillis(); + } + private static final class CredentialKeys { static final String ACCESS_KEY_ID = "access_key_id"; static final String SECRET_ACCESS_KEY = "secret_access_key"; diff --git a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java index b5b5a08011..e41c95d561 100644 --- a/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java +++ b/multiapps-controller-persistence/src/test/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorageTest.java @@ -2,6 +2,7 @@ import org.cloudfoundry.multiapps.controller.persistence.model.FileEntry; import org.cloudfoundry.multiapps.controller.persistence.model.ImmutableFileEntry; +import org.cloudfoundry.multiapps.controller.persistence.monitoring.UploadDurationTracker; import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreConstants; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -62,16 +63,19 @@ class AwsS3ObjectStoreFileStorageTest { @Mock private FileContentProcessor fileContentProcessor; + @Mock + private UploadDurationTracker uploadDurationTracker; + private AwsS3ObjectStoreFileStorage fileStorage; private final InputStream inputStream = new ByteArrayInputStream(new byte[] {}); private static final String TEST_SPACE_ID = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_SPACE_ID_2 = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_ID = UUID.randomUUID() - .toString(); + .toString(); private static final String TEST_ID_2 = UUID.randomUUID() - .toString(); + .toString(); private static final String NAMESPACE = "namespace"; private static final String NAMESPACE_2 = "namespace_2"; @@ -80,7 +84,7 @@ void setUp() throws Exception { MockitoAnnotations.openMocks(this) .close(); - fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME)) { + fileStorage = new AwsS3ObjectStoreFileStorage(Map.of("bucket", BUCKET_NAME), uploadDurationTracker) { @Override protected S3Client createS3Client(Map credentials) { @@ -136,7 +140,7 @@ void testGetFileEntriesWithoutContentWithoutMatches() { } @Test - void testExistsInObjectStoreWhenFileExists() { + void testExistsInObjectStoreWhenFileExists() { when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(HeadObjectResponse.builder() .build()); FileEntry fileEntry = createFileEntry(TEST_SPACE_ID, TEST_ID); @@ -232,7 +236,7 @@ void testDeleteFilesBySpaceAndNamespaceWithOneMatch() { @Test void testDeleteFilesModifiedBefore() { LocalDateTime oldModified = FILE_TIMESTAMP - .minusMinutes(15); + .minusMinutes(15); setupListObjectsWithKeys(TEST_ID, TEST_ID_2); setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, oldModified); @@ -250,7 +254,7 @@ void testDeleteFilesModifiedBeforeWithNoOldFiles() { setupHeadObjectWithMetadata(TEST_ID, TEST_SPACE_ID, NAMESPACE, FILE_TIMESTAMP); LocalDateTime cutoff = FILE_TIMESTAMP - .minusDays(1); + .minusDays(1); int deletedCount = fileStorage.deleteFilesModifiedBefore(cutoff); assertEquals(0, deletedCount); @@ -532,7 +536,7 @@ void testBuildClientOverrideConfigSetsApiCallAttemptTimeout() { assertTrue(config.apiCallAttemptTimeout() .isPresent()); assertEquals(ObjectStoreConstants.OBJECT_STORE_TOTAL_TIMEOUT_CONFIG_IN_MINUTES, config.apiCallAttemptTimeout() - .get()); + .get()); } @Test @@ -544,8 +548,8 @@ void testBuildClientOverrideConfigSetsRetryStrategy() { assertInstanceOf(StandardRetryStrategy.class, config.retryStrategy() .get()); assertEquals(ObjectStoreConstants.OBJECT_STORE_MAX_ATTEMPTS_CONFIG, config.retryStrategy() - .get() - .maxAttempts()); + .get() + .maxAttempts()); } @Test @@ -555,7 +559,7 @@ void testCreateS3ClientWithValidCredentials() { "bucket", "test-bucket", "host", "s3.amazonaws.com", "region", "eu-central-1"); - AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials); + AwsS3ObjectStoreFileStorage storage = new AwsS3ObjectStoreFileStorage(credentials, uploadDurationTracker); assertNotNull(storage); storage.destroy(); From ce767724268a09ced3391ddf401b7163c8b3d7bc Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Sun, 2 Aug 2026 22:49:49 +0300 Subject: [PATCH 3/4] Fixing spring bean problems and imports LMCROSSITXSADEPLOY-3054 --- .../persistence/services/AwsS3ObjectStoreFileStorage.java | 1 - .../web/configuration/factory/ObjectStoreSelectorFactory.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java index 0647856341..53e813a877 100644 --- a/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java +++ b/multiapps-controller-persistence/src/main/java/org/cloudfoundry/multiapps/controller/persistence/services/AwsS3ObjectStoreFileStorage.java @@ -27,7 +27,6 @@ import org.cloudfoundry.multiapps.controller.persistence.util.ObjectStoreMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.cglib.core.Local; import org.springframework.http.MediaType; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; diff --git a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java index d4a4d27284..ceeb850bf2 100644 --- a/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java +++ b/multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/configuration/factory/ObjectStoreSelectorFactory.java @@ -49,8 +49,8 @@ public ObjectStoreSelectorFactory(String serviceName, EnvironmentServicesFinder this.serviceName = serviceName; this.environmentServicesFinder = environmentServicesFinder; this.applicationConfiguration = applicationConfiguration; - this.selectedObjectStore = doSelect(); this.uploadDurationTracker = uploadDurationTracker; + this.selectedObjectStore = doSelect(); } public FileStorage fileStorage() { From f0681925c1e6d3182e0ca14fe814cd3f46fa5fe3 Mon Sep 17 00:00:00 2001 From: Krasimir Kargov Date: Mon, 3 Aug 2026 17:58:03 +0300 Subject: [PATCH 4/4] Fix alphabetical order of netty LMCROSSITXSADEPLOY-3054 --- multiapps-controller-persistence/src/main/java/module-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multiapps-controller-persistence/src/main/java/module-info.java b/multiapps-controller-persistence/src/main/java/module-info.java index b6a21467c0..99c890ab64 100644 --- a/multiapps-controller-persistence/src/main/java/module-info.java +++ b/multiapps-controller-persistence/src/main/java/module-info.java @@ -47,6 +47,7 @@ requires google.cloud.core; requires google.cloud.nio; requires google.cloud.storage; + requires io.netty.handler; requires jakarta.xml.bind; requires jakarta.annotation; requires jakarta.inject; @@ -74,5 +75,4 @@ requires software.amazon.awssdk.retries.api; requires static java.compiler; requires static org.immutables.value; - requires io.netty.handler; }