Do not return database locks during health check

Author: IvanBorislavovDimitrov

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java

@@ -98,7 +98,6 @@ public final class Messages {
     public static final String ENVIRONMENT_VARIABLE_IS_NOT_SET_USING_DEFAULT = "Environment variable \"{0}\" is not set. Using default \"{1}\"...";
     public static final String OPTIONAL_RESOURCE_IS_NOT_SERVICE = "Optional resource \"{0}\" it will be not created because it''s not a service";
     public static final String SERVICE_IS_NOT_ACTIVE = "Service \"{0}\" is inactive and will not be processed";
-    public static final String DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE = "Detected increased number of processes waiting for locks: \"{0}\" for instance: \"{1}\"";
     public static final String DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE_0_GETTING_THE_LOCKS = "Detected increased number of processes waiting for locks for instance {0}. Getting the locks...";
 
     public static final String INVALID_VCAP_APPLICATION = "Invalid VCAP_APPLICATION \"{0}\"";

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculator.java

@@ -14,8 +14,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.function.Consumer;
-import java.util.function.Supplier;
-
 import jakarta.inject.Inject;
 import jakarta.inject.Named;
 import org.cloudfoundry.multiapps.common.SLException;
@@ -27,9 +25,7 @@
 import org.cloudfoundry.multiapps.controller.core.application.health.model.ImmutableApplicationHealthResult;
 import org.cloudfoundry.multiapps.controller.core.model.CachedObject;
 import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration;
-import org.cloudfoundry.multiapps.controller.core.util.ApplicationInstanceNameUtil;
 import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService;
-import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService;
 import org.cloudfoundry.multiapps.controller.persistence.services.FileStorage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -50,7 +46,6 @@ public class ApplicationHealthCalculator {
     private final FileStorage objectStoreFileStorage;
     private final ApplicationConfiguration applicationConfiguration;
     private final DatabaseHealthService databaseHealthService;
-    private final DatabaseMonitoringService databaseMonitoringService;
     private final DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer;
 
     private final CachedObject<Boolean> objectStoreFileStorageHealthCache = new CachedObject<>(
@@ -77,12 +72,10 @@ public class ApplicationHealthCalculator {
     @Inject
     public ApplicationHealthCalculator(@Autowired(required = false) FileStorage objectStoreFileStorage,
                                        ApplicationConfiguration applicationConfiguration, DatabaseHealthService databaseHealthService,
-                                       DatabaseMonitoringService databaseMonitoringService,
                                        DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) {
         this.objectStoreFileStorage = objectStoreFileStorage;
         this.applicationConfiguration = applicationConfiguration;
         this.databaseHealthService = databaseHealthService;
-        this.databaseMonitoringService = databaseMonitoringService;
         this.databaseWaitingLocksAnalyzer = databaseWaitingLocksAnalyzer;
         scheduleRegularHealthUpdate();
     }
@@ -155,16 +148,10 @@ public ResponseEntity<ApplicationHealthResult> calculateApplicationHealth() {
             LOGGER.warn(
                 MessageFormat.format(Messages.DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE_0_GETTING_THE_LOCKS,
                                      applicationConfiguration.getApplicationInstanceIndex()));
-            long countOfProcessesWaitingForLocks = resilientOperationExecutor.execute(
-                (Supplier<Long>) () -> databaseMonitoringService.getProcessesWaitingForLocks(
-                    ApplicationInstanceNameUtil.buildApplicationInstanceTemplate(applicationConfiguration)));
-            LOGGER.warn(MessageFormat.format(Messages.DETECTED_INCREASED_NUMBER_OF_PROCESSES_WAITING_FOR_LOCKS_FOR_INSTANCE,
-                                             countOfProcessesWaitingForLocks, applicationConfiguration.getApplicationInstanceIndex()));
             return ResponseEntity.ok(ImmutableApplicationHealthResult.builder() // TODO: Make this return 503 instead of 200 when the
                                                                      // detection is trustworthy
                                                                      .status(ApplicationHealthResult.Status.DOWN)
                                                                      .hasIncreasedLocks(true)
-                                                                     .countOfProcessesWaitingForLocks(countOfProcessesWaitingForLocks)
                                                                      .build());
         }
         return ResponseEntity.ok(ImmutableApplicationHealthResult.builder()

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/database/DatabaseWaitingLocksAnalyzer.java

@@ -7,7 +7,8 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
-
+import jakarta.inject.Inject;
+import jakarta.inject.Named;
 import org.cloudfoundry.multiapps.controller.core.Messages;
 import org.cloudfoundry.multiapps.controller.core.model.CachedObject;
 import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration;
@@ -16,9 +17,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import jakarta.inject.Inject;
-import jakarta.inject.Named;
-
 @Named
 public class DatabaseWaitingLocksAnalyzer {
 
@@ -59,7 +57,8 @@ private void deleteObsoleteSamples() {
     }
 
     private void takeLocksSample() {
-        waitingLocksSamples.add(new CachedObject<>(databaseMonitoringService.getProcessesWaitingForLocks(ApplicationInstanceNameUtil.buildApplicationInstanceTemplate(applicationConfiguration)),
+        waitingLocksSamples.add(new CachedObject<>(databaseMonitoringService.getProcessesWaitingForLocks(
+            ApplicationInstanceNameUtil.buildApplicationInstanceTemplate(applicationConfiguration)),
                                                    MAXIMUM_VALIDITY_OF_LOCKS_SAMPLE_IN_MINUTES));
     }
 
@@ -73,10 +72,11 @@ public synchronized boolean hasIncreasedDbLocks() {
         double calculatedIncreasingOrEqualIndex = calculateIncreasingOrEqualIndex();
         boolean hasIncreasedLocks = calculatedIncreasingOrEqualIndex >= MAXIMAL_ACCEPTABLE_INCREMENTAL_LOCKS_DEVIATION_INDEX
             && checkIfLastOneThirdOfSequenceHasIncreasedOrIsEqualComparedToFirstOneThird(minimumRequiredSamplesCount);
-        if (shouldLogValues()) {
+        if (hasIncreasedLocks || shouldLogValues()) {
             LOGGER.info(MessageFormat.format(Messages.VALUES_IN_INSTANCE_IN_THE_WAITING_FOR_LOCKS_SAMPLES,
                                              applicationConfiguration.getApplicationInstanceIndex(), waitingLocksSamples.stream()
-                                                                                                                        .map(CachedObject::get)
+                                                                                                                        .map(
+                                                                                                                            CachedObject::get)
                                                                                                                         .toList()));
         }
         return hasIncreasedLocks;

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/application/health/model/ApplicationHealthResult.java

@@ -2,7 +2,6 @@
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import org.cloudfoundry.multiapps.controller.client.facade.Nullable;
 import org.immutables.value.Value;
 
 @Value.Immutable
@@ -12,9 +11,6 @@ public interface ApplicationHealthResult {
 
     Status getStatus();
 
-    @Nullable
-    Long countOfProcessesWaitingForLocks();
-
     Boolean hasIncreasedLocks();
 
     enum Status {

multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/application/health/ApplicationHealthCalculatorTest.java

@@ -6,7 +6,6 @@
 import org.cloudfoundry.multiapps.controller.core.application.health.model.ApplicationHealthResult;
 import org.cloudfoundry.multiapps.controller.core.util.ApplicationConfiguration;
 import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseHealthService;
-import org.cloudfoundry.multiapps.controller.persistence.services.DatabaseMonitoringService;
 import org.cloudfoundry.multiapps.controller.persistence.services.JCloudsObjectStoreFileStorage;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -15,7 +14,6 @@
 import org.mockito.MockitoAnnotations;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
-
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -29,8 +27,6 @@ class ApplicationHealthCalculatorTest {
     @Mock
     private DatabaseHealthService databaseHealthService;
     @Mock
-    private DatabaseMonitoringService databaseMonitoringService;
-    @Mock
     private DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer;
 
     private ApplicationHealthCalculator applicationHealthCalculator;
@@ -44,7 +40,6 @@ void setUp() throws Exception {
         applicationHealthCalculator = new ApplicationHealthCalculatorMock(jCloudsObjectStoreFileStorage,
                                                                           applicationConfiguration,
                                                                           databaseHealthService,
-                                                                          databaseMonitoringService,
                                                                           databaseWaitingLocksAnalyzer);
     }
 
@@ -105,7 +100,6 @@ void testSuccessfulUpdateWithMissingObjectStore() {
         applicationHealthCalculator = new ApplicationHealthCalculatorMock(null,
                                                                           applicationConfiguration,
                                                                           databaseHealthService,
-                                                                          databaseMonitoringService,
                                                                           databaseWaitingLocksAnalyzer);
         applicationHealthCalculator.updateHealthStatus();
         ResponseEntity<ApplicationHealthResult> applicationHealthResultResponseEntity = applicationHealthCalculator.calculateApplicationHealth();
@@ -120,12 +114,10 @@ private static class ApplicationHealthCalculatorMock extends ApplicationHealthCa
         public ApplicationHealthCalculatorMock(JCloudsObjectStoreFileStorage JCloudsObjectStoreFileStorage,
                                                ApplicationConfiguration applicationConfiguration,
                                                DatabaseHealthService databaseHealthService,
-                                               DatabaseMonitoringService databaseMonitoringService,
                                                DatabaseWaitingLocksAnalyzer databaseWaitingLocksAnalyzer) {
             super(JCloudsObjectStoreFileStorage,
                   applicationConfiguration,
                   databaseHealthService,
-                  databaseMonitoringService,
                   databaseWaitingLocksAnalyzer);
         }