[WIP] feat: naive performance test and handling results storing

.github/scripts/performance-to-markdown.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+import json
+import sys
+import os
+
+def convert_to_markdown(json_file):
+    """Convert performance test JSON results to markdown format"""
+    try:
+        with open(json_file, 'r') as f:
+            data = json.load(f)
+    except FileNotFoundError:
+        return "## Performance Test Results\n\nNo performance test results found."
+    except json.JSONDecodeError:
+        return "## Performance Test Results\n\nError parsing performance test results."
+
+    markdown = "## Performance Test Results\n\n"
+
+    if 'summaries' not in data or not data['summaries']:
+        return markdown + "No test summaries available."
+
+    for summary in data['summaries']:
+        name = summary.get('name', 'Unknown Test')
+        duration = summary.get('duration', 0)
+        processors = summary.get('numberOfProcessors', 0)
+        max_memory = summary.get('maxMemory', 0)
+
+        # Convert memory from bytes to GB
+        max_memory_gb = max_memory / (1024 ** 3) if max_memory > 0 else 0
+
+        markdown += f"### {name}\n\n"
+        markdown += "| Metric | Value |\n"
+        markdown += "|--------|-------|\n"
+        markdown += f"| Duration | {duration} ms |\n"
+        markdown += f"| Processors | {processors} |\n"
+        markdown += f"| Max Memory | {max_memory_gb:.2f} GB |\n"
+        markdown += "\n"
+
+    return markdown
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: performance-to-markdown.py <json_file>")
+        sys.exit(1)
+
+    json_file = sys.argv[1]
+    markdown = convert_to_markdown(json_file)
+    print(markdown)

.github/workflows/integration-tests.yml

@@ -55,3 +55,10 @@ jobs:
           echo "Using profile: ${it_profile}"
           ./mvnw ${MAVEN_ARGS} -T1C -B install -DskipTests -Pno-apt --file pom.xml
           ./mvnw ${MAVEN_ARGS} -T1C -B package -P${it_profile} -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml
+
+      - name: Upload performance test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: performance-results-java${{ inputs.java-version }}-k8s${{ inputs.kube-version }}-${{ inputs.http-client }}
+          path: operator-framework/target/performance_test_result.json
+          if-no-files-found: ignore

.github/workflows/pr.yml

@@ -32,3 +32,51 @@ jobs:
 
   build:
     uses: ./.github/workflows/build.yml
+
+  performance_report:
+    name: Post Performance Results
+    runs-on: ubuntu-latest
+    needs: build
+    if: always()
+    permissions:
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Download all performance artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: performance-results-*
+          path: performance-results
+          merge-multiple: true
+
+      - name: Check for performance results
+        id: check_results
+        run: |
+          if [ -d "performance-results" ] && [ "$(ls -A performance-results/*.json 2>/dev/null)" ]; then
+            echo "has_results=true" >> $GITHUB_OUTPUT
+          else
+            echo "has_results=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Convert performance results to markdown
+        if: steps.check_results.outputs.has_results == 'true'
+        id: convert
+        run: |
+          echo "# Performance Test Results" > comment.md
+          echo "" >> comment.md
+          for file in performance-results/*.json; do
+            if [ -f "$file" ]; then
+              echo "Processing $file"
+              python3 .github/scripts/performance-to-markdown.py "$file" >> comment.md
+              echo "" >> comment.md
+            fi
+          done
+
+      - name: Post PR comment
+        if: steps.check_results.outputs.has_results == 'true' && github.event_name == 'pull_request'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          path: comment.md
+          recreate: true
+

operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.baseapi.performance;
+
+import java.util.List;
+
+public class PerformanceTestResult {
+
+  private List<PerformanceTestSummary> summaries;
+
+  public List<PerformanceTestSummary> getSummaries() {
+    return summaries;
+  }
+
+  public void setSummaries(List<PerformanceTestSummary> summaries) {
+    this.summaries = summaries;
+  }
+}

operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.baseapi.performance;
+
+public class PerformanceTestSummary {
+
+  private String name;
+
+  // data about the machine
+  private int numberOfProcessors;
+  private long maxMemory;
+  private long duration;
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public int getNumberOfProcessors() {
+    return numberOfProcessors;
+  }
+
+  public void setNumberOfProcessors(int numberOfProcessors) {
+    this.numberOfProcessors = numberOfProcessors;
+  }
+
+  public long getMaxMemory() {
+    return maxMemory;
+  }
+
+  public void setMaxMemory(long maxMemory) {
+    this.maxMemory = maxMemory;
+  }
+
+  public long getDuration() {
+    return duration;
+  }
+
+  public void setDuration(long duration) {
+    this.duration = duration;
+  }
+}

operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java

@@ -0,0 +1,197 @@
+/*
+ * Copyright Java Operator SDK Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.javaoperatorsdk.operator.baseapi.performance;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
+import io.fabric8.kubernetes.client.informers.ResourceEventHandler;
+import io.fabric8.kubernetes.client.informers.SharedIndexInformer;
+import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension;
+import io.vertx.core.impl.ConcurrentHashSet;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class SimplePerformanceTestIT {
+
+  private static final Logger log = LoggerFactory.getLogger(SimplePerformanceTestIT.class);
+  public static final String INITIAL_VALUE = "initialValue";
+  public static final String RESOURCE_NAME_PREFIX = "resource";
+  public static final String INDEX = "index";
+
+  @RegisterExtension
+  LocallyRunOperatorExtension extension =
+      LocallyRunOperatorExtension.builder()
+          .withReconciler(new SimplePerformanceTestReconciler())
+          .build();
+
+  final int WARM_UP_RESOURCE_NUMBER = 10;
+  final int TEST_RESOURCE_NUMBER = 150;
+
+  ExecutorService executor = Executors.newFixedThreadPool(TEST_RESOURCE_NUMBER);
+
+  @Test
+  void simpleNaivePerformanceTest() {
+    var processors = Runtime.getRuntime().availableProcessors();
+    long maxMemory = Runtime.getRuntime().maxMemory();
+    log.info("Running performance test with memory: {} and processors: {}", maxMemory, processors);
+
+    var primaryInformer =
+        extension
+            .getKubernetesClient()
+            .resources(SimplePerformanceTestResource.class)
+            .inNamespace(extension.getNamespace())
+            .inform();
+
+    var statusChecker =
+        new StatusChecker(INITIAL_VALUE, 0, WARM_UP_RESOURCE_NUMBER, primaryInformer);
+    createResources(0, WARM_UP_RESOURCE_NUMBER, INITIAL_VALUE);
+    statusChecker.waitUntilAllInStatus();
+
+    long startTime = System.currentTimeMillis();
+    statusChecker =
+        new StatusChecker(
+            INITIAL_VALUE, WARM_UP_RESOURCE_NUMBER, TEST_RESOURCE_NUMBER, primaryInformer);
+    createResources(WARM_UP_RESOURCE_NUMBER, TEST_RESOURCE_NUMBER, INITIAL_VALUE);
+    statusChecker.waitUntilAllInStatus();
+    var duration = System.currentTimeMillis() - startTime;
+
+    log.info("Create duration: {}", duration);
+    saveResults(duration);
+  }
+
+  private void saveResults(long duration) {
+    try {
+      var result = new PerformanceTestResult();
+      var summary = new PerformanceTestSummary();
+      result.setSummaries(List.of(summary));
+      summary.setName("Naive performance test");
+      summary.setDuration(duration);
+      summary.setNumberOfProcessors(Runtime.getRuntime().availableProcessors());
+      summary.setMaxMemory(Runtime.getRuntime().maxMemory());
+      var objectMapper = new ObjectMapper();
+      objectMapper.writeValue(new File("target/performance_test_result.json"), result);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void createResources(int startIndex, int number, String value) {
+    try {
+      List<Callable<Void>> callables = new ArrayList<>(number);
+
+      for (int i = startIndex; i < startIndex + number; i++) {
+        var res = new SimplePerformanceTestResource();
+        res.setMetadata(
+            new ObjectMetaBuilder()
+                .withAnnotations(Map.of(INDEX, "" + i))
+                .withName(RESOURCE_NAME_PREFIX + i)
+                .build());
+        res.setSpec(new SimplePerformanceTestSpec());
+        res.getSpec().setValue(value);
+        callables.add(
+            () -> {
+              extension.create(res);
+              return null;
+            });
+      }
+      var futures = executor.invokeAll(callables);
+      for (var future : futures) {
+        future.get();
+      }
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  static class StatusChecker {
+    private final String expectedStatus; // null indicates deleted
+    private final Set<Integer> remaining = new ConcurrentHashSet<>();
+
+    StatusChecker(
+        String expectedStatus,
+        int startIndex,
+        int number,
+        SharedIndexInformer<SimplePerformanceTestResource> primaryInformer) {
+      this.expectedStatus = expectedStatus;
+      for (int i = startIndex; i < startIndex + number; i++) {
+        remaining.add(i);
+      }
+      primaryInformer.addEventHandler(
+          new ResourceEventHandler<>() {
+            @Override
+            public void onAdd(SimplePerformanceTestResource obj) {
+              checkOnStatus(obj);
+            }
+
+            @Override
+            public void onUpdate(
+                SimplePerformanceTestResource oldObj, SimplePerformanceTestResource newObj) {
+              checkOnStatus(newObj);
+            }
+
+            @Override
+            public void onDelete(
+                SimplePerformanceTestResource obj, boolean deletedFinalStateUnknown) {
+              if (expectedStatus == null) {
+                synchronized (remaining) {
+                  remaining.remove(Integer.parseInt(obj.getMetadata().getAnnotations().get(INDEX)));
+                  remaining.notifyAll();
+                }
+              }
+            }
+          });
+      primaryInformer.getStore().list().forEach(this::checkOnStatus);
+    }
+
+    private void checkOnStatus(SimplePerformanceTestResource res) {
+      if (expectedStatus != null
+          && res.getStatus() != null
+          && res.getStatus().getValue().equals(expectedStatus)) {
+        synchronized (remaining) {
+          remaining.remove(Integer.parseInt(res.getMetadata().getAnnotations().get(INDEX)));
+          remaining.notifyAll();
+        }
+      }
+    }
+
+    public void waitUntilAllInStatus() {
+      synchronized (remaining) {
+        while (!remaining.isEmpty()) {
+          try {
+            remaining.wait();
+          } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new RuntimeException(e);
+          }
+        }
+      }
+    }
+  }
+}