[tests-only] test: fix tests run without remotephp

.woodpecker.star

@@ -201,13 +201,6 @@ config = {
             ],
             "skip": False,
         },
-        "accountsHashDifficulty": {
-            "skip": False,
-            "suites": [
-                "apiAccountsHashDifficulty",
-            ],
-            "accounts_hash_difficulty": "default",
-        },
         "notification": {
             "suites": [
                 "apiNotification",
@@ -234,7 +227,6 @@ config = {
             ],
             "skip": False,
             "antivirusNeeded": True,
-            "generateVirusFiles": True,
             "extraServerEnvironment": {
                 "ANTIVIRUS_SCANNER_TYPE": "clamav",
                 "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310",
@@ -301,7 +293,6 @@ config = {
             "skip": False,
             "withRemotePhp": [True],
             "antivirusNeeded": True,
-            "generateVirusFiles": True,
             "extraServerEnvironment": {
                 "ANTIVIRUS_SCANNER_TYPE": "clamav",
                 "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310",
@@ -519,16 +510,7 @@ def main(ctx):
     )
 
     test_pipelines = \
-        codestyle(ctx) + \
-        checkGherkinLint(ctx) + \
-        checkTestSuitesInExpectedFailures(ctx) + \
-        pipelinesDependsOn(buildWebCache(ctx), savePipelineNumber(ctx)) + \
-        pipelinesDependsOn(cacheBrowsers(ctx), savePipelineNumber(ctx)) + \
-        getGoBinForTesting(ctx) + \
         pipelinesDependsOn(buildOpencloudBinaryForTesting(ctx), savePipelineNumber(ctx)) + \
-        checkStarlark(ctx) + \
-        build_release_helpers + \
-        testOpencloudAndUploadResults(ctx) + \
         testPipelines(ctx)
 
     build_release_pipelines = \
@@ -667,15 +649,15 @@ def testPipelines(ctx):
         storage = "decomposed"
 
     if "skip" not in config["cs3ApiTests"] or not config["cs3ApiTests"]["skip"]:
-        pipelines += cs3ApiTests(ctx, storage, "default")
+        pipelines += cs3ApiTests(ctx, storage)
     if "skip" not in config["wopiValidatorTests"] or not config["wopiValidatorTests"]["skip"]:
-        pipelines += wopiValidatorTests(ctx, storage, "builtin", "default")
-        pipelines += wopiValidatorTests(ctx, storage, "cs3", "default")
+        pipelines += wopiValidatorTests(ctx, storage, "builtin")
+        pipelines += wopiValidatorTests(ctx, storage, "cs3")
 
-    pipelines += localApiTestPipeline(ctx)
+    pipelines = localApiTestPipeline(ctx)
     pipelines += coreApiTestPipeline(ctx)
-    pipelines += e2eTestPipeline(ctx)
-    pipelines += multiServiceE2ePipeline(ctx)
+    # pipelines += e2eTestPipeline(ctx)
+    # pipelines += multiServiceE2ePipeline(ctx)
 
     if ("skip" not in config["k6LoadTests"] or not config["k6LoadTests"]["skip"]) and ("k6-test" in ctx.build.title.lower() or ctx.build.event == "cron"):
         pipelines += pipelineDependsOn(k6LoadTests(ctx), savePipelineNumber(ctx))
@@ -1059,12 +1041,12 @@ def codestyle(ctx):
 
     return pipelines
 
-def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4):
+def cs3ApiTests(ctx, storage):
     pipeline = {
         "name": "test-cs3-API-%s" % storage,
         "steps": evaluateWorkflowStep() +
                  restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
-                 opencloudServer(storage, accounts_hash_difficulty, deploy_type = "cs3api_validator") +
+                 opencloudServer(storage, deploy_type = "cs3api_validator") +
                  [
                      {
                          "name": "cs3ApiTests",
@@ -1095,7 +1077,7 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4):
     ])
     return [pipeline]
 
-def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = 4):
+def wopiValidatorTests(ctx, storage, wopiServerType):
     testgroups = [
         "BaseWopiViewing",
         "CheckFileInfoSchema",
@@ -1173,7 +1155,7 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty =
         "steps": evaluateWorkflowStep() +
                  restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
                  waitForServices("fake-office", ["fakeoffice:8080"]) +
-                 opencloudServer(storage, accounts_hash_difficulty, deploy_type = "wopi_validator", extra_server_environment = extra_server_environment) +
+                 opencloudServer(storage, deploy_type = "wopi_validator", extra_server_environment = extra_server_environment) +
                  wopiServer +
                  waitForServices("wopi-fakeoffice", ["wopi-fakeoffice:9300"]) +
                  [
@@ -1218,29 +1200,21 @@ def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty =
 def localApiTestPipeline(ctx):
     pipelines = []
 
-    with_remote_php = [True]
-    enable_watch_fs = [False]
-    if ctx.build.event == "cron":
-        with_remote_php.append(False)
-        enable_watch_fs.append(True)
-
     defaults = {
         "suites": {},
         "skip": False,
         "extraTestEnvironment": {},
         "extraServerEnvironment": {},
         "storages": ["posix"],
-        "accounts_hash_difficulty": 4,
         "emailNeeded": False,
         "antivirusNeeded": False,
         "tikaNeeded": False,
         "federationServer": False,
         "collaborationServiceNeeded": False,
         "extraCollaborationEnvironment": {},
-        "withRemotePhp": with_remote_php,
-        "enableWatchFs": enable_watch_fs,
+        "withRemotePhp": [True],
+        "enableWatchFs": [False],
         "ldapNeeded": False,
-        "generateVirusFiles": False,
     }
 
     if "localApiTests" in config:
@@ -1255,6 +1229,15 @@ def localApiTestPipeline(ctx):
                 if "[decomposed]" in ctx.build.title.lower() or name.startswith("cli"):
                     params["storages"] = ["decomposed"]
 
+                params["withRemotePhp"] = [True, False]
+                if ctx.build.event == "cron":
+                    params["withRemotePhp"] = [True, False]
+                    params["enableWatchFs"] = [True, False]
+
+                # override withRemotePhp if specified in the suite config
+                if "withRemotePhp" in matrix:
+                    params["withRemotePhp"] = matrix["withRemotePhp"]
+
                 for storage in params["storages"]:
                     for run_with_remote_php in params["withRemotePhp"]:
                         for run_with_watch_fs_enabled in params["enableWatchFs"]:
@@ -1279,16 +1262,15 @@ def localApiTestPipeline(ctx):
                                          (waitForLdapService() if params["ldapNeeded"] else []) +
                                          opencloudServer(
                                              storage,
-                                             params["accounts_hash_difficulty"],
                                              extra_server_environment = params["extraServerEnvironment"],
                                              with_wrapper = True,
                                              tika_enabled = params["tikaNeeded"],
                                              watch_fs_enabled = run_with_watch_fs_enabled,
                                          ) +
-                                         (opencloudServer(storage, params["accounts_hash_difficulty"], deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs_enabled) if params["federationServer"] else []) +
+                                         (opencloudServer(storage, deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs_enabled) if params["federationServer"] else []) +
                                          ((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) +
                                          (openCloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) +
-                                         localApiTest(params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php, params["generateVirusFiles"]) +
+                                         localApiTest(params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php) +
                                          logRequests(),
                                 "services": (emailService() if params["emailNeeded"] else []) +
                                             (clamavService() if params["antivirusNeeded"] else []) +
@@ -1312,7 +1294,7 @@ def localApiTestPipeline(ctx):
                             pipelines.append(pipeline)
     return pipelines
 
-def localApiTest(suites, storage = "decomposed", extra_environment = {}, with_remote_php = False, generate_virus_files = False):
+def localApiTest(suites, storage = "decomposed", extra_environment = {}, with_remote_php = False):
     test_dir = "%s/tests/acceptance" % dirs["base"]
     expected_failures_file = "%s/expected-failures-%s-storage.md" % (test_dir, storage)
 
@@ -1337,11 +1319,6 @@ def localApiTest(suites, storage = "decomposed", extra_environment = {}, with_re
 
     commands = []
 
-    # Generate EICAR virus test files if needed
-    if generate_virus_files:
-        commands.append("chmod +x %s/tests/acceptance/scripts/generate-virus-files.sh" % dirs["base"])
-        commands.append("bash %s/tests/acceptance/scripts/generate-virus-files.sh" % dirs["base"])
-
     # Merge expected failures
     if not with_remote_php:
         commands.append("cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file))
@@ -1364,7 +1341,6 @@ def coreApiTestPipeline(ctx):
         "numberOfParts": 7,
         "skipExceptParts": [],
         "skip": False,
-        "accounts_hash_difficulty": 4,
     }
 
     pipelines = []
@@ -1381,10 +1357,15 @@ def coreApiTestPipeline(ctx):
         if "[decomposed]" in ctx.build.title.lower():
             params["storages"] = ["decomposed"]
 
+        params["withRemotePhp"] = [True, False]
         if ctx.build.event == "cron":
             params["withRemotePhp"] = [True, False]
             params["enableWatchFs"] = [True, False]
 
+        # override withRemotePhp if specified in the suite config
+        if "withRemotePhp" in matrix:
+            params["withRemotePhp"] = matrix["withRemotePhp"]
+
         debugParts = params["skipExceptParts"]
         debugPartsEnabled = (len(debugParts) != 0)
 
@@ -1406,7 +1387,6 @@ def coreApiTestPipeline(ctx):
                                          restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
                                          opencloudServer(
                                              storage,
-                                             params["accounts_hash_difficulty"],
                                              with_wrapper = True,
                                              watch_fs_enabled = run_with_watch_fs_enabled,
                                          ) +
@@ -2324,7 +2304,7 @@ def notifyMatrix(ctx):
 
     return result
 
-def opencloudServer(storage = "decomposed", accounts_hash_difficulty = 4, depends_on = [], deploy_type = "", extra_server_environment = {}, with_wrapper = False, tika_enabled = False, watch_fs_enabled = False):
+def opencloudServer(storage = "decomposed", depends_on = [], deploy_type = "", extra_server_environment = {}, with_wrapper = False, tika_enabled = False, watch_fs_enabled = False):
     user = "0:0"
     container_name = OC_SERVER_NAME
     environment = {
@@ -2420,13 +2400,6 @@ def opencloudServer(storage = "decomposed", accounts_hash_difficulty = 4, depend
     if watch_fs_enabled:
         environment["STORAGE_USERS_POSIX_WATCH_FS"] = True
 
-    # Pass in "default" accounts_hash_difficulty to not set this environment variable.
-    # That will allow OpenCloud to use whatever its built-in default is.
-    # Otherwise pass in a value from 4 to about 11 or 12 (default 4, for making regular tests fast)
-    # The high values cause lots of CPU to be used when hashing passwords, and really slow down the tests.
-    if accounts_hash_difficulty != "default":
-        environment["ACCOUNTS_HASH_DIFFICULTY"] = accounts_hash_difficulty
-
     for item in extra_server_environment:
         environment[item] = extra_server_environment[item]
 

tests/README.md

@@ -466,15 +466,6 @@ For antivirus running localy on Linux OS, use `ANTIVIRUS_CLAMAV_SOCKET= "/var/ru
 For antivirus running localy on Mac OS, use `ANTIVIRUS_CLAMAV_SOCKET= "/tmp/clamd.sock"`.
 For antivirus running with docker, use `ANTIVIRUS_CLAMAV_SOCKET= "tcp://host.docker.internal:3310"`
 
-#### Create Virus Files
-
-The antivirus tests require EICAR test files which are not stored in the repository
-They are generated dynamically when needed for testing.
-
-```bash
-tests/acceptance/scripts/generate-virus-files.sh
-```
-
 #### Run the Acceptance Test
 
 Run the acceptance test with the following command:

tests/acceptance/TestHelpers/HttpRequestHelper.php

@@ -62,7 +62,7 @@ public static function numRetriesOnHttpTooEarly(): int {
 
 	/**
 	 *
-	 * @param string|null $url
+	 * @param string $url
 	 * @param string|null $xRequestId
 	 * @param string|null $method
 	 * @param string|null $user
@@ -80,8 +80,8 @@ public static function numRetriesOnHttpTooEarly(): int {
 	 * @throws GuzzleException
 	 */
 	public static function sendRequestOnce(
-		?string $url,
-		?string $xRequestId,
+		string $url,
+		?string $xRequestId = null,
 		?string $method = 'GET',
 		?string $user = null,
 		?string $password = null,

tests/acceptance/TestHelpers/UploadHelper.php

@@ -195,13 +195,20 @@ public static function createFileWithText(?string $name, ?string $text): void {
 	}
 
 	/**
-	 * get the path of a file from FilesForUpload directory
+	 * get the path of the acceptance tests directory
 	 *
-	 * @param string|null $name name of the file to upload
+	 * @return string
+	 */
+	public static function getAcceptanceTestsDir(): string {
+		return \dirname(__FILE__) . "/../";
+	}
+
+	/**
+	 * get the path of the filesForUpload directory
 	 *
 	 * @return string
 	 */
-	public static function getUploadFilesDir(?string $name): string {
-		return \getenv("FILES_FOR_UPLOAD") . $name;
+	public static function getFilesForUploadDir(): string {
+		return \dirname(__FILE__) . "/../filesForUpload/";
 	}
 }

tests/acceptance/bootstrap/ChecksumContext.php

@@ -25,6 +25,7 @@
 use Psr\Http\Message\ResponseInterface;
 use TestHelpers\WebDavHelper;
 use TestHelpers\BehatHelper;
+use TestHelpers\UploadHelper;
 
 require_once 'bootstrap.php';
 
@@ -49,7 +50,7 @@ public function uploadFileToWithChecksumUsingTheAPI(
 		string  $checksum
 	): ResponseInterface {
 		$file = \file_get_contents(
-			$this->featureContext->acceptanceTestsDirLocation() . $source
+			UploadHelper::getAcceptanceTestsDir() . $source
 		);
 		return $this->featureContext->makeDavRequest(
 			$user,

tests/acceptance/bootstrap/FeatureContext.php

@@ -43,6 +43,7 @@
 use TestHelpers\SettingsHelper;
 use TestHelpers\OcConfigHelper;
 use TestHelpers\BehatHelper;
+use TestHelpers\UploadHelper;
 use Swaggest\JsonSchema\InvalidValue as JsonSchemaException;
 use Swaggest\JsonSchema\Exception\ArrayException;
 use Swaggest\JsonSchema\Exception\ConstException;
@@ -562,6 +563,38 @@ public static function setupLogDir(BeforeSuiteScope $scope): void {
 		}
 	}
 
+	/**
+	 * @BeforeScenario @antivirus
+	 *
+	 * @return void
+	 * @throws Exception
+	 */
+	public function createTestVirusFiles(): void {
+		$uploadDir = UploadHelper::getFilesForUploadDir() . 'filesWithVirus/';
+		$virusFile = $uploadDir . 'eicar.com';
+		$virusZipFile = $uploadDir . 'eicar_com.zip';
+
+		if (file_exists($virusFile) && file_exists($virusZipFile)) {
+			return;
+		}
+
+		if (!is_dir($uploadDir)) {
+			mkdir($uploadDir, 0755);
+		}
+
+		$res1 = HttpRequestHelper::sendRequestOnce('https://secure.eicar.org/eicar.com');
+		if ($res1->getStatusCode() !== 200) {
+			throw new Exception("Could not download eicar.com test virus file");
+		}
+		file_put_contents($virusFile, $res1->getBody()->getContents());
+
+		$res2 = HttpRequestHelper::sendRequestOnce('https://secure.eicar.org/eicar_com.zip');
+		file_put_contents($virusZipFile, $res2->getBody()->getContents());
+		if ($res2->getStatusCode() !== 200) {
+			throw new Exception("Could not download eicar_com.zip test virus file");
+		}
+	}
+
 	/**
 	 *
 	 * @BeforeScenario
@@ -2596,18 +2629,11 @@ public function temporaryStorageSubfolderName(): string {
 		return "work_tmp";
 	}
 
-	/**
-	 * @return string
-	 */
-	public function acceptanceTestsDirLocation(): string {
-		return \dirname(__FILE__) . "/../";
-	}
-
 	/**
 	 * @return string
 	 */
 	public function workStorageDirLocation(): string {
-		return $this->acceptanceTestsDirLocation() . $this->temporaryStorageSubfolderName() . "/";
+		return UploadHelper::getAcceptanceTestsDir() . $this->temporaryStorageSubfolderName() . "/";
 	}
 
 	/**