From 1a8e3a0be4dae1e00c4120a1e7fef7a55142e1a2 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 17 Jun 2025 11:19:17 +0300 Subject: [PATCH 01/55] bigint --- src/Database/Adapter/MariaDB.php | 4 +-- tests/e2e/Adapter/Base.php | 4 ++- tests/e2e/Adapter/Scopes/DocumentTests.php | 31 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index bab2eb267..35fbbd6d9 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -154,7 +154,7 @@ public function createCollection(string $name, array $attributes = [], array $in $collection = " CREATE TABLE {$this->getSQLTable($id)} ( - _id INT(11) UNSIGNED NOT NULL AUTO_INCREMENT, + _id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, _uid VARCHAR(255) NOT NULL, _createdAt DATETIME(3) DEFAULT NULL, _updatedAt DATETIME(3) DEFAULT NULL, @@ -185,7 +185,7 @@ public function createCollection(string $name, array $attributes = [], array $in $permissions = " CREATE TABLE {$this->getSQLTable($id . '_perms')} ( - _id int(11) UNSIGNED NOT NULL AUTO_INCREMENT, + _id BIGINT NOT NULL AUTO_INCREMENT, _type VARCHAR(12) NOT NULL, _permission VARCHAR(255) NOT NULL, _document VARCHAR(255) NOT NULL, diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index a57fe2748..3ce07ecec 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -17,8 +17,10 @@ abstract class Base extends TestCase { - use CollectionTests; use DocumentTests; + + use CollectionTests; + use AttributeTests; use IndexTests; use PermissionTests; diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 3fdbb87d7..3f86fe43d 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -21,6 +21,37 @@ trait DocumentTests { + public function testBigintSequence(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + $database->createCollection(__FUNCTION__); + + $sequence = 5_000_000_000_000_000; + + $document = $database->createDocument(__FUNCTION__, new Document([ + '$sequence' => (string)$sequence, + '$permissions' => [ + Permission::read(Role::any()), + ], + ])); + + $this->assertEquals((string)$sequence, $document->getSequence()); + + $document = $database->getDocument(__FUNCTION__, $document->getId()); + + $this->assertEquals((string)$sequence, $document->getSequence()); + + $document = $database->createDocument(__FUNCTION__, new Document([ + '$permissions' => [ + Permission::read(Role::any()), + ], + ])); + + $this->assertEquals((string)($sequence + 1), $document->getSequence()); + } + public function testCreateDocument(): Document { /** @var Database $database */ From bf598870d272809a01daa26beb316f6f6a8a0aa5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 17 Jun 2025 14:11:41 +0300 Subject: [PATCH 02/55] autoincrement --- composer.lock | 181 ++++++++++++++---------------- docker-compose.yml | 4 +- src/Database/Adapter/Postgres.php | 23 ++-- 3 files changed, 102 insertions(+), 106 deletions(-) diff --git a/composer.lock b/composer.lock index 774cd790d..a06c77f0c 100644 --- a/composer.lock +++ b/composer.lock @@ -8,16 +8,16 @@ "packages": [ { "name": "brick/math", - "version": "0.12.3", + "version": "0.13.1", "source": { "type": "git", "url": "https://github.com/brick/math.git", - "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba" + "reference": "fc7ed316430118cc7836bf45faff18d5dfc8de04" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/brick/math/zipball/866551da34e9a618e64a819ee1e01c20d8a588ba", - "reference": "866551da34e9a618e64a819ee1e01c20d8a588ba", + "url": "https://api.github.com/repos/brick/math/zipball/fc7ed316430118cc7836bf45faff18d5dfc8de04", + "reference": "fc7ed316430118cc7836bf45faff18d5dfc8de04", "shasum": "" }, "require": { @@ -56,7 +56,7 @@ ], "support": { "issues": "https://github.com/brick/math/issues", - "source": "https://github.com/brick/math/tree/0.12.3" + "source": "https://github.com/brick/math/tree/0.13.1" }, "funding": [ { @@ -64,7 +64,7 @@ "type": "github" } ], - "time": "2025-02-28T13:11:00+00:00" + "time": "2025-03-29T13:50:30+00:00" }, { "name": "composer/semver", @@ -149,16 +149,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.2", + "version": "v4.31.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced" + "reference": "2b028ce8876254e2acbeceea7d9b573faad41864" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/a4c4d8565b40b9f76debc9dfeb221412eacb8ced", - "reference": "a4c4d8565b40b9f76debc9dfeb221412eacb8ced", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/2b028ce8876254e2acbeceea7d9b573faad41864", + "reference": "2b028ce8876254e2acbeceea7d9b573faad41864", "shasum": "" }, "require": { @@ -187,9 +187,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.2" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.31.1" }, - "time": "2025-03-26T18:01:50+00:00" + "time": "2025-05-28T18:52:35+00:00" }, { "name": "nyholm/psr7", @@ -466,16 +466,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.3.0", + "version": "1.3.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c" + "reference": "8b3ca1f86d01429c73b407bf1a2075d9c187001e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c", - "reference": "19adf03d2b0f91f9e9b1c7f93db6c755c737cf6c", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/8b3ca1f86d01429c73b407bf1a2075d9c187001e", + "reference": "8b3ca1f86d01429c73b407bf1a2075d9c187001e", "shasum": "" }, "require": { @@ -526,7 +526,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-05-12T00:36:35+00:00" + "time": "2025-05-21T12:02:20+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -593,16 +593,16 @@ }, { "name": "open-telemetry/sdk", - "version": "1.4.0", + "version": "1.5.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "939d3a28395c249a763676458140dad44b3a8011" + "reference": "cd0d7367599717fc29e04eb8838ec061e6c2c657" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/939d3a28395c249a763676458140dad44b3a8011", - "reference": "939d3a28395c249a763676458140dad44b3a8011", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/cd0d7367599717fc29e04eb8838ec061e6c2c657", + "reference": "cd0d7367599717fc29e04eb8838ec061e6c2c657", "shasum": "" }, "require": { @@ -679,7 +679,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-05-07T12:32:21+00:00" + "time": "2025-05-22T02:33:34+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1158,20 +1158,20 @@ }, { "name": "ramsey/uuid", - "version": "4.7.6", + "version": "4.8.1", "source": { "type": "git", "url": "https://github.com/ramsey/uuid.git", - "reference": "91039bc1faa45ba123c4328958e620d382ec7088" + "reference": "fdf4dd4e2ff1813111bd0ad58d7a1ddbb5b56c28" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ramsey/uuid/zipball/91039bc1faa45ba123c4328958e620d382ec7088", - "reference": "91039bc1faa45ba123c4328958e620d382ec7088", + "url": "https://api.github.com/repos/ramsey/uuid/zipball/fdf4dd4e2ff1813111bd0ad58d7a1ddbb5b56c28", + "reference": "fdf4dd4e2ff1813111bd0ad58d7a1ddbb5b56c28", "shasum": "" }, "require": { - "brick/math": "^0.8.8 || ^0.9 || ^0.10 || ^0.11 || ^0.12", + "brick/math": "^0.8.8 || ^0.9 || ^0.10 || ^0.11 || ^0.12 || ^0.13", "ext-json": "*", "php": "^8.0", "ramsey/collection": "^1.2 || ^2.0" @@ -1180,26 +1180,23 @@ "rhumsaa/uuid": "self.version" }, "require-dev": { - "captainhook/captainhook": "^5.10", + "captainhook/captainhook": "^5.25", "captainhook/plugin-composer": "^5.3", - "dealerdirect/phpcodesniffer-composer-installer": "^0.7.0", - "doctrine/annotations": "^1.8", - "ergebnis/composer-normalize": "^2.15", - "mockery/mockery": "^1.3", + "dealerdirect/phpcodesniffer-composer-installer": "^1.0", + "ergebnis/composer-normalize": "^2.47", + "mockery/mockery": "^1.6", "paragonie/random-lib": "^2", - "php-mock/php-mock": "^2.2", - "php-mock/php-mock-mockery": "^1.3", - "php-parallel-lint/php-parallel-lint": "^1.1", - "phpbench/phpbench": "^1.0", - "phpstan/extension-installer": "^1.1", - "phpstan/phpstan": "^1.8", - "phpstan/phpstan-mockery": "^1.1", - "phpstan/phpstan-phpunit": "^1.1", - "phpunit/phpunit": "^8.5 || ^9", - "ramsey/composer-repl": "^1.4", - "slevomat/coding-standard": "^8.4", - "squizlabs/php_codesniffer": "^3.5", - "vimeo/psalm": "^4.9" + "php-mock/php-mock": "^2.6", + "php-mock/php-mock-mockery": "^1.5", + "php-parallel-lint/php-parallel-lint": "^1.4.0", + "phpbench/phpbench": "^1.2.14", + "phpstan/extension-installer": "^1.4", + "phpstan/phpstan": "^2.1", + "phpstan/phpstan-mockery": "^2.0", + "phpstan/phpstan-phpunit": "^2.0", + "phpunit/phpunit": "^9.6", + "slevomat/coding-standard": "^8.18", + "squizlabs/php_codesniffer": "^3.13" }, "suggest": { "ext-bcmath": "Enables faster math with arbitrary-precision integers using BCMath.", @@ -1234,32 +1231,22 @@ ], "support": { "issues": "https://github.com/ramsey/uuid/issues", - "source": "https://github.com/ramsey/uuid/tree/4.7.6" + "source": "https://github.com/ramsey/uuid/tree/4.8.1" }, - "funding": [ - { - "url": "https://github.com/ramsey", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/ramsey/uuid", - "type": "tidelift" - } - ], - "time": "2024-04-27T21:32:50+00:00" + "time": "2025-06-01T06:28:46+00:00" }, { "name": "symfony/deprecation-contracts", - "version": "v3.5.1", + "version": "v3.6.0", "source": { "type": "git", "url": "https://github.com/symfony/deprecation-contracts.git", - "reference": "74c71c939a79f7d5bf3c1ce9f5ea37ba0114c6f6" + "reference": "63afe740e99a13ba87ec199bb07bbdee937a5b62" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/74c71c939a79f7d5bf3c1ce9f5ea37ba0114c6f6", - "reference": "74c71c939a79f7d5bf3c1ce9f5ea37ba0114c6f6", + "url": "https://api.github.com/repos/symfony/deprecation-contracts/zipball/63afe740e99a13ba87ec199bb07bbdee937a5b62", + "reference": "63afe740e99a13ba87ec199bb07bbdee937a5b62", "shasum": "" }, "require": { @@ -1272,7 +1259,7 @@ "name": "symfony/contracts" }, "branch-alias": { - "dev-main": "3.5-dev" + "dev-main": "3.6-dev" } }, "autoload": { @@ -1297,7 +1284,7 @@ "description": "A generic function and convention to trigger deprecation notices", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/deprecation-contracts/tree/v3.5.1" + "source": "https://github.com/symfony/deprecation-contracts/tree/v3.6.0" }, "funding": [ { @@ -1313,20 +1300,20 @@ "type": "tidelift" } ], - "time": "2024-09-25T14:20:29+00:00" + "time": "2024-09-25T14:21:43+00:00" }, { "name": "symfony/http-client", - "version": "v7.2.4", + "version": "v7.3.0", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "78981a2ffef6437ed92d4d7e2a86a82f256c6dc6" + "reference": "57e4fb86314015a695a750ace358d07a7e37b8a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/78981a2ffef6437ed92d4d7e2a86a82f256c6dc6", - "reference": "78981a2ffef6437ed92d4d7e2a86a82f256c6dc6", + "url": "https://api.github.com/repos/symfony/http-client/zipball/57e4fb86314015a695a750ace358d07a7e37b8a9", + "reference": "57e4fb86314015a695a750ace358d07a7e37b8a9", "shasum": "" }, "require": { @@ -1392,7 +1379,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.2.4" + "source": "https://github.com/symfony/http-client/tree/v7.3.0" }, "funding": [ { @@ -1408,20 +1395,20 @@ "type": "tidelift" } ], - "time": "2025-02-13T10:27:23+00:00" + "time": "2025-05-02T08:23:16+00:00" }, { "name": "symfony/http-client-contracts", - "version": "v3.5.2", + "version": "v3.6.0", "source": { "type": "git", "url": "https://github.com/symfony/http-client-contracts.git", - "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645" + "reference": "75d7043853a42837e68111812f4d964b01e5101c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/ee8d807ab20fcb51267fdace50fbe3494c31e645", - "reference": "ee8d807ab20fcb51267fdace50fbe3494c31e645", + "url": "https://api.github.com/repos/symfony/http-client-contracts/zipball/75d7043853a42837e68111812f4d964b01e5101c", + "reference": "75d7043853a42837e68111812f4d964b01e5101c", "shasum": "" }, "require": { @@ -1434,7 +1421,7 @@ "name": "symfony/contracts" }, "branch-alias": { - "dev-main": "3.5-dev" + "dev-main": "3.6-dev" } }, "autoload": { @@ -1470,7 +1457,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/http-client-contracts/tree/v3.5.2" + "source": "https://github.com/symfony/http-client-contracts/tree/v3.6.0" }, "funding": [ { @@ -1486,7 +1473,7 @@ "type": "tidelift" } ], - "time": "2024-12-07T08:49:48+00:00" + "time": "2025-04-29T11:18:49+00:00" }, { "name": "symfony/polyfill-mbstring", @@ -1647,16 +1634,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.5.1", + "version": "v3.6.0", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "e53260aabf78fb3d63f8d79d69ece59f80d5eda0" + "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/e53260aabf78fb3d63f8d79d69ece59f80d5eda0", - "reference": "e53260aabf78fb3d63f8d79d69ece59f80d5eda0", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/f021b05a130d35510bd6b25fe9053c2a8a15d5d4", + "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4", "shasum": "" }, "require": { @@ -1674,7 +1661,7 @@ "name": "symfony/contracts" }, "branch-alias": { - "dev-main": "3.5-dev" + "dev-main": "3.6-dev" } }, "autoload": { @@ -1710,7 +1697,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.5.1" + "source": "https://github.com/symfony/service-contracts/tree/v3.6.0" }, "funding": [ { @@ -1726,7 +1713,7 @@ "type": "tidelift" } ], - "time": "2024-09-25T14:20:29+00:00" + "time": "2025-04-25T09:37:31+00:00" }, { "name": "tbachert/spi", @@ -1880,16 +1867,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.19", + "version": "0.33.20", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0" + "reference": "e1c7ab4e0b5b0a9a70256b1e00912e101e76a131" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", - "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", + "url": "https://api.github.com/repos/utopia-php/http/zipball/e1c7ab4e0b5b0a9a70256b1e00912e101e76a131", + "reference": "e1c7ab4e0b5b0a9a70256b1e00912e101e76a131", "shasum": "" }, "require": { @@ -1921,9 +1908,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.19" + "source": "https://github.com/utopia-php/http/tree/0.33.20" }, - "time": "2025-03-06T11:37:49+00:00" + "time": "2025-05-18T23:51:21+00:00" }, { "name": "utopia-php/pools", @@ -2498,16 +2485,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.25", + "version": "1.12.27", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "e310849a19e02b8bfcbb63147f495d8f872dd96f" + "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/e310849a19e02b8bfcbb63147f495d8f872dd96f", - "reference": "e310849a19e02b8bfcbb63147f495d8f872dd96f", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3a6e423c076ab39dfedc307e2ac627ef579db162", + "reference": "3a6e423c076ab39dfedc307e2ac627ef579db162", "shasum": "" }, "require": { @@ -2552,7 +2539,7 @@ "type": "github" } ], - "time": "2025-04-27T12:20:45+00:00" + "time": "2025-05-21T20:51:45+00:00" }, { "name": "phpunit/php-code-coverage", @@ -4131,7 +4118,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { @@ -4139,6 +4126,6 @@ "ext-pdo": "*", "ext-mbstring": "*" }, - "platform-dev": {}, + "platform-dev": [], "plugin-api-version": "2.6.0" } diff --git a/docker-compose.yml b/docker-compose.yml index e7861f69e..591ea0a7e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: - database postgres: - image: postgres:16.4 + image: postgres:17.5 container_name: utopia-postgres networks: - database @@ -39,7 +39,7 @@ services: POSTGRES_PASSWORD: password postgres-mirror: - image: postgres:16.4 + image: postgres:17.5 container_name: utopia-postgres-mirror networks: - database diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 37d2edeec..aa184d4d7 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -231,14 +231,13 @@ public function createCollection(string $name, array $attributes = [], array $in $sqlTenant = $this->sharedTables ? '_tenant INTEGER DEFAULT NULL,' : ''; $collection = " CREATE TABLE {$this->getSQLTable($id)} ( - _id SERIAL NOT NULL, + _id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, _uid VARCHAR(255) NOT NULL, ". $sqlTenant ." \"_createdAt\" TIMESTAMP(3) DEFAULT NULL, \"_updatedAt\" TIMESTAMP(3) DEFAULT NULL, - _permissions TEXT DEFAULT NULL, " . \implode(' ', $attributeStrings) . " - PRIMARY KEY (_id) + _permissions TEXT DEFAULT NULL ); "; @@ -261,13 +260,12 @@ public function createCollection(string $name, array $attributes = [], array $in $permissions = " CREATE TABLE {$this->getSQLTable($id . '_perms')} ( - _id SERIAL NOT NULL, + _id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, _tenant INTEGER DEFAULT NULL, _type VARCHAR(12) NOT NULL, _permission VARCHAR(255) NOT NULL, - _document VARCHAR(255) NOT NULL, - PRIMARY KEY (_id) - ); + _document VARCHAR(255) NOT NULL + ); "; if ($this->sharedTables) { @@ -1029,6 +1027,17 @@ public function createDocument(string $collection, Document $document): Document try { $this->execute($stmt); $lastInsertedId = $this->getPDO()->lastInsertId(); + + if (!empty($document->getSequence())) { + $this->getPDO()->exec(" + SELECT setval( + pg_get_serial_sequence('{$this->getSQLTable($name)}', '_id'), + (SELECT MAX(_id) FROM {$this->getSQLTable($name)}), + true + ); + "); + } + // Sequence can be manually set as well $document['$sequence'] ??= $lastInsertedId; From b566721e9784053e025a53ecbe5df373dc958dda Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 17 Jun 2025 14:26:36 +0300 Subject: [PATCH 03/55] Fix tests --- tests/e2e/Adapter/Base.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 3ce07ecec..a57fe2748 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -17,10 +17,8 @@ abstract class Base extends TestCase { - use DocumentTests; - use CollectionTests; - + use DocumentTests; use AttributeTests; use IndexTests; use PermissionTests; From 6924ad95baf879d8fec4a5c82f893f2df961ecd3 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 17 Jun 2025 16:21:24 +0300 Subject: [PATCH 04/55] Move higher --- src/Database/Adapter/Postgres.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index aa184d4d7..e42d835a4 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1026,7 +1026,6 @@ public function createDocument(string $collection, Document $document): Document try { $this->execute($stmt); - $lastInsertedId = $this->getPDO()->lastInsertId(); if (!empty($document->getSequence())) { $this->getPDO()->exec(" @@ -1037,7 +1036,8 @@ public function createDocument(string $collection, Document $document): Document ); "); } - + + $lastInsertedId = $this->getPDO()->lastInsertId(); // Sequence can be manually set as well $document['$sequence'] ??= $lastInsertedId; From 302d51a28adf625f030545e0a9f206852c877b92 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 17 Jun 2025 16:38:24 +0300 Subject: [PATCH 05/55] lint --- src/Database/Adapter/Postgres.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index e42d835a4..c91035b4b 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1036,7 +1036,7 @@ public function createDocument(string $collection, Document $document): Document ); "); } - + $lastInsertedId = $this->getPDO()->lastInsertId(); // Sequence can be manually set as well $document['$sequence'] ??= $lastInsertedId; From 51c9b78395593e27418a8db6ee9e7483249a20bb Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 21 Jul 2025 11:33:08 +0300 Subject: [PATCH 06/55] Add UNSIGNED --- src/Database/Adapter/MariaDB.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 35fbbd6d9..51d881f65 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -185,7 +185,7 @@ public function createCollection(string $name, array $attributes = [], array $in $permissions = " CREATE TABLE {$this->getSQLTable($id . '_perms')} ( - _id BIGINT NOT NULL AUTO_INCREMENT, + _id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, _type VARCHAR(12) NOT NULL, _permission VARCHAR(255) NOT NULL, _document VARCHAR(255) NOT NULL, From 115e341afd778605bdb1a7e8d93ba7b544602525 Mon Sep 17 00:00:00 2001 From: fogelito Date: Wed, 10 Sep 2025 16:41:43 +0300 Subject: [PATCH 07/55] Spatial --- phpunit.xml | 3 +- src/Database/Database.php | 54 +++++++++++++++++++ tests/e2e/Adapter/Base.php | 12 ++--- tests/e2e/Adapter/Scopes/SpatialTests.php | 66 +++++++++++------------ 4 files changed, 94 insertions(+), 41 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 2a0531cfd..7469c5341 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,8 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false" -> + stopOnFailure="true"> ./tests/unit diff --git a/src/Database/Database.php b/src/Database/Database.php index 2fec661a1..62ee486ef 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -477,6 +477,57 @@ function (?string $value) { return DateTime::formatTz($value); } ); + + self::addFilter( + 'point', + function (mixed $value) { + return "POINT({$value[0]} {$value[1]})"; + }, + function (?array $value) { + return $value; + } + ); + + self::addFilter( + 'polygon', + function (mixed $value) { + // Check if this is a single ring (flat array of points) or multiple rings + $isSingleRing = count($value) > 0 && is_array($value[0]) && + count($value[0]) === 2 && is_numeric($value[0][0]) && is_numeric($value[0][1]); + + if ($isSingleRing) { + // Convert single ring format [[x1,y1], [x2,y2], ...] to multi-ring format + $value = [$value]; + } + + $rings = []; + foreach ($value as $ring) { + $points = []; + foreach ($ring as $point) { + $points[] = "{$point[0]} {$point[1]}"; + } + $rings[] = '(' . implode(', ', $points) . ')'; + } + return 'POLYGON(' . implode(', ', $rings) . ')'; + }, + function (?array $value) { + return $value; + } + ); + + self::addFilter( + 'linestring', + function (mixed $value) { + $points = []; + foreach ($value as $point) { + $points[] = "{$point[0]} {$point[1]}"; + } + return 'LINESTRING(' . implode(', ', $points) . ')'; + }, + function (?array $value) { + return $value; + } + ); } /** @@ -1889,6 +1940,9 @@ protected function getRequiredFilters(?string $type): array { return match ($type) { self::VAR_DATETIME => ['datetime'], + self::VAR_POINT => ['point'], + self::VAR_POLYGON => ['polygon'], + self::VAR_LINESTRING => ['linestring'], default => [], }; } diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 37ad7cce3..481704ce6 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -18,12 +18,12 @@ abstract class Base extends TestCase { - use CollectionTests; - use DocumentTests; - use AttributeTests; - use IndexTests; - use PermissionTests; - use RelationshipTests; +// use CollectionTests; +// use DocumentTests; +// use AttributeTests; +// use IndexTests; +// use PermissionTests; +// use RelationshipTests; use SpatialTests; use GeneralTests; diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 3d93d233e..2df7981a6 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -74,7 +74,7 @@ public function testSpatialCollection(): void $this->assertIsArray($col->getAttribute('indexes')); $this->assertCount(2, $col->getAttribute('indexes')); - $database->createAttribute($collectionName, 'attribute3', Database::VAR_POINT, 0, true); + $database->createAttribute($collectionName, 'attribute3', Database::VAR_POINT, 0, true, filters: ['point']); $database->createIndex($collectionName, ID::custom("index3"), Database::INDEX_SPATIAL, ['attribute3']); $col = $database->getCollection($collectionName); @@ -102,9 +102,9 @@ public function testSpatialTypeDocuments(): void $database->createCollection($collectionName); // Create spatial attributes using createAttribute method - $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true, filters: ['point'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true, filters: ['linestring'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true, filters: ['polygon'])); // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'point_spatial', Database::INDEX_SPATIAL, ['pointAttr'])); @@ -231,7 +231,7 @@ public function testSpatialRelationshipOneToOne(): void $database->createCollection('building'); $database->createAttribute('location', 'name', Database::VAR_STRING, 255, true); - $database->createAttribute('location', 'coordinates', Database::VAR_POINT, 0, true); + $database->createAttribute('location', 'coordinates', Database::VAR_POINT, 0, true, filters: ['point']); $database->createAttribute('building', 'name', Database::VAR_STRING, 255, true); $database->createAttribute('building', 'area', Database::VAR_STRING, 255, true); @@ -336,9 +336,9 @@ public function testSpatialAttributes(): void $database->createCollection($collectionName); $required = $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true; - $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $required)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $required)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $required)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $required, filters: ['point'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $required, filters: ['linestring'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $required, filters: ['polygon'])); // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'idx_point', Database::INDEX_SPATIAL, ['pointAttr'])); @@ -387,7 +387,7 @@ public function testSpatialOneToMany(): void $database->createAttribute($parent, 'name', Database::VAR_STRING, 255, true); $database->createAttribute($child, 'name', Database::VAR_STRING, 255, true); - $database->createAttribute($child, 'coord', Database::VAR_POINT, 0, true); + $database->createAttribute($child, 'coord', Database::VAR_POINT, 0, true, filters: ['point']); $database->createIndex($child, 'coord_spatial', Database::INDEX_SPATIAL, ['coord']); $database->createRelationship( @@ -499,7 +499,7 @@ public function testSpatialManyToOne(): void $database->createAttribute($parent, 'name', Database::VAR_STRING, 255, true); $database->createAttribute($child, 'name', Database::VAR_STRING, 255, true); - $database->createAttribute($child, 'coord', Database::VAR_POINT, 0, true); + $database->createAttribute($child, 'coord', Database::VAR_POINT, 0, true, filters: ['point']); $database->createIndex($child, 'coord_spatial', Database::INDEX_SPATIAL, ['coord']); $database->createRelationship( @@ -603,10 +603,10 @@ public function testSpatialManyToMany(): void $database->createCollection($b); $database->createAttribute($a, 'name', Database::VAR_STRING, 255, true); - $database->createAttribute($a, 'home', Database::VAR_POINT, 0, true); + $database->createAttribute($a, 'home', Database::VAR_POINT, 0, true, filters: ['point']); $database->createIndex($a, 'home_spatial', Database::INDEX_SPATIAL, ['home']); $database->createAttribute($b, 'title', Database::VAR_STRING, 255, true); - $database->createAttribute($b, 'area', Database::VAR_POLYGON, 0, true); + $database->createAttribute($b, 'area', Database::VAR_POLYGON, 0, true, filters: ['polygon']); $database->createIndex($b, 'area_spatial', Database::INDEX_SPATIAL, ['area']); $database->createRelationship( @@ -705,7 +705,7 @@ public function testSpatialIndex(): void $collectionName = 'spatial_index_'; try { $database->createCollection($collectionName); - $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true); + $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true, filters: ['point']); $this->assertEquals(true, $database->createIndex($collectionName, 'loc_spatial', Database::INDEX_SPATIAL, ['loc'])); $collection = $database->getCollection($collectionName); @@ -766,7 +766,7 @@ public function testSpatialIndex(): void $collOrderIndex = 'spatial_idx_order_index_' . uniqid(); try { $database->createCollection($collOrderIndex); - $database->createAttribute($collOrderIndex, 'loc', Database::VAR_POINT, 0, true); + $database->createAttribute($collOrderIndex, 'loc', Database::VAR_POINT, 0, true, filters: ['point']); if ($orderSupported) { $this->assertTrue($database->createIndex($collOrderIndex, 'idx_loc', Database::INDEX_SPATIAL, ['loc'], [], [Database::ORDER_DESC])); } else { @@ -826,7 +826,7 @@ public function testSpatialIndex(): void $collNullIndex = 'spatial_idx_null_index_' . uniqid(); try { $database->createCollection($collNullIndex); - $database->createAttribute($collNullIndex, 'loc', Database::VAR_POINT, 0, false); + $database->createAttribute($collNullIndex, 'loc', Database::VAR_POINT, 0, false, filters: ['point']); if ($nullSupported) { $this->assertTrue($database->createIndex($collNullIndex, 'idx_loc', Database::INDEX_SPATIAL, ['loc'])); } else { @@ -855,12 +855,12 @@ public function testComplexGeometricShapes(): void $database->createCollection($collectionName); // Create spatial attributes for different geometric shapes - $this->assertEquals(true, $database->createAttribute($collectionName, 'rectangle', Database::VAR_POLYGON, 0, true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'square', Database::VAR_POLYGON, 0, true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'triangle', Database::VAR_POLYGON, 0, true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'circle_center', Database::VAR_POINT, 0, true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'complex_polygon', Database::VAR_POLYGON, 0, true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'multi_linestring', Database::VAR_LINESTRING, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'rectangle', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'square', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'triangle', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'circle_center', Database::VAR_POINT, 0, true, filters: ['point'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'complex_polygon', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'multi_linestring', Database::VAR_LINESTRING, 0, true, filters: ['linestring'])); // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'idx_rectangle', Database::INDEX_SPATIAL, ['rectangle'])); @@ -1285,9 +1285,9 @@ public function testSpatialQueryCombinations(): void $database->createCollection($collectionName); // Create spatial attributes - $this->assertEquals(true, $database->createAttribute($collectionName, 'location', Database::VAR_POINT, 0, true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'area', Database::VAR_POLYGON, 0, true)); - $this->assertEquals(true, $database->createAttribute($collectionName, 'route', Database::VAR_LINESTRING, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'location', Database::VAR_POINT, 0, true, filters: ['point'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'area', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'route', Database::VAR_LINESTRING, 0, true, filters: ['linestring'])); $this->assertEquals(true, $database->createAttribute($collectionName, 'name', Database::VAR_STRING, 255, true)); // Create spatial indexes @@ -1719,8 +1719,8 @@ public function testSptialAggregation(): void // Create collection with spatial and numeric attributes $database->createCollection($collectionName); $database->createAttribute($collectionName, 'name', Database::VAR_STRING, 255, true); - $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true); - $database->createAttribute($collectionName, 'area', Database::VAR_POLYGON, 0, true); + $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true, filters: ['point']); + $database->createAttribute($collectionName, 'area', Database::VAR_POLYGON, 0, true, filters: ['polygon']); $database->createAttribute($collectionName, 'score', Database::VAR_INTEGER, 0, true); // Spatial indexes @@ -1808,21 +1808,21 @@ public function testUpdateSpatialAttributes(): void // 0) Disallow creation of spatial attributes with size or array try { - $database->createAttribute($collectionName, 'geom_bad_size', Database::VAR_POINT, 10, true); + $database->createAttribute($collectionName, 'geom_bad_size', Database::VAR_POINT, 10, true, filters: ['point']); $this->fail('Expected DatabaseException when creating spatial attribute with non-zero size'); } catch (\Throwable $e) { $this->assertInstanceOf(Exception::class, $e); } try { - $database->createAttribute($collectionName, 'geom_bad_array', Database::VAR_POINT, 0, true, array: true); + $database->createAttribute($collectionName, 'geom_bad_array', Database::VAR_POINT, 0, true, array: true, filters: ['point']); $this->fail('Expected DatabaseException when creating spatial attribute with array=true'); } catch (\Throwable $e) { $this->assertInstanceOf(Exception::class, $e); } // Create a single spatial attribute (required=true) - $this->assertEquals(true, $database->createAttribute($collectionName, 'geom', Database::VAR_POINT, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'geom', Database::VAR_POINT, 0, true, filters: ['point'])); $this->assertEquals(true, $database->createIndex($collectionName, 'idx_geom', Database::INDEX_SPATIAL, ['geom'])); // 1) Disallow size and array updates on spatial attributes: expect DatabaseException @@ -1893,9 +1893,9 @@ public function testSpatialAttributeDefaults(): void $database->createCollection($collectionName); // Create spatial attributes with defaults and no indexes to avoid nullability/index constraints - $this->assertEquals(true, $database->createAttribute($collectionName, 'pt', Database::VAR_POINT, 0, false, [1.0, 2.0])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'ln', Database::VAR_LINESTRING, 0, false, [[0.0, 0.0], [1.0, 1.0]])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'pg', Database::VAR_POLYGON, 0, false, [[[0.0, 0.0], [0.0, 2.0], [2.0, 2.0], [0.0, 0.0]]])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pt', Database::VAR_POINT, 0, false, [1.0, 2.0], filters: ['point'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'ln', Database::VAR_LINESTRING, 0, false, [[0.0, 0.0], [1.0, 1.0]], filters: ['linestring'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pg', Database::VAR_POLYGON, 0, false, [[[0.0, 0.0], [0.0, 2.0], [2.0, 2.0], [0.0, 0.0]]], filters: ['polygon'])); // Create non-spatial attributes (mix of defaults and no defaults) $this->assertEquals(true, $database->createAttribute($collectionName, 'title', Database::VAR_STRING, 255, false, 'Untitled')); @@ -2100,7 +2100,7 @@ public function testSpatialDistanceInMeter(): void $collectionName = 'spatial_distance_meters_'; try { $database->createCollection($collectionName); - $this->assertEquals(true, $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true, filters: ['point'])); $this->assertEquals(true, $database->createIndex($collectionName, 'idx_loc', Database::INDEX_SPATIAL, ['loc'])); // Two points roughly ~1000 meters apart by latitude delta (~0.009 deg ≈ 1km) From 6f8bdf9d936b7d11db95dd48fd3abac011a15dce Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 10:39:05 +0300 Subject: [PATCH 08/55] Postgres point --- src/Database/Adapter.php | 4 ++ src/Database/Adapter/Postgres.php | 19 ++++- src/Database/Adapter/SQL.php | 10 +++ src/Database/Database.php | 31 ++++----- src/Database/Validator/Spatial.php | 1 + tests/e2e/Adapter/Scopes/SpatialTests.php | 84 +++++++++++++---------- 6 files changed, 94 insertions(+), 55 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 25ed510a5..f7b832143 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1285,4 +1285,8 @@ abstract public function getTenantQuery(string $collection, string $alias = ''): * @return bool */ abstract protected function execute(mixed $stmt): bool; + + abstract protected function encodePoint(array $point): mixed; + abstract protected function decodePoint(mixed $data): array; + } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index f666d1184..cacf72610 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -993,7 +993,7 @@ public function createDocument(Document $collection, Document $document): Docume INSERT INTO {$this->getSQLTable($name)} ({$columns} \"_uid\") VALUES ({$columnNames} :_uid) "; - +var_dump($sql); $sql = $this->trigger(Database::EVENT_DOCUMENT_CREATE, $sql); $stmt = $this->getPDO()->prepare($sql); @@ -2002,4 +2002,21 @@ public function getSupportForSpatialAxisOrder(): bool { return false; } + + public function encodePoint(array $point): string + { + return "SRID=4326;POINT({$point[0]} {$point[1]})";// EWKT + } + + public function decodePoint(mixed $data): array + { + $ewkt = str_replace('SRID=4326;', '', $data); + + // Expect format "POINT(x y)" + if (preg_match('/POINT\(([-\d\.]+) ([-\d\.]+)\)/', $ewkt, $matches)) { + return [(float)$matches[1], (float)$matches[2]]; + } + + throw new Exception("Invalid EWKT format: $ewkt"); + } } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 5876858e8..d5757c3ee 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2710,4 +2710,14 @@ public function getSpatialTypeFromWKT(string $wkt): string } return strtolower(trim(substr($wkt, 0, $pos))); } + + public function encodePoint(array $point): string + { + return "POINT({$point[0]} {$point[1]})"; + } + + public function decodePoint(mixed $data): array + { + return $data; + } } diff --git a/src/Database/Database.php b/src/Database/Database.php index abb0bff2c..36a3031f0 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -481,29 +481,27 @@ function (?string $value) { self::addFilter( Database::VAR_POINT, - /** - * @param mixed $value - * @return mixed - */ function (mixed $value) { - if (!is_array($value)) { - return $value; + if ($value === null) { + return null; } try { - return self::encodeSpatialData($value, Database::VAR_POINT); + //return self::encodeSpatialData($value, Database::VAR_POINT); + return $this->adapter->encodePoint($value); } catch (\Throwable) { - return $value; + throw new StructureException('Invalid point'); } }, - /** - * @param string|null $value - * @return string|null - */ function (?string $value) { - if (!is_string($value)) { - return $value; + if ($value === null) { + return null; + } + try { + //return self::decodeSpatialData($value); + return $this->adapter->decodePoint($value); + } catch (\Throwable) { + throw new StructureException('Invalid point'); } - return self::decodeSpatialData($value); } ); self::addFilter( @@ -1972,9 +1970,6 @@ protected function getRequiredFilters(?string $type): array { return match ($type) { self::VAR_DATETIME => ['datetime'], - self::VAR_POINT => ['point'], - self::VAR_POLYGON => ['polygon'], - self::VAR_LINESTRING => ['linestring'], default => [], }; } diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index 912f05b2b..f08216b80 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -135,6 +135,7 @@ protected function validatePolygon(array $value): bool public static function isWKTString(string $value): bool { $value = trim($value); + $value = str_replace('SRID=4326;', '', $value); return (bool) preg_match('/^(POINT|LINESTRING|POLYGON)\s*\(/i', $value); } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 2f28590f3..3fbf95b92 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -76,7 +76,7 @@ public function testSpatialCollection(): void $this->assertIsArray($col->getAttribute('indexes')); $this->assertCount(2, $col->getAttribute('indexes')); - $database->createAttribute($collectionName, 'attribute3', Database::VAR_POINT, 0, true, filters: ['point']); + $database->createAttribute($collectionName, 'attribute3', Database::VAR_POINT, 0, true); $database->createIndex($collectionName, ID::custom("index3"), Database::INDEX_SPATIAL, ['attribute3']); $col = $database->getCollection($collectionName); @@ -104,14 +104,26 @@ public function testSpatialTypeDocuments(): void $database->createCollection($collectionName); // Create spatial attributes using createAttribute method - $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true, filters: ['point'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true, filters: ['linestring'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true, filters: ['polygon'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); + //$this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); + //$this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'point_spatial', Database::INDEX_SPATIAL, ['pointAttr'])); - $this->assertEquals(true, $database->createIndex($collectionName, 'line_spatial', Database::INDEX_SPATIAL, ['lineAttr'])); - $this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); + //$this->assertEquals(true, $database->createIndex($collectionName, 'line_spatial', Database::INDEX_SPATIAL, ['lineAttr'])); + //$this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); + + // Create test document + $doc1 = new Document([ + '$id' => 'doc1', + 'pointAttr' => [5.0, 5.0], + '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] + ]); + $createdDoc = $database->createDocument($collectionName, $doc1); + $this->assertInstanceOf(Document::class, $createdDoc); + $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); + $this->assertEquals('222','2'); + // Create test document $doc1 = new Document([ @@ -233,7 +245,7 @@ public function testSpatialTypeDocuments(): void $this->assertEquals('doc1', $result[0]->getId(), sprintf('Incorrect document for distance %s on polyAttr', $queryType)); } } finally { - $database->deleteCollection($collectionName); + //$database->deleteCollection($collectionName); } } @@ -251,7 +263,7 @@ public function testSpatialRelationshipOneToOne(): void $database->createCollection('building'); $database->createAttribute('location', 'name', Database::VAR_STRING, 255, true); - $database->createAttribute('location', 'coordinates', Database::VAR_POINT, 0, true, filters: ['point']); + $database->createAttribute('location', 'coordinates', Database::VAR_POINT, 0, true); $database->createAttribute('building', 'name', Database::VAR_STRING, 255, true); $database->createAttribute('building', 'area', Database::VAR_STRING, 255, true); @@ -356,9 +368,9 @@ public function testSpatialAttributes(): void $database->createCollection($collectionName); $required = $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true; - $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $required, filters: ['point'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $required, filters: ['linestring'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $required, filters: ['polygon'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $required)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $required)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $required)); // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'idx_point', Database::INDEX_SPATIAL, ['pointAttr'])); @@ -407,7 +419,7 @@ public function testSpatialOneToMany(): void $database->createAttribute($parent, 'name', Database::VAR_STRING, 255, true); $database->createAttribute($child, 'name', Database::VAR_STRING, 255, true); - $database->createAttribute($child, 'coord', Database::VAR_POINT, 0, true, filters: ['point']); + $database->createAttribute($child, 'coord', Database::VAR_POINT, 0, true); $database->createIndex($child, 'coord_spatial', Database::INDEX_SPATIAL, ['coord']); $database->createRelationship( @@ -519,7 +531,7 @@ public function testSpatialManyToOne(): void $database->createAttribute($parent, 'name', Database::VAR_STRING, 255, true); $database->createAttribute($child, 'name', Database::VAR_STRING, 255, true); - $database->createAttribute($child, 'coord', Database::VAR_POINT, 0, true, filters: ['point']); + $database->createAttribute($child, 'coord', Database::VAR_POINT, 0, true); $database->createIndex($child, 'coord_spatial', Database::INDEX_SPATIAL, ['coord']); $database->createRelationship( @@ -623,10 +635,10 @@ public function testSpatialManyToMany(): void $database->createCollection($b); $database->createAttribute($a, 'name', Database::VAR_STRING, 255, true); - $database->createAttribute($a, 'home', Database::VAR_POINT, 0, true, filters: ['point']); + $database->createAttribute($a, 'home', Database::VAR_POINT, 0, true); $database->createIndex($a, 'home_spatial', Database::INDEX_SPATIAL, ['home']); $database->createAttribute($b, 'title', Database::VAR_STRING, 255, true); - $database->createAttribute($b, 'area', Database::VAR_POLYGON, 0, true, filters: ['polygon']); + $database->createAttribute($b, 'area', Database::VAR_POLYGON, 0, true); $database->createIndex($b, 'area_spatial', Database::INDEX_SPATIAL, ['area']); $database->createRelationship( @@ -725,7 +737,7 @@ public function testSpatialIndex(): void $collectionName = 'spatial_index_'; try { $database->createCollection($collectionName); - $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true, filters: ['point']); + $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true); $this->assertEquals(true, $database->createIndex($collectionName, 'loc_spatial', Database::INDEX_SPATIAL, ['loc'])); $collection = $database->getCollection($collectionName); @@ -786,7 +798,7 @@ public function testSpatialIndex(): void $collOrderIndex = 'spatial_idx_order_index_' . uniqid(); try { $database->createCollection($collOrderIndex); - $database->createAttribute($collOrderIndex, 'loc', Database::VAR_POINT, 0, true, filters: ['point']); + $database->createAttribute($collOrderIndex, 'loc', Database::VAR_POINT, 0, true); if ($orderSupported) { $this->assertTrue($database->createIndex($collOrderIndex, 'idx_loc', Database::INDEX_SPATIAL, ['loc'], [], [Database::ORDER_DESC])); } else { @@ -846,7 +858,7 @@ public function testSpatialIndex(): void $collNullIndex = 'spatial_idx_null_index_' . uniqid(); try { $database->createCollection($collNullIndex); - $database->createAttribute($collNullIndex, 'loc', Database::VAR_POINT, 0, false, filters: ['point']); + $database->createAttribute($collNullIndex, 'loc', Database::VAR_POINT, 0, false); if ($nullSupported) { $this->assertTrue($database->createIndex($collNullIndex, 'idx_loc', Database::INDEX_SPATIAL, ['loc'])); } else { @@ -922,12 +934,12 @@ public function testComplexGeometricShapes(): void $database->createCollection($collectionName); // Create spatial attributes for different geometric shapes - $this->assertEquals(true, $database->createAttribute($collectionName, 'rectangle', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'square', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'triangle', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'circle_center', Database::VAR_POINT, 0, true, filters: ['point'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'complex_polygon', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'multi_linestring', Database::VAR_LINESTRING, 0, true, filters: ['linestring'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'rectangle', Database::VAR_POLYGON, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'square', Database::VAR_POLYGON, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'triangle', Database::VAR_POLYGON, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'circle_center', Database::VAR_POINT, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'complex_polygon', Database::VAR_POLYGON, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'multi_linestring', Database::VAR_LINESTRING, 0, true)); // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'idx_rectangle', Database::INDEX_SPATIAL, ['rectangle'])); @@ -1352,9 +1364,9 @@ public function testSpatialQueryCombinations(): void $database->createCollection($collectionName); // Create spatial attributes - $this->assertEquals(true, $database->createAttribute($collectionName, 'location', Database::VAR_POINT, 0, true, filters: ['point'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'area', Database::VAR_POLYGON, 0, true, filters: ['polygon'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'route', Database::VAR_LINESTRING, 0, true, filters: ['linestring'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'location', Database::VAR_POINT, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'area', Database::VAR_POLYGON, 0, true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'route', Database::VAR_LINESTRING, 0, true)); $this->assertEquals(true, $database->createAttribute($collectionName, 'name', Database::VAR_STRING, 255, true)); // Create spatial indexes @@ -1783,8 +1795,8 @@ public function testSptialAggregation(): void // Create collection with spatial and numeric attributes $database->createCollection($collectionName); $database->createAttribute($collectionName, 'name', Database::VAR_STRING, 255, true); - $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true, filters: ['point']); - $database->createAttribute($collectionName, 'area', Database::VAR_POLYGON, 0, true, filters: ['polygon']); + $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true); + $database->createAttribute($collectionName, 'area', Database::VAR_POLYGON, 0, true); $database->createAttribute($collectionName, 'score', Database::VAR_INTEGER, 0, true); // Spatial indexes @@ -1872,21 +1884,21 @@ public function testUpdateSpatialAttributes(): void // 0) Disallow creation of spatial attributes with size or array try { - $database->createAttribute($collectionName, 'geom_bad_size', Database::VAR_POINT, 10, true, filters: ['point']); + $database->createAttribute($collectionName, 'geom_bad_size', Database::VAR_POINT, 10, true); $this->fail('Expected DatabaseException when creating spatial attribute with non-zero size'); } catch (\Throwable $e) { $this->assertInstanceOf(Exception::class, $e); } try { - $database->createAttribute($collectionName, 'geom_bad_array', Database::VAR_POINT, 0, true, array: true, filters: ['point']); + $database->createAttribute($collectionName, 'geom_bad_array', Database::VAR_POINT, 0, true, array: true); $this->fail('Expected DatabaseException when creating spatial attribute with array=true'); } catch (\Throwable $e) { $this->assertInstanceOf(Exception::class, $e); } // Create a single spatial attribute (required=true) - $this->assertEquals(true, $database->createAttribute($collectionName, 'geom', Database::VAR_POINT, 0, true, filters: ['point'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'geom', Database::VAR_POINT, 0, true)); $this->assertEquals(true, $database->createIndex($collectionName, 'idx_geom', Database::INDEX_SPATIAL, ['geom'])); // 1) Disallow size and array updates on spatial attributes: expect DatabaseException @@ -1957,9 +1969,9 @@ public function testSpatialAttributeDefaults(): void $database->createCollection($collectionName); // Create spatial attributes with defaults and no indexes to avoid nullability/index constraints - $this->assertEquals(true, $database->createAttribute($collectionName, 'pt', Database::VAR_POINT, 0, false, [1.0, 2.0], filters: ['point'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'ln', Database::VAR_LINESTRING, 0, false, [[0.0, 0.0], [1.0, 1.0]], filters: ['linestring'])); - $this->assertEquals(true, $database->createAttribute($collectionName, 'pg', Database::VAR_POLYGON, 0, false, [[[0.0, 0.0], [0.0, 2.0], [2.0, 2.0], [0.0, 0.0]]], filters: ['polygon'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pt', Database::VAR_POINT, 0, false, [1.0, 2.0])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'ln', Database::VAR_LINESTRING, 0, false, [[0.0, 0.0], [1.0, 1.0]])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'pg', Database::VAR_POLYGON, 0, false, [[[0.0, 0.0], [0.0, 2.0], [2.0, 2.0], [0.0, 0.0]]])); // Create non-spatial attributes (mix of defaults and no defaults) $this->assertEquals(true, $database->createAttribute($collectionName, 'title', Database::VAR_STRING, 255, false, 'Untitled')); @@ -2164,7 +2176,7 @@ public function testSpatialDistanceInMeter(): void $collectionName = 'spatial_distance_meters_'; try { $database->createCollection($collectionName); - $this->assertEquals(true, $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true, filters: ['point'])); + $this->assertEquals(true, $database->createAttribute($collectionName, 'loc', Database::VAR_POINT, 0, true)); $this->assertEquals(true, $database->createIndex($collectionName, 'idx_loc', Database::INDEX_SPATIAL, ['loc'])); // Two points roughly ~1000 meters apart by latitude delta (~0.009 deg ≈ 1km) From 4ef5206c193505a412d1b4ed872d6e9c62882ceb Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 14:34:54 +0300 Subject: [PATCH 09/55] decodePoint --- src/Database/Adapter.php | 2 +- src/Database/Adapter/MariaDB.php | 2 +- src/Database/Adapter/Postgres.php | 21 ++++++++++---- src/Database/Adapter/SQL.php | 35 ++++++++++++++++++++--- src/Database/Database.php | 32 ++++++++++++++++++--- src/Database/Validator/Spatial.php | 8 +++++- tests/e2e/Adapter/Scopes/SpatialTests.php | 8 +++++- 7 files changed, 91 insertions(+), 17 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index f7b832143..2d273160e 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1287,6 +1287,6 @@ abstract public function getTenantQuery(string $collection, string $alias = ''): abstract protected function execute(mixed $stmt): bool; abstract protected function encodePoint(array $point): mixed; - abstract protected function decodePoint(mixed $data): array; + abstract protected function decodePoint(mixed $wkb): array; } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index c78d6637c..77b3647e5 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -864,7 +864,7 @@ public function createDocument(Document $collection, Document $document): Docume INSERT INTO {$this->getSQLTable($name)} ({$columns} _uid) VALUES ({$columnNames} :_uid) "; - +var_dump($sql); $sql = $this->trigger(Database::EVENT_DOCUMENT_CREATE, $sql); $stmt = $this->getPDO()->prepare($sql); diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index cacf72610..94ad08c42 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2005,18 +2005,29 @@ public function getSupportForSpatialAxisOrder(): bool public function encodePoint(array $point): string { - return "SRID=4326;POINT({$point[0]} {$point[1]})";// EWKT + return "POINT({$point[0]} {$point[1]})";// EWKT + //return "SRID=4326;POINT({$point[0]} {$point[1]})";// EWKT } - public function decodePoint(mixed $data): array + public function decodePoint(mixed $wkb): array { - $ewkt = str_replace('SRID=4326;', '', $data); + //$wkb = str_replace('SRID=4326;', '', $wkb); // Remove if was added in encode // Expect format "POINT(x y)" - if (preg_match('/POINT\(([-\d\.]+) ([-\d\.]+)\)/', $ewkt, $matches)) { + if (preg_match('/POINT\(([-\d\.]+) ([-\d\.]+)\)/', $wkb, $matches)) { return [(float)$matches[1], (float)$matches[2]]; } - throw new Exception("Invalid EWKT format: $ewkt"); + $bin = hex2bin($wkb); + + $isLE = ord($bin[0]) === 1; + $type = unpack($isLE ? 'V' : 'N', substr($bin, 1, 4))[1]; + $offset = 5 + (($type & 0x20000000) ? 4 : 0); + + $fmt = $isLE ? 'e' : 'E'; // little vs big endian double + $x = unpack($fmt, substr($bin, $offset, 8))[1]; + $y = unpack($fmt, substr($bin, $offset + 8, 8))[1]; + + return [(float)$x, (float)$y]; } } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index d5757c3ee..eb304f471 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -359,14 +359,14 @@ public function getDocument(Document $collection, string $id, array $queries = [ $forUpdate = $forUpdate ? 'FOR UPDATE' : ''; $alias = Query::DEFAULT_ALIAS; - + $spatialAttributes=[]; $sql = " SELECT {$this->getAttributeProjection($selections, $alias, $spatialAttributes)} FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} WHERE {$this->quote($alias)}.{$this->quote('_uid')} = :_uid {$this->getTenantQuery($collection, $alias)} "; - +var_dump($sql); if ($this->getSupportForUpdateLock()) { $sql .= " {$forUpdate}"; } @@ -2716,8 +2716,35 @@ public function encodePoint(array $point): string return "POINT({$point[0]} {$point[1]})"; } - public function decodePoint(mixed $data): array + public function decodePoint(mixed $wkb): array { - return $data; + var_dump($wkb); + + if (str_starts_with(strtoupper($wkb), 'POINT(')) { + $start = strpos($wkb, '(') + 1; + $end = strrpos($wkb, ')'); + $inside = substr($wkb, $start, $end - $start); + + $coords = explode(' ', trim($inside)); + return [(float)$coords[0], (float)$coords[1]]; + } + + // MySQL SRID-aware WKB layout: + // 1 byte = endian (1 = little endian) + // 4 bytes = type + SRID flag + // 4 bytes = SRID + // 16 bytes = X,Y coordinates (double each, little endian) + + $byteOrder = ord($wkb[0]); + $littleEndian = ($byteOrder === 1); + + // Skip 1 + 4 + 4 = 9 bytes to get coordinates + $coordsBin = substr($wkb, 9, 16); + + // Unpack doubles + $format = $littleEndian ? 'd2' : 'd2'; // little-endian doubles + $coords = unpack($format, $coordsBin); + + return [$coords[1], $coords[2]]; } } diff --git a/src/Database/Database.php b/src/Database/Database.php index 36a3031f0..782f93602 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -486,8 +486,8 @@ function (mixed $value) { return null; } try { - //return self::encodeSpatialData($value, Database::VAR_POINT); - return $this->adapter->encodePoint($value); + return self::encodeSpatialData($value, Database::VAR_POINT); + //return $this->adapter->encodePoint($value); } catch (\Throwable) { throw new StructureException('Invalid point'); } @@ -496,11 +496,18 @@ function (?string $value) { if ($value === null) { return null; } + var_dump('shmuel'); + var_dump($value); + + /** + * Validate array point + */ + try { //return self::decodeSpatialData($value); return $this->adapter->decodePoint($value); - } catch (\Throwable) { - throw new StructureException('Invalid point'); + } catch (\Throwable $th) { + throw new StructureException($th->getMessage()); } } ); @@ -1323,6 +1330,19 @@ public function delete(?string $database = null): bool */ public function createCollection(string $id, array $attributes = [], array $indexes = [], ?array $permissions = null, bool $documentSecurity = true): Document { + foreach ($attributes as &$attribute) { + if (in_array($attribute['type'], Database::SPATIAL_TYPES)) { + $existingFilters = $attribute['filters'] ?? []; + if (!is_array($existingFilters)) { + $existingFilters = [$existingFilters]; + } + $attribute['filters'] = array_values( + array_unique(array_merge($existingFilters, [$attribute['type']])) + ); + } + } + unset($attribute); + $permissions ??= [ Permission::create(Role::any()), ]; @@ -1679,6 +1699,10 @@ public function createAttribute(string $collection, string $id, string $type, in if ($collection->isEmpty()) { throw new NotFoundException('Collection not found'); } + if (in_array($type, Database::SPATIAL_TYPES)) { + $filters[] = $type; + $filters = array_unique($filters); + } $attribute = $this->validateAttribute( $collection, diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index f08216b80..cc3ea83d4 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -134,8 +134,14 @@ protected function validatePolygon(array $value): bool */ public static function isWKTString(string $value): bool { + + /** + * We need to decode the value first + */ + + // return true; + $value = trim($value); - $value = str_replace('SRID=4326;', '', $value); return (bool) preg_match('/^(POINT|LINESTRING|POLYGON)\s*\(/i', $value); } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 3fbf95b92..3cdeb9216 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -109,7 +109,7 @@ public function testSpatialTypeDocuments(): void //$this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); // Create spatial indexes - $this->assertEquals(true, $database->createIndex($collectionName, 'point_spatial', Database::INDEX_SPATIAL, ['pointAttr'])); + //$this->assertEquals(true, $database->createIndex($collectionName, 'point_spatial', Database::INDEX_SPATIAL, ['pointAttr'])); //$this->assertEquals(true, $database->createIndex($collectionName, 'line_spatial', Database::INDEX_SPATIAL, ['lineAttr'])); //$this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); @@ -120,11 +120,17 @@ public function testSpatialTypeDocuments(): void '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] ]); $createdDoc = $database->createDocument($collectionName, $doc1); + $this->assertInstanceOf(Document::class, $createdDoc); + $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); + + $createdDoc = $database->getDocument($collectionName, 'doc1'); + $this->assertInstanceOf(Document::class, $createdDoc); $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); $this->assertEquals('222','2'); + // Create test document $doc1 = new Document([ '$id' => 'doc1', From 37db0b1984dccfbe9ca76f8523b6650ddddc4a43 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 14:37:55 +0300 Subject: [PATCH 10/55] same decode --- src/Database/Adapter/Postgres.php | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 94ad08c42..12bdc6ce4 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2013,9 +2013,13 @@ public function decodePoint(mixed $wkb): array { //$wkb = str_replace('SRID=4326;', '', $wkb); // Remove if was added in encode - // Expect format "POINT(x y)" - if (preg_match('/POINT\(([-\d\.]+) ([-\d\.]+)\)/', $wkb, $matches)) { - return [(float)$matches[1], (float)$matches[2]]; + if (str_starts_with(strtoupper($wkb), 'POINT(')) { + $start = strpos($wkb, '(') + 1; + $end = strrpos($wkb, ')'); + $inside = substr($wkb, $start, $end - $start); + + $coords = explode(' ', trim($inside)); + return [(float)$coords[0], (float)$coords[1]]; } $bin = hex2bin($wkb); From d01fdbceed6588a4cecc4b92fb65ad5d28398705 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 15:49:33 +0300 Subject: [PATCH 11/55] decodeLinestring --- src/Database/Adapter.php | 5 ++- src/Database/Adapter/SQL.php | 46 +++++++++++++++++++++++ src/Database/Database.php | 27 +++++++------ tests/e2e/Adapter/Scopes/SpatialTests.php | 12 +++--- 4 files changed, 69 insertions(+), 21 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 2d273160e..b7f8378b6 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1286,7 +1286,8 @@ abstract public function getTenantQuery(string $collection, string $alias = ''): */ abstract protected function execute(mixed $stmt): bool; - abstract protected function encodePoint(array $point): mixed; - abstract protected function decodePoint(mixed $wkb): array; + abstract public function encodePoint(array $point): mixed; + abstract public function decodePoint(string $wkb): array; + abstract public function decodeLinestring(string $wkb): array; } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index eb304f471..6c6004539 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2747,4 +2747,50 @@ public function decodePoint(mixed $wkb): array return [$coords[1], $coords[2]]; } + + public function decodeLinestring(string $wkb): array + { + if (str_starts_with(strtoupper($wkb), 'LINESTRING(')) { + $start = strpos($wkb, '(') + 1; + $end = strrpos($wkb, ')'); + $inside = substr($wkb, $start, $end - $start); + + $points = explode(',', $inside); + return array_map(function ($point) { + $coords = explode(' ', trim($point)); + return [(float)$coords[0], (float)$coords[1]]; + }, $points); + } + + var_dump($wkb); + + $isLE = ord($wkb[0]) === 1; // little-endian? + $type = unpack($isLE ? 'V' : 'N', substr($wkb, 1, 4))[1]; + + // Check for SRID flag (0x20000000) + $hasSRID = ($type & 0x20000000) !== 0; + $geomType = $type & 0xFFFF; // Mask lower 16 bits to get actual type + + if ($geomType !== 2) { // 2 = LINESTRING + throw new \RuntimeException("Not a LINESTRING geometry type, got {$geomType}"); + } + + $offset = 5 + ($hasSRID ? 4 : 0); // skip endian + type + optional SRID + + // Number of points (4 bytes) + $numPoints = unpack($isLE ? 'V' : 'N', substr($wkb, $offset, 4))[1]; + $offset += 4; + + $points = []; + $fmt = $isLE ? 'e' : 'E'; // little/big endian double + + for ($i = 0; $i < $numPoints; $i++) { + $x = unpack($fmt, substr($wkb, $offset, 8))[1]; + $y = unpack($fmt, substr($wkb, $offset + 8, 8))[1]; + $points[] = [(float)$x, (float)$y]; + $offset += 16; + } + + return $points; + } } diff --git a/src/Database/Database.php b/src/Database/Database.php index 782f93602..266aa912b 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -496,27 +496,21 @@ function (?string $value) { if ($value === null) { return null; } - var_dump('shmuel'); - var_dump($value); /** - * Validate array point + * todo:validate array point */ try { - //return self::decodeSpatialData($value); return $this->adapter->decodePoint($value); } catch (\Throwable $th) { throw new StructureException($th->getMessage()); } } ); + self::addFilter( Database::VAR_LINESTRING, - /** - * @param mixed $value - * @return mixed - */ function (mixed $value) { if (!is_array($value)) { return $value; @@ -527,17 +521,22 @@ function (mixed $value) { return $value; } }, - /** - * @param string|null $value - * @return string|null - */ + function (?string $value) { if (is_null($value)) { - return $value; + return null; + } + + try { + //return self::decodeSpatialData($value); + return $this->adapter->decodeLinestring($value); + } catch (\Throwable $th) { + var_dump($th); + throw new StructureException($th->getMessage()); } - return self::decodeSpatialData($value); } ); + self::addFilter( Database::VAR_POLYGON, /** diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 3cdeb9216..fab44ed24 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -105,32 +105,34 @@ public function testSpatialTypeDocuments(): void // Create spatial attributes using createAttribute method $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); - //$this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); //$this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); // Create spatial indexes - //$this->assertEquals(true, $database->createIndex($collectionName, 'point_spatial', Database::INDEX_SPATIAL, ['pointAttr'])); - //$this->assertEquals(true, $database->createIndex($collectionName, 'line_spatial', Database::INDEX_SPATIAL, ['lineAttr'])); + $this->assertEquals(true, $database->createIndex($collectionName, 'point_spatial', Database::INDEX_SPATIAL, ['pointAttr'])); + $this->assertEquals(true, $database->createIndex($collectionName, 'line_spatial', Database::INDEX_SPATIAL, ['lineAttr'])); //$this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); // Create test document $doc1 = new Document([ '$id' => 'doc1', 'pointAttr' => [5.0, 5.0], + 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] ]); + $createdDoc = $database->createDocument($collectionName, $doc1); $this->assertInstanceOf(Document::class, $createdDoc); $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); + $this->assertEquals([[1.0, 2.0], [3.0, 4.0]], $createdDoc->getAttribute('lineAttr')); $createdDoc = $database->getDocument($collectionName, 'doc1'); - $this->assertInstanceOf(Document::class, $createdDoc); $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); + $this->assertEquals([[1.0, 2.0], [3.0, 4.0]], $createdDoc->getAttribute('lineAttr')); $this->assertEquals('222','2'); - // Create test document $doc1 = new Document([ '$id' => 'doc1', From 6f27ff9ff91c3a151f2c1d7635355ccff04b4249 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 14 Sep 2025 15:53:59 +0300 Subject: [PATCH 12/55] Remove commented-out test descriptions for new NOT query types in QueryTest.php to clean up the code. --- tests/unit/QueryTest.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unit/QueryTest.php b/tests/unit/QueryTest.php index c48755cb2..188d1873c 100644 --- a/tests/unit/QueryTest.php +++ b/tests/unit/QueryTest.php @@ -86,7 +86,6 @@ public function testCreate(): void $this->assertEquals('title', $query->getAttribute()); $this->assertEquals([], $query->getValues()); - // Test new NOT query types $query = Query::notContains('tags', ['test', 'example']); $this->assertEquals(Query::TYPE_NOT_CONTAINS, $query->getMethod()); @@ -206,7 +205,6 @@ public function testParse(): void $this->assertEquals('score', $query->getAttribute()); $this->assertEquals(8.5, $query->getValues()[0]); - // Test new NOT query types parsing $query = Query::parse(Query::notContains('tags', ['unwanted', 'spam'])->toString()); $this->assertEquals('notContains', $query->getMethod()); $this->assertEquals('tags', $query->getAttribute()); @@ -431,7 +429,6 @@ public function testIsMethod(): void public function testNewQueryTypesInTypesArray(): void { - // Test that all new query types are included in the TYPES array $this->assertContains(Query::TYPE_NOT_CONTAINS, Query::TYPES); $this->assertContains(Query::TYPE_NOT_SEARCH, Query::TYPES); $this->assertContains(Query::TYPE_NOT_STARTS_WITH, Query::TYPES); From 1f8cdb201a08b665dfb8285ecadf72585f268553 Mon Sep 17 00:00:00 2001 From: shimon Date: Sun, 14 Sep 2025 15:55:03 +0300 Subject: [PATCH 13/55] Enhance MongoDB query capabilities by adding support for NOT operators in the Mongo adapter. Updated the DocumentTests to include comprehensive tests for notSearch, notStartsWith, notEndsWith, and notBetween functionalities, ensuring proper handling of these new query types. Adjusted docker-compose to include the MongoDB client file for development. --- docker-compose.yml | 2 +- src/Database/Adapter/Mongo.php | 44 +- tests/e2e/Adapter/Scopes/DocumentTests.php | 494 ++++++++++----------- 3 files changed, 287 insertions(+), 253 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8e571d3c9..1948e8b00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - ./dev/xdebug.ini:/usr/local/etc/php/conf.d/xdebug.ini - /var/run/docker.sock:/var/run/docker.sock - ./docker-compose.yml:/usr/src/code/docker-compose.yml - #- ./vendor/utopia-php/mongo/src/Client.php:/usr/src/code/vendor/utopia-php/mongo/src/Client.php + - ./vendor/utopia-php/mongo/src/Client.php:/usr/src/code/vendor/utopia-php/mongo/src/Client.php environment: PHP_IDE_CONFIG: serverName=tests diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index ad47a8185..097800bd6 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -37,6 +37,8 @@ class Mongo extends Adapter '$and', '$match', '$regex', + '$not', + '$nor', ]; protected Client $client; @@ -1580,7 +1582,6 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $response = $this->client->find($name, $filters, $options); $results = $response->cursor->firstBatch ?? []; - // Process first batch foreach ($results as $result) { $record = $this->replaceChars('_', '$', (array)$result); @@ -1972,7 +1973,7 @@ protected function buildFilter(Query $query): array : $query->getValues()[0] ), }; - + $filter = []; if ($operator == '$eq' && \is_array($value)) { @@ -1982,14 +1983,36 @@ protected function buildFilter(Query $query): array } elseif ($operator == '$in') { if ($query->getMethod() === Query::TYPE_CONTAINS && !$query->onArray()) { $filter[$attribute]['$regex'] = new Regex(".*{$this->escapeWildcards($value)}.*", 'i'); + } elseif ($query->getMethod() === Query::TYPE_NOT_CONTAINS && !$query->onArray()) { + $filter[$attribute] = ['$not' => new Regex(".*{$this->escapeWildcards($value)}.*", 'i')]; } else { $filter[$attribute]['$in'] = $query->getValues(); } } elseif ($operator == '$search') { - $filter['$text'][$operator] = $value; + if ($query->getMethod() === Query::TYPE_NOT_SEARCH) { + // MongoDB doesn't support negating $text expressions directly + // Use regex as fallback for NOT search while keeping fulltext for positive search + if (empty($value)) { + // Empty search term should return all documents (no documents contain empty string) + // Don't add any filter - this will match all documents + } else { + // Escape special regex characters and create a pattern that matches the search term as substring + $escapedValue = preg_quote($value, '/'); + $filter[$attribute] = ['$not' => new Regex(".*{$escapedValue}.*", 'i')]; + } + } else { + $filter['$text'][$operator] = $value; + } } elseif ($operator === Query::TYPE_BETWEEN) { $filter[$attribute]['$lte'] = $value[1]; $filter[$attribute]['$gte'] = $value[0]; + } elseif ($operator === Query::TYPE_NOT_BETWEEN) { + $filter['$or'] = [ + [$attribute => ['$lt' => $value[0]]], + [$attribute => ['$gt' => $value[1]]] + ]; + } elseif ($operator === '$regex' && in_array($query->getMethod(), [Query::TYPE_NOT_STARTS_WITH, Query::TYPE_NOT_ENDS_WITH])) { + $filter[$attribute] = ['$not' => new Regex($value, 'i')]; } else { $filter[$attribute][$operator] = $value; } @@ -2017,13 +2040,18 @@ protected function getQueryOperator(string $operator): string Query::TYPE_GREATER => '$gt', Query::TYPE_GREATER_EQUAL => '$gte', Query::TYPE_CONTAINS => '$in', + Query::TYPE_NOT_CONTAINS => '$in', Query::TYPE_SEARCH => '$search', + Query::TYPE_NOT_SEARCH => '$search', Query::TYPE_BETWEEN => 'between', + Query::TYPE_NOT_BETWEEN => 'notBetween', Query::TYPE_STARTS_WITH, - Query::TYPE_ENDS_WITH => '$regex', + Query::TYPE_NOT_STARTS_WITH, + Query::TYPE_ENDS_WITH, + Query::TYPE_NOT_ENDS_WITH => '$regex', Query::TYPE_OR => '$or', Query::TYPE_AND => '$and', - default => throw new DatabaseException('Unknown operator:' . $operator . '. Must be one of ' . Query::TYPE_EQUAL . ', ' . Query::TYPE_NOT_EQUAL . ', ' . Query::TYPE_LESSER . ', ' . Query::TYPE_LESSER_EQUAL . ', ' . Query::TYPE_GREATER . ', ' . Query::TYPE_GREATER_EQUAL . ', ' . Query::TYPE_IS_NULL . ', ' . Query::TYPE_IS_NOT_NULL . ', ' . Query::TYPE_BETWEEN . ', ' . Query::TYPE_CONTAINS . ', ' . Query::TYPE_SEARCH . ', ' . Query::TYPE_SELECT), + default => throw new DatabaseException('Unknown operator:' . $operator . '. Must be one of ' . Query::TYPE_EQUAL . ', ' . Query::TYPE_NOT_EQUAL . ', ' . Query::TYPE_LESSER . ', ' . Query::TYPE_LESSER_EQUAL . ', ' . Query::TYPE_GREATER . ', ' . Query::TYPE_GREATER_EQUAL . ', ' . Query::TYPE_IS_NULL . ', ' . Query::TYPE_IS_NOT_NULL . ', ' . Query::TYPE_BETWEEN . ', ' . Query::TYPE_NOT_BETWEEN . ', ' . Query::TYPE_STARTS_WITH . ', ' . Query::TYPE_NOT_STARTS_WITH . ', ' . Query::TYPE_ENDS_WITH . ', ' . Query::TYPE_NOT_ENDS_WITH . ', ' . Query::TYPE_CONTAINS . ', ' . Query::TYPE_NOT_CONTAINS . ', ' . Query::TYPE_SEARCH . ', ' . Query::TYPE_NOT_SEARCH . ', ' . Query::TYPE_SELECT), }; } @@ -2033,9 +2061,15 @@ protected function getQueryValue(string $method, mixed $value): mixed case Query::TYPE_STARTS_WITH: $value = $this->escapeWildcards($value); return $value . '.*'; + case Query::TYPE_NOT_STARTS_WITH: + $value = $this->escapeWildcards($value); + return $value . '.*'; case Query::TYPE_ENDS_WITH: $value = $this->escapeWildcards($value); return '.*' . $value; + case Query::TYPE_NOT_ENDS_WITH: + $value = $this->escapeWildcards($value); + return '.*' . $value; default: return $value; } diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 24e3b173a..89b777fbf 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -3310,253 +3310,253 @@ public function testFindNotContains(): void } } - // public function testFindNotSearch(): void - // { - // /** @var Database $database */ - // $database = static::getDatabase(); - - // // Only test if fulltext search is supported - // if ($this->getDatabase()->getAdapter()->getSupportForFulltextIndex()) { - // // Ensure fulltext index exists (may already exist from previous tests) - // try { - // $database->createIndex('movies', 'name', Database::INDEX_FULLTEXT, ['name']); - // } catch (Throwable $e) { - // // Index may already exist, ignore duplicate error - // if (!str_contains($e->getMessage(), 'already exists')) { - // throw $e; - // } - // } - - // // Test notSearch - should return documents that don't match the search term - // $documents = $database->find('movies', [ - // Query::notSearch('name', 'captain'), - // ]); - - // $this->assertEquals(4, count($documents)); // All movies except the 2 with 'captain' in name - - // // Test notSearch with term that doesn't exist - should return all documents - // $documents = $database->find('movies', [ - // Query::notSearch('name', 'nonexistent'), - // ]); - - // $this->assertEquals(6, count($documents)); - - // // Test notSearch with partial term - // if ($this->getDatabase()->getAdapter()->getSupportForFulltextWildCardIndex()) { - // $documents = $database->find('movies', [ - // Query::notSearch('name', 'cap'), - // ]); - - // $this->assertEquals(4, count($documents)); // All movies except those matching 'cap' - // } - - // // Test notSearch with empty string - should return all documents - // $documents = $database->find('movies', [ - // Query::notSearch('name', ''), - // ]); - // $this->assertEquals(6, count($documents)); // All movies since empty search matches nothing - - // // Test notSearch combined with other filters - // $documents = $database->find('movies', [ - // Query::notSearch('name', 'captain'), - // Query::lessThan('year', 2010) - // ]); - // $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-captain movies before 2010 - - // // Test notSearch with special characters - // $documents = $database->find('movies', [ - // Query::notSearch('name', '@#$%'), - // ]); - // $this->assertEquals(6, count($documents)); // All movies since special chars don't match - // } - - // $this->assertEquals(true, true); // Test must do an assertion - // } - - // public function testFindNotStartsWith(): void - // { - // /** @var Database $database */ - // $database = static::getDatabase(); - - // // Test notStartsWith - should return documents that don't start with 'Work' - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'Work'), - // ]); - - // $this->assertEquals(4, count($documents)); // All movies except the 2 starting with 'Work' - - // // Test notStartsWith with non-existent prefix - should return all documents - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'NonExistent'), - // ]); - - // $this->assertEquals(6, count($documents)); - - // // Test notStartsWith with wildcard characters (should treat them literally) - // if ($this->getDatabase()->getAdapter() instanceof SQL) { - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', '%ork'), - // ]); - // } else { - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', '.*ork'), - // ]); - // } - - // $this->assertEquals(6, count($documents)); // Should return all since no movie starts with these patterns - - // // Test notStartsWith with empty string - should return no documents (all strings start with empty) - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', ''), - // ]); - // $this->assertEquals(0, count($documents)); // No documents since all strings start with empty string - - // // Test notStartsWith with single character - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'C'), - // ]); - // $this->assertGreaterThanOrEqual(4, count($documents)); // Movies not starting with 'C' - - // // Test notStartsWith with case sensitivity (may be case-insensitive depending on DB) - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'work'), // lowercase vs 'Work' - // ]); - // $this->assertGreaterThanOrEqual(4, count($documents)); // May match case-insensitively - - // // Test notStartsWith combined with other queries - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'Work'), - // Query::equal('year', [2006]) - // ]); - // $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-Work movies from 2006 - // } - - // public function testFindNotEndsWith(): void - // { - // /** @var Database $database */ - // $database = static::getDatabase(); - - // // Test notEndsWith - should return documents that don't end with 'Marvel' - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'Marvel'), - // ]); - - // $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'Marvel' - - // // Test notEndsWith with non-existent suffix - should return all documents - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'NonExistent'), - // ]); - - // $this->assertEquals(6, count($documents)); - - // // Test notEndsWith with partial suffix - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'vel'), - // ]); - - // $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'vel' (from 'Marvel') - - // // Test notEndsWith with empty string - should return no documents (all strings end with empty) - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', ''), - // ]); - // $this->assertEquals(0, count($documents)); // No documents since all strings end with empty string - - // // Test notEndsWith with single character - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'l'), - // ]); - // $this->assertGreaterThanOrEqual(5, count($documents)); // Movies not ending with 'l' - - // // Test notEndsWith with case sensitivity (may be case-insensitive depending on DB) - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'marvel'), // lowercase vs 'Marvel' - // ]); - // $this->assertGreaterThanOrEqual(5, count($documents)); // May match case-insensitively - - // // Test notEndsWith combined with limit - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'Marvel'), - // Query::limit(3) - // ]); - // $this->assertEquals(3, count($documents)); // Limited to 3 results - // $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies - // } - - // public function testFindNotBetween(): void - // { - // /** @var Database $database */ - // $database = static::getDatabase(); - // - // // Test notBetween with price range - should return documents outside the range - // $documents = $database->find('movies', [ - // Query::notBetween('price', 25.94, 25.99), - // ]); - // $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range - // - // // Test notBetween with range that includes no documents - should return all documents - // $documents = $database->find('movies', [ - // Query::notBetween('price', 30, 35), - // ]); - // $this->assertEquals(6, count($documents)); - // - // // Test notBetween with date range - // $documents = $database->find('movies', [ - // Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), - // ]); - // $this->assertEquals(0, count($documents)); // No movies outside this wide date range - // - // // Test notBetween with narrower date range - // $documents = $database->find('movies', [ - // Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), - // ]); - // $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - // - // // Test notBetween with updated date range - // $documents = $database->find('movies', [ - // Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), - // ]); - // $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - // - // // Test notBetween with year range (integer values) - // $documents = $database->find('movies', [ - // Query::notBetween('year', 2005, 2007), - // ]); - // $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range - // - // // Test notBetween with reversed range (start > end) - should still work - // $documents = $database->find('movies', [ - // Query::notBetween('price', 25.99, 25.94), // Note: reversed order - // ]); - // $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully - // - // // Test notBetween with same start and end values - // $documents = $database->find('movies', [ - // Query::notBetween('year', 2006, 2006), - // ]); - // $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 - // - // // Test notBetween combined with other filters - // $documents = $database->find('movies', [ - // Query::notBetween('price', 25.94, 25.99), - // Query::orderDesc('year'), - // Query::limit(2) - // ]); - // $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range - // - // // Test notBetween with extreme ranges - // $documents = $database->find('movies', [ - // Query::notBetween('year', -1000, 1000), // Very wide range - // ]); - // $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range - // - // // Test notBetween with float precision - // $documents = $database->find('movies', [ - // Query::notBetween('price', 25.945, 25.955), // Very narrow range - // ]); - // $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range - // } + public function testFindNotSearch(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Only test if fulltext search is supported + if ($this->getDatabase()->getAdapter()->getSupportForFulltextIndex()) { + // Ensure fulltext index exists (may already exist from previous tests) + try { + $database->createIndex('movies', 'name', Database::INDEX_FULLTEXT, ['name']); + } catch (Throwable $e) { + // Index may already exist, ignore duplicate error + if (!str_contains($e->getMessage(), 'already exists')) { + throw $e; + } + } + + // Test notSearch - should return documents that don't match the search term + $documents = $database->find('movies', [ + Query::notSearch('name', 'captain'), + ]); + + $this->assertEquals(4, count($documents)); // All movies except the 2 with 'captain' in name + + // Test notSearch with term that doesn't exist - should return all documents + $documents = $database->find('movies', [ + Query::notSearch('name', 'nonexistent'), + ]); + + $this->assertEquals(6, count($documents)); + + // Test notSearch with partial term + if ($this->getDatabase()->getAdapter()->getSupportForFulltextWildCardIndex()) { + $documents = $database->find('movies', [ + Query::notSearch('name', 'cap'), + ]); + + $this->assertEquals(4, count($documents)); // All movies except those matching 'cap' + } + + // Test notSearch with empty string - should return all documents + $documents = $database->find('movies', [ + Query::notSearch('name', ''), + ]); + $this->assertEquals(6, count($documents)); // All movies since empty search matches nothing + + // Test notSearch combined with other filters + $documents = $database->find('movies', [ + Query::notSearch('name', 'captain'), + Query::lessThan('year', 2010) + ]); + $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-captain movies before 2010 + + // Test notSearch with special characters + $documents = $database->find('movies', [ + Query::notSearch('name', '@#$%'), + ]); + $this->assertEquals(6, count($documents)); // All movies since special chars don't match + } + + $this->assertEquals(true, true); // Test must do an assertion + } + + public function testFindNotStartsWith(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Test notStartsWith - should return documents that don't start with 'Work' + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'Work'), + ]); + + $this->assertEquals(4, count($documents)); // All movies except the 2 starting with 'Work' + + // Test notStartsWith with non-existent prefix - should return all documents + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'NonExistent'), + ]); + + $this->assertEquals(6, count($documents)); + + // Test notStartsWith with wildcard characters (should treat them literally) + if ($this->getDatabase()->getAdapter() instanceof SQL) { + $documents = $database->find('movies', [ + Query::notStartsWith('name', '%ork'), + ]); + } else { + $documents = $database->find('movies', [ + Query::notStartsWith('name', '.*ork'), + ]); + } + + $this->assertEquals(6, count($documents)); // Should return all since no movie starts with these patterns + + // Test notStartsWith with empty string - should return no documents (all strings start with empty) + $documents = $database->find('movies', [ + Query::notStartsWith('name', ''), + ]); + $this->assertEquals(0, count($documents)); // No documents since all strings start with empty string + + // Test notStartsWith with single character + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'C'), + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // Movies not starting with 'C' + + // Test notStartsWith with case sensitivity (may be case-insensitive depending on DB) + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'work'), // lowercase vs 'Work' + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // May match case-insensitively + + // Test notStartsWith combined with other queries + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'Work'), + Query::equal('year', [2006]) + ]); + $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-Work movies from 2006 + } + + public function testFindNotEndsWith(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Test notEndsWith - should return documents that don't end with 'Marvel' + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'Marvel'), + ]); + + $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'Marvel' + + // Test notEndsWith with non-existent suffix - should return all documents + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'NonExistent'), + ]); + + $this->assertEquals(6, count($documents)); + + // Test notEndsWith with partial suffix + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'vel'), + ]); + + $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'vel' (from 'Marvel') + + // Test notEndsWith with empty string - should return no documents (all strings end with empty) + $documents = $database->find('movies', [ + Query::notEndsWith('name', ''), + ]); + $this->assertEquals(0, count($documents)); // No documents since all strings end with empty string + + // Test notEndsWith with single character + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'l'), + ]); + $this->assertGreaterThanOrEqual(5, count($documents)); // Movies not ending with 'l' + + // Test notEndsWith with case sensitivity (may be case-insensitive depending on DB) + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'marvel'), // lowercase vs 'Marvel' + ]); + $this->assertGreaterThanOrEqual(5, count($documents)); // May match case-insensitively + + // Test notEndsWith combined with limit + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'Marvel'), + Query::limit(3) + ]); + $this->assertEquals(3, count($documents)); // Limited to 3 results + $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies + } + + public function testFindNotBetween(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Test notBetween with price range - should return documents outside the range + $documents = $database->find('movies', [ + Query::notBetween('price', 25.94, 25.99), + ]); + $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range + + // Test notBetween with range that includes no documents - should return all documents + $documents = $database->find('movies', [ + Query::notBetween('price', 30, 35), + ]); + $this->assertEquals(6, count($documents)); + + // Test notBetween with date range + $documents = $database->find('movies', [ + Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), + ]); + $this->assertEquals(0, count($documents)); // No movies outside this wide date range + + // Test notBetween with narrower date range + $documents = $database->find('movies', [ + Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), + ]); + $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + + // Test notBetween with updated date range + $documents = $database->find('movies', [ + Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), + ]); + $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + + // Test notBetween with year range (integer values) + $documents = $database->find('movies', [ + Query::notBetween('year', 2005, 2007), + ]); + $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range + + // Test notBetween with reversed range (start > end) - should still work + $documents = $database->find('movies', [ + Query::notBetween('price', 25.99, 25.94), // Note: reversed order + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully + + // Test notBetween with same start and end values + $documents = $database->find('movies', [ + Query::notBetween('year', 2006, 2006), + ]); + $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 + + // Test notBetween combined with other filters + $documents = $database->find('movies', [ + Query::notBetween('price', 25.94, 25.99), + Query::orderDesc('year'), + Query::limit(2) + ]); + $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range + + // Test notBetween with extreme ranges + $documents = $database->find('movies', [ + Query::notBetween('year', -1000, 1000), // Very wide range + ]); + $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range + + // Test notBetween with float precision + $documents = $database->find('movies', [ + Query::notBetween('price', 25.945, 25.955), // Very narrow range + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range + } public function testFindSelect(): void { From f25ede91e4b148830290d17ec5be8e8f517616bc Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 16:17:29 +0300 Subject: [PATCH 14/55] polygon --- src/Database/Adapter.php | 2 +- src/Database/Adapter/SQL.php | 85 ++++++++++++++++++++++++++++-------- src/Database/Database.php | 19 ++++---- 3 files changed, 77 insertions(+), 29 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index b7f8378b6..6bd2c6d8e 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1289,5 +1289,5 @@ abstract protected function execute(mixed $stmt): bool; abstract public function encodePoint(array $point): mixed; abstract public function decodePoint(string $wkb): array; abstract public function decodeLinestring(string $wkb): array; - + abstract public function decodePolygon(string $wkb): array; } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 6c6004539..568dcde84 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2764,33 +2764,82 @@ public function decodeLinestring(string $wkb): array var_dump($wkb); - $isLE = ord($wkb[0]) === 1; // little-endian? - $type = unpack($isLE ? 'V' : 'N', substr($wkb, 1, 4))[1]; + // Skip 1 byte (endianness) + 4 bytes (type) + 4 bytes (SRID) + $offset = 9; - // Check for SRID flag (0x20000000) - $hasSRID = ($type & 0x20000000) !== 0; - $geomType = $type & 0xFFFF; // Mask lower 16 bits to get actual type - - if ($geomType !== 2) { // 2 = LINESTRING - throw new \RuntimeException("Not a LINESTRING geometry type, got {$geomType}"); - } - - $offset = 5 + ($hasSRID ? 4 : 0); // skip endian + type + optional SRID - - // Number of points (4 bytes) - $numPoints = unpack($isLE ? 'V' : 'N', substr($wkb, $offset, 4))[1]; + // Number of points (4 bytes little-endian) + $numPoints = unpack('V', substr($wkb, $offset, 4))[1]; $offset += 4; $points = []; - $fmt = $isLE ? 'e' : 'E'; // little/big endian double - for ($i = 0; $i < $numPoints; $i++) { - $x = unpack($fmt, substr($wkb, $offset, 8))[1]; - $y = unpack($fmt, substr($wkb, $offset + 8, 8))[1]; + $x = unpack('d', substr($wkb, $offset, 8))[1]; + $y = unpack('d', substr($wkb, $offset + 8, 8))[1]; $points[] = [(float)$x, (float)$y]; $offset += 16; } return $points; } + + public function decodePolygon(string $wkb): array + { + // POLYGON((x1,y1),(x2,y2)) + if (str_starts_with($wkb, 'POLYGON((')) { + $start = strpos($wkb, '((') + 2; + $end = strrpos($wkb, '))'); + $inside = substr($wkb, $start, $end - $start); + + $rings = explode('),(', $inside); + return array_map(function ($ring) { + $points = explode(',', $ring); + return array_map(function ($point) { + $coords = explode(' ', trim($point)); + return [(float)$coords[0], (float)$coords[1]]; + }, $points); + }, $rings); + } + + var_dump($wkb); + + if (strlen($wkb) < 9) { + throw new \RuntimeException('WKB too short to be a POLYGON'); + } + + $byteOrder = ord($wkb[0]); + if ($byteOrder !== 1) { + throw new \RuntimeException('Only little-endian WKB supported'); + } + + // Type + SRID flag + $typeInt = unpack('V', substr($wkb, 1, 4))[1]; + $hasSRID = ($typeInt & 0x20000000) === 0x20000000; + $geomType = $typeInt & 0x0FFFFFFF; + + if ($geomType !== 3) { // 3 = POLYGON + throw new \RuntimeException("Not a POLYGON geometry type, got {$geomType}"); + } + + $offset = 5 + ($hasSRID ? 4 : 0); // Skip endian + type + optional SRID + $format = 'd'; // little-endian double + + $numRings = unpack('V', substr($wkb, $offset, 4))[1]; + $offset += 4; + + $polygon = []; + for ($r = 0; $r < $numRings; $r++) { + $numPoints = unpack('V', substr($wkb, $offset, 4))[1]; + $offset += 4; + + $ring = []; + for ($i = 0; $i < $numPoints; $i++) { + $pt = unpack($format . '2', substr($wkb, $offset, 16)); + $ring[] = [(float)$pt[1], (float)$pt[2]]; + $offset += 16; + } + $polygon[] = $ring; + } + + return $polygon; + } } diff --git a/src/Database/Database.php b/src/Database/Database.php index 266aa912b..3540f1646 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -539,10 +539,6 @@ function (?string $value) { self::addFilter( Database::VAR_POLYGON, - /** - * @param mixed $value - * @return mixed - */ function (mixed $value) { if (!is_array($value)) { return $value; @@ -553,15 +549,18 @@ function (mixed $value) { return $value; } }, - /** - * @param string|null $value - * @return string|null - */ function (?string $value) { if (is_null($value)) { - return $value; + return null; + } + + try { + //return self::decodeSpatialData($value); + return $this->adapter->decodePolygon($value); + } catch (\Throwable $th) { + var_dump($th); + throw new StructureException($th->getMessage()); } - return self::decodeSpatialData($value); } ); } From 9e9d50a1dc19d176f64ebf550df9f95baf3356aa Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 16:57:43 +0300 Subject: [PATCH 15/55] Postgres linestring --- src/Database/Adapter/Postgres.php | 59 +++++++++++++++++++++++ src/Database/Adapter/SQL.php | 39 ++++++++++----- src/Database/Database.php | 1 - tests/e2e/Adapter/Scopes/SpatialTests.php | 13 ++++- 4 files changed, 96 insertions(+), 16 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 12bdc6ce4..ed9adb4e1 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2034,4 +2034,63 @@ public function decodePoint(mixed $wkb): array return [(float)$x, (float)$y]; } + + public function decodeLinestring(mixed $wkb): array + { + if (str_starts_with(strtoupper($wkb), 'LINESTRING(')) { + $start = strpos($wkb, '(') + 1; + $end = strrpos($wkb, ')'); + $inside = substr($wkb, $start, $end - $start); + + $points = explode(',', $inside); + return array_map(function ($point) { + $coords = explode(' ', trim($point)); + return [(float)$coords[0], (float)$coords[1]]; + }, $points); + } + + var_dump($wkb); + + if (ctype_xdigit($wkb)) { + $wkb = hex2bin($wkb); + } + + if (strlen($wkb) < 9) { + throw new DatabaseException("WKB too short to be a valid geometry"); + } + + $byteOrder = ord($wkb[0]); + if ($byteOrder === 0) { + throw new DatabaseException("Big-endian WKB not supported"); + } elseif ($byteOrder !== 1) { + throw new DatabaseException("Invalid byte order in WKB"); + } + + // Type + SRID flag + $typeField = unpack('V', substr($wkb, 1, 4))[1]; + $geomType = $typeField & 0xFF; + $hasSRID = ($typeField & 0x20000000) !== 0; + + if ($geomType !== 2) { // 2 = LINESTRING + throw new DatabaseException("Not a LINESTRING geometry type, got {$geomType}"); + } + + $offset = 5; + if ($hasSRID) { + $offset += 4; + } + + $numPoints = unpack('V', substr($wkb, $offset, 4))[1]; + $offset += 4; + + $points = []; + for ($i = 0; $i < $numPoints; $i++) { + $x = unpack('e', substr($wkb, $offset, 8))[1]; $offset += 8; + $y = unpack('e', substr($wkb, $offset, 8))[1]; $offset += 8; + $points[] = [(float)$x, (float)$y]; + } + + return $points; + } + } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 568dcde84..e959e105a 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2802,44 +2802,57 @@ public function decodePolygon(string $wkb): array var_dump($wkb); + // Convert HEX to binary if needed + if (ctype_xdigit($wkb) && strlen($wkb) % 2 === 0) { + $wkb = hex2bin($wkb); + if ($wkb === false) { + throw new \RuntimeException("Invalid HEX WKB"); + } + } + if (strlen($wkb) < 9) { - throw new \RuntimeException('WKB too short to be a POLYGON'); + throw new \RuntimeException("WKB too short"); } $byteOrder = ord($wkb[0]); if ($byteOrder !== 1) { - throw new \RuntimeException('Only little-endian WKB supported'); + throw new \RuntimeException("Only little-endian WKB supported"); } // Type + SRID flag $typeInt = unpack('V', substr($wkb, 1, 4))[1]; - $hasSRID = ($typeInt & 0x20000000) === 0x20000000; - $geomType = $typeInt & 0x0FFFFFFF; + $hasSrid = ($typeInt & 0x20000000) !== 0; + $geomType = $typeInt & 0xFF; if ($geomType !== 3) { // 3 = POLYGON throw new \RuntimeException("Not a POLYGON geometry type, got {$geomType}"); } - $offset = 5 + ($hasSRID ? 4 : 0); // Skip endian + type + optional SRID - $format = 'd'; // little-endian double + $offset = 5 + ($hasSrid ? 4 : 0); + // Number of rings $numRings = unpack('V', substr($wkb, $offset, 4))[1]; $offset += 4; - $polygon = []; + $rings = []; + for ($r = 0; $r < $numRings; $r++) { + // Number of points in this ring $numPoints = unpack('V', substr($wkb, $offset, 4))[1]; $offset += 4; - $ring = []; - for ($i = 0; $i < $numPoints; $i++) { - $pt = unpack($format . '2', substr($wkb, $offset, 16)); - $ring[] = [(float)$pt[1], (float)$pt[2]]; + $points = []; + for ($p = 0; $p < $numPoints; $p++) { + $x = unpack('d', substr($wkb, $offset, 8))[1]; + $y = unpack('d', substr($wkb, $offset + 8, 8))[1]; + $points[] = [(float)$x, (float)$y]; $offset += 16; } - $polygon[] = $ring; + + $rings[] = $points; } - return $polygon; + return $rings; + } } diff --git a/src/Database/Database.php b/src/Database/Database.php index 3540f1646..5656b2c3e 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -531,7 +531,6 @@ function (?string $value) { //return self::decodeSpatialData($value); return $this->adapter->decodeLinestring($value); } catch (\Throwable $th) { - var_dump($th); throw new StructureException($th->getMessage()); } } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index fab44ed24..e7a5661de 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -106,18 +106,27 @@ public function testSpatialTypeDocuments(): void // Create spatial attributes using createAttribute method $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); - //$this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); + // $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'point_spatial', Database::INDEX_SPATIAL, ['pointAttr'])); $this->assertEquals(true, $database->createIndex($collectionName, 'line_spatial', Database::INDEX_SPATIAL, ['lineAttr'])); - //$this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); + // $this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); // Create test document $doc1 = new Document([ '$id' => 'doc1', 'pointAttr' => [5.0, 5.0], 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], +// 'polyAttr' => [ +// [ +// [0.0, 0.0], +// [0.0, 10.0], +// [10.0, 10.0], +// [10.0, 0.0], +// [0.0, 0.0] +// ] +// ], '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] ]); From 4e195f82f650c687d3d72e0fdf879c015c946198 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 17:17:38 +0300 Subject: [PATCH 16/55] Postgres polygon --- src/Database/Adapter/Postgres.php | 71 +++++++++++++++++++++++ src/Database/Database.php | 1 - tests/e2e/Adapter/Scopes/SpatialTests.php | 17 ++---- 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index ed9adb4e1..7e12492df 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2093,4 +2093,75 @@ public function decodeLinestring(mixed $wkb): array return $points; } + public function decodePolygon(string $wkb): array + { + // POLYGON((x1,y1),(x2,y2)) + if (str_starts_with($wkb, 'POLYGON((')) { + $start = strpos($wkb, '((') + 2; + $end = strrpos($wkb, '))'); + $inside = substr($wkb, $start, $end - $start); + + $rings = explode('),(', $inside); + return array_map(function ($ring) { + $points = explode(',', $ring); + return array_map(function ($point) { + $coords = explode(' ', trim($point)); + return [(float)$coords[0], (float)$coords[1]]; + }, $points); + }, $rings); + } + + var_dump($wkb); + + // Convert hex string to binary if needed + if (preg_match('/^[0-9a-fA-F]+$/', $wkb)) { + $wkb = hex2bin($wkb); + if ($wkb === false) { + throw new \RuntimeException("Invalid hex WKB"); + } + } + + if (strlen($wkb) < 9) { + throw new \RuntimeException("WKB too short"); + } + + $byteOrder = ord($wkb[0]); + $isLE = $byteOrder === 1; // assume little-endian + $uInt32 = 'V'; // little-endian 32-bit unsigned + $uDouble = 'd'; // little-endian double + + $typeInt = unpack($uInt32, substr($wkb, 1, 4))[1]; + $hasSrid = ($typeInt & 0x20000000) !== 0; + $geomType = $typeInt & 0xFF; + + if ($geomType !== 3) { // 3 = POLYGON + throw new \RuntimeException("Not a POLYGON geometry type, got {$geomType}"); + } + + $offset = 5; + if ($hasSrid) { + $offset += 4; + } + + // Number of rings + $numRings = unpack($uInt32, substr($wkb, $offset, 4))[1]; + $offset += 4; + + $rings = []; + for ($r = 0; $r < $numRings; $r++) { + $numPoints = unpack($uInt32, substr($wkb, $offset, 4))[1]; + $offset += 4; + $points = []; + for ($i = 0; $i < $numPoints; $i++) { + $x = unpack($uDouble, substr($wkb, $offset, 8))[1]; + $y = unpack($uDouble, substr($wkb, $offset + 8, 8))[1]; + $points[] = [(float)$x, (float)$y]; + $offset += 16; + } + $rings[] = $points; + } + + return $rings; // array of rings, each ring is array of [x,y] + } + } diff --git a/src/Database/Database.php b/src/Database/Database.php index 5656b2c3e..9b6f9d98e 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -557,7 +557,6 @@ function (?string $value) { //return self::decodeSpatialData($value); return $this->adapter->decodePolygon($value); } catch (\Throwable $th) { - var_dump($th); throw new StructureException($th->getMessage()); } } diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index e7a5661de..257b57156 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -106,27 +106,19 @@ public function testSpatialTypeDocuments(): void // Create spatial attributes using createAttribute method $this->assertEquals(true, $database->createAttribute($collectionName, 'pointAttr', Database::VAR_POINT, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); $this->assertEquals(true, $database->createAttribute($collectionName, 'lineAttr', Database::VAR_LINESTRING, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); - // $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); + $this->assertEquals(true, $database->createAttribute($collectionName, 'polyAttr', Database::VAR_POLYGON, 0, $database->getAdapter()->getSupportForSpatialIndexNull() ? false : true)); // Create spatial indexes $this->assertEquals(true, $database->createIndex($collectionName, 'point_spatial', Database::INDEX_SPATIAL, ['pointAttr'])); $this->assertEquals(true, $database->createIndex($collectionName, 'line_spatial', Database::INDEX_SPATIAL, ['lineAttr'])); - // $this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); + $this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); // Create test document $doc1 = new Document([ '$id' => 'doc1', 'pointAttr' => [5.0, 5.0], 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], -// 'polyAttr' => [ -// [ -// [0.0, 0.0], -// [0.0, 10.0], -// [10.0, 10.0], -// [10.0, 0.0], -// [0.0, 0.0] -// ] -// ], + 'polyAttr' => [[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [10.0, 0.0], [0.0, 0.0]]], '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] ]); @@ -134,11 +126,14 @@ public function testSpatialTypeDocuments(): void $this->assertInstanceOf(Document::class, $createdDoc); $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); $this->assertEquals([[1.0, 2.0], [3.0, 4.0]], $createdDoc->getAttribute('lineAttr')); + $this->assertEquals([[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [10.0, 0.0], [0.0, 0.0]]], $createdDoc->getAttribute('polyAttr')); $createdDoc = $database->getDocument($collectionName, 'doc1'); $this->assertInstanceOf(Document::class, $createdDoc); $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); $this->assertEquals([[1.0, 2.0], [3.0, 4.0]], $createdDoc->getAttribute('lineAttr')); + $this->assertEquals([[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [10.0, 0.0], [0.0, 0.0]]], $createdDoc->getAttribute('polyAttr')); + $this->assertEquals('222','2'); From bb4b3e48ed40e32a656c964bbf59e5858f2ce81b Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 17:40:16 +0300 Subject: [PATCH 17/55] Mysql polygon --- src/Database/Adapter/SQL.php | 43 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index e959e105a..f73b03d8b 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2802,54 +2802,59 @@ public function decodePolygon(string $wkb): array var_dump($wkb); - // Convert HEX to binary if needed - if (ctype_xdigit($wkb) && strlen($wkb) % 2 === 0) { - $wkb = hex2bin($wkb); + // Convert HEX string to binary if needed + if (str_starts_with($wkb, '0x') || ctype_xdigit($wkb)) { + $wkb = hex2bin(str_starts_with($wkb, '0x') ? substr($wkb, 2) : $wkb); if ($wkb === false) { - throw new \RuntimeException("Invalid HEX WKB"); + throw new DatabaseException('Invalid hex WKB'); } } - if (strlen($wkb) < 9) { - throw new \RuntimeException("WKB too short"); + if (strlen($wkb) < 21) { + throw new DatabaseException('WKB too short to be a POLYGON'); } - $byteOrder = ord($wkb[0]); + // MySQL SRID-aware WKB layout: 4 bytes SRID prefix + $offset = 4; + + $byteOrder = ord($wkb[$offset]); if ($byteOrder !== 1) { - throw new \RuntimeException("Only little-endian WKB supported"); + throw new DatabaseException('Only little-endian WKB supported'); } + $offset += 1; - // Type + SRID flag - $typeInt = unpack('V', substr($wkb, 1, 4))[1]; - $hasSrid = ($typeInt & 0x20000000) !== 0; - $geomType = $typeInt & 0xFF; + $type = unpack('V', substr($wkb, $offset, 4))[1]; + $hasSRID = ($type & 0x20000000) === 0x20000000; + $geomType = $type & 0xFF; + $offset += 4; if ($geomType !== 3) { // 3 = POLYGON - throw new \RuntimeException("Not a POLYGON geometry type, got {$geomType}"); + throw new DatabaseException("Not a POLYGON geometry type, got {$geomType}"); } - $offset = 5 + ($hasSrid ? 4 : 0); + // Skip SRID in type flag if present + if ($hasSRID) { + $offset += 4; + } - // Number of rings $numRings = unpack('V', substr($wkb, $offset, 4))[1]; $offset += 4; $rings = []; for ($r = 0; $r < $numRings; $r++) { - // Number of points in this ring $numPoints = unpack('V', substr($wkb, $offset, 4))[1]; $offset += 4; + $ring = []; - $points = []; for ($p = 0; $p < $numPoints; $p++) { $x = unpack('d', substr($wkb, $offset, 8))[1]; $y = unpack('d', substr($wkb, $offset + 8, 8))[1]; - $points[] = [(float)$x, (float)$y]; + $ring[] = [(float)$x, (float)$y]; $offset += 16; } - $rings[] = $points; + $rings[] = $ring; } return $rings; From b94dada5fa5f28d4fd1913718b07e8fbe81cf4f9 Mon Sep 17 00:00:00 2001 From: fogelito Date: Sun, 14 Sep 2025 17:58:34 +0300 Subject: [PATCH 18/55] clean var_dump --- src/Database/Adapter.php | 1 - src/Database/Adapter/SQL.php | 10 ----- tests/e2e/Adapter/Scopes/SpatialTests.php | 48 +++++++---------------- 3 files changed, 14 insertions(+), 45 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 6bd2c6d8e..831b87064 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1286,7 +1286,6 @@ abstract public function getTenantQuery(string $collection, string $alias = ''): */ abstract protected function execute(mixed $stmt): bool; - abstract public function encodePoint(array $point): mixed; abstract public function decodePoint(string $wkb): array; abstract public function decodeLinestring(string $wkb): array; abstract public function decodePolygon(string $wkb): array; diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index f73b03d8b..aabe0bf4d 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2711,20 +2711,12 @@ public function getSpatialTypeFromWKT(string $wkt): string return strtolower(trim(substr($wkt, 0, $pos))); } - public function encodePoint(array $point): string - { - return "POINT({$point[0]} {$point[1]})"; - } - public function decodePoint(mixed $wkb): array { - var_dump($wkb); - if (str_starts_with(strtoupper($wkb), 'POINT(')) { $start = strpos($wkb, '(') + 1; $end = strrpos($wkb, ')'); $inside = substr($wkb, $start, $end - $start); - $coords = explode(' ', trim($inside)); return [(float)$coords[0], (float)$coords[1]]; } @@ -2800,8 +2792,6 @@ public function decodePolygon(string $wkb): array }, $rings); } - var_dump($wkb); - // Convert HEX string to binary if needed if (str_starts_with($wkb, '0x') || ctype_xdigit($wkb)) { $wkb = hex2bin(str_starts_with($wkb, '0x') ? substr($wkb, 2) : $wkb); diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 257b57156..e9839b7de 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -113,49 +113,29 @@ public function testSpatialTypeDocuments(): void $this->assertEquals(true, $database->createIndex($collectionName, 'line_spatial', Database::INDEX_SPATIAL, ['lineAttr'])); $this->assertEquals(true, $database->createIndex($collectionName, 'poly_spatial', Database::INDEX_SPATIAL, ['polyAttr'])); + $point = [5.0, 5.0]; + $linestring = [[1.0, 2.0], [3.0, 4.0]]; + $polygon = [[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [10.0, 0.0], [0.0, 0.0]]]; + // Create test document $doc1 = new Document([ '$id' => 'doc1', - 'pointAttr' => [5.0, 5.0], - 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], - 'polyAttr' => [[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [10.0, 0.0], [0.0, 0.0]]], + 'pointAttr' => $point, + 'lineAttr' => $linestring, + 'polyAttr' => $polygon, '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] ]); - $createdDoc = $database->createDocument($collectionName, $doc1); $this->assertInstanceOf(Document::class, $createdDoc); - $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); - $this->assertEquals([[1.0, 2.0], [3.0, 4.0]], $createdDoc->getAttribute('lineAttr')); - $this->assertEquals([[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [10.0, 0.0], [0.0, 0.0]]], $createdDoc->getAttribute('polyAttr')); + $this->assertEquals($point, $createdDoc->getAttribute('pointAttr')); + $this->assertEquals($linestring, $createdDoc->getAttribute('lineAttr')); + $this->assertEquals($polygon, $createdDoc->getAttribute('polyAttr')); $createdDoc = $database->getDocument($collectionName, 'doc1'); $this->assertInstanceOf(Document::class, $createdDoc); - $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); - $this->assertEquals([[1.0, 2.0], [3.0, 4.0]], $createdDoc->getAttribute('lineAttr')); - $this->assertEquals([[[0.0, 0.0], [0.0, 10.0], [10.0, 10.0], [10.0, 0.0], [0.0, 0.0]]], $createdDoc->getAttribute('polyAttr')); - - $this->assertEquals('222','2'); - - - // Create test document - $doc1 = new Document([ - '$id' => 'doc1', - 'pointAttr' => [5.0, 5.0], - 'lineAttr' => [[1.0, 2.0], [3.0, 4.0]], - 'polyAttr' => [ - [ - [0.0, 0.0], - [0.0, 10.0], - [10.0, 10.0], - [10.0, 0.0], - [0.0, 0.0] - ] - ], - '$permissions' => [Permission::update(Role::any()), Permission::read(Role::any())] - ]); - $createdDoc = $database->createDocument($collectionName, $doc1); - $this->assertInstanceOf(Document::class, $createdDoc); - $this->assertEquals([5.0, 5.0], $createdDoc->getAttribute('pointAttr')); + $this->assertEquals($point, $createdDoc->getAttribute('pointAttr')); + $this->assertEquals($linestring, $createdDoc->getAttribute('lineAttr')); + $this->assertEquals($polygon, $createdDoc->getAttribute('polyAttr')); // Update spatial data $doc1->setAttribute('pointAttr', [6.0, 6.0]); @@ -257,7 +237,7 @@ public function testSpatialTypeDocuments(): void $this->assertEquals('doc1', $result[0]->getId(), sprintf('Incorrect document for distance %s on polyAttr', $queryType)); } } finally { - //$database->deleteCollection($collectionName); + $database->deleteCollection($collectionName); } } From 7f7dfbb2eeead078c0b848008112f8cc32a915c5 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 08:28:18 +0300 Subject: [PATCH 19/55] Remove decodeSpatialData method --- src/Database/Database.php | 54 ------------------------------ src/Database/Validator/Spatial.php | 7 ---- 2 files changed, 61 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 9b6f9d98e..23de3f3c5 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -554,7 +554,6 @@ function (?string $value) { } try { - //return self::decodeSpatialData($value); return $this->adapter->decodePolygon($value); } catch (\Throwable $th) { throw new StructureException($th->getMessage()); @@ -7324,57 +7323,4 @@ protected function encodeSpatialData(mixed $value, string $type): string throw new DatabaseException('Unknown spatial type: ' . $type); } } - - /** - * Decode spatial data from WKT (Well-Known Text) format to array format - * - * @param string $wkt - * @return array - * @throws DatabaseException - */ - public function decodeSpatialData(string $wkt): array - { - $upper = strtoupper($wkt); - - // POINT(x y) - if (str_starts_with($upper, 'POINT(')) { - $start = strpos($wkt, '(') + 1; - $end = strrpos($wkt, ')'); - $inside = substr($wkt, $start, $end - $start); - - $coords = explode(' ', trim($inside)); - return [(float)$coords[0], (float)$coords[1]]; - } - - // LINESTRING(x1 y1, x2 y2, ...) - if (str_starts_with($upper, 'LINESTRING(')) { - $start = strpos($wkt, '(') + 1; - $end = strrpos($wkt, ')'); - $inside = substr($wkt, $start, $end - $start); - - $points = explode(',', $inside); - return array_map(function ($point) { - $coords = explode(' ', trim($point)); - return [(float)$coords[0], (float)$coords[1]]; - }, $points); - } - - // POLYGON((x1,y1),(x2,y2)) - if (str_starts_with($upper, 'POLYGON((')) { - $start = strpos($wkt, '((') + 2; - $end = strrpos($wkt, '))'); - $inside = substr($wkt, $start, $end - $start); - - $rings = explode('),(', $inside); - return array_map(function ($ring) { - $points = explode(',', $ring); - return array_map(function ($point) { - $coords = explode(' ', trim($point)); - return [(float)$coords[0], (float)$coords[1]]; - }, $points); - }, $rings); - } - - return [$wkt]; - } } diff --git a/src/Database/Validator/Spatial.php b/src/Database/Validator/Spatial.php index cc3ea83d4..912f05b2b 100644 --- a/src/Database/Validator/Spatial.php +++ b/src/Database/Validator/Spatial.php @@ -134,13 +134,6 @@ protected function validatePolygon(array $value): bool */ public static function isWKTString(string $value): bool { - - /** - * We need to decode the value first - */ - - // return true; - $value = trim($value); return (bool) preg_match('/^(POINT|LINESTRING|POLYGON)\s*\(/i', $value); } From 773df87a340ad921a7e560cc31679246ca6481b8 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 08:31:50 +0300 Subject: [PATCH 20/55] Remove try catch --- src/Database/Adapter/Postgres.php | 6 ------ src/Database/Database.php | 29 ++++------------------------- 2 files changed, 4 insertions(+), 31 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 7e12492df..cb2b71aca 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2003,12 +2003,6 @@ public function getSupportForSpatialAxisOrder(): bool return false; } - public function encodePoint(array $point): string - { - return "POINT({$point[0]} {$point[1]})";// EWKT - //return "SRID=4326;POINT({$point[0]} {$point[1]})";// EWKT - } - public function decodePoint(mixed $wkb): array { //$wkb = str_replace('SRID=4326;', '', $wkb); // Remove if was added in encode diff --git a/src/Database/Database.php b/src/Database/Database.php index 23de3f3c5..428360046 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -460,7 +460,7 @@ function (mixed $value) { */ function (mixed $value) { if (is_null($value)) { - return; + return null; } try { $value = new \DateTime($value); @@ -487,7 +487,6 @@ function (mixed $value) { } try { return self::encodeSpatialData($value, Database::VAR_POINT); - //return $this->adapter->encodePoint($value); } catch (\Throwable) { throw new StructureException('Invalid point'); } @@ -496,16 +495,7 @@ function (?string $value) { if ($value === null) { return null; } - - /** - * todo:validate array point - */ - - try { - return $this->adapter->decodePoint($value); - } catch (\Throwable $th) { - throw new StructureException($th->getMessage()); - } + return $this->adapter->decodePoint($value); } ); @@ -526,13 +516,7 @@ function (?string $value) { if (is_null($value)) { return null; } - - try { - //return self::decodeSpatialData($value); - return $this->adapter->decodeLinestring($value); - } catch (\Throwable $th) { - throw new StructureException($th->getMessage()); - } + return $this->adapter->decodeLinestring($value); } ); @@ -552,12 +536,7 @@ function (?string $value) { if (is_null($value)) { return null; } - - try { - return $this->adapter->decodePolygon($value); - } catch (\Throwable $th) { - throw new StructureException($th->getMessage()); - } + return $this->adapter->decodePolygon($value); } ); } From f6bd630d6a5e84e6fbd5024a098f45a74669062e Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 08:40:23 +0300 Subject: [PATCH 21/55] Add hints --- src/Database/Database.php | 33 +++++++++++++++++++---- tests/e2e/Adapter/Scopes/SpatialTests.php | 1 + 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 428360046..3e99dfb92 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -481,16 +481,24 @@ function (?string $value) { self::addFilter( Database::VAR_POINT, + /** + * @param mixed $value + * @return mixed + */ function (mixed $value) { - if ($value === null) { - return null; + if (!is_array($value)) { + return $value; } try { - return self::encodeSpatialData($value, Database::VAR_POINT); + return self::encodeSpatialData($value, Database::VAR_POINT); } catch (\Throwable) { - throw new StructureException('Invalid point'); + return $value; } }, + /** + * @param string|null $value + * @return string|null + */ function (?string $value) { if ($value === null) { return null; @@ -501,6 +509,10 @@ function (?string $value) { self::addFilter( Database::VAR_LINESTRING, + /** + * @param mixed $value + * @return mixed + */ function (mixed $value) { if (!is_array($value)) { return $value; @@ -511,7 +523,10 @@ function (mixed $value) { return $value; } }, - + /** + * @param string|null $value + * @return string|null + */ function (?string $value) { if (is_null($value)) { return null; @@ -522,6 +537,10 @@ function (?string $value) { self::addFilter( Database::VAR_POLYGON, + /** + * @param mixed $value + * @return mixed + */ function (mixed $value) { if (!is_array($value)) { return $value; @@ -532,6 +551,10 @@ function (mixed $value) { return $value; } }, + /** + * @param string|null $value + * @return string|null + */ function (?string $value) { if (is_null($value)) { return null; diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index e9839b7de..0aaf01928 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -2101,6 +2101,7 @@ public function testInvalidSpatialTypes(): void ])); $this->fail("Expected StructureException for invalid point"); } catch (\Throwable $th) { + var_dump($th); $this->assertInstanceOf(StructureException::class, $th); } From cc972d2b897f79c75e64876b792a77a7c472cc0e Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 16:20:59 +0300 Subject: [PATCH 22/55] dbg --- src/Database/Adapter/MariaDB.php | 2 +- src/Database/Adapter/Postgres.php | 6 +----- src/Database/Adapter/SQL.php | 4 +--- src/Database/Database.php | 6 +++--- tests/e2e/Adapter/Base.php | 12 ++++++------ tests/e2e/Adapter/Scopes/SpatialTests.php | 1 - 6 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 77b3647e5..c78d6637c 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -864,7 +864,7 @@ public function createDocument(Document $collection, Document $document): Docume INSERT INTO {$this->getSQLTable($name)} ({$columns} _uid) VALUES ({$columnNames} :_uid) "; -var_dump($sql); + $sql = $this->trigger(Database::EVENT_DOCUMENT_CREATE, $sql); $stmt = $this->getPDO()->prepare($sql); diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index cb2b71aca..f7bb3c29a 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -993,7 +993,7 @@ public function createDocument(Document $collection, Document $document): Docume INSERT INTO {$this->getSQLTable($name)} ({$columns} \"_uid\") VALUES ({$columnNames} :_uid) "; -var_dump($sql); + $sql = $this->trigger(Database::EVENT_DOCUMENT_CREATE, $sql); $stmt = $this->getPDO()->prepare($sql); @@ -2043,8 +2043,6 @@ public function decodeLinestring(mixed $wkb): array }, $points); } - var_dump($wkb); - if (ctype_xdigit($wkb)) { $wkb = hex2bin($wkb); } @@ -2105,8 +2103,6 @@ public function decodePolygon(string $wkb): array }, $rings); } - var_dump($wkb); - // Convert hex string to binary if needed if (preg_match('/^[0-9a-fA-F]+$/', $wkb)) { $wkb = hex2bin($wkb); diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index aabe0bf4d..1f734c332 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -366,7 +366,7 @@ public function getDocument(Document $collection, string $id, array $queries = [ WHERE {$this->quote($alias)}.{$this->quote('_uid')} = :_uid {$this->getTenantQuery($collection, $alias)} "; -var_dump($sql); + if ($this->getSupportForUpdateLock()) { $sql .= " {$forUpdate}"; } @@ -2754,8 +2754,6 @@ public function decodeLinestring(string $wkb): array }, $points); } - var_dump($wkb); - // Skip 1 byte (endianness) + 4 bytes (type) + 4 bytes (SRID) $offset = 9; diff --git a/src/Database/Database.php b/src/Database/Database.php index 3e99dfb92..584a4d942 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -497,7 +497,7 @@ function (mixed $value) { }, /** * @param string|null $value - * @return string|null + * @return array|null */ function (?string $value) { if ($value === null) { @@ -525,7 +525,7 @@ function (mixed $value) { }, /** * @param string|null $value - * @return string|null + * @return array|null */ function (?string $value) { if (is_null($value)) { @@ -553,7 +553,7 @@ function (mixed $value) { }, /** * @param string|null $value - * @return string|null + * @return array|null */ function (?string $value) { if (is_null($value)) { diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 481704ce6..37ad7cce3 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -18,12 +18,12 @@ abstract class Base extends TestCase { -// use CollectionTests; -// use DocumentTests; -// use AttributeTests; -// use IndexTests; -// use PermissionTests; -// use RelationshipTests; + use CollectionTests; + use DocumentTests; + use AttributeTests; + use IndexTests; + use PermissionTests; + use RelationshipTests; use SpatialTests; use GeneralTests; diff --git a/tests/e2e/Adapter/Scopes/SpatialTests.php b/tests/e2e/Adapter/Scopes/SpatialTests.php index 0aaf01928..e9839b7de 100644 --- a/tests/e2e/Adapter/Scopes/SpatialTests.php +++ b/tests/e2e/Adapter/Scopes/SpatialTests.php @@ -2101,7 +2101,6 @@ public function testInvalidSpatialTypes(): void ])); $this->fail("Expected StructureException for invalid point"); } catch (\Throwable $th) { - var_dump($th); $this->assertInstanceOf(StructureException::class, $th); } From 1b9b095e534cf6454d9e79eecb024d0fb74d8772 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 16:22:31 +0300 Subject: [PATCH 23/55] formatting --- src/Database/Adapter/Postgres.php | 6 ++++-- src/Database/Adapter/SQL.php | 2 +- src/Database/Database.php | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index f7bb3c29a..4b4c2155a 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2077,8 +2077,10 @@ public function decodeLinestring(mixed $wkb): array $points = []; for ($i = 0; $i < $numPoints; $i++) { - $x = unpack('e', substr($wkb, $offset, 8))[1]; $offset += 8; - $y = unpack('e', substr($wkb, $offset, 8))[1]; $offset += 8; + $x = unpack('e', substr($wkb, $offset, 8))[1]; + $offset += 8; + $y = unpack('e', substr($wkb, $offset, 8))[1]; + $offset += 8; $points[] = [(float)$x, (float)$y]; } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 1f734c332..0134634a7 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -359,7 +359,7 @@ public function getDocument(Document $collection, string $id, array $queries = [ $forUpdate = $forUpdate ? 'FOR UPDATE' : ''; $alias = Query::DEFAULT_ALIAS; - $spatialAttributes=[]; + $spatialAttributes = []; $sql = " SELECT {$this->getAttributeProjection($selections, $alias, $spatialAttributes)} FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} diff --git a/src/Database/Database.php b/src/Database/Database.php index 584a4d942..90a6d378f 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -460,7 +460,7 @@ function (mixed $value) { */ function (mixed $value) { if (is_null($value)) { - return null; + return; } try { $value = new \DateTime($value); From 9a0e28517a3a34c0142a119146ebfe097894242d Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 16:33:42 +0300 Subject: [PATCH 24/55] signature --- src/Database/Adapter/Postgres.php | 4 +--- src/Database/Adapter/SQL.php | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 4b4c2155a..38807e442 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2003,10 +2003,8 @@ public function getSupportForSpatialAxisOrder(): bool return false; } - public function decodePoint(mixed $wkb): array + public function decodePoint(string $wkb): array { - //$wkb = str_replace('SRID=4326;', '', $wkb); // Remove if was added in encode - if (str_starts_with(strtoupper($wkb), 'POINT(')) { $start = strpos($wkb, '(') + 1; $end = strrpos($wkb, ')'); diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 0134634a7..b74d468b7 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2711,7 +2711,7 @@ public function getSpatialTypeFromWKT(string $wkt): string return strtolower(trim(substr($wkt, 0, $pos))); } - public function decodePoint(mixed $wkb): array + public function decodePoint(string $wkb): array { if (str_starts_with(strtoupper($wkb), 'POINT(')) { $start = strpos($wkb, '(') + 1; From 5962c7b3642bf1a8f0c91a12258849ed66facf66 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 15 Sep 2025 16:35:10 +0300 Subject: [PATCH 25/55] Enhance MongoDB adapter by adding support for the '$nin' operator and updating the handling of 'notContains' queries. Updated the getSupportForQueryContains method to return true, enabling support for query contains functionality. Refactored DocumentTests to include comprehensive tests for the notBetween functionality, ensuring accurate results across various scenarios. --- src/Database/Adapter/Mongo.php | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 097800bd6..050659926 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -31,6 +31,7 @@ class Mongo extends Adapter '$gt', '$gte', '$in', + '$nin', '$text', '$search', '$or', @@ -1973,7 +1974,7 @@ protected function buildFilter(Query $query): array : $query->getValues()[0] ), }; - + $filter = []; if ($operator == '$eq' && \is_array($value)) { @@ -1983,11 +1984,15 @@ protected function buildFilter(Query $query): array } elseif ($operator == '$in') { if ($query->getMethod() === Query::TYPE_CONTAINS && !$query->onArray()) { $filter[$attribute]['$regex'] = new Regex(".*{$this->escapeWildcards($value)}.*", 'i'); - } elseif ($query->getMethod() === Query::TYPE_NOT_CONTAINS && !$query->onArray()) { - $filter[$attribute] = ['$not' => new Regex(".*{$this->escapeWildcards($value)}.*", 'i')]; } else { $filter[$attribute]['$in'] = $query->getValues(); } + } elseif ($operator === 'notContains') { + if (!$query->onArray()) { + $filter[$attribute] = ['$not' => new Regex(".*{$this->escapeWildcards($value)}.*", 'i')]; + } else { + $filter[$attribute]['$nin'] = $query->getValues(); + } } elseif ($operator == '$search') { if ($query->getMethod() === Query::TYPE_NOT_SEARCH) { // MongoDB doesn't support negating $text expressions directly @@ -2040,7 +2045,7 @@ protected function getQueryOperator(string $operator): string Query::TYPE_GREATER => '$gt', Query::TYPE_GREATER_EQUAL => '$gte', Query::TYPE_CONTAINS => '$in', - Query::TYPE_NOT_CONTAINS => '$in', + Query::TYPE_NOT_CONTAINS => 'notContains', Query::TYPE_SEARCH => '$search', Query::TYPE_NOT_SEARCH => '$search', Query::TYPE_BETWEEN => 'between', @@ -2266,7 +2271,7 @@ public function getSupportForFulltextWildcardIndex(): bool */ public function getSupportForQueryContains(): bool { - return false; + return true; } /** From 5d2c0c3ba53218ceb0c3a0c11df87efbf7f862c6 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 16:38:47 +0300 Subject: [PATCH 26/55] fix Pool adapter --- src/Database/Adapter/Pool.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 4d95611e1..3a4faa56a 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -549,4 +549,19 @@ public function getSupportForSpatialAxisOrder(): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } + + public function decodePoint(string $wkb): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function decodeLinestring(string $wkb): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } + + public function decodePolygon(string $wkb): array + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } } From 7d61019c31f5958a854485d3373537b255b14b7d Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 15 Sep 2025 16:40:29 +0300 Subject: [PATCH 27/55] linter --- tests/e2e/Adapter/Scopes/DocumentTests.php | 148 ++++++++++----------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 89b777fbf..25d4bf0ba 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -3340,7 +3340,7 @@ public function testFindNotSearch(): void ]); $this->assertEquals(6, count($documents)); - + // Test notSearch with partial term if ($this->getDatabase()->getAdapter()->getSupportForFulltextWildCardIndex()) { $documents = $database->find('movies', [ @@ -3484,79 +3484,79 @@ public function testFindNotEndsWith(): void $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies } - public function testFindNotBetween(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - // Test notBetween with price range - should return documents outside the range - $documents = $database->find('movies', [ - Query::notBetween('price', 25.94, 25.99), - ]); - $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range - - // Test notBetween with range that includes no documents - should return all documents - $documents = $database->find('movies', [ - Query::notBetween('price', 30, 35), - ]); - $this->assertEquals(6, count($documents)); - - // Test notBetween with date range - $documents = $database->find('movies', [ - Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), - ]); - $this->assertEquals(0, count($documents)); // No movies outside this wide date range - - // Test notBetween with narrower date range - $documents = $database->find('movies', [ - Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), - ]); - $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - - // Test notBetween with updated date range - $documents = $database->find('movies', [ - Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), - ]); - $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - - // Test notBetween with year range (integer values) - $documents = $database->find('movies', [ - Query::notBetween('year', 2005, 2007), - ]); - $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range - - // Test notBetween with reversed range (start > end) - should still work - $documents = $database->find('movies', [ - Query::notBetween('price', 25.99, 25.94), // Note: reversed order - ]); - $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully - - // Test notBetween with same start and end values - $documents = $database->find('movies', [ - Query::notBetween('year', 2006, 2006), - ]); - $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 - - // Test notBetween combined with other filters - $documents = $database->find('movies', [ - Query::notBetween('price', 25.94, 25.99), - Query::orderDesc('year'), - Query::limit(2) - ]); - $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range - - // Test notBetween with extreme ranges - $documents = $database->find('movies', [ - Query::notBetween('year', -1000, 1000), // Very wide range - ]); - $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range - - // Test notBetween with float precision - $documents = $database->find('movies', [ - Query::notBetween('price', 25.945, 25.955), // Very narrow range - ]); - $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range - } + public function testFindNotBetween(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Test notBetween with price range - should return documents outside the range + $documents = $database->find('movies', [ + Query::notBetween('price', 25.94, 25.99), + ]); + $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range + + // Test notBetween with range that includes no documents - should return all documents + $documents = $database->find('movies', [ + Query::notBetween('price', 30, 35), + ]); + $this->assertEquals(6, count($documents)); + + // Test notBetween with date range + $documents = $database->find('movies', [ + Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), + ]); + $this->assertEquals(0, count($documents)); // No movies outside this wide date range + + // Test notBetween with narrower date range + $documents = $database->find('movies', [ + Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), + ]); + $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + + // Test notBetween with updated date range + $documents = $database->find('movies', [ + Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), + ]); + $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + + // Test notBetween with year range (integer values) + $documents = $database->find('movies', [ + Query::notBetween('year', 2005, 2007), + ]); + $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range + + // Test notBetween with reversed range (start > end) - should still work + $documents = $database->find('movies', [ + Query::notBetween('price', 25.99, 25.94), // Note: reversed order + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully + + // Test notBetween with same start and end values + $documents = $database->find('movies', [ + Query::notBetween('year', 2006, 2006), + ]); + $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 + + // Test notBetween combined with other filters + $documents = $database->find('movies', [ + Query::notBetween('price', 25.94, 25.99), + Query::orderDesc('year'), + Query::limit(2) + ]); + $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range + + // Test notBetween with extreme ranges + $documents = $database->find('movies', [ + Query::notBetween('year', -1000, 1000), // Very wide range + ]); + $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range + + // Test notBetween with float precision + $documents = $database->find('movies', [ + Query::notBetween('price', 25.945, 25.955), // Very narrow range + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range + } public function testFindSelect(): void { From 298196da298d64776a2e8daa610041ebd8f209b7 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 15 Sep 2025 16:41:35 +0300 Subject: [PATCH 28/55] Clarify comment in Mongo adapter regarding handling of empty search terms to improve code readability. --- src/Database/Adapter/Mongo.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 050659926..f05eb2789 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1998,8 +1998,7 @@ protected function buildFilter(Query $query): array // MongoDB doesn't support negating $text expressions directly // Use regex as fallback for NOT search while keeping fulltext for positive search if (empty($value)) { - // Empty search term should return all documents (no documents contain empty string) - // Don't add any filter - this will match all documents + // If value is not passed, don't add any filter - this will match all documents } else { // Escape special regex characters and create a pattern that matches the search term as substring $escapedValue = preg_quote($value, '/'); From 87af182984ce5e3d97e3f0e4602d98894edc0145 Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 15 Sep 2025 16:42:10 +0300 Subject: [PATCH 29/55] link comment --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1948e8b00..8e571d3c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - ./dev/xdebug.ini:/usr/local/etc/php/conf.d/xdebug.ini - /var/run/docker.sock:/var/run/docker.sock - ./docker-compose.yml:/usr/src/code/docker-compose.yml - - ./vendor/utopia-php/mongo/src/Client.php:/usr/src/code/vendor/utopia-php/mongo/src/Client.php + #- ./vendor/utopia-php/mongo/src/Client.php:/usr/src/code/vendor/utopia-php/mongo/src/Client.php environment: PHP_IDE_CONFIG: serverName=tests From 943c42e98f1036b05d550d73ec9651db894435ea Mon Sep 17 00:00:00 2001 From: shimon Date: Mon, 15 Sep 2025 16:59:30 +0300 Subject: [PATCH 30/55] Implement cursor paging in Mongo adapter with default batch size for improved performance on large result sets. --- src/Database/Adapter/Mongo.php | 48 ++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index dcb920144..82a5c7432 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -41,6 +41,11 @@ class Mongo extends Adapter protected Client $client; + /** + * Default batch size for cursor operations + */ + private const DEFAULT_BATCH_SIZE = 1000; + //protected ?int $timeout = null; /** @@ -1277,15 +1282,43 @@ public function getSequences(string $collection, array $documents): array $filters['_tenant'] = $this->getTenantFilters($collection, $documentTenants); } try { - $results = $this->client->find($name, $filters, ['projection' => ['_uid' => 1, '_id' => 1]]); + // Use cursor paging for large result sets + $options = [ + 'projection' => ['_uid' => 1, '_id' => 1], + 'batchSize' => self::DEFAULT_BATCH_SIZE + ]; + + $response = $this->client->find($name, $filters, $options); + $results = $response->cursor->firstBatch ?? []; + + // Process first batch + foreach ($results as $result) { + $sequences[$result->_uid] = (string)$result->_id; + } + + // Get cursor ID for subsequent batches + $cursorId = $response->cursor->id ?? null; + + // Continue fetching with getMore + while ($cursorId && $cursorId !== 0) { + $moreResponse = $this->client->getMore($cursorId, $name, self::DEFAULT_BATCH_SIZE); + $moreResults = $moreResponse->cursor->nextBatch ?? []; + + if (empty($moreResults)) { + break; + } + + foreach ($moreResults as $result) { + $sequences[$result->_uid] = (string)$result->_id; + } + + // Update cursor ID for next iteration + $cursorId = $moreResponse->cursor->id ?? null; + } } catch (MongoException $e) { throw $this->processException($e); } - foreach ($results->cursor->firstBatch as $result) { - $sequences[$result->_uid] = (string)$result->_id; - } - foreach ($documents as $document) { if (isset($sequences[$document->getId()])) { $document['$sequence'] = $sequences[$document->getId()]; @@ -1575,8 +1608,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 try { // Use proper cursor iteration with reasonable batch size - $batchSize = 1000; - $options['batchSize'] = $batchSize; + $options['batchSize'] = self::DEFAULT_BATCH_SIZE; $response = $this->client->find($name, $filters, $options); $results = $response->cursor->firstBatch ?? []; @@ -1597,7 +1629,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 break; } - $moreResponse = $this->client->getMore($cursorId, $name, $batchSize); + $moreResponse = $this->client->getMore($cursorId, $name, self::DEFAULT_BATCH_SIZE); $moreResults = $moreResponse->cursor->nextBatch ?? []; if (empty($moreResults)) { From 44cc3f34be4bcc9d1ea5fec778566c6ded037d2c Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 17:10:39 +0300 Subject: [PATCH 31/55] formatting --- src/Database/Adapter.php | 20 ++++++++++++ src/Database/Adapter/SQL.php | 59 ++++++++++++++++++++++++++++-------- tests/e2e/Adapter/Base.php | 12 ++++---- 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 831b87064..5cd7c8a41 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1286,7 +1286,27 @@ abstract public function getTenantQuery(string $collection, string $alias = ''): */ abstract protected function execute(mixed $stmt): bool; + /** + * Decode a WKB or textual POINT into [x, y] + * + * @param string $wkb + * @return float[] Array with two elements: [x, y] + */ abstract public function decodePoint(string $wkb): array; + + /** + * Decode a WKB or textual LINESTRING into [[x1, y1], [x2, y2], ...] + * + * @param string $wkb + * @return float[][] Array of points, each as [x, y] + */ abstract public function decodeLinestring(string $wkb): array; + + /** + * Decode a WKB or textual POLYGON into [[[x1, y1], [x2, y2], ...], ...] + * + * @param string $wkb + * @return float[][][] Array of rings, each ring is an array of points [x, y] + */ abstract public function decodePolygon(string $wkb): array; } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index b74d468b7..e2a8f775d 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2737,7 +2737,11 @@ public function decodePoint(string $wkb): array $format = $littleEndian ? 'd2' : 'd2'; // little-endian doubles $coords = unpack($format, $coordsBin); - return [$coords[1], $coords[2]]; + if ($coords === false || !isset($coords[1], $coords[2])) { + throw new DatabaseException('Invalid WKB for POINT: cannot unpack coordinates'); + } + + return [(float)$coords[1], (float)$coords[2]]; } public function decodeLinestring(string $wkb): array @@ -2758,14 +2762,24 @@ public function decodeLinestring(string $wkb): array $offset = 9; // Number of points (4 bytes little-endian) - $numPoints = unpack('V', substr($wkb, $offset, 4))[1]; + $numPointsArr = unpack('V', substr($wkb, $offset, 4)); + if ($numPointsArr === false || !isset($numPointsArr[1])) { + throw new DatabaseException('Invalid WKB: cannot unpack number of points'); + } + + $numPoints = $numPointsArr[1]; $offset += 4; $points = []; for ($i = 0; $i < $numPoints; $i++) { - $x = unpack('d', substr($wkb, $offset, 8))[1]; - $y = unpack('d', substr($wkb, $offset + 8, 8))[1]; - $points[] = [(float)$x, (float)$y]; + $xArr = unpack('d', substr($wkb, $offset, 8)); + $yArr = unpack('d', substr($wkb, $offset + 8, 8)); + + if ($xArr === false || !isset($xArr[1]) || $yArr === false || !isset($yArr[1])) { + throw new DatabaseException('Invalid WKB: cannot unpack point coordinates'); + } + + $points[] = [(float)$xArr[1], (float)$yArr[1]]; $offset += 16; } @@ -2811,7 +2825,12 @@ public function decodePolygon(string $wkb): array } $offset += 1; - $type = unpack('V', substr($wkb, $offset, 4))[1]; + $typeArr = unpack('V', substr($wkb, $offset, 4)); + if ($typeArr === false || !isset($typeArr[1])) { + throw new DatabaseException('Invalid WKB: cannot unpack geometry type'); + } + + $type = $typeArr[1]; $hasSRID = ($type & 0x20000000) === 0x20000000; $geomType = $type & 0xFF; $offset += 4; @@ -2825,20 +2844,37 @@ public function decodePolygon(string $wkb): array $offset += 4; } - $numRings = unpack('V', substr($wkb, $offset, 4))[1]; + $numRingsArr = unpack('V', substr($wkb, $offset, 4)); + + if ($numRingsArr === false || !isset($numRingsArr[1])) { + throw new DatabaseException('Invalid WKB: cannot unpack number of rings'); + } + + $numRings = $numRingsArr[1]; $offset += 4; $rings = []; for ($r = 0; $r < $numRings; $r++) { - $numPoints = unpack('V', substr($wkb, $offset, 4))[1]; + $numPointsArr = unpack('V', substr($wkb, $offset, 4)); + + if ($numPointsArr === false || !isset($numPointsArr[1])) { + throw new DatabaseException('Invalid WKB: cannot unpack number of points'); + } + + $numPoints = $numPointsArr[1]; $offset += 4; $ring = []; for ($p = 0; $p < $numPoints; $p++) { - $x = unpack('d', substr($wkb, $offset, 8))[1]; - $y = unpack('d', substr($wkb, $offset + 8, 8))[1]; - $ring[] = [(float)$x, (float)$y]; + $xArr = unpack('d', substr($wkb, $offset, 8)); + $yArr = unpack('d', substr($wkb, $offset + 8, 8)); + + if ($xArr === false || $yArr === false || !isset($xArr[1], $yArr[1])) { + throw new DatabaseException('Invalid WKB: cannot unpack point coordinates'); + } + + $ring[] = [(float)$xArr[1], (float)$yArr[1]]; $offset += 16; } @@ -2846,6 +2882,5 @@ public function decodePolygon(string $wkb): array } return $rings; - } } diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 37ad7cce3..481704ce6 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -18,12 +18,12 @@ abstract class Base extends TestCase { - use CollectionTests; - use DocumentTests; - use AttributeTests; - use IndexTests; - use PermissionTests; - use RelationshipTests; +// use CollectionTests; +// use DocumentTests; +// use AttributeTests; +// use IndexTests; +// use PermissionTests; +// use RelationshipTests; use SpatialTests; use GeneralTests; From 7849c6c0a393f38cb1cf5644e991c8a4f7700ec1 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 17:56:52 +0300 Subject: [PATCH 32/55] formatting --- src/Database/Adapter/Postgres.php | 135 +++++++++++++++++++++++++----- 1 file changed, 114 insertions(+), 21 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 38807e442..c3524d36b 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2015,16 +2015,41 @@ public function decodePoint(string $wkb): array } $bin = hex2bin($wkb); + if ($bin === false) { + throw new \RuntimeException('Invalid hex WKB string'); + } $isLE = ord($bin[0]) === 1; - $type = unpack($isLE ? 'V' : 'N', substr($bin, 1, 4))[1]; + $bytes = substr($bin, 1, 4); + if (strlen($bytes) < 4) { + throw new \RuntimeException('WKB too short to read type'); + } + + $unpacked = unpack($isLE ? 'V' : 'N', $bytes); + if ($unpacked === false) { + throw new \RuntimeException('Failed to unpack type from WKB'); + } + + $type = $unpacked[1]; $offset = 5 + (($type & 0x20000000) ? 4 : 0); $fmt = $isLE ? 'e' : 'E'; // little vs big endian double - $x = unpack($fmt, substr($bin, $offset, 8))[1]; - $y = unpack($fmt, substr($bin, $offset + 8, 8))[1]; - return [(float)$x, (float)$y]; + $unpacked = unpack($fmt, substr($bin, $offset, 8)); + if ($unpacked === false) { + throw new \RuntimeException('Failed to unpack double from WKB'); + } + + $x = (float)$unpacked[1]; + + $unpackedY = unpack($fmt, substr($bin, $offset + 8, 8)); + if ($unpackedY === false) { + throw new \RuntimeException('Failed to unpack Y coordinate from WKB'); + } + + $y = (float)$unpackedY[1]; + + return [$x, $y]; } public function decodeLinestring(mixed $wkb): array @@ -2043,26 +2068,36 @@ public function decodeLinestring(mixed $wkb): array if (ctype_xdigit($wkb)) { $wkb = hex2bin($wkb); + if ($wkb === false) { + throw new \RuntimeException("Failed to convert hex WKB to binary."); + } } if (strlen($wkb) < 9) { - throw new DatabaseException("WKB too short to be a valid geometry"); + throw new \RuntimeException("WKB too short to be a valid geometry"); } $byteOrder = ord($wkb[0]); if ($byteOrder === 0) { - throw new DatabaseException("Big-endian WKB not supported"); + throw new \RuntimeException("Big-endian WKB not supported"); } elseif ($byteOrder !== 1) { - throw new DatabaseException("Invalid byte order in WKB"); + throw new \RuntimeException("Invalid byte order in WKB"); } // Type + SRID flag - $typeField = unpack('V', substr($wkb, 1, 4))[1]; + $typeFieldBytes = substr($wkb, 1, 4); + $typeField = unpack('V', $typeFieldBytes); + + if ($typeField === false) { + throw new \RuntimeException('Failed to unpack the type field from WKB.'); + } + + $typeField = $typeField[1]; $geomType = $typeField & 0xFF; $hasSRID = ($typeField & 0x20000000) !== 0; if ($geomType !== 2) { // 2 = LINESTRING - throw new DatabaseException("Not a LINESTRING geometry type, got {$geomType}"); + throw new \RuntimeException("Not a LINESTRING geometry type, got {$geomType}"); } $offset = 5; @@ -2070,16 +2105,39 @@ public function decodeLinestring(mixed $wkb): array $offset += 4; } - $numPoints = unpack('V', substr($wkb, $offset, 4))[1]; + $numPointsBytes = substr($wkb, $offset, 4); + $numPointsUnpacked = unpack('V', $numPointsBytes); + + if ($numPointsUnpacked === false || !isset($numPointsUnpacked[1])) { + throw new \RuntimeException("Failed to unpack number of points at offset {$offset}."); + } + + $numPoints = $numPointsUnpacked[1]; $offset += 4; $points = []; for ($i = 0; $i < $numPoints; $i++) { - $x = unpack('e', substr($wkb, $offset, 8))[1]; + $xBytes = substr($wkb, $offset, 8); + $xUnpacked = unpack('e', $xBytes); + + if ($xUnpacked === false) { + throw new \RuntimeException("Failed to unpack X coordinate at offset {$offset}."); + } + + $x = (float) $xUnpacked[1]; + $offset += 8; - $y = unpack('e', substr($wkb, $offset, 8))[1]; + $yBytes = substr($wkb, $offset, 8); + $yUnpacked = unpack('e', $yBytes); + + if ($yUnpacked === false || !isset($yUnpacked[1])) { + throw new \RuntimeException("Failed to unpack Y coordinate at offset {$offset}."); + } + + $y = (float) $yUnpacked[1]; + $offset += 8; - $points[] = [(float)$x, (float)$y]; + $points[] = [$x, $y]; } return $points; @@ -2115,12 +2173,17 @@ public function decodePolygon(string $wkb): array throw new \RuntimeException("WKB too short"); } - $byteOrder = ord($wkb[0]); - $isLE = $byteOrder === 1; // assume little-endian $uInt32 = 'V'; // little-endian 32-bit unsigned $uDouble = 'd'; // little-endian double - $typeInt = unpack($uInt32, substr($wkb, 1, 4))[1]; + $bytes = substr($wkb, 1, 4); + $unpacked = unpack($uInt32, $bytes); + + if ($unpacked === false || !isset($unpacked[1])) { + throw new \RuntimeException('Failed to unpack type field from WKB.'); + } + + $typeInt = (int) $unpacked[1]; $hasSrid = ($typeInt & 0x20000000) !== 0; $geomType = $typeInt & 0xFF; @@ -2134,18 +2197,48 @@ public function decodePolygon(string $wkb): array } // Number of rings - $numRings = unpack($uInt32, substr($wkb, $offset, 4))[1]; + $bytes = substr($wkb, $offset, 4); + $unpacked = unpack($uInt32, $bytes); + + if ($unpacked === false || !isset($unpacked[1])) { + throw new \RuntimeException('Failed to unpack number of rings from WKB.'); + } + + $numRings = (int) $unpacked[1]; $offset += 4; $rings = []; for ($r = 0; $r < $numRings; $r++) { - $numPoints = unpack($uInt32, substr($wkb, $offset, 4))[1]; + $bytes = substr($wkb, $offset, 4); + $unpacked = unpack($uInt32, $bytes); + + if ($unpacked === false || !isset($unpacked[1])) { + throw new \RuntimeException('Failed to unpack number of points from WKB.'); + } + + $numPoints = (int) $unpacked[1]; $offset += 4; $points = []; for ($i = 0; $i < $numPoints; $i++) { - $x = unpack($uDouble, substr($wkb, $offset, 8))[1]; - $y = unpack($uDouble, substr($wkb, $offset + 8, 8))[1]; - $points[] = [(float)$x, (float)$y]; + $bytes = substr($wkb, $offset, 8); + $unpacked = unpack($uDouble, $bytes); + + if ($unpacked === false) { + throw new \RuntimeException('Failed to unpack X coordinate from WKB.'); + } + + $x = (float) $unpacked[1]; + + $bytes = substr($wkb, $offset + 8, 8); + $unpacked = unpack($uDouble, $bytes); + + if ($unpacked === false || !isset($unpacked[1])) { + throw new \RuntimeException('Failed to unpack Y coordinate from WKB.'); + } + + $y = (float) $unpacked[1]; + + $points[] = [$x, $y]; $offset += 16; } $rings[] = $points; From 2aed9dc6ff1004ec80b79934a1c87b683f714ad9 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 18:05:54 +0300 Subject: [PATCH 33/55] fix getAttributeProjection --- src/Database/Adapter.php | 4 ++-- src/Database/Adapter/Pool.php | 8 +------ src/Database/Adapter/SQL.php | 43 +++++------------------------------ 3 files changed, 9 insertions(+), 46 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 5cd7c8a41..bac76fc73 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1153,9 +1153,9 @@ abstract public function getKeywords(): array; * * @param array $selections * @param string $prefix - * @return mixed + * @return string */ - abstract protected function getAttributeProjection(array $selections, string $prefix): mixed; + abstract protected function getAttributeProjection(array $selections, string $prefix): string; /** * Get all selected attributes from queries diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 3a4faa56a..c2cd363ba 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -470,13 +470,7 @@ public function getKeywords(): array return $this->delegate(__FUNCTION__, \func_get_args()); } - /** - * @param array $selections - * @param string $prefix - * @param array $spatialAttributes - * @return mixed - */ - protected function getAttributeProjection(array $selections, string $prefix, array $spatialAttributes = []): mixed + protected function getAttributeProjection(array $selections, string $prefix): string { return $this->delegate(__FUNCTION__, \func_get_args()); } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index e2a8f775d..0fc1440e6 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -350,7 +350,6 @@ public function deleteAttribute(string $collection, string $id, bool $array = fa */ public function getDocument(Document $collection, string $id, array $queries = [], bool $forUpdate = false): Document { - $spatialAttributes = $this->getSpatialAttributes($collection); $collection = $collection->getId(); $name = $this->filter($collection); @@ -359,9 +358,9 @@ public function getDocument(Document $collection, string $id, array $queries = [ $forUpdate = $forUpdate ? 'FOR UPDATE' : ''; $alias = Query::DEFAULT_ALIAS; - $spatialAttributes = []; + $sql = " - SELECT {$this->getAttributeProjection($selections, $alias, $spatialAttributes)} + SELECT {$this->getAttributeProjection($selections, $alias)} FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} WHERE {$this->quote($alias)}.{$this->quote('_uid')} = :_uid {$this->getTenantQuery($collection, $alias)} @@ -1877,37 +1876,13 @@ public function getTenantQuery( * * @param array $selections * @param string $prefix - * @param array $spatialAttributes - * @return mixed + * @return string * @throws Exception */ - protected function getAttributeProjection(array $selections, string $prefix, array $spatialAttributes = []): mixed + protected function getAttributeProjection(array $selections, string $prefix): string { if (empty($selections) || \in_array('*', $selections)) { - if (empty($spatialAttributes)) { - return "{$this->quote($prefix)}.*"; - } - - $projections = []; - $projections[] = "{$this->quote($prefix)}.*"; - - $internalColumns = ['_id', '_uid', '_createdAt', '_updatedAt', '_permissions']; - if ($this->sharedTables) { - $internalColumns[] = '_tenant'; - } - foreach ($internalColumns as $col) { - $projections[] = "{$this->quote($prefix)}.{$this->quote($col)}"; - } - - foreach ($spatialAttributes as $spatialAttr) { - $filteredAttr = $this->filter($spatialAttr); - $quotedAttr = $this->quote($filteredAttr); - $axisOrder = $this->getSupportForSpatialAxisOrder() ? ', ' . $this->getSpatialAxisOrderSpec() : ''; - $projections[] = "ST_AsText({$this->quote($prefix)}.{$quotedAttr} {$axisOrder} ) AS {$quotedAttr}"; - } - - - return implode(', ', $projections); + return "{$this->quote($prefix)}.*"; } // Handle specific selections with spatial conversion where needed @@ -1929,13 +1904,7 @@ protected function getAttributeProjection(array $selections, string $prefix, arr foreach ($selections as $selection) { $filteredSelection = $this->filter($selection); $quotedSelection = $this->quote($filteredSelection); - - if (in_array($selection, $spatialAttributes)) { - $axisOrder = $this->getSupportForSpatialAxisOrder() ? ', ' . $this->getSpatialAxisOrderSpec() : ''; - $projections[] = "ST_AsText({$this->quote($prefix)}.{$quotedSelection} {$axisOrder}) AS {$quotedSelection}"; - } else { - $projections[] = "{$this->quote($prefix)}.{$quotedSelection}"; - } + $projections[] = "{$this->quote($prefix)}.{$quotedSelection}"; } return \implode(',', $projections); From 4ed19c6ae9bd2cd2f71349ec8eee3f336d183cf7 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 18:09:09 +0300 Subject: [PATCH 34/55] Runn tests --- tests/e2e/Adapter/Base.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 481704ce6..37ad7cce3 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -18,12 +18,12 @@ abstract class Base extends TestCase { -// use CollectionTests; -// use DocumentTests; -// use AttributeTests; -// use IndexTests; -// use PermissionTests; -// use RelationshipTests; + use CollectionTests; + use DocumentTests; + use AttributeTests; + use IndexTests; + use PermissionTests; + use RelationshipTests; use SpatialTests; use GeneralTests; From ee7fbc1db35bbb285bd87623a09b741cca7bcb95 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 18:12:35 +0300 Subject: [PATCH 35/55] decode polygon --- src/Database/Adapter/SQL.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 0fc1440e6..53e643e64 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2837,13 +2837,20 @@ public function decodePolygon(string $wkb): array for ($p = 0; $p < $numPoints; $p++) { $xArr = unpack('d', substr($wkb, $offset, 8)); - $yArr = unpack('d', substr($wkb, $offset + 8, 8)); + if ($xArr === false) { + throw new \RuntimeException('Failed to unpack X coordinate from WKB.'); + } + + $x = (float) $xArr[1]; - if ($xArr === false || $yArr === false || !isset($xArr[1], $yArr[1])) { - throw new DatabaseException('Invalid WKB: cannot unpack point coordinates'); + $yArr = unpack('d', substr($wkb, $offset + 8, 8)); + if ($yArr === false) { + throw new \RuntimeException('Failed to unpack Y coordinate from WKB.'); } - $ring[] = [(float)$xArr[1], (float)$yArr[1]]; + $y = (float) $yArr[1]; + + $ring[] = [$x, $y]; $offset += 16; } From 2648ef909bfeb99718a2f8111a6ac84693ba9263 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 18:17:29 +0300 Subject: [PATCH 36/55] remove $spatialAttributes --- src/Database/Adapter/SQL.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 53e643e64..3fceac621 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2348,7 +2348,6 @@ protected function getAttributeType(string $attributeName, array $attributes): ? */ public function find(Document $collection, array $queries = [], ?int $limit = 25, ?int $offset = null, array $orderAttributes = [], array $orderTypes = [], array $cursor = [], string $cursorDirection = Database::CURSOR_AFTER, string $forPermission = Database::PERMISSION_READ): array { - $spatialAttributes = $this->getSpatialAttributes($collection); $attributes = $collection->getAttribute('attributes', []); $collection = $collection->getId(); @@ -2456,7 +2455,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $sql = " - SELECT {$this->getAttributeProjection($selections, $alias, $spatialAttributes)} + SELECT {$this->getAttributeProjection($selections, $alias)} FROM {$this->getSQLTable($name)} AS {$this->quote($alias)} {$sqlWhere} {$sqlOrder} From a0b963a59d87d11b8b1d20f89109c279204c28c0 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 18:28:33 +0300 Subject: [PATCH 37/55] unpack --- phpunit.xml | 2 +- src/Database/Adapter/Postgres.php | 90 ++++++++++++------------------- 2 files changed, 35 insertions(+), 57 deletions(-) diff --git a/phpunit.xml b/phpunit.xml index 7469c5341..8ba994793 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,7 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="true"> + stopOnFailure="false"> ./tests/unit diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index c3524d36b..0a9c76bd0 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2020,34 +2020,29 @@ public function decodePoint(string $wkb): array } $isLE = ord($bin[0]) === 1; - $bytes = substr($bin, 1, 4); - if (strlen($bytes) < 4) { - throw new \RuntimeException('WKB too short to read type'); - } - - $unpacked = unpack($isLE ? 'V' : 'N', $bytes); - if ($unpacked === false) { + $type = unpack($isLE ? 'V' : 'N', substr($bin, 1, 4)); + if ($type === false) { throw new \RuntimeException('Failed to unpack type from WKB'); } - $type = $unpacked[1]; + $type = $type[1]; $offset = 5 + (($type & 0x20000000) ? 4 : 0); $fmt = $isLE ? 'e' : 'E'; // little vs big endian double - $unpacked = unpack($fmt, substr($bin, $offset, 8)); - if ($unpacked === false) { + $x = unpack($fmt, substr($bin, $offset, 8)); + if ($x === false) { throw new \RuntimeException('Failed to unpack double from WKB'); } - $x = (float)$unpacked[1]; + $x = (float)$x[1]; - $unpackedY = unpack($fmt, substr($bin, $offset + 8, 8)); - if ($unpackedY === false) { + $y = unpack($fmt, substr($bin, $offset + 8, 8)); + if ($y === false) { throw new \RuntimeException('Failed to unpack Y coordinate from WKB'); } - $y = (float)$unpackedY[1]; + $y = (float)$y[1]; return [$x, $y]; } @@ -2085,9 +2080,7 @@ public function decodeLinestring(mixed $wkb): array } // Type + SRID flag - $typeFieldBytes = substr($wkb, 1, 4); - $typeField = unpack('V', $typeFieldBytes); - + $typeField = unpack('V', substr($wkb, 1, 4)); if ($typeField === false) { throw new \RuntimeException('Failed to unpack the type field from WKB.'); } @@ -2105,36 +2098,31 @@ public function decodeLinestring(mixed $wkb): array $offset += 4; } - $numPointsBytes = substr($wkb, $offset, 4); - $numPointsUnpacked = unpack('V', $numPointsBytes); - - if ($numPointsUnpacked === false || !isset($numPointsUnpacked[1])) { + $numPoints = unpack('V', substr($wkb, $offset, 4)); + if ($numPoints === false) { throw new \RuntimeException("Failed to unpack number of points at offset {$offset}."); } - $numPoints = $numPointsUnpacked[1]; + $numPoints = $numPoints[1]; $offset += 4; $points = []; for ($i = 0; $i < $numPoints; $i++) { - $xBytes = substr($wkb, $offset, 8); - $xUnpacked = unpack('e', $xBytes); - - if ($xUnpacked === false) { + $x = unpack('e', substr($wkb, $offset, 8)); + if ($x === false) { throw new \RuntimeException("Failed to unpack X coordinate at offset {$offset}."); } - $x = (float) $xUnpacked[1]; + $x = (float) $x[1]; $offset += 8; - $yBytes = substr($wkb, $offset, 8); - $yUnpacked = unpack('e', $yBytes); - if ($yUnpacked === false || !isset($yUnpacked[1])) { + $y = unpack('e', substr($wkb, $offset, 8)); + if ($y === false) { throw new \RuntimeException("Failed to unpack Y coordinate at offset {$offset}."); } - $y = (float) $yUnpacked[1]; + $y = (float) $y[1]; $offset += 8; $points[] = [$x, $y]; @@ -2176,14 +2164,12 @@ public function decodePolygon(string $wkb): array $uInt32 = 'V'; // little-endian 32-bit unsigned $uDouble = 'd'; // little-endian double - $bytes = substr($wkb, 1, 4); - $unpacked = unpack($uInt32, $bytes); - - if ($unpacked === false || !isset($unpacked[1])) { + $typeInt = unpack($uInt32, substr($wkb, 1, 4)); + if ($typeInt === false) { throw new \RuntimeException('Failed to unpack type field from WKB.'); } - $typeInt = (int) $unpacked[1]; + $typeInt = (int) $typeInt[1]; $hasSrid = ($typeInt & 0x20000000) !== 0; $geomType = $typeInt & 0xFF; @@ -2197,46 +2183,38 @@ public function decodePolygon(string $wkb): array } // Number of rings - $bytes = substr($wkb, $offset, 4); - $unpacked = unpack($uInt32, $bytes); - - if ($unpacked === false || !isset($unpacked[1])) { + $numRings = unpack($uInt32, substr($wkb, $offset, 4)); + if ($numRings === false) { throw new \RuntimeException('Failed to unpack number of rings from WKB.'); } - $numRings = (int) $unpacked[1]; + $numRings = (int) $numRings[1]; $offset += 4; $rings = []; for ($r = 0; $r < $numRings; $r++) { - $bytes = substr($wkb, $offset, 4); - $unpacked = unpack($uInt32, $bytes); - - if ($unpacked === false || !isset($unpacked[1])) { + $numPoints = unpack($uInt32, substr($wkb, $offset, 4)); + if ($numPoints === false) { throw new \RuntimeException('Failed to unpack number of points from WKB.'); } - $numPoints = (int) $unpacked[1]; + $numPoints = (int) $numPoints[1]; $offset += 4; $points = []; for ($i = 0; $i < $numPoints; $i++) { - $bytes = substr($wkb, $offset, 8); - $unpacked = unpack($uDouble, $bytes); - - if ($unpacked === false) { + $x = unpack($uDouble, substr($wkb, $offset, 8)); + if ($x === false) { throw new \RuntimeException('Failed to unpack X coordinate from WKB.'); } - $x = (float) $unpacked[1]; - - $bytes = substr($wkb, $offset + 8, 8); - $unpacked = unpack($uDouble, $bytes); + $x = (float) $x[1]; - if ($unpacked === false || !isset($unpacked[1])) { + $y = unpack($uDouble, substr($wkb, $offset + 8, 8)); + if ($y === false) { throw new \RuntimeException('Failed to unpack Y coordinate from WKB.'); } - $y = (float) $unpacked[1]; + $y = (float) $y[1]; $points[] = [$x, $y]; $offset += 16; From cfe9b848b047575db6f0712b985e69e0bcfdf66d Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 15 Sep 2025 18:29:27 +0300 Subject: [PATCH 38/55] stopOnFailure --- phpunit.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/phpunit.xml b/phpunit.xml index 8ba994793..2a0531cfd 100755 --- a/phpunit.xml +++ b/phpunit.xml @@ -7,7 +7,8 @@ convertNoticesToExceptions="true" convertWarningsToExceptions="true" processIsolation="false" - stopOnFailure="false"> + stopOnFailure="false" +> ./tests/unit From 89a5ec6e4191da899b292b2e7292c0da527e99aa Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 16 Sep 2025 09:44:48 +0300 Subject: [PATCH 39/55] Revert autoincrement set to default behaviour --- src/Database/Adapter/Postgres.php | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 9d7e4dcaf..90be239da 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1043,17 +1043,6 @@ public function createDocument(Document $collection, Document $document): Docume try { $this->execute($stmt); - - if (!empty($document->getSequence())) { - $this->getPDO()->exec(" - SELECT setval( - pg_get_serial_sequence('{$this->getSQLTable($name)}', '_id'), - (SELECT MAX(_id) FROM {$this->getSQLTable($name)}), - true - ); - "); - } - $lastInsertedId = $this->getPDO()->lastInsertId(); // Sequence can be manually set as well $document['$sequence'] ??= $lastInsertedId; From 6c5b7df99d66c64c18baff335acf8ea0669ff7e7 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 16 Sep 2025 09:51:32 +0300 Subject: [PATCH 40/55] test find --- tests/e2e/Adapter/Scopes/DocumentTests.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 09a47ffff..200dfa871 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -41,16 +41,10 @@ public function testBigintSequence(): void $this->assertEquals((string)$sequence, $document->getSequence()); $document = $database->getDocument(__FUNCTION__, $document->getId()); - $this->assertEquals((string)$sequence, $document->getSequence()); - $document = $database->createDocument(__FUNCTION__, new Document([ - '$permissions' => [ - Permission::read(Role::any()), - ], - ])); - - $this->assertEquals((string)($sequence + 1), $document->getSequence()); + $document = $database->findOne(__FUNCTION__, [Query::equal('$sequence', [(string)$sequence])]); + $this->assertEquals((string)$sequence, $document->getSequence()); } public function testCreateDocument(): Document From a1a52f633cda9e061a669cd40289a1be5cb9e8b0 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 16 Sep 2025 12:19:30 +0300 Subject: [PATCH 41/55] fix decode point --- src/Database/Adapter/SQL.php | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 3fceac621..f750627f9 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2689,24 +2689,35 @@ public function decodePoint(string $wkb): array return [(float)$coords[0], (float)$coords[1]]; } - // MySQL SRID-aware WKB layout: - // 1 byte = endian (1 = little endian) - // 4 bytes = type + SRID flag - // 4 bytes = SRID - // 16 bytes = X,Y coordinates (double each, little endian) + /** + * [0..3] SRID (4 bytes, little-endian) + * [4] Byte order (1 = little-endian, 0 = big-endian) + * [5..8] Geometry type (with SRID flag bit) + * [9..] Geometry payload (coordinates, etc.) + */ - $byteOrder = ord($wkb[0]); + if (strlen($wkb) < 25) { + throw new DatabaseException('Invalid WKB: too short for POINT'); + } + + // 4 bytes SRID first → skip to byteOrder at offset 4 + $byteOrder = ord($wkb[4]); $littleEndian = ($byteOrder === 1); - // Skip 1 + 4 + 4 = 9 bytes to get coordinates - $coordsBin = substr($wkb, 9, 16); + if (!$littleEndian) { + throw new DatabaseException('Only little-endian WKB supported'); + } - // Unpack doubles - $format = $littleEndian ? 'd2' : 'd2'; // little-endian doubles - $coords = unpack($format, $coordsBin); + // After SRID (4) + byteOrder (1) + type (4) = 9 bytes + $coordsBin = substr($wkb, 9, 16); + if (strlen($coordsBin) !== 16) { + throw new DatabaseException('Invalid WKB: missing coordinate bytes'); + } + // Unpack two doubles + $coords = unpack('d2', $coordsBin); if ($coords === false || !isset($coords[1], $coords[2])) { - throw new DatabaseException('Invalid WKB for POINT: cannot unpack coordinates'); + throw new DatabaseException('Invalid WKB: failed to unpack coordinates'); } return [(float)$coords[1], (float)$coords[2]]; From c349c08f8e59c6da7df6ca49c4175dc23cf3b4e8 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 16 Sep 2025 12:34:55 +0300 Subject: [PATCH 42/55] DatabaseException --- src/Database/Adapter/Postgres.php | 42 +++++++++++++++---------------- src/Database/Adapter/SQL.php | 4 +-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 3caee828f..5e6f99194 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2014,13 +2014,13 @@ public function decodePoint(string $wkb): array $bin = hex2bin($wkb); if ($bin === false) { - throw new \RuntimeException('Invalid hex WKB string'); + throw new DatabaseException('Invalid hex WKB string'); } $isLE = ord($bin[0]) === 1; $type = unpack($isLE ? 'V' : 'N', substr($bin, 1, 4)); if ($type === false) { - throw new \RuntimeException('Failed to unpack type from WKB'); + throw new DatabaseException('Failed to unpack type from WKB'); } $type = $type[1]; @@ -2030,14 +2030,14 @@ public function decodePoint(string $wkb): array $x = unpack($fmt, substr($bin, $offset, 8)); if ($x === false) { - throw new \RuntimeException('Failed to unpack double from WKB'); + throw new DatabaseException('Failed to unpack double from WKB'); } $x = (float)$x[1]; $y = unpack($fmt, substr($bin, $offset + 8, 8)); if ($y === false) { - throw new \RuntimeException('Failed to unpack Y coordinate from WKB'); + throw new DatabaseException('Failed to unpack Y coordinate from WKB'); } $y = (float)$y[1]; @@ -2062,25 +2062,25 @@ public function decodeLinestring(mixed $wkb): array if (ctype_xdigit($wkb)) { $wkb = hex2bin($wkb); if ($wkb === false) { - throw new \RuntimeException("Failed to convert hex WKB to binary."); + throw new DatabaseException("Failed to convert hex WKB to binary."); } } if (strlen($wkb) < 9) { - throw new \RuntimeException("WKB too short to be a valid geometry"); + throw new DatabaseException("WKB too short to be a valid geometry"); } $byteOrder = ord($wkb[0]); if ($byteOrder === 0) { - throw new \RuntimeException("Big-endian WKB not supported"); + throw new DatabaseException("Big-endian WKB not supported"); } elseif ($byteOrder !== 1) { - throw new \RuntimeException("Invalid byte order in WKB"); + throw new DatabaseException("Invalid byte order in WKB"); } // Type + SRID flag $typeField = unpack('V', substr($wkb, 1, 4)); if ($typeField === false) { - throw new \RuntimeException('Failed to unpack the type field from WKB.'); + throw new DatabaseException('Failed to unpack the type field from WKB.'); } $typeField = $typeField[1]; @@ -2088,7 +2088,7 @@ public function decodeLinestring(mixed $wkb): array $hasSRID = ($typeField & 0x20000000) !== 0; if ($geomType !== 2) { // 2 = LINESTRING - throw new \RuntimeException("Not a LINESTRING geometry type, got {$geomType}"); + throw new DatabaseException("Not a LINESTRING geometry type, got {$geomType}"); } $offset = 5; @@ -2098,7 +2098,7 @@ public function decodeLinestring(mixed $wkb): array $numPoints = unpack('V', substr($wkb, $offset, 4)); if ($numPoints === false) { - throw new \RuntimeException("Failed to unpack number of points at offset {$offset}."); + throw new DatabaseException("Failed to unpack number of points at offset {$offset}."); } $numPoints = $numPoints[1]; @@ -2108,7 +2108,7 @@ public function decodeLinestring(mixed $wkb): array for ($i = 0; $i < $numPoints; $i++) { $x = unpack('e', substr($wkb, $offset, 8)); if ($x === false) { - throw new \RuntimeException("Failed to unpack X coordinate at offset {$offset}."); + throw new DatabaseException("Failed to unpack X coordinate at offset {$offset}."); } $x = (float) $x[1]; @@ -2117,7 +2117,7 @@ public function decodeLinestring(mixed $wkb): array $y = unpack('e', substr($wkb, $offset, 8)); if ($y === false) { - throw new \RuntimeException("Failed to unpack Y coordinate at offset {$offset}."); + throw new DatabaseException("Failed to unpack Y coordinate at offset {$offset}."); } $y = (float) $y[1]; @@ -2151,12 +2151,12 @@ public function decodePolygon(string $wkb): array if (preg_match('/^[0-9a-fA-F]+$/', $wkb)) { $wkb = hex2bin($wkb); if ($wkb === false) { - throw new \RuntimeException("Invalid hex WKB"); + throw new DatabaseException("Invalid hex WKB"); } } if (strlen($wkb) < 9) { - throw new \RuntimeException("WKB too short"); + throw new DatabaseException("WKB too short"); } $uInt32 = 'V'; // little-endian 32-bit unsigned @@ -2164,7 +2164,7 @@ public function decodePolygon(string $wkb): array $typeInt = unpack($uInt32, substr($wkb, 1, 4)); if ($typeInt === false) { - throw new \RuntimeException('Failed to unpack type field from WKB.'); + throw new DatabaseException('Failed to unpack type field from WKB.'); } $typeInt = (int) $typeInt[1]; @@ -2172,7 +2172,7 @@ public function decodePolygon(string $wkb): array $geomType = $typeInt & 0xFF; if ($geomType !== 3) { // 3 = POLYGON - throw new \RuntimeException("Not a POLYGON geometry type, got {$geomType}"); + throw new DatabaseException("Not a POLYGON geometry type, got {$geomType}"); } $offset = 5; @@ -2183,7 +2183,7 @@ public function decodePolygon(string $wkb): array // Number of rings $numRings = unpack($uInt32, substr($wkb, $offset, 4)); if ($numRings === false) { - throw new \RuntimeException('Failed to unpack number of rings from WKB.'); + throw new DatabaseException('Failed to unpack number of rings from WKB.'); } $numRings = (int) $numRings[1]; @@ -2193,7 +2193,7 @@ public function decodePolygon(string $wkb): array for ($r = 0; $r < $numRings; $r++) { $numPoints = unpack($uInt32, substr($wkb, $offset, 4)); if ($numPoints === false) { - throw new \RuntimeException('Failed to unpack number of points from WKB.'); + throw new DatabaseException('Failed to unpack number of points from WKB.'); } $numPoints = (int) $numPoints[1]; @@ -2202,14 +2202,14 @@ public function decodePolygon(string $wkb): array for ($i = 0; $i < $numPoints; $i++) { $x = unpack($uDouble, substr($wkb, $offset, 8)); if ($x === false) { - throw new \RuntimeException('Failed to unpack X coordinate from WKB.'); + throw new DatabaseException('Failed to unpack X coordinate from WKB.'); } $x = (float) $x[1]; $y = unpack($uDouble, substr($wkb, $offset + 8, 8)); if ($y === false) { - throw new \RuntimeException('Failed to unpack Y coordinate from WKB.'); + throw new DatabaseException('Failed to unpack Y coordinate from WKB.'); } $y = (float) $y[1]; diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index f750627f9..600f047f1 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -2848,14 +2848,14 @@ public function decodePolygon(string $wkb): array for ($p = 0; $p < $numPoints; $p++) { $xArr = unpack('d', substr($wkb, $offset, 8)); if ($xArr === false) { - throw new \RuntimeException('Failed to unpack X coordinate from WKB.'); + throw new DatabaseException('Failed to unpack X coordinate from WKB.'); } $x = (float) $xArr[1]; $yArr = unpack('d', substr($wkb, $offset + 8, 8)); if ($yArr === false) { - throw new \RuntimeException('Failed to unpack Y coordinate from WKB.'); + throw new DatabaseException('Failed to unpack Y coordinate from WKB.'); } $y = (float) $yArr[1]; From 96e8a5e648ac5d4f631f73c82ad6ffc384190781 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 16 Sep 2025 12:39:26 +0300 Subject: [PATCH 43/55] Postgres update point --- src/Database/Adapter/Postgres.php | 42 +++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 5e6f99194..abff332d3 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2017,30 +2017,46 @@ public function decodePoint(string $wkb): array throw new DatabaseException('Invalid hex WKB string'); } + if (strlen($bin) < 13) { // 1 byte endian + 4 bytes type + 8 bytes for X + throw new DatabaseException('WKB too short'); + } + $isLE = ord($bin[0]) === 1; - $type = unpack($isLE ? 'V' : 'N', substr($bin, 1, 4)); - if ($type === false) { + +// Type (4 bytes) + $typeBytes = substr($bin, 1, 4); + if (strlen($typeBytes) !== 4) { + throw new DatabaseException('Failed to extract type bytes from WKB'); + } + + $typeArr = unpack($isLE ? 'V' : 'N', $typeBytes); + if ($typeArr === false || !isset($typeArr[1])) { throw new DatabaseException('Failed to unpack type from WKB'); } + $type = $typeArr[1]; - $type = $type[1]; + // Offset to coordinates (skip SRID if present) $offset = 5 + (($type & 0x20000000) ? 4 : 0); - $fmt = $isLE ? 'e' : 'E'; // little vs big endian double - - $x = unpack($fmt, substr($bin, $offset, 8)); - if ($x === false) { - throw new DatabaseException('Failed to unpack double from WKB'); + if (strlen($bin) < $offset + 16) { // 16 bytes for X,Y + throw new DatabaseException('WKB too short for coordinates'); } - $x = (float)$x[1]; + $fmt = $isLE ? 'e' : 'E'; // little vs big endian double - $y = unpack($fmt, substr($bin, $offset + 8, 8)); - if ($y === false) { - throw new DatabaseException('Failed to unpack Y coordinate from WKB'); + // X coordinate + $xArr = unpack($fmt, substr($bin, $offset, 8)); + if ($xArr === false || !isset($xArr[1])) { + throw new DatabaseException('Failed to unpack X coordinate'); } + $x = (float)$xArr[1]; - $y = (float)$y[1]; + // Y coordinate + $yArr = unpack($fmt, substr($bin, $offset + 8, 8)); + if ($yArr === false || !isset($yArr[1])) { + throw new DatabaseException('Failed to unpack Y coordinate'); + } + $y = (float)$yArr[1]; return [$x, $y]; } From 09be1c0902fe00ca94ad0707f82c2cf606f67018 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 16 Sep 2025 12:41:28 +0300 Subject: [PATCH 44/55] formatting --- src/Database/Adapter/Postgres.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index abff332d3..25f9ffc34 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2023,7 +2023,7 @@ public function decodePoint(string $wkb): array $isLE = ord($bin[0]) === 1; -// Type (4 bytes) + // Type (4 bytes) $typeBytes = substr($bin, 1, 4); if (strlen($typeBytes) !== 4) { throw new DatabaseException('Failed to extract type bytes from WKB'); From fcc550b33052fbe2d926031573774a02857821cd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 17 Sep 2025 00:19:19 +1200 Subject: [PATCH 45/55] Enable reconnection + retry --- src/Database/PDO.php | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Database/PDO.php b/src/Database/PDO.php index 069ef88f8..ebcc8bbfa 100644 --- a/src/Database/PDO.php +++ b/src/Database/PDO.php @@ -3,6 +3,7 @@ namespace Utopia\Database; use InvalidArgumentException; +use Utopia\CLI\Console; /** * A PDO wrapper that forwards method calls to the internal PDO instance. @@ -41,7 +42,24 @@ public function __construct( */ public function __call(string $method, array $args): mixed { - return $this->pdo->{$method}(...$args); + try { + return $this->pdo->{$method}(...$args); + } catch (\Throwable $e) { + if (Connection::hasError($e)) { + Console::warning('[Database] Lost connection detected. Reconnecting...'); + + // Attempt to reconnect + $this->reconnect(); + + // If we're not in a transaction, also retry the query + // In a transaction we can't retry as it would lead to data integrity issues + if (!$this->pdo->inTransaction()) { + return $this->pdo->{$method}(...$args); + } + } + + throw $e; + } } /** From 6496e63aa2bdf30ce9b6f91f84f347b43fabc2ee Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 17 Sep 2025 00:21:09 +1200 Subject: [PATCH 46/55] Enable reconnect test --- tests/unit/PDOTest.php | 76 +++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/unit/PDOTest.php b/tests/unit/PDOTest.php index a17de1eb3..45e9a12a2 100644 --- a/tests/unit/PDOTest.php +++ b/tests/unit/PDOTest.php @@ -41,44 +41,44 @@ public function testMethodCallIsForwardedToPDO(): void $this->assertSame($pdoStatementMock, $result); } - // public function testLostConnectionRetriesCall(): void - // { - // $dsn = 'sqlite::memory:'; - // $pdoWrapper = $this->getMockBuilder(PDO::class) - // ->setConstructorArgs([$dsn, null, null, []]) - // ->onlyMethods(['reconnect']) - // ->getMock(); - // - // $pdoMock = $this->getMockBuilder(\PDO::class) - // ->disableOriginalConstructor() - // ->getMock(); - // $pdoStatementMock = $this->getMockBuilder(\PDOStatement::class) - // ->disableOriginalConstructor() - // ->getMock(); - // - // $pdoMock->expects($this->exactly(2)) - // ->method('query') - // ->with('SELECT 1') - // ->will($this->onConsecutiveCalls( - // $this->throwException(new \Exception("Lost connection")), - // $pdoStatementMock - // )); - // - // $reflection = new ReflectionClass($pdoWrapper); - // $pdoProperty = $reflection->getProperty('pdo'); - // $pdoProperty->setAccessible(true); - // $pdoProperty->setValue($pdoWrapper, $pdoMock); - // - // $pdoWrapper->expects($this->once()) - // ->method('reconnect') - // ->willReturnCallback(function () use ($pdoWrapper, $pdoMock, $pdoProperty) { - // $pdoProperty->setValue($pdoWrapper, $pdoMock); - // }); - // - // $result = $pdoWrapper->query('SELECT 1'); - // - // $this->assertSame($pdoStatementMock, $result); - // } + public function testLostConnectionRetriesCall(): void + { + $dsn = 'sqlite::memory:'; + $pdoWrapper = $this->getMockBuilder(PDO::class) + ->setConstructorArgs([$dsn, null, null, []]) + ->onlyMethods(['reconnect']) + ->getMock(); + + $pdoMock = $this->getMockBuilder(\PDO::class) + ->disableOriginalConstructor() + ->getMock(); + $pdoStatementMock = $this->getMockBuilder(\PDOStatement::class) + ->disableOriginalConstructor() + ->getMock(); + + $pdoMock->expects($this->exactly(2)) + ->method('query') + ->with('SELECT 1') + ->will($this->onConsecutiveCalls( + $this->throwException(new \Exception("Lost connection")), + $pdoStatementMock + )); + + $reflection = new ReflectionClass($pdoWrapper); + $pdoProperty = $reflection->getProperty('pdo'); + $pdoProperty->setAccessible(true); + $pdoProperty->setValue($pdoWrapper, $pdoMock); + + $pdoWrapper->expects($this->once()) + ->method('reconnect') + ->willReturnCallback(function () use ($pdoWrapper, $pdoMock, $pdoProperty) { + $pdoProperty->setValue($pdoWrapper, $pdoMock); + }); + + $result = $pdoWrapper->query('SELECT 1'); + + $this->assertSame($pdoStatementMock, $result); + } public function testNonLostConnectionExceptionIsRethrown(): void { From 8f2840d55dcf71bdbb7ddd1ea1cfad64008bf0a5 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 17 Sep 2025 00:56:31 +1200 Subject: [PATCH 47/55] Check in transaction before reconnect --- src/Database/PDO.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Database/PDO.php b/src/Database/PDO.php index ebcc8bbfa..c00f7b073 100644 --- a/src/Database/PDO.php +++ b/src/Database/PDO.php @@ -48,12 +48,14 @@ public function __call(string $method, array $args): mixed if (Connection::hasError($e)) { Console::warning('[Database] Lost connection detected. Reconnecting...'); + $inTransaction = $this->pdo->inTransaction(); + // Attempt to reconnect $this->reconnect(); - // If we're not in a transaction, also retry the query - // In a transaction we can't retry as it would lead to data integrity issues - if (!$this->pdo->inTransaction()) { + // If we weren't in a transaction, also retry the query + // In a transaction we can't retry as the state is attached to the previous connection + if (!$inTransaction) { return $this->pdo->{$method}(...$args); } } From 9a046f232524582dd84b0e3be9d6bac34444366c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Wed, 17 Sep 2025 01:25:47 +1200 Subject: [PATCH 48/55] Log connection error message --- src/Database/PDO.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Database/PDO.php b/src/Database/PDO.php index c00f7b073..245b0dfad 100644 --- a/src/Database/PDO.php +++ b/src/Database/PDO.php @@ -46,6 +46,7 @@ public function __call(string $method, array $args): mixed return $this->pdo->{$method}(...$args); } catch (\Throwable $e) { if (Connection::hasError($e)) { + Console::warning('[Database] ' . $e->getMessage()); Console::warning('[Database] Lost connection detected. Reconnecting...'); $inTransaction = $this->pdo->inTransaction(); From f4f7453959f1925386ce211b6df9a49f83faa22b Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 16 Sep 2025 16:32:14 +0300 Subject: [PATCH 49/55] Refactor Key and UID validators to use a constant for maximum length, improving maintainability and consistency in validation messages. --- src/Database/Adapter/Mongo.php | 58 ++++++++++++++++++++++++++++------ src/Database/Validator/Key.php | 11 +++++-- src/Database/Validator/UID.php | 2 +- 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 82a5c7432..708e36c34 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -31,12 +31,15 @@ class Mongo extends Adapter '$gt', '$gte', '$in', + '$nin', '$text', '$search', '$or', '$and', '$match', '$regex', + '$not', + '$nor', ]; protected Client $client; @@ -1301,7 +1304,7 @@ public function getSequences(string $collection, array $documents): array // Continue fetching with getMore while ($cursorId && $cursorId !== 0) { - $moreResponse = $this->client->getMore($cursorId, $name, self::DEFAULT_BATCH_SIZE); + $moreResponse = $this->client->getMore((int)$cursorId, $name, self::DEFAULT_BATCH_SIZE); $moreResults = $moreResponse->cursor->nextBatch ?? []; if (empty($moreResults)) { @@ -1313,7 +1316,7 @@ public function getSequences(string $collection, array $documents): array } // Update cursor ID for next iteration - $cursorId = $moreResponse->cursor->id ?? null; + $cursorId = (int)($moreResponse->cursor->id ?? 0); } } catch (MongoException $e) { throw $this->processException($e); @@ -1612,7 +1615,6 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 $response = $this->client->find($name, $filters, $options); $results = $response->cursor->firstBatch ?? []; - // Process first batch foreach ($results as $result) { $record = $this->replaceChars('_', '$', (array)$result); @@ -1628,8 +1630,8 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 if (!\is_null($limit) && count($found) >= $limit) { break; } - - $moreResponse = $this->client->getMore($cursorId, $name, self::DEFAULT_BATCH_SIZE); + + $moreResponse = $this->client->getMore((int)$cursorId, $name, self::DEFAULT_BATCH_SIZE); $moreResults = $moreResponse->cursor->nextBatch ?? []; if (empty($moreResults)) { @@ -1646,7 +1648,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 } } - $cursorId = $moreResponse->cursor->id ?? 0; + $cursorId = (int)($moreResponse->cursor->id ?? 0); } } catch (MongoException $e) { @@ -2017,11 +2019,36 @@ protected function buildFilter(Query $query): array } else { $filter[$attribute]['$in'] = $query->getValues(); } + } elseif ($operator === 'notContains') { + if (!$query->onArray()) { + $filter[$attribute] = ['$not' => new Regex(".*{$this->escapeWildcards($value)}.*", 'i')]; + } else { + $filter[$attribute]['$nin'] = $query->getValues(); + } } elseif ($operator == '$search') { - $filter['$text'][$operator] = $value; + if ($query->getMethod() === Query::TYPE_NOT_SEARCH) { + // MongoDB doesn't support negating $text expressions directly + // Use regex as fallback for NOT search while keeping fulltext for positive search + if (empty($value)) { + // If value is not passed, don't add any filter - this will match all documents + } else { + // Escape special regex characters and create a pattern that matches the search term as substring + $escapedValue = preg_quote($value, '/'); + $filter[$attribute] = ['$not' => new Regex(".*{$escapedValue}.*", 'i')]; + } + } else { + $filter['$text'][$operator] = $value; + } } elseif ($operator === Query::TYPE_BETWEEN) { $filter[$attribute]['$lte'] = $value[1]; $filter[$attribute]['$gte'] = $value[0]; + } elseif ($operator === Query::TYPE_NOT_BETWEEN) { + $filter['$or'] = [ + [$attribute => ['$lt' => $value[0]]], + [$attribute => ['$gt' => $value[1]]] + ]; + } elseif ($operator === '$regex' && in_array($query->getMethod(), [Query::TYPE_NOT_STARTS_WITH, Query::TYPE_NOT_ENDS_WITH])) { + $filter[$attribute] = ['$not' => new Regex($value, 'i')]; } else { $filter[$attribute][$operator] = $value; } @@ -2049,13 +2076,18 @@ protected function getQueryOperator(string $operator): string Query::TYPE_GREATER => '$gt', Query::TYPE_GREATER_EQUAL => '$gte', Query::TYPE_CONTAINS => '$in', + Query::TYPE_NOT_CONTAINS => 'notContains', Query::TYPE_SEARCH => '$search', + Query::TYPE_NOT_SEARCH => '$search', Query::TYPE_BETWEEN => 'between', + Query::TYPE_NOT_BETWEEN => 'notBetween', Query::TYPE_STARTS_WITH, - Query::TYPE_ENDS_WITH => '$regex', + Query::TYPE_NOT_STARTS_WITH, + Query::TYPE_ENDS_WITH, + Query::TYPE_NOT_ENDS_WITH => '$regex', Query::TYPE_OR => '$or', Query::TYPE_AND => '$and', - default => throw new DatabaseException('Unknown operator:' . $operator . '. Must be one of ' . Query::TYPE_EQUAL . ', ' . Query::TYPE_NOT_EQUAL . ', ' . Query::TYPE_LESSER . ', ' . Query::TYPE_LESSER_EQUAL . ', ' . Query::TYPE_GREATER . ', ' . Query::TYPE_GREATER_EQUAL . ', ' . Query::TYPE_IS_NULL . ', ' . Query::TYPE_IS_NOT_NULL . ', ' . Query::TYPE_BETWEEN . ', ' . Query::TYPE_CONTAINS . ', ' . Query::TYPE_SEARCH . ', ' . Query::TYPE_SELECT), + default => throw new DatabaseException('Unknown operator:' . $operator . '. Must be one of ' . Query::TYPE_EQUAL . ', ' . Query::TYPE_NOT_EQUAL . ', ' . Query::TYPE_LESSER . ', ' . Query::TYPE_LESSER_EQUAL . ', ' . Query::TYPE_GREATER . ', ' . Query::TYPE_GREATER_EQUAL . ', ' . Query::TYPE_IS_NULL . ', ' . Query::TYPE_IS_NOT_NULL . ', ' . Query::TYPE_BETWEEN . ', ' . Query::TYPE_NOT_BETWEEN . ', ' . Query::TYPE_STARTS_WITH . ', ' . Query::TYPE_NOT_STARTS_WITH . ', ' . Query::TYPE_ENDS_WITH . ', ' . Query::TYPE_NOT_ENDS_WITH . ', ' . Query::TYPE_CONTAINS . ', ' . Query::TYPE_NOT_CONTAINS . ', ' . Query::TYPE_SEARCH . ', ' . Query::TYPE_NOT_SEARCH . ', ' . Query::TYPE_SELECT), }; } @@ -2065,9 +2097,15 @@ protected function getQueryValue(string $method, mixed $value): mixed case Query::TYPE_STARTS_WITH: $value = $this->escapeWildcards($value); return $value . '.*'; + case Query::TYPE_NOT_STARTS_WITH: + $value = $this->escapeWildcards($value); + return $value . '.*'; case Query::TYPE_ENDS_WITH: $value = $this->escapeWildcards($value); return '.*' . $value; + case Query::TYPE_NOT_ENDS_WITH: + $value = $this->escapeWildcards($value); + return '.*' . $value; default: return $value; } @@ -2264,7 +2302,7 @@ public function getSupportForFulltextWildcardIndex(): bool */ public function getSupportForQueryContains(): bool { - return false; + return true; } /** diff --git a/src/Database/Validator/Key.php b/src/Database/Validator/Key.php index 920ff7b92..1d180c38c 100644 --- a/src/Database/Validator/Key.php +++ b/src/Database/Validator/Key.php @@ -8,10 +8,15 @@ class Key extends Validator { protected bool $allowInternal = false; // If true, you keys starting with $ are allowed + /** + * Maximum length for Key validation + */ + protected const KEY_MAX_LENGTH = 255; + /** * @var string */ - protected string $message = 'Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char'; + protected string $message = 'Parameter must contain at most ' . self::KEY_MAX_LENGTH . ' chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char'; /** * Get Description. @@ -76,8 +81,8 @@ public function isValid($value): bool if (\preg_match('/[^A-Za-z0-9\_\-\.]/', $value)) { return false; } - // At most 36 chars - if (\mb_strlen($value) > 36) { + // At most KEY_MAX_LENGTH chars + if (\mb_strlen($value) > self::KEY_MAX_LENGTH) { return false; } diff --git a/src/Database/Validator/UID.php b/src/Database/Validator/UID.php index 34d466e34..45971da66 100644 --- a/src/Database/Validator/UID.php +++ b/src/Database/Validator/UID.php @@ -13,6 +13,6 @@ class UID extends Key */ public function getDescription(): string { - return 'UID must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, and underscore. Can\'t start with a leading underscore'; + return 'UID must contain at most ' . self::KEY_MAX_LENGTH . ' chars. Valid chars are a-z, A-Z, 0-9, and underscore. Can\'t start with a leading underscore'; } } From 00723f2b972a8cfcb770b072e39387bd79692027 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 16 Sep 2025 18:45:20 +0300 Subject: [PATCH 50/55] Refactor Mongo adapter to ensure cursor IDs are cast to integers for consistency and improve regex handling for 'notStartsWith' and 'notEndsWith' queries. --- src/Database/Adapter/Mongo.php | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index d0fb250f2..fac64d76f 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1304,7 +1304,7 @@ public function getSequences(string $collection, array $documents): array // Continue fetching with getMore while ($cursorId && $cursorId !== 0) { - $moreResponse = $this->client->getMore($cursorId, $name, self::DEFAULT_BATCH_SIZE); + $moreResponse = $this->client->getMore((int)$cursorId, $name, self::DEFAULT_BATCH_SIZE); $moreResults = $moreResponse->cursor->nextBatch ?? []; if (empty($moreResults)) { @@ -1316,7 +1316,7 @@ public function getSequences(string $collection, array $documents): array } // Update cursor ID for next iteration - $cursorId = $moreResponse->cursor->id ?? null; + $cursorId = (int)($moreResponse->cursor->id ?? 0); } } catch (MongoException $e) { throw $this->processException($e); @@ -1630,8 +1630,8 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 if (!\is_null($limit) && count($found) >= $limit) { break; } - - $moreResponse = $this->client->getMore($cursorId, $name, self::DEFAULT_BATCH_SIZE); + + $moreResponse = $this->client->getMore((int)$cursorId, $name, self::DEFAULT_BATCH_SIZE); $moreResults = $moreResponse->cursor->nextBatch ?? []; if (empty($moreResults)) { @@ -1648,7 +1648,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 } } - $cursorId = $moreResponse->cursor->id ?? 0; + $cursorId = (int)($moreResponse->cursor->id ?? 0); } } catch (MongoException $e) { @@ -2047,8 +2047,10 @@ protected function buildFilter(Query $query): array [$attribute => ['$lt' => $value[0]]], [$attribute => ['$gt' => $value[1]]] ]; - } elseif ($operator === '$regex' && in_array($query->getMethod(), [Query::TYPE_NOT_STARTS_WITH, Query::TYPE_NOT_ENDS_WITH])) { - $filter[$attribute] = ['$not' => new Regex($value, 'i')]; + } elseif ($operator === '$regex' && $query->getMethod() === Query::TYPE_NOT_STARTS_WITH) { + $filter[$attribute] = ['$not' => new Regex('^' . $value, 'i')]; + } elseif ($operator === '$regex' && $query->getMethod() === Query::TYPE_NOT_ENDS_WITH) { + $filter[$attribute] = ['$not' => new Regex($value . '$', 'i')]; } else { $filter[$attribute][$operator] = $value; } From ef36ef67aac106e1a1c7476812fac7b33d4370c4 Mon Sep 17 00:00:00 2001 From: shimon Date: Tue, 16 Sep 2025 18:48:02 +0300 Subject: [PATCH 51/55] Refactor Key and UID validators to use a constant for maximum length, improving maintainability and consistency in validation messages. --- src/Database/Validator/Key.php | 11 ++++++++--- src/Database/Validator/UID.php | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Database/Validator/Key.php b/src/Database/Validator/Key.php index 920ff7b92..1d180c38c 100644 --- a/src/Database/Validator/Key.php +++ b/src/Database/Validator/Key.php @@ -8,10 +8,15 @@ class Key extends Validator { protected bool $allowInternal = false; // If true, you keys starting with $ are allowed + /** + * Maximum length for Key validation + */ + protected const KEY_MAX_LENGTH = 255; + /** * @var string */ - protected string $message = 'Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char'; + protected string $message = 'Parameter must contain at most ' . self::KEY_MAX_LENGTH . ' chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char'; /** * Get Description. @@ -76,8 +81,8 @@ public function isValid($value): bool if (\preg_match('/[^A-Za-z0-9\_\-\.]/', $value)) { return false; } - // At most 36 chars - if (\mb_strlen($value) > 36) { + // At most KEY_MAX_LENGTH chars + if (\mb_strlen($value) > self::KEY_MAX_LENGTH) { return false; } diff --git a/src/Database/Validator/UID.php b/src/Database/Validator/UID.php index 34d466e34..45971da66 100644 --- a/src/Database/Validator/UID.php +++ b/src/Database/Validator/UID.php @@ -13,6 +13,6 @@ class UID extends Key */ public function getDescription(): string { - return 'UID must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, and underscore. Can\'t start with a leading underscore'; + return 'UID must contain at most ' . self::KEY_MAX_LENGTH . ' chars. Valid chars are a-z, A-Z, 0-9, and underscore. Can\'t start with a leading underscore'; } } From f37e22a145d0a3d68b952213d541eb58c36fbfd7 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 18 Sep 2025 16:38:36 +0300 Subject: [PATCH 52/55] sync with main --- src/Database/Adapter.php | 4 +- src/Database/Adapter/Mongo.php | 41 ++ src/Database/Adapter/Pool.php | 2 +- src/Database/Adapter/SQL.php | 4 +- tests/e2e/Adapter/Scopes/DocumentTests.php | 497 +++++++++++---------- 5 files changed, 296 insertions(+), 252 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 42e880a8d..5f5846146 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1153,9 +1153,9 @@ abstract public function getKeywords(): array; * * @param array $selections * @param string $prefix - * @return string + * @return mixed */ - abstract protected function getAttributeProjection(array $selections, string $prefix): string; + abstract protected function getAttributeProjection(array $selections, string $prefix): mixed; /** * Get all selected attributes from queries diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 708e36c34..1e8a0006e 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -2698,4 +2698,45 @@ public function getTenantFilters( return ['$in' => $values]; } + + public function decodePoint(string $wkb): array + { + return []; + } + + /** + * Decode a WKB or textual LINESTRING into [[x1, y1], [x2, y2], ...] + * + * @param string $wkb + * @return float[][] Array of points, each as [x, y] + */ + public function decodeLinestring(string $wkb): array + { + return []; + } + + /** + * Decode a WKB or textual POLYGON into [[[x1, y1], [x2, y2], ...], ...] + * + * @param string $wkb + * @return float[][][] Array of rings, each ring is an array of points [x, y] + */ + public function decodePolygon(string $wkb): array + { + return []; + } + + /** + * Get the query to check for tenant when in shared tables mode + * + * @param string $collection The collection being queried + * @param string $alias The alias of the parent collection if in a subquery + * @return string + */ + public function getTenantQuery(string $collection, string $alias = ''): string + { + return ''; + } + + } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 33ffeda79..4c43f8536 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -470,7 +470,7 @@ public function getKeywords(): array return $this->delegate(__FUNCTION__, \func_get_args()); } - protected function getAttributeProjection(array $selections, string $prefix): string + protected function getAttributeProjection(array $selections, string $prefix): mixed { return $this->delegate(__FUNCTION__, \func_get_args()); } diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index 248cb0428..e6a77478e 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -1906,10 +1906,10 @@ public function getTenantQuery( * * @param array $selections * @param string $prefix - * @return string + * @return mixed * @throws Exception */ - protected function getAttributeProjection(array $selections, string $prefix): string + protected function getAttributeProjection(array $selections, string $prefix): mixed { if (empty($selections) || \in_array('*', $selections)) { return "{$this->quote($prefix)}.*"; diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index b4c524564..3b313ef23 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -30,6 +30,9 @@ public function testBigintSequence(): void $database->createCollection(__FUNCTION__); $sequence = 5_000_000_000_000_000; + if ($database->getAdapter()->getIdAttributeType() == Database::VAR_UUID7) { + $sequence = '01995753-881b-78cf-9506-2cffecf8f227'; + } $document = $database->createDocument(__FUNCTION__, new Document([ '$sequence' => (string)$sequence, @@ -3335,253 +3338,253 @@ public function testFindNotContains(): void } } - // public function testFindNotSearch(): void - // { - // /** @var Database $database */ - // $database = static::getDatabase(); - - // // Only test if fulltext search is supported - // if ($this->getDatabase()->getAdapter()->getSupportForFulltextIndex()) { - // // Ensure fulltext index exists (may already exist from previous tests) - // try { - // $database->createIndex('movies', 'name', Database::INDEX_FULLTEXT, ['name']); - // } catch (Throwable $e) { - // // Index may already exist, ignore duplicate error - // if (!str_contains($e->getMessage(), 'already exists')) { - // throw $e; - // } - // } - - // // Test notSearch - should return documents that don't match the search term - // $documents = $database->find('movies', [ - // Query::notSearch('name', 'captain'), - // ]); - - // $this->assertEquals(4, count($documents)); // All movies except the 2 with 'captain' in name - - // // Test notSearch with term that doesn't exist - should return all documents - // $documents = $database->find('movies', [ - // Query::notSearch('name', 'nonexistent'), - // ]); - - // $this->assertEquals(6, count($documents)); - - // // Test notSearch with partial term - // if ($this->getDatabase()->getAdapter()->getSupportForFulltextWildCardIndex()) { - // $documents = $database->find('movies', [ - // Query::notSearch('name', 'cap'), - // ]); - - // $this->assertEquals(4, count($documents)); // All movies except those matching 'cap' - // } - - // // Test notSearch with empty string - should return all documents - // $documents = $database->find('movies', [ - // Query::notSearch('name', ''), - // ]); - // $this->assertEquals(6, count($documents)); // All movies since empty search matches nothing - - // // Test notSearch combined with other filters - // $documents = $database->find('movies', [ - // Query::notSearch('name', 'captain'), - // Query::lessThan('year', 2010) - // ]); - // $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-captain movies before 2010 - - // // Test notSearch with special characters - // $documents = $database->find('movies', [ - // Query::notSearch('name', '@#$%'), - // ]); - // $this->assertEquals(6, count($documents)); // All movies since special chars don't match - // } - - // $this->assertEquals(true, true); // Test must do an assertion - // } - - // public function testFindNotStartsWith(): void - // { - // /** @var Database $database */ - // $database = static::getDatabase(); - - // // Test notStartsWith - should return documents that don't start with 'Work' - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'Work'), - // ]); - - // $this->assertEquals(4, count($documents)); // All movies except the 2 starting with 'Work' - - // // Test notStartsWith with non-existent prefix - should return all documents - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'NonExistent'), - // ]); - - // $this->assertEquals(6, count($documents)); - - // // Test notStartsWith with wildcard characters (should treat them literally) - // if ($this->getDatabase()->getAdapter() instanceof SQL) { - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', '%ork'), - // ]); - // } else { - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', '.*ork'), - // ]); - // } - - // $this->assertEquals(6, count($documents)); // Should return all since no movie starts with these patterns - - // // Test notStartsWith with empty string - should return no documents (all strings start with empty) - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', ''), - // ]); - // $this->assertEquals(0, count($documents)); // No documents since all strings start with empty string - - // // Test notStartsWith with single character - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'C'), - // ]); - // $this->assertGreaterThanOrEqual(4, count($documents)); // Movies not starting with 'C' - - // // Test notStartsWith with case sensitivity (may be case-insensitive depending on DB) - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'work'), // lowercase vs 'Work' - // ]); - // $this->assertGreaterThanOrEqual(4, count($documents)); // May match case-insensitively - - // // Test notStartsWith combined with other queries - // $documents = $database->find('movies', [ - // Query::notStartsWith('name', 'Work'), - // Query::equal('year', [2006]) - // ]); - // $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-Work movies from 2006 - // } - - // public function testFindNotEndsWith(): void - // { - // /** @var Database $database */ - // $database = static::getDatabase(); - - // // Test notEndsWith - should return documents that don't end with 'Marvel' - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'Marvel'), - // ]); - - // $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'Marvel' - - // // Test notEndsWith with non-existent suffix - should return all documents - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'NonExistent'), - // ]); - - // $this->assertEquals(6, count($documents)); - - // // Test notEndsWith with partial suffix - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'vel'), - // ]); - - // $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'vel' (from 'Marvel') - - // // Test notEndsWith with empty string - should return no documents (all strings end with empty) - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', ''), - // ]); - // $this->assertEquals(0, count($documents)); // No documents since all strings end with empty string - - // // Test notEndsWith with single character - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'l'), - // ]); - // $this->assertGreaterThanOrEqual(5, count($documents)); // Movies not ending with 'l' - - // // Test notEndsWith with case sensitivity (may be case-insensitive depending on DB) - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'marvel'), // lowercase vs 'Marvel' - // ]); - // $this->assertGreaterThanOrEqual(5, count($documents)); // May match case-insensitively - - // // Test notEndsWith combined with limit - // $documents = $database->find('movies', [ - // Query::notEndsWith('name', 'Marvel'), - // Query::limit(3) - // ]); - // $this->assertEquals(3, count($documents)); // Limited to 3 results - // $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies - // } - - // public function testFindNotBetween(): void - // { - // /** @var Database $database */ - // $database = static::getDatabase(); - // - // // Test notBetween with price range - should return documents outside the range - // $documents = $database->find('movies', [ - // Query::notBetween('price', 25.94, 25.99), - // ]); - // $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range - // - // // Test notBetween with range that includes no documents - should return all documents - // $documents = $database->find('movies', [ - // Query::notBetween('price', 30, 35), - // ]); - // $this->assertEquals(6, count($documents)); - // - // // Test notBetween with date range - // $documents = $database->find('movies', [ - // Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), - // ]); - // $this->assertEquals(0, count($documents)); // No movies outside this wide date range - // - // // Test notBetween with narrower date range - // $documents = $database->find('movies', [ - // Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), - // ]); - // $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - // - // // Test notBetween with updated date range - // $documents = $database->find('movies', [ - // Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), - // ]); - // $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - // - // // Test notBetween with year range (integer values) - // $documents = $database->find('movies', [ - // Query::notBetween('year', 2005, 2007), - // ]); - // $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range - // - // // Test notBetween with reversed range (start > end) - should still work - // $documents = $database->find('movies', [ - // Query::notBetween('price', 25.99, 25.94), // Note: reversed order - // ]); - // $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully - // - // // Test notBetween with same start and end values - // $documents = $database->find('movies', [ - // Query::notBetween('year', 2006, 2006), - // ]); - // $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 - // - // // Test notBetween combined with other filters - // $documents = $database->find('movies', [ - // Query::notBetween('price', 25.94, 25.99), - // Query::orderDesc('year'), - // Query::limit(2) - // ]); - // $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range - // - // // Test notBetween with extreme ranges - // $documents = $database->find('movies', [ - // Query::notBetween('year', -1000, 1000), // Very wide range - // ]); - // $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range - // - // // Test notBetween with float precision - // $documents = $database->find('movies', [ - // Query::notBetween('price', 25.945, 25.955), // Very narrow range - // ]); - // $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range - // } + public function testFindNotSearch(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Only test if fulltext search is supported + if ($this->getDatabase()->getAdapter()->getSupportForFulltextIndex()) { + // Ensure fulltext index exists (may already exist from previous tests) + try { + $database->createIndex('movies', 'name', Database::INDEX_FULLTEXT, ['name']); + } catch (Throwable $e) { + // Index may already exist, ignore duplicate error + if (!str_contains($e->getMessage(), 'already exists')) { + throw $e; + } + } + + // Test notSearch - should return documents that don't match the search term + $documents = $database->find('movies', [ + Query::notSearch('name', 'captain'), + ]); + + $this->assertEquals(4, count($documents)); // All movies except the 2 with 'captain' in name + + // Test notSearch with term that doesn't exist - should return all documents + $documents = $database->find('movies', [ + Query::notSearch('name', 'nonexistent'), + ]); + + $this->assertEquals(6, count($documents)); + + // Test notSearch with partial term + if ($this->getDatabase()->getAdapter()->getSupportForFulltextWildCardIndex()) { + $documents = $database->find('movies', [ + Query::notSearch('name', 'cap'), + ]); + + $this->assertEquals(4, count($documents)); // All movies except those matching 'cap' + } + + // Test notSearch with empty string - should return all documents + $documents = $database->find('movies', [ + Query::notSearch('name', ''), + ]); + $this->assertEquals(6, count($documents)); // All movies since empty search matches nothing + + // Test notSearch combined with other filters + $documents = $database->find('movies', [ + Query::notSearch('name', 'captain'), + Query::lessThan('year', 2010) + ]); + $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-captain movies before 2010 + + // Test notSearch with special characters + $documents = $database->find('movies', [ + Query::notSearch('name', '@#$%'), + ]); + $this->assertEquals(6, count($documents)); // All movies since special chars don't match + } + + $this->assertEquals(true, true); // Test must do an assertion + } + + public function testFindNotStartsWith(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Test notStartsWith - should return documents that don't start with 'Work' + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'Work'), + ]); + + $this->assertEquals(4, count($documents)); // All movies except the 2 starting with 'Work' + + // Test notStartsWith with non-existent prefix - should return all documents + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'NonExistent'), + ]); + + $this->assertEquals(6, count($documents)); + + // Test notStartsWith with wildcard characters (should treat them literally) + if ($this->getDatabase()->getAdapter() instanceof SQL) { + $documents = $database->find('movies', [ + Query::notStartsWith('name', '%ork'), + ]); + } else { + $documents = $database->find('movies', [ + Query::notStartsWith('name', '.*ork'), + ]); + } + + $this->assertEquals(6, count($documents)); // Should return all since no movie starts with these patterns + + // Test notStartsWith with empty string - should return no documents (all strings start with empty) + $documents = $database->find('movies', [ + Query::notStartsWith('name', ''), + ]); + $this->assertEquals(0, count($documents)); // No documents since all strings start with empty string + + // Test notStartsWith with single character + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'C'), + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // Movies not starting with 'C' + + // Test notStartsWith with case sensitivity (may be case-insensitive depending on DB) + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'work'), // lowercase vs 'Work' + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // May match case-insensitively + + // Test notStartsWith combined with other queries + $documents = $database->find('movies', [ + Query::notStartsWith('name', 'Work'), + Query::equal('year', [2006]) + ]); + $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-Work movies from 2006 + } + + public function testFindNotEndsWith(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Test notEndsWith - should return documents that don't end with 'Marvel' + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'Marvel'), + ]); + + $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'Marvel' + + // Test notEndsWith with non-existent suffix - should return all documents + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'NonExistent'), + ]); + + $this->assertEquals(6, count($documents)); + + // Test notEndsWith with partial suffix + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'vel'), + ]); + + $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'vel' (from 'Marvel') + + // Test notEndsWith with empty string - should return no documents (all strings end with empty) + $documents = $database->find('movies', [ + Query::notEndsWith('name', ''), + ]); + $this->assertEquals(0, count($documents)); // No documents since all strings end with empty string + + // Test notEndsWith with single character + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'l'), + ]); + $this->assertGreaterThanOrEqual(5, count($documents)); // Movies not ending with 'l' + + // Test notEndsWith with case sensitivity (may be case-insensitive depending on DB) + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'marvel'), // lowercase vs 'Marvel' + ]); + $this->assertGreaterThanOrEqual(5, count($documents)); // May match case-insensitively + + // Test notEndsWith combined with limit + $documents = $database->find('movies', [ + Query::notEndsWith('name', 'Marvel'), + Query::limit(3) + ]); + $this->assertEquals(3, count($documents)); // Limited to 3 results + $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies + } + + public function testFindNotBetween(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Test notBetween with price range - should return documents outside the range + $documents = $database->find('movies', [ + Query::notBetween('price', 25.94, 25.99), + ]); + $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range + + // Test notBetween with range that includes no documents - should return all documents + $documents = $database->find('movies', [ + Query::notBetween('price', 30, 35), + ]); + $this->assertEquals(6, count($documents)); + + // Test notBetween with date range + $documents = $database->find('movies', [ + Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), + ]); + $this->assertEquals(0, count($documents)); // No movies outside this wide date range + + // Test notBetween with narrower date range + $documents = $database->find('movies', [ + Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), + ]); + $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + + // Test notBetween with updated date range + $documents = $database->find('movies', [ + Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), + ]); + $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + + // Test notBetween with year range (integer values) + $documents = $database->find('movies', [ + Query::notBetween('year', 2005, 2007), + ]); + $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range + + // Test notBetween with reversed range (start > end) - should still work + $documents = $database->find('movies', [ + Query::notBetween('price', 25.99, 25.94), // Note: reversed order + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully + + // Test notBetween with same start and end values + $documents = $database->find('movies', [ + Query::notBetween('year', 2006, 2006), + ]); + $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 + + // Test notBetween combined with other filters + $documents = $database->find('movies', [ + Query::notBetween('price', 25.94, 25.99), + Query::orderDesc('year'), + Query::limit(2) + ]); + $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range + + // Test notBetween with extreme ranges + $documents = $database->find('movies', [ + Query::notBetween('year', -1000, 1000), // Very wide range + ]); + $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range + + // Test notBetween with float precision + $documents = $database->find('movies', [ + Query::notBetween('price', 25.945, 25.955), // Very narrow range + ]); + $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range + } public function testFindSelect(): void { From cc841a45de684b8109297378cb204e2b65396be5 Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 18 Sep 2025 17:22:45 +0300 Subject: [PATCH 53/55] remove inversion --- src/Database/Adapter/Mongo.php | 46 +++------------------------------- 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 1e8a0006e..b94e0005a 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -31,15 +31,12 @@ class Mongo extends Adapter '$gt', '$gte', '$in', - '$nin', '$text', '$search', '$or', '$and', '$match', '$regex', - '$not', - '$nor', ]; protected Client $client; @@ -2019,36 +2016,11 @@ protected function buildFilter(Query $query): array } else { $filter[$attribute]['$in'] = $query->getValues(); } - } elseif ($operator === 'notContains') { - if (!$query->onArray()) { - $filter[$attribute] = ['$not' => new Regex(".*{$this->escapeWildcards($value)}.*", 'i')]; - } else { - $filter[$attribute]['$nin'] = $query->getValues(); - } } elseif ($operator == '$search') { - if ($query->getMethod() === Query::TYPE_NOT_SEARCH) { - // MongoDB doesn't support negating $text expressions directly - // Use regex as fallback for NOT search while keeping fulltext for positive search - if (empty($value)) { - // If value is not passed, don't add any filter - this will match all documents - } else { - // Escape special regex characters and create a pattern that matches the search term as substring - $escapedValue = preg_quote($value, '/'); - $filter[$attribute] = ['$not' => new Regex(".*{$escapedValue}.*", 'i')]; - } - } else { - $filter['$text'][$operator] = $value; - } + $filter['$text'][$operator] = $value; } elseif ($operator === Query::TYPE_BETWEEN) { $filter[$attribute]['$lte'] = $value[1]; $filter[$attribute]['$gte'] = $value[0]; - } elseif ($operator === Query::TYPE_NOT_BETWEEN) { - $filter['$or'] = [ - [$attribute => ['$lt' => $value[0]]], - [$attribute => ['$gt' => $value[1]]] - ]; - } elseif ($operator === '$regex' && in_array($query->getMethod(), [Query::TYPE_NOT_STARTS_WITH, Query::TYPE_NOT_ENDS_WITH])) { - $filter[$attribute] = ['$not' => new Regex($value, 'i')]; } else { $filter[$attribute][$operator] = $value; } @@ -2076,18 +2048,13 @@ protected function getQueryOperator(string $operator): string Query::TYPE_GREATER => '$gt', Query::TYPE_GREATER_EQUAL => '$gte', Query::TYPE_CONTAINS => '$in', - Query::TYPE_NOT_CONTAINS => 'notContains', Query::TYPE_SEARCH => '$search', - Query::TYPE_NOT_SEARCH => '$search', Query::TYPE_BETWEEN => 'between', - Query::TYPE_NOT_BETWEEN => 'notBetween', Query::TYPE_STARTS_WITH, - Query::TYPE_NOT_STARTS_WITH, - Query::TYPE_ENDS_WITH, - Query::TYPE_NOT_ENDS_WITH => '$regex', + Query::TYPE_ENDS_WITH => '$regex', Query::TYPE_OR => '$or', Query::TYPE_AND => '$and', - default => throw new DatabaseException('Unknown operator:' . $operator . '. Must be one of ' . Query::TYPE_EQUAL . ', ' . Query::TYPE_NOT_EQUAL . ', ' . Query::TYPE_LESSER . ', ' . Query::TYPE_LESSER_EQUAL . ', ' . Query::TYPE_GREATER . ', ' . Query::TYPE_GREATER_EQUAL . ', ' . Query::TYPE_IS_NULL . ', ' . Query::TYPE_IS_NOT_NULL . ', ' . Query::TYPE_BETWEEN . ', ' . Query::TYPE_NOT_BETWEEN . ', ' . Query::TYPE_STARTS_WITH . ', ' . Query::TYPE_NOT_STARTS_WITH . ', ' . Query::TYPE_ENDS_WITH . ', ' . Query::TYPE_NOT_ENDS_WITH . ', ' . Query::TYPE_CONTAINS . ', ' . Query::TYPE_NOT_CONTAINS . ', ' . Query::TYPE_SEARCH . ', ' . Query::TYPE_NOT_SEARCH . ', ' . Query::TYPE_SELECT), + default => throw new DatabaseException('Unknown operator:' . $operator . '. Must be one of ' . Query::TYPE_EQUAL . ', ' . Query::TYPE_NOT_EQUAL . ', ' . Query::TYPE_LESSER . ', ' . Query::TYPE_LESSER_EQUAL . ', ' . Query::TYPE_GREATER . ', ' . Query::TYPE_GREATER_EQUAL . ', ' . Query::TYPE_IS_NULL . ', ' . Query::TYPE_IS_NOT_NULL . ', ' . Query::TYPE_BETWEEN . ', ' . Query::TYPE_CONTAINS . ', ' . Query::TYPE_SEARCH . ', ' . Query::TYPE_SELECT), }; } @@ -2097,15 +2064,9 @@ protected function getQueryValue(string $method, mixed $value): mixed case Query::TYPE_STARTS_WITH: $value = $this->escapeWildcards($value); return $value . '.*'; - case Query::TYPE_NOT_STARTS_WITH: - $value = $this->escapeWildcards($value); - return $value . '.*'; case Query::TYPE_ENDS_WITH: $value = $this->escapeWildcards($value); return '.*' . $value; - case Query::TYPE_NOT_ENDS_WITH: - $value = $this->escapeWildcards($value); - return '.*' . $value; default: return $value; } @@ -2591,7 +2552,6 @@ public function getKeywords(): array protected function processException(Exception $e): \Exception { - // Timeout if ($e->getCode() === 50) { return new Timeout('Query timed out', $e->getCode(), $e); From 09793afeea4232b89ab3d58ad38c4ad03a5d7b4e Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 18 Sep 2025 17:35:03 +0300 Subject: [PATCH 54/55] remove inversion --- src/Database/Adapter/Mongo.php | 28 +- tests/e2e/Adapter/Scopes/DocumentTests.php | 618 ++++++++++----------- 2 files changed, 323 insertions(+), 323 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index b94e0005a..39ccc5583 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1627,7 +1627,7 @@ public function find(Document $collection, array $queries = [], ?int $limit = 25 if (!\is_null($limit) && count($found) >= $limit) { break; } - + $moreResponse = $this->client->getMore((int)$cursorId, $name, self::DEFAULT_BATCH_SIZE); $moreResults = $moreResponse->cursor->nextBatch ?? []; @@ -2659,10 +2659,10 @@ public function getTenantFilters( return ['$in' => $values]; } - public function decodePoint(string $wkb): array - { - return []; - } + public function decodePoint(string $wkb): array + { + return []; + } /** * Decode a WKB or textual LINESTRING into [[x1, y1], [x2, y2], ...] @@ -2670,10 +2670,10 @@ public function decodePoint(string $wkb): array * @param string $wkb * @return float[][] Array of points, each as [x, y] */ - public function decodeLinestring(string $wkb): array - { - return []; - } + public function decodeLinestring(string $wkb): array + { + return []; + } /** * Decode a WKB or textual POLYGON into [[[x1, y1], [x2, y2], ...], ...] @@ -2681,10 +2681,10 @@ public function decodeLinestring(string $wkb): array * @param string $wkb * @return float[][][] Array of rings, each ring is an array of points [x, y] */ - public function decodePolygon(string $wkb): array - { - return []; - } + public function decodePolygon(string $wkb): array + { + return []; + } /** * Get the query to check for tenant when in shared tables mode @@ -2695,7 +2695,7 @@ public function decodePolygon(string $wkb): array */ public function getTenantQuery(string $collection, string $alias = ''): string { - return ''; + return ''; } diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 3b313ef23..2e4aa4b29 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -3276,315 +3276,315 @@ public function testFindEndsWith(): void $this->assertEquals(1, count($documents)); } - public function testFindNotContains(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - if (!$database->getAdapter()->getSupportForQueryContains()) { - $this->expectNotToPerformAssertions(); - return; - } - - // Test notContains with array attributes - should return documents that don't contain specified genres - $documents = $database->find('movies', [ - Query::notContains('genres', ['comics']) - ]); - - $this->assertEquals(4, count($documents)); // All movies except the 2 with 'comics' genre - - // Test notContains with multiple values (AND logic - exclude documents containing ANY of these) - $documents = $database->find('movies', [ - Query::notContains('genres', ['comics', 'kids']), - ]); - - $this->assertEquals(2, count($documents)); // Movies that have neither 'comics' nor 'kids' - - // Test notContains with non-existent genre - should return all documents - $documents = $database->find('movies', [ - Query::notContains('genres', ['non-existent']), - ]); - - $this->assertEquals(6, count($documents)); - - // Test notContains with string attribute (substring search) - $documents = $database->find('movies', [ - Query::notContains('name', ['Captain']) - ]); - $this->assertEquals(4, count($documents)); // All movies except those containing 'Captain' - - // Test notContains combined with other queries (AND logic) - $documents = $database->find('movies', [ - Query::notContains('genres', ['comics']), - Query::greaterThan('year', 2000) - ]); - $this->assertLessThanOrEqual(4, count($documents)); // Subset of movies without 'comics' and after 2000 - - // Test notContains with case sensitivity - $documents = $database->find('movies', [ - Query::notContains('genres', ['COMICS']) // Different case - ]); - $this->assertEquals(6, count($documents)); // All movies since case doesn't match - - // Test error handling for invalid attribute type - try { - $database->find('movies', [ - Query::notContains('price', [10.5]), - ]); - $this->fail('Failed to throw exception'); - } catch (Throwable $e) { - $this->assertEquals('Invalid query: Cannot query notContains on attribute "price" because it is not an array or string.', $e->getMessage()); - $this->assertTrue($e instanceof DatabaseException); - } - } - - public function testFindNotSearch(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - // Only test if fulltext search is supported - if ($this->getDatabase()->getAdapter()->getSupportForFulltextIndex()) { - // Ensure fulltext index exists (may already exist from previous tests) - try { - $database->createIndex('movies', 'name', Database::INDEX_FULLTEXT, ['name']); - } catch (Throwable $e) { - // Index may already exist, ignore duplicate error - if (!str_contains($e->getMessage(), 'already exists')) { - throw $e; - } - } - - // Test notSearch - should return documents that don't match the search term - $documents = $database->find('movies', [ - Query::notSearch('name', 'captain'), - ]); - - $this->assertEquals(4, count($documents)); // All movies except the 2 with 'captain' in name - - // Test notSearch with term that doesn't exist - should return all documents - $documents = $database->find('movies', [ - Query::notSearch('name', 'nonexistent'), - ]); - - $this->assertEquals(6, count($documents)); - - // Test notSearch with partial term - if ($this->getDatabase()->getAdapter()->getSupportForFulltextWildCardIndex()) { - $documents = $database->find('movies', [ - Query::notSearch('name', 'cap'), - ]); - - $this->assertEquals(4, count($documents)); // All movies except those matching 'cap' - } - - // Test notSearch with empty string - should return all documents - $documents = $database->find('movies', [ - Query::notSearch('name', ''), - ]); - $this->assertEquals(6, count($documents)); // All movies since empty search matches nothing - - // Test notSearch combined with other filters - $documents = $database->find('movies', [ - Query::notSearch('name', 'captain'), - Query::lessThan('year', 2010) - ]); - $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-captain movies before 2010 - - // Test notSearch with special characters - $documents = $database->find('movies', [ - Query::notSearch('name', '@#$%'), - ]); - $this->assertEquals(6, count($documents)); // All movies since special chars don't match - } - - $this->assertEquals(true, true); // Test must do an assertion - } - - public function testFindNotStartsWith(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - // Test notStartsWith - should return documents that don't start with 'Work' - $documents = $database->find('movies', [ - Query::notStartsWith('name', 'Work'), - ]); - - $this->assertEquals(4, count($documents)); // All movies except the 2 starting with 'Work' - - // Test notStartsWith with non-existent prefix - should return all documents - $documents = $database->find('movies', [ - Query::notStartsWith('name', 'NonExistent'), - ]); - - $this->assertEquals(6, count($documents)); - - // Test notStartsWith with wildcard characters (should treat them literally) - if ($this->getDatabase()->getAdapter() instanceof SQL) { - $documents = $database->find('movies', [ - Query::notStartsWith('name', '%ork'), - ]); - } else { - $documents = $database->find('movies', [ - Query::notStartsWith('name', '.*ork'), - ]); - } - - $this->assertEquals(6, count($documents)); // Should return all since no movie starts with these patterns - - // Test notStartsWith with empty string - should return no documents (all strings start with empty) - $documents = $database->find('movies', [ - Query::notStartsWith('name', ''), - ]); - $this->assertEquals(0, count($documents)); // No documents since all strings start with empty string - - // Test notStartsWith with single character - $documents = $database->find('movies', [ - Query::notStartsWith('name', 'C'), - ]); - $this->assertGreaterThanOrEqual(4, count($documents)); // Movies not starting with 'C' - - // Test notStartsWith with case sensitivity (may be case-insensitive depending on DB) - $documents = $database->find('movies', [ - Query::notStartsWith('name', 'work'), // lowercase vs 'Work' - ]); - $this->assertGreaterThanOrEqual(4, count($documents)); // May match case-insensitively - - // Test notStartsWith combined with other queries - $documents = $database->find('movies', [ - Query::notStartsWith('name', 'Work'), - Query::equal('year', [2006]) - ]); - $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-Work movies from 2006 - } - - public function testFindNotEndsWith(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - // Test notEndsWith - should return documents that don't end with 'Marvel' - $documents = $database->find('movies', [ - Query::notEndsWith('name', 'Marvel'), - ]); - - $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'Marvel' - - // Test notEndsWith with non-existent suffix - should return all documents - $documents = $database->find('movies', [ - Query::notEndsWith('name', 'NonExistent'), - ]); - - $this->assertEquals(6, count($documents)); - - // Test notEndsWith with partial suffix - $documents = $database->find('movies', [ - Query::notEndsWith('name', 'vel'), - ]); - - $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'vel' (from 'Marvel') - - // Test notEndsWith with empty string - should return no documents (all strings end with empty) - $documents = $database->find('movies', [ - Query::notEndsWith('name', ''), - ]); - $this->assertEquals(0, count($documents)); // No documents since all strings end with empty string - - // Test notEndsWith with single character - $documents = $database->find('movies', [ - Query::notEndsWith('name', 'l'), - ]); - $this->assertGreaterThanOrEqual(5, count($documents)); // Movies not ending with 'l' - - // Test notEndsWith with case sensitivity (may be case-insensitive depending on DB) - $documents = $database->find('movies', [ - Query::notEndsWith('name', 'marvel'), // lowercase vs 'Marvel' - ]); - $this->assertGreaterThanOrEqual(5, count($documents)); // May match case-insensitively - - // Test notEndsWith combined with limit - $documents = $database->find('movies', [ - Query::notEndsWith('name', 'Marvel'), - Query::limit(3) - ]); - $this->assertEquals(3, count($documents)); // Limited to 3 results - $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies - } - - public function testFindNotBetween(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - // Test notBetween with price range - should return documents outside the range - $documents = $database->find('movies', [ - Query::notBetween('price', 25.94, 25.99), - ]); - $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range - - // Test notBetween with range that includes no documents - should return all documents - $documents = $database->find('movies', [ - Query::notBetween('price', 30, 35), - ]); - $this->assertEquals(6, count($documents)); - - // Test notBetween with date range - $documents = $database->find('movies', [ - Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), - ]); - $this->assertEquals(0, count($documents)); // No movies outside this wide date range - - // Test notBetween with narrower date range - $documents = $database->find('movies', [ - Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), - ]); - $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - - // Test notBetween with updated date range - $documents = $database->find('movies', [ - Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), - ]); - $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range - - // Test notBetween with year range (integer values) - $documents = $database->find('movies', [ - Query::notBetween('year', 2005, 2007), - ]); - $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range - - // Test notBetween with reversed range (start > end) - should still work - $documents = $database->find('movies', [ - Query::notBetween('price', 25.99, 25.94), // Note: reversed order - ]); - $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully - - // Test notBetween with same start and end values - $documents = $database->find('movies', [ - Query::notBetween('year', 2006, 2006), - ]); - $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 - - // Test notBetween combined with other filters - $documents = $database->find('movies', [ - Query::notBetween('price', 25.94, 25.99), - Query::orderDesc('year'), - Query::limit(2) - ]); - $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range - - // Test notBetween with extreme ranges - $documents = $database->find('movies', [ - Query::notBetween('year', -1000, 1000), // Very wide range - ]); - $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range - - // Test notBetween with float precision - $documents = $database->find('movies', [ - Query::notBetween('price', 25.945, 25.955), // Very narrow range - ]); - $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range - } + // public function testFindNotContains(): void + // { + // /** @var Database $database */ + // $database = static::getDatabase(); + // + // if (!$database->getAdapter()->getSupportForQueryContains()) { + // $this->expectNotToPerformAssertions(); + // return; + // } + // + // // Test notContains with array attributes - should return documents that don't contain specified genres + // $documents = $database->find('movies', [ + // Query::notContains('genres', ['comics']) + // ]); + // + // $this->assertEquals(4, count($documents)); // All movies except the 2 with 'comics' genre + // + // // Test notContains with multiple values (AND logic - exclude documents containing ANY of these) + // $documents = $database->find('movies', [ + // Query::notContains('genres', ['comics', 'kids']), + // ]); + // + // $this->assertEquals(2, count($documents)); // Movies that have neither 'comics' nor 'kids' + // + // // Test notContains with non-existent genre - should return all documents + // $documents = $database->find('movies', [ + // Query::notContains('genres', ['non-existent']), + // ]); + // + // $this->assertEquals(6, count($documents)); + // + // // Test notContains with string attribute (substring search) + // $documents = $database->find('movies', [ + // Query::notContains('name', ['Captain']) + // ]); + // $this->assertEquals(4, count($documents)); // All movies except those containing 'Captain' + // + // // Test notContains combined with other queries (AND logic) + // $documents = $database->find('movies', [ + // Query::notContains('genres', ['comics']), + // Query::greaterThan('year', 2000) + // ]); + // $this->assertLessThanOrEqual(4, count($documents)); // Subset of movies without 'comics' and after 2000 + // + // // Test notContains with case sensitivity + // $documents = $database->find('movies', [ + // Query::notContains('genres', ['COMICS']) // Different case + // ]); + // $this->assertEquals(6, count($documents)); // All movies since case doesn't match + // + // // Test error handling for invalid attribute type + // try { + // $database->find('movies', [ + // Query::notContains('price', [10.5]), + // ]); + // $this->fail('Failed to throw exception'); + // } catch (Throwable $e) { + // $this->assertEquals('Invalid query: Cannot query notContains on attribute "price" because it is not an array or string.', $e->getMessage()); + // $this->assertTrue($e instanceof DatabaseException); + // } + // } + // + // public function testFindNotSearch(): void + // { + // /** @var Database $database */ + // $database = static::getDatabase(); + // + // // Only test if fulltext search is supported + // if ($this->getDatabase()->getAdapter()->getSupportForFulltextIndex()) { + // // Ensure fulltext index exists (may already exist from previous tests) + // try { + // $database->createIndex('movies', 'name', Database::INDEX_FULLTEXT, ['name']); + // } catch (Throwable $e) { + // // Index may already exist, ignore duplicate error + // if (!str_contains($e->getMessage(), 'already exists')) { + // throw $e; + // } + // } + // + // // Test notSearch - should return documents that don't match the search term + // $documents = $database->find('movies', [ + // Query::notSearch('name', 'captain'), + // ]); + // + // $this->assertEquals(4, count($documents)); // All movies except the 2 with 'captain' in name + // + // // Test notSearch with term that doesn't exist - should return all documents + // $documents = $database->find('movies', [ + // Query::notSearch('name', 'nonexistent'), + // ]); + // + // $this->assertEquals(6, count($documents)); + // + // // Test notSearch with partial term + // if ($this->getDatabase()->getAdapter()->getSupportForFulltextWildCardIndex()) { + // $documents = $database->find('movies', [ + // Query::notSearch('name', 'cap'), + // ]); + // + // $this->assertEquals(4, count($documents)); // All movies except those matching 'cap' + // } + // + // // Test notSearch with empty string - should return all documents + // $documents = $database->find('movies', [ + // Query::notSearch('name', ''), + // ]); + // $this->assertEquals(6, count($documents)); // All movies since empty search matches nothing + // + // // Test notSearch combined with other filters + // $documents = $database->find('movies', [ + // Query::notSearch('name', 'captain'), + // Query::lessThan('year', 2010) + // ]); + // $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-captain movies before 2010 + // + // // Test notSearch with special characters + // $documents = $database->find('movies', [ + // Query::notSearch('name', '@#$%'), + // ]); + // $this->assertEquals(6, count($documents)); // All movies since special chars don't match + // } + // + // $this->assertEquals(true, true); // Test must do an assertion + // } + // + // public function testFindNotStartsWith(): void + // { + // /** @var Database $database */ + // $database = static::getDatabase(); + // + // // Test notStartsWith - should return documents that don't start with 'Work' + // $documents = $database->find('movies', [ + // Query::notStartsWith('name', 'Work'), + // ]); + // + // $this->assertEquals(4, count($documents)); // All movies except the 2 starting with 'Work' + // + // // Test notStartsWith with non-existent prefix - should return all documents + // $documents = $database->find('movies', [ + // Query::notStartsWith('name', 'NonExistent'), + // ]); + // + // $this->assertEquals(6, count($documents)); + // + // // Test notStartsWith with wildcard characters (should treat them literally) + // if ($this->getDatabase()->getAdapter() instanceof SQL) { + // $documents = $database->find('movies', [ + // Query::notStartsWith('name', '%ork'), + // ]); + // } else { + // $documents = $database->find('movies', [ + // Query::notStartsWith('name', '.*ork'), + // ]); + // } + // + // $this->assertEquals(6, count($documents)); // Should return all since no movie starts with these patterns + // + // // Test notStartsWith with empty string - should return no documents (all strings start with empty) + // $documents = $database->find('movies', [ + // Query::notStartsWith('name', ''), + // ]); + // $this->assertEquals(0, count($documents)); // No documents since all strings start with empty string + // + // // Test notStartsWith with single character + // $documents = $database->find('movies', [ + // Query::notStartsWith('name', 'C'), + // ]); + // $this->assertGreaterThanOrEqual(4, count($documents)); // Movies not starting with 'C' + // + // // Test notStartsWith with case sensitivity (may be case-insensitive depending on DB) + // $documents = $database->find('movies', [ + // Query::notStartsWith('name', 'work'), // lowercase vs 'Work' + // ]); + // $this->assertGreaterThanOrEqual(4, count($documents)); // May match case-insensitively + // + // // Test notStartsWith combined with other queries + // $documents = $database->find('movies', [ + // Query::notStartsWith('name', 'Work'), + // Query::equal('year', [2006]) + // ]); + // $this->assertLessThanOrEqual(4, count($documents)); // Subset of non-Work movies from 2006 + // } + // + // public function testFindNotEndsWith(): void + // { + // /** @var Database $database */ + // $database = static::getDatabase(); + // + // // Test notEndsWith - should return documents that don't end with 'Marvel' + // $documents = $database->find('movies', [ + // Query::notEndsWith('name', 'Marvel'), + // ]); + // + // $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'Marvel' + // + // // Test notEndsWith with non-existent suffix - should return all documents + // $documents = $database->find('movies', [ + // Query::notEndsWith('name', 'NonExistent'), + // ]); + // + // $this->assertEquals(6, count($documents)); + // + // // Test notEndsWith with partial suffix + // $documents = $database->find('movies', [ + // Query::notEndsWith('name', 'vel'), + // ]); + // + // $this->assertEquals(5, count($documents)); // All movies except the 1 ending with 'vel' (from 'Marvel') + // + // // Test notEndsWith with empty string - should return no documents (all strings end with empty) + // $documents = $database->find('movies', [ + // Query::notEndsWith('name', ''), + // ]); + // $this->assertEquals(0, count($documents)); // No documents since all strings end with empty string + // + // // Test notEndsWith with single character + // $documents = $database->find('movies', [ + // Query::notEndsWith('name', 'l'), + // ]); + // $this->assertGreaterThanOrEqual(5, count($documents)); // Movies not ending with 'l' + // + // // Test notEndsWith with case sensitivity (may be case-insensitive depending on DB) + // $documents = $database->find('movies', [ + // Query::notEndsWith('name', 'marvel'), // lowercase vs 'Marvel' + // ]); + // $this->assertGreaterThanOrEqual(5, count($documents)); // May match case-insensitively + // + // // Test notEndsWith combined with limit + // $documents = $database->find('movies', [ + // Query::notEndsWith('name', 'Marvel'), + // Query::limit(3) + // ]); + // $this->assertEquals(3, count($documents)); // Limited to 3 results + // $this->assertLessThanOrEqual(5, count($documents)); // But still excluding Marvel movies + // } + // + // public function testFindNotBetween(): void + // { + // /** @var Database $database */ + // $database = static::getDatabase(); + // + // // Test notBetween with price range - should return documents outside the range + // $documents = $database->find('movies', [ + // Query::notBetween('price', 25.94, 25.99), + // ]); + // $this->assertEquals(4, count($documents)); // All movies except the 2 in the price range + // + // // Test notBetween with range that includes no documents - should return all documents + // $documents = $database->find('movies', [ + // Query::notBetween('price', 30, 35), + // ]); + // $this->assertEquals(6, count($documents)); + // + // // Test notBetween with date range + // $documents = $database->find('movies', [ + // Query::notBetween('$createdAt', '1975-12-06', '2050-12-06'), + // ]); + // $this->assertEquals(0, count($documents)); // No movies outside this wide date range + // + // // Test notBetween with narrower date range + // $documents = $database->find('movies', [ + // Query::notBetween('$createdAt', '2000-01-01', '2001-01-01'), + // ]); + // $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + // + // // Test notBetween with updated date range + // $documents = $database->find('movies', [ + // Query::notBetween('$updatedAt', '2000-01-01T00:00:00.000+00:00', '2001-01-01T00:00:00.000+00:00'), + // ]); + // $this->assertEquals(6, count($documents)); // All movies should be outside this narrow range + // + // // Test notBetween with year range (integer values) + // $documents = $database->find('movies', [ + // Query::notBetween('year', 2005, 2007), + // ]); + // $this->assertLessThanOrEqual(6, count($documents)); // Movies outside 2005-2007 range + // + // // Test notBetween with reversed range (start > end) - should still work + // $documents = $database->find('movies', [ + // Query::notBetween('price', 25.99, 25.94), // Note: reversed order + // ]); + // $this->assertGreaterThanOrEqual(4, count($documents)); // Should handle reversed range gracefully + // + // // Test notBetween with same start and end values + // $documents = $database->find('movies', [ + // Query::notBetween('year', 2006, 2006), + // ]); + // $this->assertGreaterThanOrEqual(5, count($documents)); // All movies except those from exactly 2006 + // + // // Test notBetween combined with other filters + // $documents = $database->find('movies', [ + // Query::notBetween('price', 25.94, 25.99), + // Query::orderDesc('year'), + // Query::limit(2) + // ]); + // $this->assertEquals(2, count($documents)); // Limited results, ordered, excluding price range + // + // // Test notBetween with extreme ranges + // $documents = $database->find('movies', [ + // Query::notBetween('year', -1000, 1000), // Very wide range + // ]); + // $this->assertLessThanOrEqual(6, count($documents)); // Movies outside this range + // + // // Test notBetween with float precision + // $documents = $database->find('movies', [ + // Query::notBetween('price', 25.945, 25.955), // Very narrow range + // ]); + // $this->assertGreaterThanOrEqual(4, count($documents)); // Most movies should be outside this narrow range + // } public function testFindSelect(): void { From 901a7d82347ab23b5e6f8576a9f9d60503fb9b1a Mon Sep 17 00:00:00 2001 From: shimon Date: Thu, 18 Sep 2025 18:02:21 +0300 Subject: [PATCH 55/55] remove inversion --- src/Database/Validator/Label.php | 2 +- tests/unit/Validator/KeyTest.php | 8 +++---- tests/unit/Validator/LabelTest.php | 8 +++---- tests/unit/Validator/PermissionsTest.php | 27 ++++++++++++------------ 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/Database/Validator/Label.php b/src/Database/Validator/Label.php index 6cc4f031f..6c6cb8f4a 100644 --- a/src/Database/Validator/Label.php +++ b/src/Database/Validator/Label.php @@ -4,7 +4,7 @@ class Label extends Key { - protected string $message = 'Value must be a valid string between 1 and 36 chars containing only alphanumeric chars'; + protected string $message = 'Value must be a valid string between 1 and 255 chars containing only alphanumeric chars'; /** * Is valid. diff --git a/tests/unit/Validator/KeyTest.php b/tests/unit/Validator/KeyTest.php index ca85ae56b..e09ef402e 100644 --- a/tests/unit/Validator/KeyTest.php +++ b/tests/unit/Validator/KeyTest.php @@ -66,11 +66,9 @@ public function testValues(): void $this->assertEquals(false, $this->object->isValid('as+5dasdasdas')); $this->assertEquals(false, $this->object->isValid('as=5dasdasdas')); - // At most 36 chars - $this->assertEquals(true, $this->object->isValid('socialAccountForYoutubeSubscribersss')); - $this->assertEquals(false, $this->object->isValid('socialAccountForYoutubeSubscriberssss')); - $this->assertEquals(true, $this->object->isValid('5f058a89258075f058a89258075f058t9214')); - $this->assertEquals(false, $this->object->isValid('5f058a89258075f058a89258075f058tx9214')); + // At most 255 chars + $this->assertEquals(true, $this->object->isValid(str_repeat('a', 255))); + $this->assertEquals(false, $this->object->isValid(str_repeat('a', 256))); // Internal keys $validator = new Key(true); diff --git a/tests/unit/Validator/LabelTest.php b/tests/unit/Validator/LabelTest.php index c3eef2fb4..3d9bf7576 100644 --- a/tests/unit/Validator/LabelTest.php +++ b/tests/unit/Validator/LabelTest.php @@ -58,10 +58,8 @@ public function testValues(): void $this->assertEquals(false, $this->object->isValid('as+5dasdasdas')); $this->assertEquals(false, $this->object->isValid('as=5dasdasdas')); - // At most 36 chars - $this->assertEquals(true, $this->object->isValid('socialAccountForYoutubeSubscribersss')); - $this->assertEquals(false, $this->object->isValid('socialAccountForYoutubeSubscriberssss')); - $this->assertEquals(true, $this->object->isValid('5f058a89258075f058a89258075f058t9214')); - $this->assertEquals(false, $this->object->isValid('5f058a89258075f058a89258075f058tx9214')); + // At most 255 chars + $this->assertEquals(true, $this->object->isValid(str_repeat('a', 255))); + $this->assertEquals(false, $this->object->isValid(str_repeat('a', 256))); } } diff --git a/tests/unit/Validator/PermissionsTest.php b/tests/unit/Validator/PermissionsTest.php index bc03fb201..505e69dec 100644 --- a/tests/unit/Validator/PermissionsTest.php +++ b/tests/unit/Validator/PermissionsTest.php @@ -248,24 +248,25 @@ public function testInvalidPermissions(): void // team:$value, member:$value and user:$value must have valid Key for $value // No leading special chars $this->assertFalse($object->isValid([Permission::read(Role::user('_1234'))])); - $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); $this->assertFalse($object->isValid([Permission::read(Role::team('-1234'))])); - $this->assertEquals('Role "team" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "team" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); $this->assertFalse($object->isValid([Permission::read(Role::member('.1234'))])); - $this->assertEquals('Role "member" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "member" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); // No unsupported special characters $this->assertFalse($object->isValid([Permission::read(Role::user('12$4'))])); - $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); $this->assertFalse($object->isValid([Permission::read(Role::user('12&4'))])); - $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); $this->assertFalse($object->isValid([Permission::read(Role::user('ab(124'))])); - $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); - // Shorter than 36 chars - $this->assertTrue($object->isValid([Permission::read(Role::user(ID::custom('aaaaaaaabbbbbbbbccccccccddddddddeeee')))])); - $this->assertFalse($object->isValid([Permission::read(Role::user(ID::custom('aaaaaaaabbbbbbbbccccccccddddddddeeeee')))])); - $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + // Shorter than 255 chars + + $this->assertTrue($object->isValid([Permission::read(Role::user(ID::custom(str_repeat('a', 255))))])); + $this->assertFalse($object->isValid([Permission::read(Role::user(ID::custom(str_repeat('a', 256))))])); + $this->assertEquals('Role "user" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); // Permission role must begin with one of: member, role, team, user $this->assertFalse($object->isValid(['update("memmber:1234")'])); @@ -277,7 +278,7 @@ public function testInvalidPermissions(): void // Team permission $this->assertFalse($object->isValid([Permission::read(Role::team(ID::custom('_abcd')))])); - $this->assertEquals('Role "team" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "team" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); $this->assertFalse($object->isValid([Permission::read(Role::team(ID::custom('abcd/')))])); $this->assertEquals('Dimension must not be empty', $object->getDescription()); $this->assertFalse($object->isValid([Permission::read(Role::team(ID::custom(''), 'abcd'))])); @@ -287,9 +288,9 @@ public function testInvalidPermissions(): void $this->assertFalse($object->isValid([Permission::read(Role::team(ID::custom('abcd'), 'e/fgh'))])); $this->assertEquals('Only one dimension can be provided', $object->getDescription()); $this->assertFalse($object->isValid([Permission::read(Role::team(ID::custom('ab&cd3'), 'efgh'))])); - $this->assertEquals('Role "team" identifier value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "team" identifier value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); $this->assertFalse($object->isValid([Permission::read(Role::team(ID::custom('abcd'), 'ef*gh'))])); - $this->assertEquals('Role "team" dimension value is invalid: Parameter must contain at most 36 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); + $this->assertEquals('Role "team" dimension value is invalid: Parameter must contain at most 255 chars. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char', $object->getDescription()); // Permission-list length must be valid $object = new Permissions(100);