From b6616b3764f6a582a75b24b96e8c3586c71ea78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=B9=A4=E8=A1=A2?= Date: Mon, 25 May 2026 15:49:26 +0800 Subject: [PATCH] [cli] paimon java cli --- docs/docs/program-api/java-cli.md | 886 ++++++++++++++++++ paimon-cli/bin/install.sh | 208 ++++ paimon-cli/bin/paimon | 89 ++ paimon-cli/pom.xml | 240 +++++ .../java/org/apache/paimon/cli/CliArgs.java | 108 +++ .../java/org/apache/paimon/cli/CliConfig.java | 89 ++ .../java/org/apache/paimon/cli/Command.java | 46 + .../org/apache/paimon/cli/CommandContext.java | 58 ++ .../java/org/apache/paimon/cli/PaimonCli.java | 156 +++ .../apache/paimon/cli/PredicateParser.java | 344 +++++++ .../org/apache/paimon/cli/RowPrinter.java | 212 +++++ .../cli/commands/AlterDatabaseCommand.java | 105 +++ .../cli/commands/AlterTableCommand.java | 195 ++++ .../paimon/cli/commands/BranchCommand.java | 155 +++ .../cli/commands/CreateDatabaseCommand.java | 67 ++ .../cli/commands/CreateTableCommand.java | 252 +++++ .../cli/commands/DropDatabaseCommand.java | 77 ++ .../paimon/cli/commands/DropTableCommand.java | 69 ++ .../cli/commands/ExpireSnapshotsCommand.java | 76 ++ .../paimon/cli/commands/ExplainCommand.java | 296 ++++++ .../cli/commands/FullTextSearchCommand.java | 244 +++++ .../cli/commands/GetDatabaseCommand.java | 85 ++ .../cli/commands/ListDatabasesCommand.java | 55 ++ .../cli/commands/ListPartitionsCommand.java | 148 +++ .../cli/commands/ListTablesCommand.java | 61 ++ .../cli/commands/OrphanCleanCommand.java | 138 +++ .../paimon/cli/commands/ReadCommand.java | 242 +++++ .../cli/commands/RenameTableCommand.java | 60 ++ .../paimon/cli/commands/RollbackCommand.java | 75 ++ .../paimon/cli/commands/SchemaCommand.java | 185 ++++ .../paimon/cli/commands/SnapshotCommand.java | 63 ++ .../paimon/cli/commands/TagCommand.java | 105 +++ .../paimon/cli/commands/WriteCommand.java | 335 +++++++ .../paimon/cli/sql/PaimonCalciteTable.java | 194 ++++ .../apache/paimon/cli/sql/PaimonSchema.java | 117 +++ .../paimon/cli/sql/PaimonTypeMapping.java | 168 ++++ .../apache/paimon/cli/sql/ParseResult.java | 86 ++ .../apache/paimon/cli/sql/RexToPredicate.java | 272 ++++++ .../org/apache/paimon/cli/sql/SqlCommand.java | 147 +++ .../apache/paimon/cli/sql/SqlExecutor.java | 215 +++++ .../org/apache/paimon/cli/sql/SqlParser.java | 112 +++ .../services/org.apache.paimon.cli.Command | 39 + .../org/apache/paimon/cli/CliArgsTest.java | 113 +++ .../org/apache/paimon/cli/CliConfigTest.java | 70 ++ .../org/apache/paimon/cli/PaimonCliTest.java | 79 ++ .../cli/commands/AlterTableCommandTest.java | 175 ++++ .../cli/commands/BranchCommandTest.java | 125 +++ .../cli/commands/DatabaseCommandTest.java | 137 +++ .../cli/commands/ExplainCommandTest.java | 175 ++++ .../cli/commands/OrphanCleanCommandTest.java | 137 +++ .../paimon/cli/commands/ReadCommandTest.java | 169 ++++ .../cli/commands/SchemaCommandTest.java | 141 +++ .../cli/commands/SnapshotCommandTest.java | 175 ++++ .../paimon/cli/commands/TableCommandTest.java | 182 ++++ .../paimon/cli/commands/WriteCommandTest.java | 164 ++++ .../apache/paimon/cli/sql/SqlCommandTest.java | 297 ++++++ .../apache/paimon/cli/sql/SqlParserTest.java | 211 +++++ pom.xml | 1 + 58 files changed, 9225 insertions(+) create mode 100644 docs/docs/program-api/java-cli.md create mode 100755 paimon-cli/bin/install.sh create mode 100755 paimon-cli/bin/paimon create mode 100644 paimon-cli/pom.xml create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/CliArgs.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/CliConfig.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/Command.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/CommandContext.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/PaimonCli.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/PredicateParser.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/RowPrinter.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/AlterDatabaseCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/AlterTableCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/BranchCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/CreateDatabaseCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/CreateTableCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/DropDatabaseCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/DropTableCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/ExpireSnapshotsCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/ExplainCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/FullTextSearchCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/GetDatabaseCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListDatabasesCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListPartitionsCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListTablesCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/OrphanCleanCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/ReadCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/RenameTableCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/RollbackCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/SchemaCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/SnapshotCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/TagCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/commands/WriteCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonCalciteTable.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonSchema.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonTypeMapping.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/sql/ParseResult.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/sql/RexToPredicate.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlCommand.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlExecutor.java create mode 100644 paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlParser.java create mode 100644 paimon-cli/src/main/resources/META-INF/services/org.apache.paimon.cli.Command create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/CliArgsTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/CliConfigTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/PaimonCliTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/AlterTableCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/BranchCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/DatabaseCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/ExplainCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/OrphanCleanCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/ReadCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/SchemaCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/SnapshotCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/TableCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/commands/WriteCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/sql/SqlCommandTest.java create mode 100644 paimon-cli/src/test/java/org/apache/paimon/cli/sql/SqlParserTest.java diff --git a/docs/docs/program-api/java-cli.md b/docs/docs/program-api/java-cli.md new file mode 100644 index 000000000000..71e39b3f1c30 --- /dev/null +++ b/docs/docs/program-api/java-cli.md @@ -0,0 +1,886 @@ +--- +title: "Java CLI" +weight: 4 +type: docs +aliases: +- /program-api/java-cli.html +--- + + +# Java CLI + +Paimon provides a Java-based command-line interface (CLI) for interacting with Paimon catalogs and tables. +The CLI supports reading, writing, and managing tables directly from the command line without requiring a compute engine like Flink or Spark. + +## Installation + +### Quick Install (from source) + +```shell +cd paimon-cli +./bin/install.sh +``` + +This builds the fat jar and installs the `paimon` command to `/usr/local/bin`. You can specify a custom prefix: + +```shell +./bin/install.sh --prefix ~/.local +``` + +### Development Mode + +After building, use the bundled launcher directly without installing: + +```shell +cd paimon-cli +mvn clean package -DskipTests +./bin/paimon --help +``` + +## Basic Usage + +Before using the CLI, create a catalog configuration file. By default, the CLI looks for a `paimon.yaml` file in the current directory. + +**Filesystem Catalog:** + +```yaml +metastore: filesystem +warehouse: /path/to/warehouse +``` + +**REST Catalog:** + +```yaml +metastore: rest +uri: http://localhost:8080 +warehouse: catalog_name +``` + +**Hive Catalog:** + +```yaml +metastore: hive +warehouse: /path/to/warehouse +uri: thrift://localhost:9083 +``` + +Run a command: + +```shell +paimon [options] +``` + +**Global Options:** + +- `-c, --config PATH`: Path to catalog configuration file (default: `paimon.yaml`) +- `-h, --help`: Show help message and exit +- `-v, --version`: Show version and exit + +## Catalog Commands + +### list-databases + +List all databases in the catalog. + +```shell +paimon list-databases +``` + +Output: +``` +default +mydb +analytics +``` + +### list-tables + +List all tables in a database. + +```shell +paimon list-tables mydb +``` + +Output: +``` +orders +products +users +``` + +## Table Commands + +### schema + +Display table schema information including columns, types, primary keys, and partition keys. + +```shell +paimon schema mydb.users +``` + +**Options:** + +- `-f, --format`: Output format: `table` (default) or `json` + +Output (table format): +``` +Table: mydb.users + + Column Type Nullable + ------------------------------ ----------------------------------- -------- + id INT NOT NULL NO + name STRING YES + age INT YES + city STRING YES + +Primary keys: id +Partition keys: city + +Options: + bucket = 4 + changelog-producer = input +``` + +Output (json format): +```shell +paimon schema mydb.users --format json +``` + +```json +{"fields":[{"id":0,"name":"id","type":"INT NOT NULL"},{"id":1,"name":"name","type":"STRING"},{"id":2,"name":"age","type":"INT"},{"id":3,"name":"city","type":"STRING"}],"partitionKeys":["city"],"primaryKeys":["id"],"options":{"bucket":"4","changelog-producer":"input"},"comment":"User information table"} +``` + +The JSON output matches the `create-table --schema` input format, enabling round-trip `schema → create-table` workflows. + +### read + +Read data from a Paimon table and display it in tabular format. + +```shell +paimon read mydb.users +``` + +**Options:** + +- `-s, --select`: Select specific columns to read (comma-separated) +- `-w, --where`: Filter condition (SQL-like syntax) +- `-l, --limit`: Maximum number of rows to display (default: 100) +- `-f, --format`: Output format: `tsv` (default), `csv`, or `json` + +**Examples:** + +```shell +# Read with limit +paimon read mydb.users --limit 50 + +# Read specific columns +paimon read mydb.users --select id,name,age + +# Combine select and limit +paimon read mydb.users -s id,name -l 10 + +# Output as JSON (one JSON object per line) +paimon read mydb.users --format json + +# Output as CSV +paimon read mydb.users --format csv + +# Filter with WHERE clause +paimon read mydb.users --where "age > 18 AND city = 'Beijing'" + +# Combined filtering and projection +paimon read mydb.users -s name,age -w "age BETWEEN 20 AND 30" -l 50 +``` + +**WHERE Syntax:** + +The `--where` option supports SQL-like filter expressions: + +| Operator | Example | +|---|---| +| `=`, `!=`, `<>` | `age = 25`, `city != 'Beijing'` | +| `<`, `>`, `<=`, `>=` | `age > 18`, `price <= 100.0` | +| `IS NULL`, `IS NOT NULL` | `email IS NOT NULL` | +| `IN (...)` | `city IN ('Beijing', 'Shanghai')` | +| `NOT IN (...)` | `status NOT IN (0, -1)` | +| `BETWEEN ... AND ...` | `age BETWEEN 20 AND 30` | +| `LIKE` | `name LIKE 'A%'` | +| `AND`, `OR` | `age > 18 AND city = 'Beijing'` | +| Parentheses | `(a = 1 OR b = 2) AND c > 0` | + +{{< hint info >}} +The WHERE filter leverages Paimon's scan-level predicate pushdown for partition pruning and file skipping. This provides significant performance benefits for partitioned tables. +{{< /hint >}} + +Output (default tsv format): +``` +id name age city +1 Alice 25 Beijing +2 Bob 30 Shanghai +3 Charlie 35 Guangzhou +``` + +Output (json format): +```json +{"id":1,"name":"Alice","age":25,"city":"Beijing"} +{"id":2,"name":"Bob","age":30,"city":"Shanghai"} +{"id":3,"name":"Charlie","age":35,"city":"Guangzhou"} +``` + +### write + +Write data from a local file into a Paimon table. + +```shell +paimon write data.csv mydb.users +``` + +**Options:** + +- `-f, --format`: Input format: `csv` (default) or `json` +- `-d, --delimiter`: CSV delimiter (default: `,`) +- `-e, --encoding`: File encoding (default: `utf-8`) + +**Examples:** + +```shell +# Write from CSV +paimon write users.csv mydb.users + +# Write from JSON +paimon write users.json mydb.users --format json + +# Write from TSV (tab-separated) +paimon write data.tsv mydb.users --delimiter "\t" + +# Specify encoding +paimon write data.csv mydb.users --encoding gbk +``` + +**CSV Format:** + +The CSV file may optionally include a header row matching the table's column names. If a header is detected, it is skipped automatically. + +```csv +id,name,age,city +1,Alice,25,Beijing +2,Bob,30,Shanghai +3,Charlie,35,Guangzhou +``` + +**JSON Format:** + +The JSON file must be an array of objects with keys matching the table column names. + +```json +[ + {"id": 1, "name": "Alice", "age": 25, "city": "Beijing"}, + {"id": 2, "name": "Bob", "age": 30, "city": "Shanghai"} +] +``` + +Output: +``` +Successfully wrote 3 rows into 'mydb.users'. +``` + +### create-table + +Create a new table from a schema JSON file. + +```shell +paimon create-table mydb.users --schema schema.json +``` + +**Options:** + +- `-s, --schema`: Path to schema JSON file — **Required** +- `-i, --ignore-if-exists`: Do not raise error if table already exists + +**Schema JSON Format:** + +```json +{ + "fields": [ + {"name": "id", "type": "BIGINT"}, + {"name": "name", "type": "STRING"}, + {"name": "age", "type": "INT"}, + {"name": "city", "type": "STRING"}, + {"name": "created_at", "type": "TIMESTAMP"} + ], + "partitionKeys": ["city"], + "primaryKeys": ["id"], + "options": { + "bucket": "4", + "changelog-producer": "input" + }, + "comment": "User information table" +} +``` + +**Supported Data Types:** + +| Type | Description | +|---|---| +| `BOOLEAN` | Boolean value | +| `TINYINT`, `SMALLINT`, `INT`, `BIGINT` | Integer types | +| `FLOAT`, `DOUBLE` | Floating point types | +| `DECIMAL(p, s)` | Fixed-point decimal | +| `CHAR(n)`, `VARCHAR(n)`, `STRING` | String types | +| `DATE` | Date without time | +| `TIMESTAMP`, `TIMESTAMP(p)` | Timestamp with precision | +| `BINARY`, `VARBINARY`, `BYTES` | Binary types | + +### drop-table + +Drop a table from the catalog. + +```shell +paimon drop-table mydb.old_table +``` + +**Options:** + +- `-i, --ignore-if-not-exists`: Do not raise error if table does not exist + +Output: +``` +Table 'mydb.old_table' dropped successfully. +``` + +### rename-table + +Rename a table in the catalog. + +```shell +paimon rename-table mydb.old_name mydb.new_name +``` + +Output: +``` +Table 'mydb.old_name' renamed to 'mydb.new_name' successfully. +``` + +### alter-table + +Alter a table's schema or options. Supports multiple sub-actions. + +**Set/Remove Options:** + +```shell +# Set a table option +paimon alter-table mydb.users set-option --key bucket --value 8 + +# Remove a table option +paimon alter-table mydb.users remove-option --key changelog-producer +``` + +**Column Operations:** + +```shell +# Add a new column +paimon alter-table mydb.users add-column --name email --type STRING + +# Add column with comment and position +paimon alter-table mydb.users add-column --name score --type DOUBLE --comment "User score" --after age + +# Add column at the beginning +paimon alter-table mydb.users add-column --name row_id --type BIGINT --first + +# Drop a column +paimon alter-table mydb.users drop-column --name old_field + +# Rename a column +paimon alter-table mydb.users rename-column --name city --new-name region + +# Alter column type or comment +paimon alter-table mydb.users alter-column --name age --type BIGINT +paimon alter-table mydb.users alter-column --name age --comment "User age in years" +paimon alter-table mydb.users alter-column --name age --after name +``` + +**Update Table Comment:** + +```shell +paimon alter-table mydb.users update-comment --comment "Updated user table" +``` + +**Options:** + +- `-i, --ignore-if-not-exists`: Do not raise error if table does not exist + +### snapshot + +Display the latest snapshot information of a table as JSON. + +```shell +paimon snapshot mydb.users +``` + +Output: +```json +{"version":3,"id":5,"schemaId":0,"baseManifestList":"manifest-list-...","deltaManifestList":"manifest-list-...","changelogManifestList":null,"commitUser":"...","commitIdentifier":4,"commitKind":"APPEND","timeMillis":1700000000000,"logOffsets":{},"totalRecordCount":1000,"deltaRecordCount":200,"changelogRecordCount":0,"watermark":-9223372036854775808} +``` + +### list-partitions + +List all partitions of a table with summary statistics. + +```shell +paimon list-partitions mydb.events +``` + +Output: +``` +Partition RecordCount FileSizeInBytes FileCount LastFileCreationTime +dt=2024-01-01 15000 4521983 3 2024-01-02T00:05:00 +dt=2024-01-02 22000 6812045 5 2024-01-03T00:05:00 +dt=2024-01-03 18500 5604211 4 2024-01-04T00:05:00 +``` + +### expire-snapshots + +Expire old snapshots to reclaim storage. + +```shell +paimon expire-snapshots mydb.users +``` + +**Options:** + +- `--retain-max`: Maximum number of snapshots to retain (default: 50) +- `--retain-min`: Minimum number of snapshots to retain (default: 10) + +```shell +paimon expire-snapshots mydb.users --retain-max 20 --retain-min 5 +``` + +### tag + +Manage table tags (named snapshots). + +```shell +# Create a tag from the latest snapshot +paimon tag mydb.users create --name release-v1 + +# Create a tag from a specific snapshot +paimon tag mydb.users create --name release-v1 --snapshot 5 + +# Delete a tag +paimon tag mydb.users delete --name release-v1 + +# List all tags +paimon tag mydb.users list +``` + +### rollback + +Roll back a table to a previous snapshot or tag. + +```shell +# Rollback to a snapshot +paimon rollback mydb.users --snapshot 5 + +# Rollback to a tag +paimon rollback mydb.users --tag release-v1 +``` + +### explain + +Show the scan plan of a query without reading any data files. Useful for previewing the pruning effect of a predicate. + +```shell +paimon explain mydb.events +``` + +**Options:** + +- `-s, --select`: Columns to project (comma-separated) +- `-w, --where`: Filter condition (SQL-like syntax) +- `-l, --limit`: Row limit to push down + +**Examples:** + +```shell +# Full table scan plan +paimon explain mydb.events + +# Push filter and projection +paimon explain mydb.events --where "dt = '2024-01-01' AND id > 100" -s dt,id,val +``` + +Output: +``` +== Paimon Scan Plan == +Table: mydb.events (PK, HASH_FIXED) +Snapshot: 5 (schema 0) +Predicate: dt = '2024-01-01' AND id > 100 +Projection: [dt, id, val] +Limit: + +Splits: 3 + raw-convertible: 3 / 3 + with DV: 0 / 3 + files/split: min=1 max=2 avg=1.33 + size/split: min=2.6 KiB p50=4.1 KiB p95=5.2 KiB max=5.2 KiB +Files: 4 +Total size: 15.1 KiB +Estimated rows: 150 (merged: 148) +Level histogram: L0=2 L1=2 +Deletion files: 0 +Partitions hit: 1 +Buckets hit: 3 +``` + +### full-text-search + +Perform full-text search on a table column using a Tantivy index. + +```shell +paimon full-text-search mydb.articles --column content --query "paimon lake" +``` + +**Options:** + +- `-c, --column`: Text column to search on (required) +- `-q, --query`: Query text to search for (required) +- `-l, --limit`: Maximum results (default: 10) +- `-s, --select`: Columns to display (comma-separated) +- `-f, --format`: Output format: `tsv` (default) / `csv` / `json` + +{{< hint info >}} +Requires `paimon-tantivy-index` on the classpath. Place the jar in the `plugins/` directory. +{{< /hint >}} + +### sql + +Execute SQL queries on Paimon tables. Supports one-shot mode and interactive REPL. + +**One-shot mode:** + +```shell +paimon sql "SELECT * FROM mydb.users WHERE age > 18 LIMIT 10" +paimon sql "SELECT id, name FROM mydb.users ORDER BY age DESC LIMIT 5" +paimon sql "SHOW DATABASES" +``` + +**Interactive REPL:** + +```shell +paimon sql +``` + +``` +Paimon SQL (type 'help' for usage, 'exit' to quit) + +paimon> USE mydb; +Using database 'mydb'. + +paimon> SHOW TABLES; +orders +users + +paimon> SELECT id, name, age FROM users + > WHERE age > 20 + > ORDER BY age DESC + > LIMIT 5; +id name age +3 Charlie 35 +2 Bob 30 +5 Eve 28 +(3 rows) + +paimon> exit +Bye! +``` + +**Supported SQL:** + +```sql +SELECT [col1, col2, ... | *] FROM [db.]table [WHERE ...] [ORDER BY col [ASC|DESC], ...] [LIMIT n] +SHOW DATABASES +SHOW TABLES [IN database] +USE database +``` + +## Database Commands + +### create-database + +Create a new database. + +```shell +paimon create-database mydb +``` + +**Options:** + +- `-i, --ignore-if-exists`: Do not raise error if database already exists + +Output: +``` +Database 'mydb' created successfully. +``` + +### drop-database + +Drop an existing database. + +```shell +paimon drop-database mydb +``` + +**Options:** + +- `-i, --ignore-if-not-exists`: Do not raise error if database does not exist +- `--cascade`: Drop all tables in the database before dropping it + +**Examples:** + +```shell +# Drop empty database +paimon drop-database mydb + +# Drop database with all tables +paimon drop-database mydb --cascade + +# Ignore if not exists +paimon drop-database mydb -i +``` + +Output: +``` +Database 'mydb' dropped successfully. +``` + +### get-database + +Show database details including options and comment. + +```shell +paimon get-database mydb +``` + +Output: +```json +{"name":"mydb","options":{"owner":"alice"},"comment":"My analytics database"} +``` + +### alter-database + +Alter database properties. + +```shell +# Set properties +paimon alter-database mydb --set owner=alice,team=data + +# Remove properties +paimon alter-database mydb --remove old_key1,old_key2 + +# Combine set and remove +paimon alter-database mydb --set owner=bob --remove deprecated_key +``` + +**Options:** + +- `--set`: Properties to set (comma-separated key=value pairs) +- `--remove`: Properties to remove (comma-separated keys) +- `-i, --ignore-if-not-exists`: Do not raise error if database does not exist + +## SPI Extension + +The Java CLI uses Java SPI (ServiceLoader) for command discovery, making it easy for downstream projects to extend with custom commands. + +### Implementing a Custom Command + +```java +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +public class MyCommand implements Command { + + @Override + public String name() { + return "my-command"; + } + + @Override + public String description() { + return "My custom command"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + // Your command logic here + System.out.println("Hello from my custom command!"); + } + + @Override + public String usage() { + return "Usage: paimon my-command [options]\n\nDescription of my command."; + } +} +``` + +### Registering the Command + +Create a file at `META-INF/services/org.apache.paimon.cli.Command` in your project's resources: + +``` +com.mycompany.MyCommand +``` + +### Using the Extension + +Place your jar on the classpath alongside paimon-cli: + +```shell +java -cp "paimon-cli.jar:my-extension.jar" org.apache.paimon.cli.PaimonCli my-command +``` + +### Customizing the Entry Point + +For a fully branded CLI, subclass `PaimonCli`: + +```java +import org.apache.paimon.cli.PaimonCli; + +public class MyCli extends PaimonCli { + + @Override + protected String programName() { + return "mycli"; + } + + @Override + protected String banner() { + return "MyCLI v1.0 - powered by Apache Paimon"; + } + + public static void main(String[] args) throws Exception { + new MyCli().run(args); + } +} +``` + +## Filesystem Support + +The CLI bundles OSS and S3 filesystem support out of the box — no extra downloads needed. Just configure `paimon.yaml` and use it. + +### Aliyun OSS + +```yaml +metastore: filesystem +warehouse: oss://my-bucket/warehouse + +fs.oss.endpoint: oss-cn-hangzhou.aliyuncs.com +fs.oss.accessKeyId: your-access-key-id +fs.oss.accessKeySecret: your-access-key-secret +``` + +Or nested YAML style: + +```yaml +metastore: filesystem +warehouse: oss://my-bucket/warehouse + +fs: + oss: + endpoint: oss-cn-hangzhou.aliyuncs.com + accessKeyId: your-access-key-id + accessKeySecret: your-access-key-secret +``` + +Then use commands as usual: + +```shell +paimon list-databases +paimon read mydb.users --limit 10 +``` + +### Amazon S3 + +```yaml +metastore: filesystem +warehouse: s3://my-bucket/warehouse + +s3.endpoint: s3.us-east-1.amazonaws.com +s3.access-key: your-access-key +s3.secret-key: your-secret-key +``` + +### HDFS + +HDFS is supported natively. Set `HADOOP_HOME` and the CLI will automatically include the Hadoop classpath: + +```shell +export HADOOP_HOME=/opt/hadoop +paimon list-databases +``` + +Or configure Hadoop options in `paimon.yaml`: + +```yaml +metastore: filesystem +warehouse: hdfs://namenode:8020/warehouse +hadoop-conf-dir: /etc/hadoop/conf +``` + +### Additional Filesystems + +For filesystems not bundled (Azure, GCS, etc.), place the corresponding `paimon-*` jar into the `plugins/` directory or set `PAIMON_PLUGINS_DIR`: + +```shell +cp paimon-azure-*.jar plugins/ +``` + +## Configuration Reference + +The `paimon.yaml` file supports all Paimon catalog options as flat key-value pairs or nested YAML: + +```yaml +# Flat style +metastore: filesystem +warehouse: /data/paimon + +# Nested style (automatically flattened to dotted keys) +fs: + oss: + endpoint: oss-cn-hangzhou.aliyuncs.com + accessKeyId: your-key + accessKeySecret: your-secret +``` + +Both styles above produce the same result. Nested keys are flattened with `.` separator (e.g., `fs.oss.endpoint`). + +### Common Options + +| Option | Description | +|---|---| +| `metastore` | Catalog type: `filesystem`, `hive`, or `rest` | +| `warehouse` | Warehouse path (local, `oss://`, `s3://`, `hdfs://`) | +| `fs.oss.endpoint` | Aliyun OSS endpoint | +| `fs.oss.accessKeyId` | Aliyun OSS access key ID | +| `fs.oss.accessKeySecret` | Aliyun OSS access key secret | +| `s3.endpoint` | S3 endpoint | +| `s3.access-key` | S3 access key | +| `s3.secret-key` | S3 secret key | +| `hadoop-conf-dir` | Path to Hadoop configuration directory | +| `uri` | Hive metastore URI (for Hive catalog) | diff --git a/paimon-cli/bin/install.sh b/paimon-cli/bin/install.sh new file mode 100755 index 000000000000..ca7740bef57e --- /dev/null +++ b/paimon-cli/bin/install.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Paimon CLI install script. +# Builds the fat jar (if needed) and installs the `paimon` command to a +# directory on PATH (default: /usr/local/bin). +# +# Usage: +# ./install.sh # install to /usr/local/bin +# ./install.sh --prefix ~/.local # install to ~/.local/bin +# ./install.sh --with oss # bundle OSS plugin +# ./install.sh --with s3 # bundle S3 plugin +# ./install.sh --with oss --with s3 # bundle both +# ./install.sh --no-build # skip Maven build, assume jar exists + +set -euo pipefail + +PREFIX="/usr/local" +DO_BUILD=true +PLUGINS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --prefix) + PREFIX="$2" + shift 2 + ;; + --with) + PLUGINS+=("$2") + shift 2 + ;; + --no-build) + DO_BUILD=false + shift + ;; + -h|--help) + echo "Usage: install.sh [--prefix DIR] [--with PLUGIN]... [--no-build]" + echo "" + echo "Options:" + echo " --prefix DIR Installation prefix (default: /usr/local)" + echo " The paimon command is placed in DIR/bin/" + echo " The jar and plugins are placed in DIR/lib/paimon-cli/" + echo " --with PLUGIN Include a filesystem plugin: oss, s3" + echo " Can be specified multiple times." + echo " --no-build Skip Maven build, use existing jar in target/" + echo "" + echo "Examples:" + echo " ./install.sh --with oss # install with Aliyun OSS support" + echo " ./install.sh --with s3 --with oss # install with S3 and OSS support" + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + exit 1 + ;; + esac +done + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +REPO_ROOT="$(dirname "$PROJECT_DIR")" + +echo "==> Paimon CLI Installer" +echo " Project: $PROJECT_DIR" +echo " Prefix: $PREFIX" +if [ ${#PLUGINS[@]} -gt 0 ]; then + echo " Plugins: ${PLUGINS[*]}" +fi +echo "" + +# Validate plugin names +for plugin in "${PLUGINS[@]+"${PLUGINS[@]}"}"; do + case "$plugin" in + oss|s3) ;; + *) + echo "Error: Unknown plugin '$plugin'. Supported: oss, s3" >&2 + exit 1 + ;; + esac +done + +# Build if needed +if [ "$DO_BUILD" = true ]; then + MODULES="-pl paimon-cli" + for plugin in "${PLUGINS[@]+"${PLUGINS[@]}"}"; do + MODULES="$MODULES,paimon-filesystems/paimon-$plugin" + done + echo "==> Building ($MODULES)..." + (cd "$REPO_ROOT" && mvn package $MODULES -am -DskipTests -q) + echo " Build complete." +fi + +# Locate the fat jar +JAR=$(find "$PROJECT_DIR/target" -name "paimon-cli-*.jar" ! -name "original-*" ! -name "*-sources*" ! -name "*-tests*" | head -1) +if [ -z "$JAR" ]; then + echo "Error: Cannot find paimon-cli jar in $PROJECT_DIR/target/" >&2 + echo "Run 'mvn package -DskipTests' first or remove --no-build flag." >&2 + exit 1 +fi + +JAR_NAME="$(basename "$JAR")" +INSTALL_BIN="$PREFIX/bin" +INSTALL_LIB="$PREFIX/lib/paimon-cli" +INSTALL_PLUGINS="$INSTALL_LIB/plugins" + +# Create directories +mkdir -p "$INSTALL_BIN" "$INSTALL_LIB" "$INSTALL_PLUGINS" + +# Copy main jar +echo "==> Installing $JAR_NAME to $INSTALL_LIB/" +cp "$JAR" "$INSTALL_LIB/$JAR_NAME" + +# Copy plugin jars +for plugin in "${PLUGINS[@]+"${PLUGINS[@]}"}"; do + PLUGIN_JAR=$(find "$REPO_ROOT/paimon-filesystems/paimon-$plugin/target" \ + -name "paimon-$plugin-*.jar" ! -name "original-*" ! -name "*-sources*" ! -name "*-tests*" 2>/dev/null | head -1) + if [ -z "$PLUGIN_JAR" ]; then + echo "Warning: Cannot find paimon-$plugin jar. Skipping." >&2 + continue + fi + PLUGIN_JAR_NAME="$(basename "$PLUGIN_JAR")" + echo "==> Installing plugin $PLUGIN_JAR_NAME to $INSTALL_PLUGINS/" + cp "$PLUGIN_JAR" "$INSTALL_PLUGINS/$PLUGIN_JAR_NAME" +done + +# Create launcher script +LAUNCHER="$INSTALL_BIN/paimon" +echo "==> Installing launcher to $LAUNCHER" + +cat > "$LAUNCHER" <<'WRAPPER' +#!/usr/bin/env bash +set -euo pipefail + +PAIMON_LIB="PLACEHOLDER_LIB" +PAIMON_JAR=$(find "$PAIMON_LIB" -maxdepth 1 -name "paimon-cli-*.jar" | head -1) + +if [ -z "$PAIMON_JAR" ]; then + echo "Error: paimon-cli jar not found in $PAIMON_LIB" >&2 + exit 1 +fi + +if [ -n "${JAVA_HOME:-}" ]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA="java" +fi + +JVM_OPTS="${PAIMON_CLI_OPTS:--Xmx512m}" + +# Build classpath: main jar + plugins + hadoop +CLASSPATH="$PAIMON_JAR" + +PLUGIN_DIR="$PAIMON_LIB/plugins" +if [ -d "$PLUGIN_DIR" ]; then + for jar in "$PLUGIN_DIR"/*.jar; do + [ -f "$jar" ] && CLASSPATH="$CLASSPATH:$jar" + done +fi + +if [ -n "${HADOOP_HOME:-}" ]; then + HADOOP_CP="$("$HADOOP_HOME/bin/hadoop" classpath 2>/dev/null || true)" + [ -n "$HADOOP_CP" ] && CLASSPATH="$CLASSPATH:$HADOOP_CP" +fi + +exec "$JAVA" $JVM_OPTS -cp "$CLASSPATH" org.apache.paimon.cli.PaimonCli "$@" +WRAPPER + +# Replace placeholder with actual lib path +sed -i.bak "s|PLACEHOLDER_LIB|$INSTALL_LIB|g" "$LAUNCHER" +rm -f "$LAUNCHER.bak" +chmod +x "$LAUNCHER" + +echo "" +echo "==> Installation complete!" +echo "" +echo " paimon command: $LAUNCHER" +echo " jar location: $INSTALL_LIB/$JAR_NAME" +if [ -d "$INSTALL_PLUGINS" ] && ls "$INSTALL_PLUGINS"/*.jar &>/dev/null; then + echo " plugins: $INSTALL_PLUGINS/" +fi +echo "" + +# Check if the install location is on PATH +if ! echo "$PATH" | tr ':' '\n' | grep -qx "$INSTALL_BIN"; then + echo " NOTE: $INSTALL_BIN is not on your PATH." + echo " Add it with:" + echo "" + echo " export PATH=\"$INSTALL_BIN:\$PATH\"" + echo "" +fi + +echo " Run 'paimon --help' to get started." diff --git a/paimon-cli/bin/paimon b/paimon-cli/bin/paimon new file mode 100755 index 000000000000..e9869a2502a2 --- /dev/null +++ b/paimon-cli/bin/paimon @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Paimon CLI launcher script. +# +# Plugin jars (e.g. paimon-oss, paimon-s3) are loaded from: +# 1. $PAIMON_HOME/plugins/ (alongside this script) +# 2. $PAIMON_PLUGINS_DIR (custom override) +# +# Environment variables: +# JAVA_HOME - Java installation directory +# HADOOP_HOME - Hadoop installation (classpath auto-appended) +# PAIMON_CLI_OPTS - JVM options (default: -Xmx512m) +# PAIMON_PLUGINS_DIR - Custom plugins directory + +set -euo pipefail + +# Resolve symlinks and find the real script location +SCRIPT="$(readlink -f "${BASH_SOURCE[0]}" 2>/dev/null || realpath "${BASH_SOURCE[0]}" 2>/dev/null || echo "${BASH_SOURCE[0]}")" +BIN_DIR="$(cd "$(dirname "$SCRIPT")" && pwd)" +PAIMON_HOME="$(dirname "$BIN_DIR")" + +# Find the fat jar +PAIMON_JAR="" +if [ -d "$PAIMON_HOME/target" ]; then + PAIMON_JAR=$(find "$PAIMON_HOME/target" -name "paimon-cli-*.jar" ! -name "*-original-*" ! -name "*-sources*" ! -name "*-tests*" | head -1) +fi + +if [ -z "$PAIMON_JAR" ] && [ -d "$PAIMON_HOME/lib" ]; then + PAIMON_JAR=$(find "$PAIMON_HOME/lib" -name "paimon-cli-*.jar" | head -1) +fi + +if [ -z "$PAIMON_JAR" ]; then + echo "Error: Cannot find paimon-cli jar." >&2 + echo "Run 'mvn package -DskipTests' in $PAIMON_HOME first, or place the jar in $PAIMON_HOME/lib/" >&2 + exit 1 +fi + +# Check Java +if [ -n "${JAVA_HOME:-}" ]; then + JAVA="$JAVA_HOME/bin/java" +else + JAVA="java" +fi + +if ! command -v "$JAVA" &>/dev/null; then + echo "Error: Java not found. Set JAVA_HOME or add java to PATH." >&2 + exit 1 +fi + +# JVM options (user can override via PAIMON_CLI_OPTS) +JVM_OPTS="${PAIMON_CLI_OPTS:--Xmx512m}" + +# Build classpath: main jar + plugins + hadoop +CLASSPATH="$PAIMON_JAR" + +# Collect plugin jars +PLUGINS_DIR="${PAIMON_PLUGINS_DIR:-$PAIMON_HOME/plugins}" +if [ -d "$PLUGINS_DIR" ]; then + for jar in "$PLUGINS_DIR"/*.jar; do + [ -f "$jar" ] && CLASSPATH="$CLASSPATH:$jar" + done +fi + +# Hadoop classpath (optional) +if [ -n "${HADOOP_HOME:-}" ]; then + HADOOP_CP="$("$HADOOP_HOME/bin/hadoop" classpath 2>/dev/null || true)" + if [ -n "$HADOOP_CP" ]; then + CLASSPATH="$CLASSPATH:$HADOOP_CP" + fi +fi + +exec "$JAVA" $JVM_OPTS -cp "$CLASSPATH" org.apache.paimon.cli.PaimonCli "$@" diff --git a/paimon-cli/pom.xml b/paimon-cli/pom.xml new file mode 100644 index 000000000000..d64d81efc19f --- /dev/null +++ b/paimon-cli/pom.xml @@ -0,0 +1,240 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-parent + 1.5-SNAPSHOT + + + paimon-cli + Paimon : CLI + jar + + + + org.apache.paimon + paimon-core + ${project.version} + + + + org.apache.paimon + paimon-format + ${project.version} + + + + org.yaml + snakeyaml + 2.0 + + + + org.apache.calcite + calcite-core + 1.37.0 + + + com.fasterxml.jackson.core + * + + + org.apache.commons + commons-lang3 + + + org.apache.commons + commons-math3 + + + com.yahoo.datasketches + * + + + net.hydromatic + aggdesigner-algorithm + + + org.apache.commons + commons-dbcp2 + + + org.apache.commons + commons-pool2 + + + commons-io + commons-io + + + com.googlecode.json-simple + json-simple + + + javax.annotation + javax.annotation-api + + + + + + com.google.protobuf + protobuf-java + 3.19.6 + + + + + org.apache.paimon + paimon-oss + ${project.version} + provided + + + + org.apache.paimon + paimon-s3 + ${project.version} + provided + + + + + org.junit.jupiter + junit-jupiter + test + + + + org.assertj + assertj-core + ${assertj.version} + test + + + + org.apache.paimon + paimon-test-utils + ${project.version} + test + + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + test + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + test + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + test + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + paimon-codegen/** + javax/annotation/**/package.html + META-INF/maven/javax.annotation/** + + + + + + org.apache.paimon:paimon-codegen + org.apache.paimon:paimon-codegen-loader + + + + + org.apache.paimon.cli.PaimonCli + + + + + + + + + + diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/CliArgs.java b/paimon-cli/src/main/java/org/apache/paimon/cli/CliArgs.java new file mode 100644 index 000000000000..5495dd094dfc --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/CliArgs.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Lightweight argument parser supporting {@code --key value}, {@code -k value} short aliases, and + * positional arguments. + */ +public class CliArgs { + + private final Map options = new HashMap<>(); + private final String[] positional; + + public CliArgs(String[] args, String... aliases) { + this(args, Collections.emptySet(), aliases); + } + + /** + * @param args raw command-line arguments + * @param booleanFlags flag names (without dashes) that take no value + * @param aliases pairs of short-to-long mappings, e.g. "-s", "--select" + */ + public CliArgs(String[] args, Set booleanFlags, String... aliases) { + Map aliasMap = new HashMap<>(); + for (int i = 0; i + 1 < aliases.length; i += 2) { + aliasMap.put(aliases[i], aliases[i + 1]); + } + + List posList = new ArrayList<>(); + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (arg.startsWith("-")) { + String key = aliasMap.getOrDefault(arg, arg); + key = key.replaceFirst("^--?", ""); + if (booleanFlags.contains(key)) { + options.put(key, "true"); + } else if (i + 1 < args.length && !args[i + 1].startsWith("-")) { + options.put(key, args[++i]); + } else { + options.put(key, "true"); + } + } else { + posList.add(arg); + } + } + positional = posList.toArray(new String[0]); + } + + @Nullable + public String get(String key) { + return options.get(key); + } + + public String getOrDefault(String key, String defaultValue) { + String value = options.get(key); + return value != null ? value : defaultValue; + } + + public String require(String key, String description) { + String value = options.get(key); + if (value == null || value.isEmpty()) { + System.err.println("Missing required option: --" + key + " (" + description + ")"); + System.exit(1); + } + return value; + } + + public String positional(int index, String description) { + if (index >= positional.length) { + System.err.println("Missing argument: " + description); + System.exit(1); + } + return positional[index]; + } + + public int positionalCount() { + return positional.length; + } + + public boolean hasHelp() { + return "true".equals(options.get("help")) || "true".equals(options.get("h")); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/CliConfig.java b/paimon-cli/src/main/java/org/apache/paimon/cli/CliConfig.java new file mode 100644 index 000000000000..b052c418ea7f --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/CliConfig.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import org.apache.paimon.options.Options; + +import org.yaml.snakeyaml.Yaml; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +/** Loads catalog configuration from a paimon.yaml file into Paimon {@link Options}. */ +public class CliConfig { + + public static final String DEFAULT_CONFIG_FILE = "paimon.yaml"; + + private CliConfig() {} + + public static Options load(String configPath) { + Path path = Paths.get(configPath); + if (!Files.exists(path)) { + System.err.println("Configuration file not found: " + configPath); + System.err.println( + "Please create a paimon.yaml file. Example:\n" + + " metastore: filesystem\n" + + " warehouse: /path/to/warehouse"); + System.exit(1); + } + + try (InputStream in = new FileInputStream(path.toFile())) { + Yaml yaml = new Yaml(); + Map yamlMap = yaml.load(in); + if (yamlMap == null || yamlMap.isEmpty()) { + System.err.println("Empty configuration file: " + configPath); + System.exit(1); + } + return toOptions(yamlMap); + } catch (FileNotFoundException e) { + System.err.println("Configuration file not found: " + configPath); + System.exit(1); + } catch (IOException e) { + System.err.println("Failed to read configuration file: " + e.getMessage()); + System.exit(1); + } + return new Options(); // unreachable + } + + @SuppressWarnings("unchecked") + private static Options toOptions(Map yamlMap) { + Options options = new Options(); + flattenMap("", yamlMap, options); + return options; + } + + @SuppressWarnings("unchecked") + private static void flattenMap(String prefix, Map map, Options options) { + for (Map.Entry entry : map.entrySet()) { + String key = prefix.isEmpty() ? entry.getKey() : prefix + "." + entry.getKey(); + Object value = entry.getValue(); + if (value instanceof Map) { + flattenMap(key, (Map) value, options); + } else if (value != null) { + options.set(key, String.valueOf(value)); + } + } + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/Command.java b/paimon-cli/src/main/java/org/apache/paimon/cli/Command.java new file mode 100644 index 000000000000..fc578f44c79b --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/Command.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +/** + * SPI interface for CLI commands. Implementations are discovered via {@link + * java.util.ServiceLoader}. External projects can register additional commands by providing a + * META-INF/services/org.apache.paimon.cli.Command file. + */ +public interface Command { + + /** Command name used on the command line (e.g. "read", "list-tables"). */ + String name(); + + /** One-line description shown in help output. */ + String description(); + + /** + * Execute the command. + * + * @param ctx provides access to Catalog and Options + * @param args remaining arguments after the command name + */ + void execute(CommandContext ctx, String[] args) throws Exception; + + /** Multi-line usage/help text. Printed when --help is passed. */ + default String usage() { + return "Usage: paimon " + name() + " [options]"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/CommandContext.java b/paimon-cli/src/main/java/org/apache/paimon/cli/CommandContext.java new file mode 100644 index 000000000000..592690a60f23 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/CommandContext.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.options.Options; + +/** + * Execution context shared across commands. Provides lazy access to the Catalog instance and raw + * Options loaded from paimon.yaml. + */ +public class CommandContext implements AutoCloseable { + + private final Options options; + private Catalog catalog; + + public CommandContext(Options options) { + this.options = options; + } + + public Options getOptions() { + return options; + } + + public Catalog getCatalog() { + if (catalog == null) { + CatalogContext ctx = CatalogContext.create(options); + catalog = CatalogFactory.createCatalog(ctx); + } + return catalog; + } + + @Override + public void close() throws Exception { + if (catalog != null) { + catalog.close(); + catalog = null; + } + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/PaimonCli.java b/paimon-cli/src/main/java/org/apache/paimon/cli/PaimonCli.java new file mode 100644 index 000000000000..f8e7fe1efbbf --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/PaimonCli.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import org.apache.paimon.options.Options; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.ServiceLoader; + +/** + * Main entry point for the Paimon CLI. Discovers {@link Command} implementations via SPI and + * dispatches to the matching command. + * + *

Usage: {@code java -jar paimon-cli.jar [--config paimon.yaml] [args...]} + * + *

Subclasses can override {@link #banner()} and {@link #programName()} to customize the CLI for + * downstream projects. + */ +public class PaimonCli { + + private final Map commands = new LinkedHashMap<>(); + + public PaimonCli() { + loadCommands(getClass().getClassLoader()); + } + + public PaimonCli(ClassLoader classLoader) { + loadCommands(classLoader); + } + + private void loadCommands(ClassLoader classLoader) { + ServiceLoader loader = ServiceLoader.load(Command.class, classLoader); + for (Command cmd : loader) { + commands.put(cmd.name(), cmd); + } + } + + /** Register an additional command programmatically. */ + public void registerCommand(Command command) { + commands.put(command.name(), command); + } + + public Map getCommands() { + return commands; + } + + protected String programName() { + return "paimon"; + } + + protected String banner() { + return "Apache Paimon CLI"; + } + + public void run(String[] args) { + String configPath = CliConfig.DEFAULT_CONFIG_FILE; + int argStart = 0; + + // Parse global options before the command name + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if (("--config".equals(arg) || "-c".equals(arg)) && i + 1 < args.length) { + configPath = args[i + 1]; + argStart = i + 2; + i++; + } else if ("--help".equals(arg) || "-h".equals(arg)) { + printUsage(); + return; + } else if ("--version".equals(arg) || "-v".equals(arg)) { + printVersion(); + return; + } else { + argStart = i; + break; + } + } + + if (argStart >= args.length) { + printUsage(); + return; + } + + String commandName = args[argStart]; + String[] subArgs = + argStart + 1 < args.length + ? Arrays.copyOfRange(args, argStart + 1, args.length) + : new String[0]; + + Command command = commands.get(commandName); + if (command == null) { + System.err.println("Unknown command: " + commandName); + printUsage(); + System.exit(1); + return; + } + + // Check for --help on the sub-command + for (String subArg : subArgs) { + if ("--help".equals(subArg) || "-h".equals(subArg)) { + System.err.println(command.usage()); + return; + } + } + + Options options = CliConfig.load(configPath); + try (CommandContext ctx = new CommandContext(options)) { + command.execute(ctx, subArgs); + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + System.exit(1); + } + } + + private void printUsage() { + System.err.println(banner()); + System.err.println(); + System.err.println("Usage: " + programName() + " [--config paimon.yaml] [args]"); + System.err.println(); + System.err.println("Commands:"); + for (Command cmd : commands.values()) { + System.err.println(String.format(" %-20s %s", cmd.name(), cmd.description())); + } + System.err.println(); + System.err.println("Global options:"); + System.err.println(" -c, --config PATH Path to paimon.yaml (default: ./paimon.yaml)"); + System.err.println(" -h, --help Show this help message"); + System.err.println(" -v, --version Show version"); + } + + private void printVersion() { + System.out.println(programName() + " (Apache Paimon CLI)"); + } + + public static void main(String[] args) { + PaimonCli cli = new PaimonCli(); + cli.run(args); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/PredicateParser.java b/paimon-cli/src/main/java/org/apache/paimon/cli/PredicateParser.java new file mode 100644 index 000000000000..f86340a8fd48 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/PredicateParser.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.RowType; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; + +/** + * Recursive descent parser for SQL-like WHERE expressions. Supports: =, !=, <>, <, >, <=, >=, IS + * NULL, IS NOT NULL, IN (...), BETWEEN ... AND ..., LIKE, AND, OR, parentheses. + */ +public class PredicateParser { + + private final RowType rowType; + private final PredicateBuilder builder; + private final List fieldNames; + + private String input; + private int pos; + + public PredicateParser(RowType rowType) { + this.rowType = rowType; + this.builder = new PredicateBuilder(rowType); + this.fieldNames = rowType.getFieldNames(); + } + + public Predicate parse(String expression) { + this.input = expression; + this.pos = 0; + Predicate result = parseOr(); + skipWhitespace(); + if (pos < input.length()) { + throw new IllegalArgumentException( + "Unexpected token at position " + pos + ": " + input.substring(pos)); + } + return result; + } + + private Predicate parseOr() { + Predicate left = parseAnd(); + while (true) { + skipWhitespace(); + if (matchKeyword("OR")) { + Predicate right = parseAnd(); + left = PredicateBuilder.or(left, right); + } else { + break; + } + } + return left; + } + + private Predicate parseAnd() { + Predicate left = parseAtom(); + while (true) { + skipWhitespace(); + if (matchKeyword("AND")) { + Predicate right = parseAtom(); + left = PredicateBuilder.and(left, right); + } else { + break; + } + } + return left; + } + + private Predicate parseAtom() { + skipWhitespace(); + if (pos < input.length() && input.charAt(pos) == '(') { + pos++; + Predicate inner = parseOr(); + skipWhitespace(); + expect(')'); + return inner; + } + + String field = parseIdentifier(); + int idx = fieldIndex(field); + DataType fieldType = rowType.getTypeAt(idx); + + skipWhitespace(); + + if (matchKeyword("IS")) { + skipWhitespace(); + if (matchKeyword("NOT")) { + skipWhitespace(); + expectKeyword("NULL"); + return builder.isNotNull(idx); + } else { + expectKeyword("NULL"); + return builder.isNull(idx); + } + } + + if (matchKeyword("NOT")) { + skipWhitespace(); + if (matchKeyword("IN")) { + List values = parseInList(fieldType); + return builder.notIn(idx, values); + } + throw new IllegalArgumentException("Expected IN after NOT at position " + pos); + } + + if (matchKeyword("IN")) { + List values = parseInList(fieldType); + return builder.in(idx, values); + } + + if (matchKeyword("BETWEEN")) { + skipWhitespace(); + Object lower = castLiteral(parseLiteral(), fieldType); + skipWhitespace(); + expectKeyword("AND"); + skipWhitespace(); + Object upper = castLiteral(parseLiteral(), fieldType); + return builder.between(idx, lower, upper); + } + + if (matchKeyword("LIKE")) { + skipWhitespace(); + Object pattern = castLiteral(parseLiteral(), fieldType); + return builder.like(idx, pattern); + } + + String op = parseOperator(); + skipWhitespace(); + Object value = castLiteral(parseLiteral(), fieldType); + + switch (op) { + case "=": + return builder.equal(idx, value); + case "!=": + case "<>": + return builder.notEqual(idx, value); + case "<": + return builder.lessThan(idx, value); + case "<=": + return builder.lessOrEqual(idx, value); + case ">": + return builder.greaterThan(idx, value); + case ">=": + return builder.greaterOrEqual(idx, value); + default: + throw new IllegalArgumentException("Unknown operator: " + op); + } + } + + private List parseInList(DataType fieldType) { + skipWhitespace(); + expect('('); + List values = new ArrayList<>(); + while (true) { + skipWhitespace(); + values.add(castLiteral(parseLiteral(), fieldType)); + skipWhitespace(); + if (pos < input.length() && input.charAt(pos) == ',') { + pos++; + } else { + break; + } + } + expect(')'); + return values; + } + + private String parseIdentifier() { + skipWhitespace(); + int start = pos; + while (pos < input.length() + && (Character.isLetterOrDigit(input.charAt(pos)) || input.charAt(pos) == '_')) { + pos++; + } + if (pos == start) { + throw new IllegalArgumentException("Expected identifier at position " + pos); + } + return input.substring(start, pos); + } + + private String parseOperator() { + skipWhitespace(); + if (pos + 1 < input.length()) { + String two = input.substring(pos, pos + 2); + if ("!=".equals(two) || "<>".equals(two) || "<=".equals(two) || ">=".equals(two)) { + pos += 2; + return two; + } + } + if (pos < input.length()) { + char c = input.charAt(pos); + if (c == '=' || c == '<' || c == '>') { + pos++; + return String.valueOf(c); + } + } + throw new IllegalArgumentException("Expected operator at position " + pos); + } + + private String parseLiteral() { + skipWhitespace(); + if (pos >= input.length()) { + throw new IllegalArgumentException("Expected literal at end of input"); + } + char c = input.charAt(pos); + if (c == '\'') { + pos++; + StringBuilder sb = new StringBuilder(); + while (pos < input.length()) { + char ch = input.charAt(pos); + if (ch == '\'') { + if (pos + 1 < input.length() && input.charAt(pos + 1) == '\'') { + sb.append('\''); + pos += 2; + } else { + pos++; + break; + } + } else { + sb.append(ch); + pos++; + } + } + return "'" + sb.toString() + "'"; + } + int start = pos; + if (c == '-' || c == '+') { + pos++; + } + while (pos < input.length() + && (Character.isDigit(input.charAt(pos)) || input.charAt(pos) == '.')) { + pos++; + } + if (pos == start) { + throw new IllegalArgumentException("Expected literal at position " + pos); + } + return input.substring(start, pos); + } + + private Object castLiteral(String literal, DataType type) { + if (literal.startsWith("'") && literal.endsWith("'")) { + String strValue = literal.substring(1, literal.length() - 1); + DataTypeRoot root = type.getTypeRoot(); + switch (root) { + case VARCHAR: + case CHAR: + return BinaryString.fromString(strValue); + case DATE: + return LocalDate.parse(strValue).toEpochDay(); + default: + return BinaryString.fromString(strValue); + } + } + DataTypeRoot root = type.getTypeRoot(); + switch (root) { + case TINYINT: + return Byte.parseByte(literal); + case SMALLINT: + return Short.parseShort(literal); + case INTEGER: + return Integer.parseInt(literal); + case BIGINT: + return Long.parseLong(literal); + case FLOAT: + return Float.parseFloat(literal); + case DOUBLE: + return Double.parseDouble(literal); + case DECIMAL: + return BigDecimal.valueOf(Double.parseDouble(literal)); + case BOOLEAN: + return Boolean.parseBoolean(literal); + default: + return BinaryString.fromString(literal); + } + } + + private int fieldIndex(String name) { + int idx = fieldNames.indexOf(name); + if (idx < 0) { + throw new IllegalArgumentException("Unknown field: " + name); + } + return idx; + } + + private void skipWhitespace() { + while (pos < input.length() && Character.isWhitespace(input.charAt(pos))) { + pos++; + } + } + + private boolean matchKeyword(String keyword) { + int savedPos = pos; + int len = keyword.length(); + if (pos + len > input.length()) { + return false; + } + if (input.substring(pos, pos + len).equalsIgnoreCase(keyword)) { + if (pos + len >= input.length() + || !Character.isLetterOrDigit(input.charAt(pos + len))) { + pos += len; + return true; + } + } + pos = savedPos; + return false; + } + + private void expectKeyword(String keyword) { + if (!matchKeyword(keyword)) { + throw new IllegalArgumentException("Expected '" + keyword + "' at position " + pos); + } + } + + private void expect(char c) { + skipWhitespace(); + if (pos >= input.length() || input.charAt(pos) != c) { + throw new IllegalArgumentException("Expected '" + c + "' at position " + pos); + } + pos++; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/RowPrinter.java b/paimon-cli/src/main/java/org/apache/paimon/cli/RowPrinter.java new file mode 100644 index 000000000000..f363a62185cd --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/RowPrinter.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import org.apache.paimon.data.InternalArray; +import org.apache.paimon.data.InternalMap; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeChecks; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; + +import java.util.List; + +/** Converts {@link InternalRow} field values to printable strings. */ +public class RowPrinter { + + private RowPrinter() {} + + public static String getFieldValue(InternalRow row, int index, DataType type) { + DataTypeRoot root = type.getTypeRoot(); + switch (root) { + case BOOLEAN: + return String.valueOf(row.getBoolean(index)); + case TINYINT: + return String.valueOf(row.getByte(index)); + case SMALLINT: + return String.valueOf(row.getShort(index)); + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + return String.valueOf(row.getInt(index)); + case BIGINT: + return String.valueOf(row.getLong(index)); + case FLOAT: + return String.valueOf(row.getFloat(index)); + case DOUBLE: + return String.valueOf(row.getDouble(index)); + case DECIMAL: + { + int precision = DataTypeChecks.getPrecision(type); + int scale = DataTypeChecks.getScale(type); + return row.getDecimal(index, precision, scale).toBigDecimal().toPlainString(); + } + case CHAR: + case VARCHAR: + return row.getString(index).toString(); + case BINARY: + case VARBINARY: + return formatBytes(row.getBinary(index)); + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + { + int precision = DataTypeChecks.getPrecision(type); + return row.getTimestamp(index, precision).toString(); + } + case ARRAY: + return formatArray(row.getArray(index), ((ArrayType) type).getElementType()); + case MAP: + case MULTISET: + return formatMap(row.getMap(index), type); + case ROW: + return formatRow( + row.getRow(index, DataTypeChecks.getFieldCount(type)), (RowType) type); + default: + return ""; + } + } + + public static boolean isNumericOrBoolean(DataTypeRoot root) { + switch (root) { + case BOOLEAN: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case FLOAT: + case DOUBLE: + case DECIMAL: + return true; + default: + return false; + } + } + + private static String formatArray(InternalArray array, DataType elementType) { + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < array.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(formatArrayElement(array, i, elementType)); + } + sb.append("]"); + return sb.toString(); + } + + private static String formatMap(InternalMap map, DataType mapType) { + DataType keyType; + DataType valueType; + if (mapType instanceof MapType) { + keyType = ((MapType) mapType).getKeyType(); + valueType = ((MapType) mapType).getValueType(); + } else { + keyType = ((MultisetType) mapType).getElementType(); + valueType = new org.apache.paimon.types.IntType(); + } + InternalArray keys = map.keyArray(); + InternalArray values = map.valueArray(); + StringBuilder sb = new StringBuilder("{"); + for (int i = 0; i < keys.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(formatArrayElement(keys, i, keyType)); + sb.append(":"); + sb.append(formatArrayElement(values, i, valueType)); + } + sb.append("}"); + return sb.toString(); + } + + private static String formatRow(InternalRow row, RowType rowType) { + List fields = rowType.getFields(); + StringBuilder sb = new StringBuilder("{"); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append(fields.get(i).name()).append(":"); + if (row.isNullAt(i)) { + sb.append("null"); + } else { + sb.append(getFieldValue(row, i, fields.get(i).type())); + } + } + sb.append("}"); + return sb.toString(); + } + + private static String formatArrayElement(InternalArray array, int index, DataType type) { + if (array.isNullAt(index)) { + return "null"; + } + switch (type.getTypeRoot()) { + case BOOLEAN: + return String.valueOf(array.getBoolean(index)); + case TINYINT: + return String.valueOf(array.getByte(index)); + case SMALLINT: + return String.valueOf(array.getShort(index)); + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + return String.valueOf(array.getInt(index)); + case BIGINT: + return String.valueOf(array.getLong(index)); + case FLOAT: + return String.valueOf(array.getFloat(index)); + case DOUBLE: + return String.valueOf(array.getDouble(index)); + case CHAR: + case VARCHAR: + return "\"" + array.getString(index).toString() + "\""; + case DECIMAL: + { + int precision = DataTypeChecks.getPrecision(type); + int scale = DataTypeChecks.getScale(type); + return array.getDecimal(index, precision, scale).toBigDecimal().toPlainString(); + } + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + { + int precision = DataTypeChecks.getPrecision(type); + return array.getTimestamp(index, precision).toString(); + } + default: + return String.valueOf(array.getString(index)); + } + } + + private static String formatBytes(byte[] data) { + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < data.length; i++) { + if (i > 0) { + sb.append(","); + } + sb.append(data[i] & 0xFF); + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/AlterDatabaseCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/AlterDatabaseCommand.java new file mode 100644 index 000000000000..ff7448737e0d --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/AlterDatabaseCommand.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.PropertyChange; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** Alters database properties. */ +public class AlterDatabaseCommand implements Command { + + private static final Set BOOLEAN_FLAGS; + + static { + Set flags = new HashSet<>(); + flags.add("ignore-if-not-exists"); + BOOLEAN_FLAGS = Collections.unmodifiableSet(flags); + } + + @Override + public String name() { + return "alter-database"; + } + + @Override + public String description() { + return "Alter database properties"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = + new CliArgs( + args, + BOOLEAN_FLAGS, + "--set", + "--set", + "--remove", + "--remove", + "-i", + "--ignore-if-not-exists"); + String database = parsed.positional(0, "DATABASE"); + boolean ignoreIfNotExists = "true".equals(parsed.get("ignore-if-not-exists")); + + String setStr = parsed.get("set"); + String removeStr = parsed.get("remove"); + + if (setStr == null && removeStr == null) { + System.err.println("Must specify --set or --remove."); + System.err.println(usage()); + return; + } + + List changes = new ArrayList<>(); + if (setStr != null) { + for (String kv : setStr.split(",")) { + String[] pair = kv.split("=", 2); + if (pair.length == 2) { + changes.add(PropertyChange.setProperty(pair[0].trim(), pair[1].trim())); + } + } + } + if (removeStr != null) { + for (String key : removeStr.split(",")) { + changes.add(PropertyChange.removeProperty(key.trim())); + } + } + + ctx.getCatalog().alterDatabase(database, changes, ignoreIfNotExists); + System.out.println("Database '" + database + "' altered successfully."); + } + + @Override + public String usage() { + return "Usage: paimon alter-database DATABASE [options]\n\n" + + "Alter database properties.\n\n" + + "Options:\n" + + " --set key1=value1,key2=value2 Set properties\n" + + " --remove key1,key2 Remove properties\n" + + " -i, --ignore-if-not-exists Do not raise error if database does not exist"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/AlterTableCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/AlterTableCommand.java new file mode 100644 index 000000000000..56dffd15c84e --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/AlterTableCommand.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.schema.SchemaChange; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** Alters a table's schema or options. */ +public class AlterTableCommand implements Command { + + private static final Set BOOLEAN_FLAGS; + + static { + Set flags = new HashSet<>(); + flags.add("ignore-if-not-exists"); + flags.add("first"); + BOOLEAN_FLAGS = Collections.unmodifiableSet(flags); + } + + @Override + public String name() { + return "alter-table"; + } + + @Override + public String description() { + return "Alter table schema or options"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = + new CliArgs( + args, + BOOLEAN_FLAGS, + "-n", + "--name", + "-t", + "--type", + "-c", + "--comment", + "-k", + "--key", + "-v", + "--value", + "-m", + "--new-name", + "--after", + "--after", + "--first", + "--first", + "-i", + "--ignore-if-not-exists"); + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String action = parsed.positional(1, "ACTION"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + boolean ignoreIfNotExists = "true".equals(parsed.get("ignore-if-not-exists")); + + List changes = new ArrayList<>(); + + switch (action) { + case "set-option": + { + String key = parsed.require("key", "option key"); + String value = parsed.require("value", "option value"); + changes.add(SchemaChange.setOption(key, value)); + break; + } + case "remove-option": + { + String key = parsed.require("key", "option key"); + changes.add(SchemaChange.removeOption(key)); + break; + } + case "add-column": + { + String colName = parsed.require("name", "column name"); + String typeName = parsed.require("type", "column type"); + String comment = parsed.get("comment"); + SchemaChange.Move move = parseMove(parsed, colName); + changes.add( + SchemaChange.addColumn( + colName, + CreateTableCommand.parseDataType(typeName), + comment, + move)); + break; + } + case "drop-column": + { + String colName = parsed.require("name", "column name"); + changes.add(SchemaChange.dropColumn(colName)); + break; + } + case "rename-column": + { + String colName = parsed.require("name", "current column name"); + String newName = parsed.require("new-name", "new column name"); + changes.add(SchemaChange.renameColumn(colName, newName)); + break; + } + case "alter-column": + { + String colName = parsed.require("name", "column name"); + String typeName = parsed.get("type"); + String comment = parsed.get("comment"); + if (typeName != null) { + changes.add( + SchemaChange.updateColumnType( + colName, CreateTableCommand.parseDataType(typeName))); + } + if (comment != null) { + changes.add(SchemaChange.updateColumnComment(colName, comment)); + } + SchemaChange.Move move = parseMove(parsed, colName); + if (move != null) { + changes.add(SchemaChange.updateColumnPosition(move)); + } + if (changes.isEmpty()) { + System.err.println( + "Must specify at least one of --type, --comment, --first, or --after."); + return; + } + break; + } + case "update-comment": + { + String comment = parsed.require("comment", "table comment"); + changes.add(SchemaChange.updateComment(comment)); + break; + } + default: + System.err.println("Unknown alter action: " + action); + System.err.println(usage()); + return; + } + + ctx.getCatalog() + .alterTable(Identifier.create(parts[0], parts[1]), changes, ignoreIfNotExists); + System.out.println("Table '" + tableId + "' altered successfully."); + } + + private static SchemaChange.Move parseMove(CliArgs parsed, String colName) { + boolean first = "true".equals(parsed.get("first")); + String after = parsed.get("after"); + if (first) { + return SchemaChange.Move.first(colName); + } else if (after != null) { + return SchemaChange.Move.after(colName, after); + } + return null; + } + + @Override + public String usage() { + return "Usage: paimon alter-table DATABASE.TABLE ACTION [options]\n\n" + + "Actions:\n" + + " set-option Set a table option (--key K --value V)\n" + + " remove-option Remove a table option (--key K)\n" + + " add-column Add a column (--name N --type T [--comment C] " + + "[--first|--after REF])\n" + + " drop-column Drop a column (--name N)\n" + + " rename-column Rename a column (--name OLD --new-name NEW)\n" + + " alter-column Alter a column (--name N [--type T] [--comment C] " + + "[--first|--after REF])\n" + + " update-comment Update table comment (--comment C)\n\n" + + "Options:\n" + + " -i, --ignore-if-not-exists Do not raise error if table does not exist"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/BranchCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/BranchCommand.java new file mode 100644 index 000000000000..fb53c829768f --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/BranchCommand.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.table.DataTable; +import org.apache.paimon.table.Table; + +import java.util.List; + +/** Manages branches on a table (create, delete, rename, list, fast-forward, merge). */ +public class BranchCommand implements Command { + + @Override + public String name() { + return "branch"; + } + + @Override + public String description() { + return "Manage table branches"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args); + String action = parsed.positional(0, "ACTION"); + String tableId = parsed.positional(1, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + Identifier identifier = Identifier.create(parts[0], parts[1]); + + switch (action) { + case "create": + executeCreate(ctx, identifier, parsed); + break; + case "delete": + executeDelete(ctx, identifier, parsed); + break; + case "rename": + executeRename(ctx, identifier, parsed); + break; + case "list": + executeList(ctx, identifier); + break; + case "fast-forward": + executeFastForward(ctx, identifier, parsed); + break; + case "merge": + executeMerge(ctx, identifier, parsed); + break; + default: + System.err.println("Unknown action: " + action); + System.err.println( + "Valid actions: create, delete, rename, list, " + "fast-forward, merge"); + } + } + + private void executeCreate(CommandContext ctx, Identifier id, CliArgs parsed) throws Exception { + Table table = ctx.getCatalog().getTable(id); + String branchName = parsed.require("name", "branch name"); + String tag = parsed.get("tag"); + if (tag != null) { + table.createBranch(branchName, tag); + System.out.println( + "Branch '" + branchName + "' created from tag '" + tag + "' on '" + id + "'."); + } else { + table.createBranch(branchName); + System.out.println("Branch '" + branchName + "' created on '" + id + "'."); + } + } + + private void executeDelete(CommandContext ctx, Identifier id, CliArgs parsed) throws Exception { + Table table = ctx.getCatalog().getTable(id); + String branchName = parsed.require("name", "branch name"); + table.deleteBranch(branchName); + System.out.println("Branch '" + branchName + "' deleted from '" + id + "'."); + } + + private void executeRename(CommandContext ctx, Identifier id, CliArgs parsed) throws Exception { + Table table = ctx.getCatalog().getTable(id); + String branchName = parsed.require("name", "branch name"); + String newName = parsed.require("new-name", "new branch name"); + table.renameBranch(branchName, newName); + System.out.println( + "Branch '" + branchName + "' renamed to '" + newName + "' on '" + id + "'."); + } + + private void executeList(CommandContext ctx, Identifier id) throws Exception { + Table table = ctx.getCatalog().getTable(id); + List branches = ((DataTable) table).branchManager().branches(); + if (branches.isEmpty()) { + System.out.println("No branches found for '" + id + "'."); + } else { + for (String branch : branches) { + System.out.println(branch); + } + } + } + + private void executeFastForward(CommandContext ctx, Identifier id, CliArgs parsed) + throws Exception { + Table table = ctx.getCatalog().getTable(id); + String branchName = parsed.require("name", "branch name"); + table.fastForward(branchName); + System.out.println("Branch '" + branchName + "' fast-forwarded to main on '" + id + "'."); + } + + private void executeMerge(CommandContext ctx, Identifier id, CliArgs parsed) throws Exception { + Table table = ctx.getCatalog().getTable(id); + String source = parsed.require("source", "source branch"); + String target = parsed.get("target"); + if (target == null) { + target = "main"; + } + table.mergeBranch(source, target); + System.out.println("Branch '" + source + "' merged into '" + target + "' on '" + id + "'."); + } + + @Override + public String usage() { + return "Usage: paimon branch DATABASE.TABLE [options]\n\n" + + "Actions:\n" + + " create Create a branch (--name NAME [--tag TAG])\n" + + " delete Delete a branch (--name NAME)\n" + + " rename Rename a branch (--name NAME --new-name NEW)\n" + + " list List all branches\n" + + " fast-forward Fast-forward branch to main (--name NAME)\n" + + " merge Merge branches (--source SRC [--target TGT])\n\n" + + "Options:\n" + + " --name NAME Branch name\n" + + " --tag TAG Create branch from this tag\n" + + " --new-name NEW New name for rename\n" + + " --source SRC Source branch for merge\n" + + " --target TGT Target branch for merge (default: main)"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/CreateDatabaseCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/CreateDatabaseCommand.java new file mode 100644 index 000000000000..2bc26c3c77d2 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/CreateDatabaseCommand.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** Creates a new database. */ +public class CreateDatabaseCommand implements Command { + + private static final Set BOOLEAN_FLAGS; + + static { + Set flags = new HashSet<>(); + flags.add("ignore-if-exists"); + BOOLEAN_FLAGS = Collections.unmodifiableSet(flags); + } + + @Override + public String name() { + return "create-database"; + } + + @Override + public String description() { + return "Create a new database"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args, BOOLEAN_FLAGS, "-i", "--ignore-if-exists"); + String database = parsed.positional(0, "DATABASE"); + boolean ignoreIfExists = "true".equals(parsed.get("ignore-if-exists")); + + ctx.getCatalog().createDatabase(database, ignoreIfExists); + System.out.println("Database '" + database + "' created successfully."); + } + + @Override + public String usage() { + return "Usage: paimon create-database DATABASE [options]\n\n" + + "Create a new database.\n\n" + + "Options:\n" + + " -i, --ignore-if-exists Do not raise error if database already exists"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/CreateTableCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/CreateTableCommand.java new file mode 100644 index 000000000000..3ebbaf093af3 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/CreateTableCommand.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Creates a new table from a schema JSON file. */ +public class CreateTableCommand implements Command { + + private static final Set BOOLEAN_FLAGS; + + static { + Set flags = new HashSet<>(); + flags.add("ignore-if-exists"); + BOOLEAN_FLAGS = Collections.unmodifiableSet(flags); + } + + @Override + public String name() { + return "create-table"; + } + + @Override + public String description() { + return "Create a new table from a schema JSON file"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = + new CliArgs(args, BOOLEAN_FLAGS, "-s", "--schema", "-i", "--ignore-if-exists"); + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + String schemaFile = parsed.require("schema", "path to schema JSON file"); + boolean ignoreIfExists = "true".equals(parsed.get("ignore-if-exists")); + + String schemaJson = readFile(schemaFile); + Schema schema = parseSchema(schemaJson); + + ctx.getCatalog().createTable(Identifier.create(parts[0], parts[1]), schema, ignoreIfExists); + System.out.println("Table '" + tableId + "' created successfully."); + } + + @Override + public String usage() { + return "Usage: paimon create-table DATABASE.TABLE --schema FILE [options]\n\n" + + "Create a new table from a schema JSON file.\n\n" + + "Schema JSON format:\n" + + " {\n" + + " \"fields\": [{\"name\": \"id\", \"type\": \"BIGINT\"}, ...],\n" + + " \"partitionKeys\": [\"dt\"],\n" + + " \"primaryKeys\": [\"id\"],\n" + + " \"options\": {\"bucket\": \"4\"}\n" + + " }\n\n" + + "Options:\n" + + " -s, --schema FILE Path to schema JSON file (required)\n" + + " -i, --ignore-if-exists Do not raise error if table already exists"; + } + + @SuppressWarnings("unchecked") + static Schema parseSchema(String json) throws Exception { + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(json); + + // Parse fields + JsonNode fieldsNode = root.get("fields"); + if (fieldsNode == null || !fieldsNode.isArray()) { + throw new IllegalArgumentException("Schema must contain 'fields' array."); + } + + Schema.Builder builder = Schema.newBuilder(); + for (JsonNode fieldNode : fieldsNode) { + String name = fieldNode.get("name").asText(); + String type = fieldNode.get("type").asText(); + builder.column(name, DataTypes.STRING()); // placeholder, will be resolved below + } + + // Build schema manually with proper types + List columnNames = new ArrayList<>(); + List columnTypes = new ArrayList<>(); + for (JsonNode fieldNode : fieldsNode) { + String name = fieldNode.get("name").asText(); + String typeStr = fieldNode.get("type").asText(); + columnNames.add(name); + columnTypes.add(parseDataType(typeStr)); + } + + // Parse partition keys + List partitionKeys = new ArrayList<>(); + JsonNode partNode = root.get("partitionKeys"); + if (partNode != null && partNode.isArray()) { + for (JsonNode n : partNode) { + partitionKeys.add(n.asText()); + } + } + + // Parse primary keys + List primaryKeys = new ArrayList<>(); + JsonNode pkNode = root.get("primaryKeys"); + if (pkNode != null && pkNode.isArray()) { + for (JsonNode n : pkNode) { + primaryKeys.add(n.asText()); + } + } + + // Parse options + Map options = new HashMap<>(); + JsonNode optNode = root.get("options"); + if (optNode != null && optNode.isObject()) { + Iterator> it = optNode.fields(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + options.put(entry.getKey(), entry.getValue().asText()); + } + } + + // Parse comment + String comment = null; + JsonNode commentNode = root.get("comment"); + if (commentNode != null && !commentNode.isNull()) { + comment = commentNode.asText(); + } + + // Build final schema + Schema.Builder schemaBuilder = Schema.newBuilder(); + for (int i = 0; i < columnNames.size(); i++) { + schemaBuilder.column(columnNames.get(i), columnTypes.get(i)); + } + schemaBuilder.partitionKeys(partitionKeys.toArray(new String[0])); + schemaBuilder.primaryKey(primaryKeys.toArray(new String[0])); + schemaBuilder.options(options); + if (comment != null) { + schemaBuilder.comment(comment); + } + return schemaBuilder.build(); + } + + static org.apache.paimon.types.DataType parseDataType(String typeStr) { + String upper = typeStr.trim().toUpperCase(); + switch (upper) { + case "BOOLEAN": + return DataTypes.BOOLEAN(); + case "TINYINT": + return DataTypes.TINYINT(); + case "SMALLINT": + return DataTypes.SMALLINT(); + case "INT": + case "INTEGER": + return DataTypes.INT(); + case "BIGINT": + return DataTypes.BIGINT(); + case "FLOAT": + return DataTypes.FLOAT(); + case "DOUBLE": + return DataTypes.DOUBLE(); + case "STRING": + return DataTypes.STRING(); + case "DATE": + return DataTypes.DATE(); + case "TIMESTAMP": + return DataTypes.TIMESTAMP(6); + case "BINARY": + return DataTypes.BINARY(1); + case "VARBINARY": + return DataTypes.VARBINARY(Integer.MAX_VALUE); + case "BYTES": + return DataTypes.BYTES(); + default: + break; + } + // DECIMAL(p, s) + if (upper.startsWith("DECIMAL")) { + String params = upper.substring("DECIMAL".length()).trim(); + if (params.startsWith("(") && params.endsWith(")")) { + String[] ps = params.substring(1, params.length() - 1).split(","); + int precision = Integer.parseInt(ps[0].trim()); + int scale = ps.length > 1 ? Integer.parseInt(ps[1].trim()) : 0; + return DataTypes.DECIMAL(precision, scale); + } + return DataTypes.DECIMAL(38, 18); + } + // VARCHAR(n) + if (upper.startsWith("VARCHAR")) { + String params = upper.substring("VARCHAR".length()).trim(); + if (params.startsWith("(") && params.endsWith(")")) { + int length = Integer.parseInt(params.substring(1, params.length() - 1).trim()); + return DataTypes.VARCHAR(length); + } + return DataTypes.STRING(); + } + // CHAR(n) + if (upper.startsWith("CHAR")) { + String params = upper.substring("CHAR".length()).trim(); + if (params.startsWith("(") && params.endsWith(")")) { + int length = Integer.parseInt(params.substring(1, params.length() - 1).trim()); + return DataTypes.CHAR(length); + } + return DataTypes.CHAR(1); + } + // TIMESTAMP(p) + if (upper.startsWith("TIMESTAMP")) { + String params = upper.substring("TIMESTAMP".length()).trim(); + if (params.startsWith("(") && params.endsWith(")")) { + int precision = Integer.parseInt(params.substring(1, params.length() - 1).trim()); + return DataTypes.TIMESTAMP(precision); + } + return DataTypes.TIMESTAMP(6); + } + return DataTypes.STRING(); + } + + private static String readFile(String path) throws IOException { + return new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.UTF_8); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/DropDatabaseCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/DropDatabaseCommand.java new file mode 100644 index 000000000000..861127327628 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/DropDatabaseCommand.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** Drops a database. */ +public class DropDatabaseCommand implements Command { + + private static final Set BOOLEAN_FLAGS; + + static { + Set flags = new HashSet<>(); + flags.add("ignore-if-not-exists"); + flags.add("cascade"); + BOOLEAN_FLAGS = Collections.unmodifiableSet(flags); + } + + @Override + public String name() { + return "drop-database"; + } + + @Override + public String description() { + return "Drop a database"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = + new CliArgs( + args, + BOOLEAN_FLAGS, + "-i", + "--ignore-if-not-exists", + "--cascade", + "--cascade"); + String database = parsed.positional(0, "DATABASE"); + boolean ignoreIfNotExists = "true".equals(parsed.get("ignore-if-not-exists")); + boolean cascade = "true".equals(parsed.get("cascade")); + + ctx.getCatalog().dropDatabase(database, ignoreIfNotExists, cascade); + System.out.println("Database '" + database + "' dropped successfully."); + } + + @Override + public String usage() { + return "Usage: paimon drop-database DATABASE [options]\n\n" + + "Drop a database.\n\n" + + "Options:\n" + + " -i, --ignore-if-not-exists Do not raise error if database does not exist\n" + + " --cascade Drop all tables before dropping database"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/DropTableCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/DropTableCommand.java new file mode 100644 index 000000000000..bb783396335c --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/DropTableCommand.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** Drops a table. */ +public class DropTableCommand implements Command { + + private static final Set BOOLEAN_FLAGS; + + static { + Set flags = new HashSet<>(); + flags.add("ignore-if-not-exists"); + BOOLEAN_FLAGS = Collections.unmodifiableSet(flags); + } + + @Override + public String name() { + return "drop-table"; + } + + @Override + public String description() { + return "Drop a table"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args, BOOLEAN_FLAGS, "-i", "--ignore-if-not-exists"); + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + boolean ignoreIfNotExists = "true".equals(parsed.get("ignore-if-not-exists")); + + ctx.getCatalog().dropTable(Identifier.create(parts[0], parts[1]), ignoreIfNotExists); + System.out.println("Table '" + tableId + "' dropped successfully."); + } + + @Override + public String usage() { + return "Usage: paimon drop-table DATABASE.TABLE [options]\n\n" + + "Drop a table.\n\n" + + "Options:\n" + + " -i, --ignore-if-not-exists Do not raise error if table does not exist"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ExpireSnapshotsCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ExpireSnapshotsCommand.java new file mode 100644 index 000000000000..c94a390ed970 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ExpireSnapshotsCommand.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.options.ExpireConfig; +import org.apache.paimon.table.ExpireSnapshots; +import org.apache.paimon.table.Table; + +/** Expires old snapshots from a table. */ +public class ExpireSnapshotsCommand implements Command { + + @Override + public String name() { + return "expire-snapshots"; + } + + @Override + public String description() { + return "Expire old snapshots from a table"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args, "-r", "--retain-max", "-m", "--retain-min"); + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + + String retainMax = parsed.get("retain-max"); + String retainMin = parsed.get("retain-min"); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + ExpireSnapshots expire = table.newExpireSnapshots(); + + ExpireConfig.Builder configBuilder = ExpireConfig.builder(); + if (retainMax != null) { + configBuilder.snapshotRetainMax(Integer.parseInt(retainMax)); + } + if (retainMin != null) { + configBuilder.snapshotRetainMin(Integer.parseInt(retainMin)); + } + expire.config(configBuilder.build()); + + System.err.println("Expiring snapshots for '" + tableId + "'..."); + int expired = expire.expire(); + System.out.println("Expired " + expired + " snapshots from '" + tableId + "'."); + } + + @Override + public String usage() { + return "Usage: paimon expire-snapshots DATABASE.TABLE [options]\n\n" + + "Expire old snapshots from a table.\n\n" + + "Options:\n" + + " -r, --retain-max N Maximum number of snapshots to retain\n" + + " -m, --retain-min N Minimum number of snapshots to retain"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ExplainCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ExplainCommand.java new file mode 100644 index 000000000000..2fb36c014ca1 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ExplainCommand.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.cli.PredicateParser; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.DataField; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** Shows the scan plan of a query without reading data. */ +public class ExplainCommand implements Command { + + @Override + public String name() { + return "explain"; + } + + @Override + public String description() { + return "Show the scan plan of a table query without reading data"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args, "-s", "--select", "-w", "--where", "-l", "--limit"); + + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + + String selectStr = parsed.get("select"); + String whereStr = parsed.get("where"); + String limitStr = parsed.get("limit"); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + ReadBuilder readBuilder = table.newReadBuilder(); + List allFields = table.rowType().getFields(); + + int[] projection = null; + if (selectStr != null && !selectStr.isEmpty()) { + String[] cols = selectStr.split(","); + projection = new int[cols.length]; + for (int i = 0; i < cols.length; i++) { + String col = cols[i].trim(); + int idx = findFieldIndex(allFields, col); + if (idx < 0) { + System.err.println("Column not found: " + col); + return; + } + projection[i] = idx; + } + readBuilder.withProjection(projection); + } + + Predicate predicate = null; + if (whereStr != null && !whereStr.isEmpty()) { + PredicateParser parser = new PredicateParser(table.rowType()); + predicate = parser.parse(whereStr); + readBuilder.withFilter(predicate); + } + + int limit = -1; + if (limitStr != null) { + limit = Integer.parseInt(limitStr); + readBuilder.withLimit(limit); + } + + TableScan scan = readBuilder.newScan(); + List splits = scan.plan().splits(); + + Optional snapshot = table.latestSnapshot(); + String tableType = inferTableType(table); + + System.out.println("== Paimon Scan Plan =="); + System.out.println("Table: " + tableId + " (" + tableType + ")"); + if (snapshot.isPresent()) { + Snapshot snap = snapshot.get(); + System.out.println( + "Snapshot: " + snap.id() + " (schema " + snap.schemaId() + ")"); + } else { + System.out.println("Snapshot: "); + } + + System.out.println("Predicate: " + (predicate != null ? whereStr : "")); + + if (projection != null) { + StringBuilder projStr = new StringBuilder("["); + for (int i = 0; i < projection.length; i++) { + if (i > 0) { + projStr.append(", "); + } + projStr.append(allFields.get(projection[i]).name()); + } + projStr.append("]"); + System.out.println("Projection: " + projStr); + } else { + System.out.println("Projection: "); + } + + System.out.println("Limit: " + (limit > 0 ? limit : "")); + + System.out.println(); + printSplitStats(splits, table); + } + + private void printSplitStats(List splits, Table table) { + int splitCount = splits.size(); + int totalFiles = 0; + long totalSize = 0; + long totalRows = 0; + long mergedRows = 0; + boolean hasMergedCount = true; + int rawConvertibleCount = 0; + int withDvCount = 0; + Map levelHistogram = new HashMap<>(); + int deletionFileCount = 0; + Set partitions = new HashSet<>(); + Set buckets = new HashSet<>(); + + List filesPerSplit = new ArrayList<>(); + List sizePerSplit = new ArrayList<>(); + + for (Split split : splits) { + if (split instanceof DataSplit) { + DataSplit ds = (DataSplit) split; + List files = ds.dataFiles(); + int fileCount = files.size(); + totalFiles += fileCount; + filesPerSplit.add(fileCount); + + long splitSize = 0; + for (DataFileMeta file : files) { + splitSize += file.fileSize(); + totalRows += file.rowCount(); + levelHistogram.merge(file.level(), 1, Integer::sum); + } + totalSize += splitSize; + sizePerSplit.add(splitSize); + + if (ds.rawConvertible()) { + rawConvertibleCount++; + } + if (ds.deletionFiles().isPresent() + && ds.deletionFiles().get().stream().anyMatch(f -> f != null)) { + withDvCount++; + deletionFileCount += + (int) ds.deletionFiles().get().stream().filter(f -> f != null).count(); + } + if (ds.mergedRowCount().isPresent()) { + mergedRows += ds.mergedRowCount().getAsLong(); + } else { + hasMergedCount = false; + } + partitions.add(ds.partition().toString()); + buckets.add(ds.bucket()); + } + } + + System.out.println("Splits: " + splitCount); + if (splitCount > 0) { + System.out.println(" raw-convertible: " + rawConvertibleCount + " / " + splitCount); + System.out.println(" with DV: " + withDvCount + " / " + splitCount); + + if (!filesPerSplit.isEmpty()) { + int minFiles = filesPerSplit.stream().mapToInt(i -> i).min().orElse(0); + int maxFiles = filesPerSplit.stream().mapToInt(i -> i).max().orElse(0); + double avgFiles = filesPerSplit.stream().mapToInt(i -> i).average().orElse(0); + System.out.printf( + " files/split: min=%d max=%d avg=%.2f%n", + minFiles, maxFiles, avgFiles); + } + + if (!sizePerSplit.isEmpty()) { + long minSize = sizePerSplit.stream().mapToLong(l -> l).min().orElse(0); + long maxSize = sizePerSplit.stream().mapToLong(l -> l).max().orElse(0); + sizePerSplit.sort(Long::compareTo); + long p50Size = sizePerSplit.get(sizePerSplit.size() / 2); + long p95Size = sizePerSplit.get((int) (sizePerSplit.size() * 0.95)); + System.out.printf( + " size/split: min=%s p50=%s p95=%s max=%s%n", + formatSize(minSize), + formatSize(p50Size), + formatSize(p95Size), + formatSize(maxSize)); + } + } + + System.out.println("Files: " + totalFiles); + System.out.println("Total size: " + formatSize(totalSize)); + String rowStr = "Estimated rows: " + totalRows; + if (hasMergedCount && mergedRows != totalRows) { + rowStr += " (merged: " + mergedRows + ")"; + } + System.out.println(rowStr); + + if (!levelHistogram.isEmpty()) { + StringBuilder lb = new StringBuilder("Level histogram: "); + List levels = new ArrayList<>(levelHistogram.keySet()); + levels.sort(Integer::compareTo); + for (int i = 0; i < levels.size(); i++) { + if (i > 0) { + lb.append(" "); + } + lb.append("L") + .append(levels.get(i)) + .append("=") + .append(levelHistogram.get(levels.get(i))); + } + System.out.println(lb); + } + + System.out.println("Deletion files: " + deletionFileCount); + System.out.println("Partitions hit: " + partitions.size()); + System.out.println("Buckets hit: " + buckets.size()); + } + + private static String inferTableType(Table table) { + List traits = new ArrayList<>(); + if (!table.primaryKeys().isEmpty()) { + traits.add("PK"); + } else { + traits.add("Append"); + } + String bucketMode = table.options().getOrDefault("bucket", "-1"); + if (!"-1".equals(bucketMode)) { + traits.add("HASH_FIXED"); + } else { + traits.add("DYNAMIC"); + } + return String.join(", ", traits); + } + + private static String formatSize(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else if (bytes < 1024 * 1024) { + return String.format("%.1f KiB", bytes / 1024.0); + } else if (bytes < 1024L * 1024 * 1024) { + return String.format("%.1f MiB", bytes / (1024.0 * 1024)); + } else { + return String.format("%.1f GiB", bytes / (1024.0 * 1024 * 1024)); + } + } + + private static int findFieldIndex(List fields, String name) { + for (int i = 0; i < fields.size(); i++) { + if (fields.get(i).name().equals(name)) { + return i; + } + } + return -1; + } + + @Override + public String usage() { + return "Usage: paimon explain DATABASE.TABLE [options]\n\n" + + "Show the scan plan without reading data.\n\n" + + "Options:\n" + + " -s, --select Columns to project (comma-separated)\n" + + " -w, --where Filter condition (SQL-like syntax)\n" + + " -l, --limit Row limit to push down"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/FullTextSearchCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/FullTextSearchCommand.java new file mode 100644 index 000000000000..4b55a7fdd7a5 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/FullTextSearchCommand.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.cli.RowPrinter; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.globalindex.GlobalIndexResult; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.FullTextSearchBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableRead; +import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.DataField; + +import java.util.ArrayList; +import java.util.List; + +/** Performs full-text search on a table with a Tantivy index. */ +public class FullTextSearchCommand implements Command { + + @Override + public String name() { + return "full-text-search"; + } + + @Override + public String description() { + return "Full-text search on a table column"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = + new CliArgs( + args, + "-c", + "--column", + "-q", + "--query", + "-l", + "--limit", + "-s", + "--select", + "-f", + "--format"); + + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + + String column = parsed.require("column", "text column to search"); + String query = parsed.require("query", "query text"); + String limitStr = parsed.get("limit"); + int limit = limitStr != null ? Integer.parseInt(limitStr) : 10; + String selectStr = parsed.get("select"); + String format = parsed.getOrDefault("format", "tsv"); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + + FullTextSearchBuilder ftsBuilder = table.newFullTextSearchBuilder(); + ftsBuilder.withTextColumn(column); + ftsBuilder.withQueryText(query); + ftsBuilder.withLimit(limit); + + GlobalIndexResult indexResult = ftsBuilder.executeLocal(); + + ReadBuilder readBuilder = table.newReadBuilder(); + List allFields = table.rowType().getFields(); + + int[] projection = null; + if (selectStr != null && !selectStr.isEmpty()) { + String[] cols = selectStr.split(","); + projection = new int[cols.length]; + for (int i = 0; i < cols.length; i++) { + String col = cols[i].trim(); + int idx = findFieldIndex(allFields, col); + if (idx < 0) { + System.err.println("Column not found: " + col); + return; + } + projection[i] = idx; + } + readBuilder.withProjection(projection); + } + + readBuilder.withLimit(limit); + + TableScan scan = readBuilder.newScan(); + scan.withGlobalIndexResult(indexResult); + List splits = scan.plan().splits(); + + if (splits.isEmpty()) { + System.err.println("No results found."); + return; + } + + TableRead tableRead = readBuilder.newRead(); + List outputFields = + projection != null ? selectFields(allFields, projection) : allFields; + + boolean isJson = "json".equals(format); + String delim = "csv".equals(format) ? "," : "\t"; + + if (!isJson) { + printHeader(outputFields, delim); + } + + int rowCount = 0; + for (Split split : splits) { + if (rowCount >= limit) { + break; + } + try (RecordReader reader = tableRead.createReader(split)) { + RecordReader.RecordIterator batch; + while ((batch = reader.readBatch()) != null) { + InternalRow row; + while ((row = batch.next()) != null) { + if (rowCount >= limit) { + break; + } + if (isJson) { + printJsonRow(row, outputFields); + } else { + printDelimitedRow(row, outputFields, delim); + } + rowCount++; + } + batch.releaseBatch(); + if (rowCount >= limit) { + break; + } + } + } + } + System.err.println("Found " + rowCount + " results."); + } + + @Override + public String usage() { + return "Usage: paimon full-text-search DATABASE.TABLE [options]\n\n" + + "Full-text search on a table column using Tantivy index.\n\n" + + "Options:\n" + + " -c, --column Text column to search on (required)\n" + + " -q, --query Query text (required)\n" + + " -l, --limit Maximum results (default: 10)\n" + + " -s, --select Columns to display (comma-separated)\n" + + " -f, --format Output format: tsv (default) / csv / json\n\n" + + "Note: Requires paimon-tantivy-index on the classpath."; + } + + private static int findFieldIndex(List fields, String name) { + for (int i = 0; i < fields.size(); i++) { + if (fields.get(i).name().equals(name)) { + return i; + } + } + return -1; + } + + private static List selectFields(List allFields, int[] projection) { + List result = new ArrayList<>(); + for (int idx : projection) { + result.add(allFields.get(idx)); + } + return result; + } + + private static void printHeader(List fields, String delim) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(delim); + } + sb.append(fields.get(i).name()); + } + System.out.println(sb.toString()); + } + + private static void printDelimitedRow(InternalRow row, List fields, String delim) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(delim); + } + if (row.isNullAt(i)) { + sb.append("NULL"); + } else { + sb.append(RowPrinter.getFieldValue(row, i, fields.get(i).type())); + } + } + System.out.println(sb.toString()); + } + + private static void printJsonRow(InternalRow row, List fields) { + StringBuilder sb = new StringBuilder("{"); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append("\"").append(fields.get(i).name()).append("\":"); + if (row.isNullAt(i)) { + sb.append("null"); + } else { + String value = RowPrinter.getFieldValue(row, i, fields.get(i).type()); + if (RowPrinter.isNumericOrBoolean(fields.get(i).type().getTypeRoot())) { + sb.append(value); + } else { + sb.append("\"").append(escapeJson(value)).append("\""); + } + } + } + sb.append("}"); + System.out.println(sb.toString()); + } + + private static String escapeJson(String value) { + return value.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/GetDatabaseCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/GetDatabaseCommand.java new file mode 100644 index 000000000000..a9fc920ac8f7 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/GetDatabaseCommand.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Database; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +import java.util.Map; + +/** Displays database information in JSON format. */ +public class GetDatabaseCommand implements Command { + + @Override + public String name() { + return "get-database"; + } + + @Override + public String description() { + return "Show database information"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args); + String database = parsed.positional(0, "DATABASE"); + + Database db = ctx.getCatalog().getDatabase(database); + + StringBuilder sb = new StringBuilder(); + sb.append("{\"name\":\"").append(escapeJson(db.name())).append("\""); + sb.append(",\"options\":{"); + Map options = db.options(); + if (options != null) { + int idx = 0; + for (Map.Entry entry : options.entrySet()) { + if (idx > 0) { + sb.append(","); + } + sb.append("\"").append(escapeJson(entry.getKey())).append("\":"); + sb.append("\"").append(escapeJson(entry.getValue())).append("\""); + idx++; + } + } + sb.append("}"); + String comment = db.comment().orElse(null); + if (comment != null) { + sb.append(",\"comment\":\"").append(escapeJson(comment)).append("\""); + } + sb.append("}"); + System.out.println(sb.toString()); + } + + @Override + public String usage() { + return "Usage: paimon get-database DATABASE\n\n" + + "Show database information in JSON format."; + } + + private static String escapeJson(String value) { + return value.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListDatabasesCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListDatabasesCommand.java new file mode 100644 index 000000000000..8bfd10554d5f --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListDatabasesCommand.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +import java.util.List; + +/** Lists all databases in the catalog. */ +public class ListDatabasesCommand implements Command { + + @Override + public String name() { + return "list-databases"; + } + + @Override + public String description() { + return "List all databases in the catalog"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + List databases = ctx.getCatalog().listDatabases(); + if (databases.isEmpty()) { + System.out.println("No databases found."); + } else { + for (String db : databases) { + System.out.println(db); + } + } + } + + @Override + public String usage() { + return "Usage: paimon list-databases\n\nList all databases in the catalog."; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListPartitionsCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListPartitionsCommand.java new file mode 100644 index 000000000000..ed9849d98fa3 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListPartitionsCommand.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.InternalRowPartitionComputer; + +import java.util.ArrayList; +import java.util.List; + +/** Lists partitions of a table with statistics. */ +public class ListPartitionsCommand implements Command { + + @Override + public String name() { + return "list-partitions"; + } + + @Override + public String description() { + return "List partitions of a table"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args, "-f", "--format"); + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + String format = parsed.getOrDefault("format", "table"); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + List partitionKeys = table.partitionKeys(); + if (partitionKeys.isEmpty()) { + System.out.println("Table '" + tableId + "' is not partitioned."); + return; + } + + RowType partType = buildPartitionType(table.rowType(), partitionKeys); + TableScan scan = table.newReadBuilder().newScan(); + List entries = scan.listPartitionEntries(); + + if (entries.isEmpty()) { + System.out.println("No partitions found for '" + tableId + "'."); + return; + } + + if ("json".equals(format)) { + printJson(entries, partType); + } else { + printTable(entries, partType); + } + } + + private void printTable(List entries, RowType partType) { + System.out.println( + String.format( + "%-40s %12s %16s %10s %20s", + "Partition", + "RecordCount", + "FileSizeInBytes", + "FileCount", + "LastFileCreationTime")); + for (PartitionEntry entry : entries) { + String partStr = + InternalRowPartitionComputer.partToSimpleString( + partType, entry.partition(), ",", 200); + System.out.println( + String.format( + "%-40s %12d %16d %10d %20d", + partStr, + entry.recordCount(), + entry.fileSizeInBytes(), + entry.fileCount(), + entry.lastFileCreationTime())); + } + } + + private void printJson(List entries, RowType partType) { + System.out.println("["); + for (int i = 0; i < entries.size(); i++) { + PartitionEntry entry = entries.get(i); + String partStr = + InternalRowPartitionComputer.partToSimpleString( + partType, entry.partition(), ",", 200); + System.out.print( + " {\"partition\":\"" + + partStr + + "\",\"recordCount\":" + + entry.recordCount() + + ",\"fileSizeInBytes\":" + + entry.fileSizeInBytes() + + ",\"fileCount\":" + + entry.fileCount() + + ",\"lastFileCreationTime\":" + + entry.lastFileCreationTime() + + "}"); + if (i < entries.size() - 1) { + System.out.println(","); + } else { + System.out.println(); + } + } + System.out.println("]"); + } + + private static RowType buildPartitionType(RowType rowType, List partitionKeys) { + List partFields = new ArrayList<>(); + for (String key : partitionKeys) { + int idx = rowType.getFieldIndex(key); + DataType type = rowType.getTypeAt(idx); + partFields.add(new DataField(partFields.size(), key, type)); + } + return new RowType(partFields); + } + + @Override + public String usage() { + return "Usage: paimon list-partitions DATABASE.TABLE [options]\n\n" + + "List partitions of a table with statistics.\n\n" + + "Options:\n" + + " -f, --format Output format: table (default) / json"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListTablesCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListTablesCommand.java new file mode 100644 index 000000000000..8e2f7db3e4b2 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ListTablesCommand.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +import java.util.Collections; +import java.util.List; + +/** Lists all tables in a database. */ +public class ListTablesCommand implements Command { + + @Override + public String name() { + return "list-tables"; + } + + @Override + public String description() { + return "List all tables in a database"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args); + String database = parsed.positional(0, "DATABASE"); + + List tables = ctx.getCatalog().listTables(database); + if (tables.isEmpty()) { + System.out.println("No tables found in database '" + database + "'."); + } else { + Collections.sort(tables); + for (String table : tables) { + System.out.println(table); + } + } + } + + @Override + public String usage() { + return "Usage: paimon list-tables DATABASE\n\nList all tables in the specified database."; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/OrphanCleanCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/OrphanCleanCommand.java new file mode 100644 index 000000000000..cd411decd8ea --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/OrphanCleanCommand.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.operation.CleanOrphanFilesResult; +import org.apache.paimon.operation.LocalOrphanFilesClean; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** Cleans orphan files from a table or database. */ +public class OrphanCleanCommand implements Command { + + @Override + public String name() { + return "orphan-clean"; + } + + @Override + public String description() { + return "Clean orphan files from a table"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args); + String tableId = parsed.positional(0, "DATABASE.TABLE or DATABASE.*"); + + boolean dryRun = "true".equals(parsed.get("dry-run")); + String olderThanStr = parsed.get("older-than"); + String parallelismStr = parsed.get("parallelism"); + + long olderThanMillis = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1); + if (olderThanStr != null) { + olderThanMillis = System.currentTimeMillis() - parseDuration(olderThanStr); + } + + Integer parallelism = parallelismStr != null ? Integer.parseInt(parallelismStr) : null; + + String database; + String tableName; + if (tableId.contains(".")) { + String[] parts = tableId.split("\\.", 2); + database = parts[0]; + tableName = parts[1]; + } else { + database = tableId; + tableName = "*"; + } + + List cleans = + LocalOrphanFilesClean.createOrphanFilesCleans( + ctx.getCatalog(), + database, + tableName, + olderThanMillis, + parallelism, + dryRun); + + long totalFiles = 0; + long totalBytes = 0; + + for (LocalOrphanFilesClean clean : cleans) { + CleanOrphanFilesResult result = clean.clean(); + totalFiles += result.getDeletedFileCount(); + totalBytes += result.getDeletedFileTotalLenInBytes(); + } + + if (dryRun) { + System.out.println( + "Dry run: would delete " + + totalFiles + + " orphan files (" + + formatBytes(totalBytes) + + ")."); + } else { + System.out.println( + "Deleted " + totalFiles + " orphan files (" + formatBytes(totalBytes) + ")."); + } + } + + private static long parseDuration(String value) { + String trimmed = value.trim().toLowerCase(); + if (trimmed.endsWith("d")) { + return TimeUnit.DAYS.toMillis(Long.parseLong(trimmed.replace("d", ""))); + } else if (trimmed.endsWith("h")) { + return TimeUnit.HOURS.toMillis(Long.parseLong(trimmed.replace("h", ""))); + } else { + return TimeUnit.DAYS.toMillis(Long.parseLong(trimmed)); + } + } + + private static String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else if (bytes < 1024 * 1024) { + return String.format("%.1f KiB", bytes / 1024.0); + } else if (bytes < 1024L * 1024 * 1024) { + return String.format("%.1f MiB", bytes / (1024.0 * 1024)); + } else { + return String.format("%.2f GiB", bytes / (1024.0 * 1024 * 1024)); + } + } + + @Override + public String usage() { + return "Usage: paimon orphan-clean DATABASE.TABLE [options]\n\n" + + "Clean orphan files that are not referenced by any snapshot.\n\n" + + "Options:\n" + + " --older-than DURATION Only delete files older than this (default: 1d)\n" + + " Format: Nd (days) or Nh (hours)\n" + + " --dry-run Show what would be deleted without deleting\n" + + " --parallelism N Thread pool size for deletion\n\n" + + "Examples:\n" + + " paimon orphan-clean mydb.users\n" + + " paimon orphan-clean mydb.users --older-than 7d --dry-run\n" + + " paimon orphan-clean mydb.* --older-than 3d"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ReadCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ReadCommand.java new file mode 100644 index 000000000000..2eb31cc5cfad --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/ReadCommand.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.cli.PredicateParser; +import org.apache.paimon.cli.RowPrinter; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableRead; +import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.DataField; + +import java.util.ArrayList; +import java.util.List; + +/** Reads data from a Paimon table. */ +public class ReadCommand implements Command { + + @Override + public String name() { + return "read"; + } + + @Override + public String description() { + return "Read data from a Paimon table"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = + new CliArgs( + args, "-s", "--select", "-l", "--limit", "-f", "--format", "-w", "--where"); + + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + + String selectStr = parsed.get("select"); + String whereStr = parsed.get("where"); + String limitStr = parsed.get("limit"); + int limit = limitStr != null ? Integer.parseInt(limitStr) : 100; + String format = parsed.getOrDefault("format", "tsv"); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + ReadBuilder readBuilder = table.newReadBuilder(); + List allFields = table.rowType().getFields(); + + int[] projection = null; + if (selectStr != null && !selectStr.isEmpty()) { + String[] cols = selectStr.split(","); + projection = new int[cols.length]; + for (int i = 0; i < cols.length; i++) { + String col = cols[i].trim(); + int idx = findFieldIndex(allFields, col); + if (idx < 0) { + System.err.println("Column not found: " + col); + System.exit(1); + } + projection[i] = idx; + } + readBuilder.withProjection(projection); + } + + if (whereStr != null && !whereStr.isEmpty()) { + PredicateParser parser = new PredicateParser(table.rowType()); + Predicate predicate = parser.parse(whereStr); + readBuilder.withFilter(predicate); + } + + if (limit > 0) { + readBuilder.withLimit(limit); + } + + TableScan scan = readBuilder.newScan(); + List splits = scan.plan().splits(); + + if (splits.isEmpty()) { + System.err.println("No data in table."); + return; + } + + TableRead tableRead = readBuilder.newRead(); + List outputFields = + projection != null ? selectFields(allFields, projection) : allFields; + + String delim = "csv".equals(format) ? "," : "\t"; + boolean isJson = "json".equals(format); + + if (!isJson) { + printDelimitedHeader(outputFields, delim); + } + + int rowCount = 0; + for (Split split : splits) { + if (limit > 0 && rowCount >= limit) { + break; + } + try (RecordReader reader = tableRead.createReader(split)) { + RecordReader.RecordIterator batch; + while ((batch = reader.readBatch()) != null) { + InternalRow row; + while ((row = batch.next()) != null) { + if (limit > 0 && rowCount >= limit) { + break; + } + if (isJson) { + printJsonRow(row, outputFields); + } else { + printDelimitedRow(row, outputFields, delim); + } + rowCount++; + } + batch.releaseBatch(); + if (limit > 0 && rowCount >= limit) { + break; + } + } + } + } + System.err.println("Read " + rowCount + " rows."); + } + + @Override + public String usage() { + return "Usage: paimon read DATABASE.TABLE [options]\n\n" + + "Read data from a Paimon table.\n\n" + + "Options:\n" + + " -s, --select Columns to read (comma-separated)\n" + + " -w, --where Filter condition (SQL-like syntax)\n" + + " -l, --limit Maximum rows to return (default: 100)\n" + + " -f, --format Output format: tsv (default) / csv / json"; + } + + private static int findFieldIndex(List fields, String name) { + for (int i = 0; i < fields.size(); i++) { + if (fields.get(i).name().equals(name)) { + return i; + } + } + return -1; + } + + private static List selectFields(List allFields, int[] projection) { + List result = new ArrayList<>(); + for (int idx : projection) { + result.add(allFields.get(idx)); + } + return result; + } + + private static void printDelimitedHeader(List fields, String delim) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(delim); + } + sb.append(fields.get(i).name()); + } + System.out.println(sb.toString()); + } + + private static void printDelimitedRow(InternalRow row, List fields, String delim) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(delim); + } + if (row.isNullAt(i)) { + sb.append("NULL"); + } else { + String value = RowPrinter.getFieldValue(row, i, fields.get(i).type()); + if (",".equals(delim) && needsCsvQuoting(value)) { + sb.append("\"").append(value.replace("\"", "\"\"")).append("\""); + } else { + sb.append(value); + } + } + } + System.out.println(sb.toString()); + } + + private static boolean needsCsvQuoting(String value) { + return value.contains(",") + || value.contains("\"") + || value.contains("\n") + || value.contains("\r"); + } + + private static void printJsonRow(InternalRow row, List fields) { + StringBuilder sb = new StringBuilder("{"); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append("\"").append(fields.get(i).name()).append("\":"); + if (row.isNullAt(i)) { + sb.append("null"); + } else { + String value = RowPrinter.getFieldValue(row, i, fields.get(i).type()); + if (RowPrinter.isNumericOrBoolean(fields.get(i).type().getTypeRoot())) { + sb.append(value); + } else { + sb.append("\"").append(escapeJson(value)).append("\""); + } + } + } + sb.append("}"); + System.out.println(sb.toString()); + } + + private static String escapeJson(String value) { + return value.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/RenameTableCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/RenameTableCommand.java new file mode 100644 index 000000000000..c5ddceeb18e0 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/RenameTableCommand.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +/** Renames a table in the catalog. */ +public class RenameTableCommand implements Command { + + @Override + public String name() { + return "rename-table"; + } + + @Override + public String description() { + return "Rename a table"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args); + String fromId = parsed.positional(0, "SOURCE_DATABASE.TABLE"); + String toId = parsed.positional(1, "TARGET_DATABASE.TABLE"); + String[] fromParts = SchemaCommand.parseTableIdentifier(fromId); + String[] toParts = SchemaCommand.parseTableIdentifier(toId); + + ctx.getCatalog() + .renameTable( + Identifier.create(fromParts[0], fromParts[1]), + Identifier.create(toParts[0], toParts[1]), + false); + System.out.println("Table '" + fromId + "' renamed to '" + toId + "' successfully."); + } + + @Override + public String usage() { + return "Usage: paimon rename-table SOURCE_DB.TABLE TARGET_DB.TABLE\n\n" + + "Rename a table in the catalog."; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/RollbackCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/RollbackCommand.java new file mode 100644 index 000000000000..2a60f76b06d7 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/RollbackCommand.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.table.Table; + +/** Rolls back a table to a specific snapshot or tag. */ +public class RollbackCommand implements Command { + + @Override + public String name() { + return "rollback"; + } + + @Override + public String description() { + return "Rollback a table to a snapshot or tag"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args, "-s", "--snapshot", "-t", "--tag"); + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + + String snapshotId = parsed.get("snapshot"); + String tagName = parsed.get("tag"); + + if (snapshotId == null && tagName == null) { + System.err.println("Must specify --snapshot ID or --tag NAME."); + System.err.println(usage()); + return; + } + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + + if (snapshotId != null) { + long sid = Long.parseLong(snapshotId); + table.rollbackTo(sid); + System.out.println("Table '" + tableId + "' rolled back to snapshot " + sid + "."); + } else { + table.rollbackTo(tagName); + System.out.println("Table '" + tableId + "' rolled back to tag '" + tagName + "'."); + } + } + + @Override + public String usage() { + return "Usage: paimon rollback DATABASE.TABLE [options]\n\n" + + "Rollback a table to a specific snapshot or tag.\n\n" + + "Options:\n" + + " -s, --snapshot ID Rollback to snapshot ID\n" + + " -t, --tag NAME Rollback to tag name"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/SchemaCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/SchemaCommand.java new file mode 100644 index 000000000000..d7dd5eb27ffc --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/SchemaCommand.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataField; + +import java.util.List; +import java.util.Map; + +/** Displays table schema information. */ +public class SchemaCommand implements Command { + + @Override + public String name() { + return "schema"; + } + + @Override + public String description() { + return "Show table schema information"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args, "-f", "--format"); + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = parseTableIdentifier(tableId); + String format = parsed.getOrDefault("format", "table"); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + List fields = table.rowType().getFields(); + List primaryKeys = table.primaryKeys(); + List partitionKeys = table.partitionKeys(); + + if ("json".equals(format)) { + printJson(table, fields, primaryKeys, partitionKeys); + } else { + printTable(tableId, table, fields, primaryKeys, partitionKeys); + } + } + + private void printJson( + Table table, + List fields, + List primaryKeys, + List partitionKeys) { + StringBuilder sb = new StringBuilder(); + sb.append("{\"fields\":["); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(","); + } + DataField f = fields.get(i); + sb.append("{\"id\":").append(f.id()); + sb.append(",\"name\":\"").append(f.name()).append("\""); + sb.append(",\"type\":\"").append(f.type().toString()).append("\"}"); + } + sb.append("],\"partitionKeys\":["); + for (int i = 0; i < partitionKeys.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append("\"").append(partitionKeys.get(i)).append("\""); + } + sb.append("],\"primaryKeys\":["); + for (int i = 0; i < primaryKeys.size(); i++) { + if (i > 0) { + sb.append(","); + } + sb.append("\"").append(primaryKeys.get(i)).append("\""); + } + sb.append("],\"options\":{"); + Map options = table.options(); + if (options != null) { + int idx = 0; + for (Map.Entry entry : options.entrySet()) { + if (idx > 0) { + sb.append(","); + } + sb.append("\"").append(escapeJson(entry.getKey())).append("\":"); + sb.append("\"").append(escapeJson(entry.getValue())).append("\""); + idx++; + } + } + sb.append("}"); + String comment = table.comment().orElse(null); + if (comment != null) { + sb.append(",\"comment\":\"").append(escapeJson(comment)).append("\""); + } + sb.append("}"); + System.out.println(sb.toString()); + } + + private void printTable( + String tableId, + Table table, + List fields, + List primaryKeys, + List partitionKeys) { + System.out.println("Table: " + tableId); + System.out.println(); + System.out.println(String.format(" %-30s %-35s %-8s", "Column", "Type", "Nullable")); + System.out.println( + String.format( + " %-30s %-35s %-8s", repeat("-", 30), repeat("-", 35), repeat("-", 8))); + for (DataField field : fields) { + String nullable = field.type().isNullable() ? "YES" : "NO"; + System.out.println( + String.format( + " %-30s %-35s %-8s", field.name(), field.type().toString(), nullable)); + } + + if (!primaryKeys.isEmpty()) { + System.out.println(); + System.out.println("Primary keys: " + String.join(", ", primaryKeys)); + } + if (!partitionKeys.isEmpty()) { + System.out.println("Partition keys: " + String.join(", ", partitionKeys)); + } + + Map options = table.options(); + if (options != null && !options.isEmpty()) { + System.out.println(); + System.out.println("Options:"); + for (Map.Entry entry : options.entrySet()) { + System.out.println(" " + entry.getKey() + " = " + entry.getValue()); + } + } + } + + @Override + public String usage() { + return "Usage: paimon schema DATABASE.TABLE [options]\n\n" + + "Show table schema, primary keys, partition keys, and options.\n\n" + + "Options:\n" + + " -f, --format Output format: table (default) / json"; + } + + static String[] parseTableIdentifier(String tableId) { + String[] parts = tableId.split("\\.", 2); + if (parts.length != 2 || parts[0].isEmpty() || parts[1].isEmpty()) { + System.err.println( + "Invalid table identifier '" + tableId + "'. Expected format: database.table"); + System.exit(1); + } + return parts; + } + + private static String repeat(String str, int count) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < count; i++) { + sb.append(str); + } + return sb.toString(); + } + + private static String escapeJson(String value) { + return value.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t"); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/SnapshotCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/SnapshotCommand.java new file mode 100644 index 000000000000..7aee853cc988 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/SnapshotCommand.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.table.Table; + +import java.util.Optional; + +/** Displays the latest snapshot information of a table. */ +public class SnapshotCommand implements Command { + + @Override + public String name() { + return "snapshot"; + } + + @Override + public String description() { + return "Show the latest snapshot of a table"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args); + String tableId = parsed.positional(0, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + Optional snapshot = table.latestSnapshot(); + if (snapshot.isPresent()) { + System.out.println(snapshot.get().toJson()); + } else { + System.out.println("No snapshot found for '" + tableId + "'."); + } + } + + @Override + public String usage() { + return "Usage: paimon snapshot DATABASE.TABLE\n\n" + + "Show the latest snapshot information of a table in JSON format."; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/TagCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/TagCommand.java new file mode 100644 index 000000000000..55f528ebf48e --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/TagCommand.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; + +import java.util.List; + +/** Manages tags on a table (create, delete, list). */ +public class TagCommand implements Command { + + @Override + public String name() { + return "tag"; + } + + @Override + public String description() { + return "Manage table tags (create, delete, list)"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = new CliArgs(args, "-s", "--snapshot"); + String action = parsed.positional(0, "ACTION (create|delete|list)"); + String tableId = parsed.positional(1, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + + switch (action) { + case "create": + { + String tagName = parsed.require("tag-name", "tag name"); + String snapshotId = parsed.get("snapshot"); + if (snapshotId != null) { + table.createTag(tagName, Long.parseLong(snapshotId)); + } else { + table.createTag(tagName); + } + System.out.println("Tag '" + tagName + "' created on '" + tableId + "'."); + break; + } + case "delete": + { + String tagName = parsed.require("tag-name", "tag name"); + table.deleteTag(tagName); + System.out.println("Tag '" + tagName + "' deleted from '" + tableId + "'."); + break; + } + case "list": + { + if (!(table instanceof FileStoreTable)) { + System.err.println("Table does not support tag listing."); + return; + } + List tags = ((FileStoreTable) table).tagManager().allTagNames(); + if (tags.isEmpty()) { + System.out.println("No tags found for '" + tableId + "'."); + } else { + for (String tag : tags) { + System.out.println(tag); + } + } + break; + } + default: + System.err.println("Unknown tag action: " + action); + System.err.println(usage()); + } + } + + @Override + public String usage() { + return "Usage: paimon tag DATABASE.TABLE [options]\n\n" + + "Actions:\n" + + " create Create a tag (--tag-name NAME [--snapshot ID])\n" + + " delete Delete a tag (--tag-name NAME)\n" + + " list List all tags\n\n" + + "Options:\n" + + " --tag-name NAME Tag name (required for create/delete)\n" + + " -s, --snapshot ID Snapshot ID (for create, defaults to latest)"; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/commands/WriteCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/WriteCommand.java new file mode 100644 index 000000000000..b2a9a6a03f5e --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/commands/WriteCommand.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CliArgs; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeChecks; +import org.apache.paimon.types.DataTypeRoot; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.math.BigDecimal; +import java.nio.charset.Charset; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** Writes data from a local file into a Paimon table. */ +public class WriteCommand implements Command { + + private static final Set BOOLEAN_FLAGS; + + static { + Set flags = new HashSet<>(); + BOOLEAN_FLAGS = Collections.unmodifiableSet(flags); + } + + @Override + public String name() { + return "write"; + } + + @Override + public String description() { + return "Write data from a local file into a Paimon table"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + CliArgs parsed = + new CliArgs( + args, + BOOLEAN_FLAGS, + "-f", + "--format", + "-d", + "--delimiter", + "-e", + "--encoding"); + + String filePath = parsed.positional(0, "FILE"); + String tableId = parsed.positional(1, "DATABASE.TABLE"); + String[] parts = SchemaCommand.parseTableIdentifier(tableId); + + String format = parsed.getOrDefault("format", "csv"); + String delimiter = parsed.getOrDefault("delimiter", ","); + String encoding = parsed.getOrDefault("encoding", "utf-8"); + + Table table = ctx.getCatalog().getTable(Identifier.create(parts[0], parts[1])); + List fields = table.rowType().getFields(); + + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + + int rowCount; + if ("json".equals(format)) { + rowCount = loadJson(filePath, encoding, fields, writer); + } else { + rowCount = loadCsv(filePath, encoding, delimiter, fields, writer); + } + + if (rowCount == 0) { + System.err.println("No data to write."); + writer.close(); + return; + } + + List messages = writer.prepareCommit(); + writer.close(); + + System.err.println("Committing " + rowCount + " rows..."); + writeBuilder.newCommit().commit(messages); + System.out.println("Successfully wrote " + rowCount + " rows into '" + tableId + "'."); + } + + @Override + public String usage() { + return "Usage: paimon write FILE DATABASE.TABLE [options]\n\n" + + "Write data from a local file into a Paimon table.\n\n" + + "Options:\n" + + " -f, --format Input format: csv (default) / json\n" + + " -d, --delimiter CSV delimiter (default: ,)\n" + + " -e, --encoding File encoding (default: utf-8)"; + } + + private static int loadCsv( + String filePath, + String encoding, + String delimiter, + List fields, + BatchTableWrite writer) + throws Exception { + Charset charset = Charset.forName(encoding); + int rowCount = 0; + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(new FileInputStream(filePath), charset))) { + String line; + boolean firstLine = true; + while ((line = reader.readLine()) != null) { + while (hasUnclosedQuote(line)) { + String nextLine = reader.readLine(); + if (nextLine == null) { + break; + } + line = line + "\n" + nextLine; + } + if (line.trim().isEmpty()) { + continue; + } + String[] values = + ",".equals(delimiter) ? parseCsvLine(line) : line.split(delimiter, -1); + if (firstLine) { + firstLine = false; + if (values.length == fields.size() && isHeaderRow(values, fields)) { + continue; + } + } + if (values.length != fields.size()) { + System.err.println( + "Skipping row (column count mismatch, expected " + + fields.size() + + ", got " + + values.length + + ")"); + continue; + } + GenericRow row = parseRow(values, fields); + writer.write(row); + rowCount++; + } + } + return rowCount; + } + + private static int loadJson( + String filePath, String encoding, List fields, BatchTableWrite writer) + throws Exception { + Charset charset = Charset.forName(encoding); + StringBuilder sb = new StringBuilder(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(new FileInputStream(filePath), charset))) { + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + } + String jsonText = sb.toString().trim(); + if (!jsonText.startsWith("[")) { + System.err.println("JSON file must be an array: [{...}, ...]"); + return 0; + } + + ObjectMapper mapper = new ObjectMapper(); + JsonNode arrayNode = mapper.readTree(jsonText); + if (!arrayNode.isArray()) { + System.err.println("JSON file must be an array: [{...}, ...]"); + return 0; + } + + int rowCount = 0; + for (JsonNode objNode : arrayNode) { + String[] values = new String[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + String fieldName = fields.get(i).name(); + JsonNode valNode = objNode.get(fieldName); + if (valNode == null || valNode.isNull()) { + values[i] = "NULL"; + } else if (valNode.isTextual()) { + values[i] = valNode.textValue(); + } else { + values[i] = valNode.toString(); + } + } + GenericRow row = parseRow(values, fields); + writer.write(row); + rowCount++; + } + return rowCount; + } + + private static GenericRow parseRow(String[] values, List fields) { + GenericRow row = new GenericRow(values.length); + for (int i = 0; i < values.length; i++) { + String value = values[i]; + if (value.isEmpty() || "NULL".equalsIgnoreCase(value)) { + row.setField(i, null); + continue; + } + row.setField(i, parseValue(value, fields.get(i).type())); + } + return row; + } + + private static Object parseValue(String value, DataType type) { + DataTypeRoot root = type.getTypeRoot(); + switch (root) { + case BOOLEAN: + return Boolean.parseBoolean(value); + case TINYINT: + return Byte.parseByte(value); + case SMALLINT: + return Short.parseShort(value); + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + return Integer.parseInt(value); + case BIGINT: + return Long.parseLong(value); + case FLOAT: + return Float.parseFloat(value); + case DOUBLE: + return Double.parseDouble(value); + case DECIMAL: + { + int precision = DataTypeChecks.getPrecision(type); + int scale = DataTypeChecks.getScale(type); + return Decimal.fromBigDecimal(new BigDecimal(value), precision, scale); + } + case CHAR: + case VARCHAR: + return BinaryString.fromString(value); + case BINARY: + case VARBINARY: + return Base64.getDecoder().decode(value); + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return Timestamp.fromLocalDateTime(LocalDateTime.parse(value.replace(" ", "T"))); + default: + return BinaryString.fromString(value); + } + } + + private static boolean hasUnclosedQuote(String line) { + boolean inQuotes = false; + for (int i = 0; i < line.length(); i++) { + char ch = line.charAt(i); + if (ch == '"') { + if (inQuotes && i + 1 < line.length() && line.charAt(i + 1) == '"') { + i++; + } else { + inQuotes = !inQuotes; + } + } + } + return inQuotes; + } + + private static String[] parseCsvLine(String line) { + List fields = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + boolean inQuotes = false; + + for (int i = 0; i < line.length(); i++) { + char ch = line.charAt(i); + if (inQuotes) { + if (ch == '"') { + if (i + 1 < line.length() && line.charAt(i + 1) == '"') { + current.append('"'); + i++; + } else { + inQuotes = false; + } + } else { + current.append(ch); + } + } else { + if (ch == '"') { + inQuotes = true; + } else if (ch == ',') { + fields.add(current.toString()); + current.setLength(0); + } else { + current.append(ch); + } + } + } + fields.add(current.toString()); + return fields.toArray(new String[0]); + } + + private static boolean isHeaderRow(String[] values, List fields) { + for (int i = 0; i < values.length; i++) { + if (!values[i].equals(fields.get(i).name())) { + return false; + } + } + return true; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonCalciteTable.java b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonCalciteTable.java new file mode 100644 index 000000000000..a50c5995f8a8 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonCalciteTable.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableRead; +import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.RowType; + +import org.apache.calcite.DataContext; +import org.apache.calcite.linq4j.AbstractEnumerable; +import org.apache.calcite.linq4j.Enumerable; +import org.apache.calcite.linq4j.Enumerator; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.schema.ProjectableFilterableTable; +import org.apache.calcite.schema.impl.AbstractTable; + +import java.util.List; + +/** Bridges a Paimon table to Calcite's query engine via ProjectableFilterableTable. */ +public class PaimonCalciteTable extends AbstractTable implements ProjectableFilterableTable { + + private final Table paimonTable; + + public PaimonCalciteTable(Table paimonTable) { + this.paimonTable = paimonTable; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) { + return PaimonTypeMapping.toCalciteRowType(typeFactory, paimonTable.rowType()); + } + + @Override + public Enumerable scan(DataContext root, List filters, int[] projects) { + ReadBuilder builder = paimonTable.newReadBuilder(); + + if (projects != null && projects.length > 0) { + builder.withProjection(projects); + } + + Predicate predicate = RexToPredicate.convert(filters, paimonTable.rowType()); + if (predicate != null) { + builder.withFilter(predicate); + } + + RowType fullRowType = paimonTable.rowType(); + List readFields; + if (projects != null && projects.length > 0) { + List allFields = fullRowType.getFields(); + readFields = new java.util.ArrayList<>(projects.length); + for (int idx : projects) { + readFields.add(allFields.get(idx)); + } + } else { + readFields = fullRowType.getFields(); + } + + return new AbstractEnumerable() { + @Override + public Enumerator enumerator() { + return new PaimonEnumerator(builder, readFields); + } + }; + } + + private static class PaimonEnumerator implements Enumerator { + + private final ReadBuilder builder; + private final List fields; + private final DataType[] fieldTypes; + + private List splits; + private int splitIndex; + private TableRead tableRead; + private RecordReader currentReader; + private RecordReader.RecordIterator currentBatch; + private Object[] current; + + PaimonEnumerator(ReadBuilder builder, List fields) { + this.builder = builder; + this.fields = fields; + this.fieldTypes = new DataType[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + this.fieldTypes[i] = fields.get(i).type(); + } + this.splitIndex = 0; + } + + @Override + public Object[] current() { + return current; + } + + @Override + public boolean moveNext() { + try { + if (splits == null) { + TableScan scan = builder.newScan(); + splits = scan.plan().splits(); + tableRead = builder.newRead(); + } + + while (true) { + if (currentBatch != null) { + InternalRow row = currentBatch.next(); + if (row != null) { + current = convertRow(row); + return true; + } + currentBatch.releaseBatch(); + currentBatch = null; + } + + if (currentReader != null) { + currentBatch = currentReader.readBatch(); + if (currentBatch != null) { + continue; + } + currentReader.close(); + currentReader = null; + } + + if (splitIndex >= splits.size()) { + return false; + } + + currentReader = tableRead.createReader(splits.get(splitIndex++)); + } + } catch (Exception e) { + throw new RuntimeException("Error reading Paimon table", e); + } + } + + @Override + public void reset() { + close(); + splits = null; + splitIndex = 0; + currentReader = null; + currentBatch = null; + current = null; + } + + @Override + public void close() { + try { + if (currentBatch != null) { + currentBatch.releaseBatch(); + currentBatch = null; + } + if (currentReader != null) { + currentReader.close(); + currentReader = null; + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private Object[] convertRow(InternalRow row) { + Object[] values = new Object[fieldTypes.length]; + for (int i = 0; i < fieldTypes.length; i++) { + values[i] = PaimonTypeMapping.getValue(row, i, fieldTypes[i]); + } + return values; + } + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonSchema.java b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonSchema.java new file mode 100644 index 000000000000..c4642d033978 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonSchema.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; + +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; + +import java.util.AbstractMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** Maps a Paimon database to a Calcite schema. Tables are resolved lazily on first access. */ +public class PaimonSchema extends AbstractSchema { + + private final Catalog catalog; + private final String database; + private final ConcurrentHashMap tableCache = new ConcurrentHashMap<>(); + private volatile List tableNames; + + public PaimonSchema(Catalog catalog, String database) { + this.catalog = catalog; + this.database = database; + } + + @Override + protected Map getTableMap() { + return new LazyTableMap(); + } + + private List tableNameList() { + if (tableNames == null) { + try { + tableNames = catalog.listTables(database); + } catch (Exception e) { + throw new RuntimeException("Failed to list tables in database: " + database, e); + } + } + return tableNames; + } + + private Table resolveTable(String name) { + return tableCache.computeIfAbsent( + name, + k -> { + try { + org.apache.paimon.table.Table paimonTable = + catalog.getTable(Identifier.create(database, k)); + return new PaimonCalciteTable(paimonTable); + } catch (Exception e) { + return null; + } + }); + } + + private class LazyTableMap extends AbstractMap { + + @Override + public Table get(Object key) { + if (!(key instanceof String)) { + return null; + } + return resolveTable((String) key); + } + + @Override + public boolean containsKey(Object key) { + if (!(key instanceof String)) { + return false; + } + return tableNameList().contains(key); + } + + @Override + public Set keySet() { + return new HashSet<>(tableNameList()); + } + + @Override + public int size() { + return tableNameList().size(); + } + + @Override + public Set> entrySet() { + Set> entries = new HashSet<>(); + for (String name : tableNameList()) { + Table t = resolveTable(name); + if (t != null) { + entries.add(new SimpleEntry<>(name, t)); + } + } + return entries; + } + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonTypeMapping.java b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonTypeMapping.java new file mode 100644 index 000000000000..e2ba94551a55 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/PaimonTypeMapping.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.Decimal; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.Timestamp; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeChecks; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.type.SqlTypeName; + +import java.util.List; + +/** Maps Paimon types to Calcite types and converts InternalRow values to Java objects. */ +public class PaimonTypeMapping { + + public static RelDataType toCalciteRowType( + RelDataTypeFactory typeFactory, RowType paimonRowType) { + List fields = paimonRowType.getFields(); + RelDataTypeFactory.Builder builder = typeFactory.builder(); + for (DataField field : fields) { + RelDataType fieldType = toCalciteType(typeFactory, field.type()); + if (field.type().isNullable()) { + fieldType = typeFactory.createTypeWithNullability(fieldType, true); + } + builder.add(field.name(), fieldType); + } + return builder.build(); + } + + public static RelDataType toCalciteType(RelDataTypeFactory typeFactory, DataType paimonType) { + switch (paimonType.getTypeRoot()) { + case BOOLEAN: + return typeFactory.createSqlType(SqlTypeName.BOOLEAN); + case TINYINT: + return typeFactory.createSqlType(SqlTypeName.TINYINT); + case SMALLINT: + return typeFactory.createSqlType(SqlTypeName.SMALLINT); + case INTEGER: + return typeFactory.createSqlType(SqlTypeName.INTEGER); + case BIGINT: + return typeFactory.createSqlType(SqlTypeName.BIGINT); + case FLOAT: + return typeFactory.createSqlType(SqlTypeName.FLOAT); + case DOUBLE: + return typeFactory.createSqlType(SqlTypeName.DOUBLE); + case DECIMAL: + { + int precision = DataTypeChecks.getPrecision(paimonType); + int scale = DataTypeChecks.getScale(paimonType); + return typeFactory.createSqlType(SqlTypeName.DECIMAL, precision, scale); + } + case CHAR: + { + int length = DataTypeChecks.getLength(paimonType); + return typeFactory.createSqlType(SqlTypeName.CHAR, Math.max(length, 1)); + } + case VARCHAR: + { + int length = DataTypeChecks.getLength(paimonType); + if (length == VarCharType.MAX_LENGTH || length == Integer.MAX_VALUE) { + return typeFactory.createSqlType(SqlTypeName.VARCHAR, 65536); + } + return typeFactory.createSqlType(SqlTypeName.VARCHAR, length); + } + case DATE: + return typeFactory.createSqlType(SqlTypeName.DATE); + case TIME_WITHOUT_TIME_ZONE: + return typeFactory.createSqlType(SqlTypeName.TIME); + case TIMESTAMP_WITHOUT_TIME_ZONE: + { + int precision = DataTypeChecks.getPrecision(paimonType); + return typeFactory.createSqlType(SqlTypeName.TIMESTAMP, precision); + } + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + { + int precision = DataTypeChecks.getPrecision(paimonType); + return typeFactory.createSqlType( + SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE, precision); + } + case BINARY: + case VARBINARY: + { + int length = DataTypeChecks.getLength(paimonType); + return typeFactory.createSqlType(SqlTypeName.VARBINARY, Math.max(length, 1)); + } + default: + return typeFactory.createSqlType(SqlTypeName.ANY); + } + } + + public static Object getValue(InternalRow row, int index, DataType type) { + if (row.isNullAt(index)) { + return null; + } + switch (type.getTypeRoot()) { + case BOOLEAN: + return row.getBoolean(index); + case TINYINT: + return row.getByte(index); + case SMALLINT: + return row.getShort(index); + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + return row.getInt(index); + case BIGINT: + return row.getLong(index); + case FLOAT: + return row.getFloat(index); + case DOUBLE: + return row.getDouble(index); + case DECIMAL: + { + int precision = DataTypeChecks.getPrecision(type); + int scale = DataTypeChecks.getScale(type); + Decimal d = row.getDecimal(index, precision, scale); + return d.toBigDecimal(); + } + case CHAR: + case VARCHAR: + { + BinaryString s = row.getString(index); + return s.toString(); + } + case BINARY: + case VARBINARY: + return row.getBinary(index); + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + { + int precision = DataTypeChecks.getPrecision(type); + Timestamp ts = row.getTimestamp(index, precision); + return ts.getMillisecond(); + } + default: + return row.getString(index).toString(); + } + } + + public static DataTypeRoot getTypeRoot(DataType type) { + return type.getTypeRoot(); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/sql/ParseResult.java b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/ParseResult.java new file mode 100644 index 000000000000..a7390e553340 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/ParseResult.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.calcite.sql.SqlNode; + +import javax.annotation.Nullable; + +/** Tagged result from {@link SqlParser}: either a Calcite SqlNode or a meta-command. */ +public class ParseResult { + + /** The type of parsed statement. */ + public enum Type { + QUERY, + SHOW_DATABASES, + SHOW_TABLES, + USE_DATABASE + } + + private final Type type; + @Nullable private final SqlNode sqlNode; + @Nullable private final String database; + @Nullable private final String originalSql; + + private ParseResult( + Type type, + @Nullable SqlNode sqlNode, + @Nullable String database, + @Nullable String originalSql) { + this.type = type; + this.sqlNode = sqlNode; + this.database = database; + this.originalSql = originalSql; + } + + public static ParseResult query(SqlNode node, String originalSql) { + return new ParseResult(Type.QUERY, node, null, originalSql); + } + + public static ParseResult showDatabases() { + return new ParseResult(Type.SHOW_DATABASES, null, null, null); + } + + public static ParseResult showTables(@Nullable String database) { + return new ParseResult(Type.SHOW_TABLES, null, database, null); + } + + public static ParseResult useDatabase(String database) { + return new ParseResult(Type.USE_DATABASE, null, database, null); + } + + public Type type() { + return type; + } + + @Nullable + public SqlNode sqlNode() { + return sqlNode; + } + + @Nullable + public String database() { + return database; + } + + @Nullable + public String originalSql() { + return originalSql; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/sql/RexToPredicate.java b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/RexToPredicate.java new file mode 100644 index 000000000000..cfc8e8250de4 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/RexToPredicate.java @@ -0,0 +1,272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.RowType; + +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.util.NlsString; + +import javax.annotation.Nullable; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +/** + * Converts Calcite RexNode filter expressions to Paimon Predicates for split-level pruning. Filters + * that cannot be converted are silently skipped (Calcite re-applies them row-by-row). + */ +public class RexToPredicate { + + @Nullable + public static Predicate convert(List filters, RowType rowType) { + if (filters == null || filters.isEmpty()) { + return null; + } + PredicateBuilder builder = new PredicateBuilder(rowType); + List predicates = new ArrayList<>(); + for (RexNode filter : filters) { + Predicate p = convertNode(filter, builder, rowType); + if (p != null) { + predicates.add(p); + } + } + if (predicates.isEmpty()) { + return null; + } + return PredicateBuilder.and(predicates); + } + + @Nullable + private static Predicate convertNode(RexNode node, PredicateBuilder builder, RowType rowType) { + if (!(node instanceof RexCall)) { + return null; + } + RexCall call = (RexCall) node; + SqlKind kind = call.getKind(); + + switch (kind) { + case AND: + return convertAnd(call, builder, rowType); + case OR: + return convertOr(call, builder, rowType); + case EQUALS: + return convertComparison(call, builder, rowType, CompOp.EQUAL); + case NOT_EQUALS: + return convertComparison(call, builder, rowType, CompOp.NOT_EQUAL); + case LESS_THAN: + return convertComparison(call, builder, rowType, CompOp.LESS_THAN); + case LESS_THAN_OR_EQUAL: + return convertComparison(call, builder, rowType, CompOp.LESS_OR_EQUAL); + case GREATER_THAN: + return convertComparison(call, builder, rowType, CompOp.GREATER_THAN); + case GREATER_THAN_OR_EQUAL: + return convertComparison(call, builder, rowType, CompOp.GREATER_OR_EQUAL); + case IS_NULL: + return convertIsNull(call, builder, true); + case IS_NOT_NULL: + return convertIsNull(call, builder, false); + default: + return null; + } + } + + @Nullable + private static Predicate convertAnd(RexCall call, PredicateBuilder builder, RowType rowType) { + List parts = new ArrayList<>(); + for (RexNode operand : call.getOperands()) { + Predicate p = convertNode(operand, builder, rowType); + if (p != null) { + parts.add(p); + } + } + if (parts.isEmpty()) { + return null; + } + return PredicateBuilder.and(parts); + } + + @Nullable + private static Predicate convertOr(RexCall call, PredicateBuilder builder, RowType rowType) { + List parts = new ArrayList<>(); + for (RexNode operand : call.getOperands()) { + Predicate p = convertNode(operand, builder, rowType); + if (p == null) { + return null; + } + parts.add(p); + } + if (parts.isEmpty()) { + return null; + } + return PredicateBuilder.or(parts); + } + + @Nullable + private static Predicate convertComparison( + RexCall call, PredicateBuilder builder, RowType rowType, CompOp op) { + List operands = call.getOperands(); + if (operands.size() != 2) { + return null; + } + + RexNode left = operands.get(0); + RexNode right = operands.get(1); + + int colIdx; + Object literal; + + if (left instanceof RexInputRef && right instanceof RexLiteral) { + colIdx = ((RexInputRef) left).getIndex(); + literal = extractLiteral((RexLiteral) right, rowType.getFields().get(colIdx)); + } else if (right instanceof RexInputRef && left instanceof RexLiteral) { + colIdx = ((RexInputRef) right).getIndex(); + literal = extractLiteral((RexLiteral) left, rowType.getFields().get(colIdx)); + op = op.flip(); + } else { + return null; + } + + if (literal == null) { + return null; + } + + switch (op) { + case EQUAL: + return builder.equal(colIdx, literal); + case NOT_EQUAL: + return builder.notEqual(colIdx, literal); + case LESS_THAN: + return builder.lessThan(colIdx, literal); + case LESS_OR_EQUAL: + return builder.lessOrEqual(colIdx, literal); + case GREATER_THAN: + return builder.greaterThan(colIdx, literal); + case GREATER_OR_EQUAL: + return builder.greaterOrEqual(colIdx, literal); + default: + return null; + } + } + + @Nullable + private static Predicate convertIsNull(RexCall call, PredicateBuilder builder, boolean isNull) { + if (call.getOperands().size() != 1) { + return null; + } + RexNode operand = call.getOperands().get(0); + if (!(operand instanceof RexInputRef)) { + return null; + } + int idx = ((RexInputRef) operand).getIndex(); + return isNull ? builder.isNull(idx) : builder.isNotNull(idx); + } + + @Nullable + private static Object extractLiteral(RexLiteral literal, DataField field) { + if (RexLiteral.isNullLiteral(literal)) { + return null; + } + DataTypeRoot root = field.type().getTypeRoot(); + switch (root) { + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + { + BigDecimal bd = literal.getValueAs(BigDecimal.class); + return bd != null ? bd.intValue() : null; + } + case BIGINT: + { + BigDecimal bd = literal.getValueAs(BigDecimal.class); + return bd != null ? bd.longValue() : null; + } + case FLOAT: + { + BigDecimal bd = literal.getValueAs(BigDecimal.class); + return bd != null ? bd.floatValue() : null; + } + case DOUBLE: + { + BigDecimal bd = literal.getValueAs(BigDecimal.class); + return bd != null ? bd.doubleValue() : null; + } + case TINYINT: + { + BigDecimal bd = literal.getValueAs(BigDecimal.class); + return bd != null ? bd.byteValue() : null; + } + case SMALLINT: + { + BigDecimal bd = literal.getValueAs(BigDecimal.class); + return bd != null ? bd.shortValue() : null; + } + case DECIMAL: + return literal.getValueAs(BigDecimal.class); + case CHAR: + case VARCHAR: + { + NlsString nls = literal.getValueAs(NlsString.class); + if (nls != null) { + return BinaryString.fromString(nls.getValue()); + } + String s = literal.getValueAs(String.class); + return s != null ? BinaryString.fromString(s) : null; + } + case BOOLEAN: + return literal.getValueAs(Boolean.class); + default: + return null; + } + } + + private enum CompOp { + EQUAL, + NOT_EQUAL, + LESS_THAN, + LESS_OR_EQUAL, + GREATER_THAN, + GREATER_OR_EQUAL; + + CompOp flip() { + switch (this) { + case LESS_THAN: + return GREATER_THAN; + case LESS_OR_EQUAL: + return GREATER_OR_EQUAL; + case GREATER_THAN: + return LESS_THAN; + case GREATER_OR_EQUAL: + return LESS_OR_EQUAL; + default: + return this; + } + } + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlCommand.java b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlCommand.java new file mode 100644 index 000000000000..3e1f2be760cd --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlCommand.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +/** SQL command supporting one-shot queries and interactive REPL mode. */ +public class SqlCommand implements Command { + + @Override + public String name() { + return "sql"; + } + + @Override + public String description() { + return "Execute SQL queries on Paimon tables"; + } + + @Override + public void execute(CommandContext ctx, String[] args) throws Exception { + SqlParser parser = new SqlParser(); + SqlExecutor executor = new SqlExecutor(); + + if (args.length > 0) { + String sql = String.join(" ", args); + executeSql(ctx, parser, executor, sql); + } else { + runRepl(ctx, parser, executor); + } + } + + private void executeSql( + CommandContext ctx, SqlParser parser, SqlExecutor executor, String sql) { + try { + ParseResult result = parser.parse(sql); + executor.execute(ctx, result); + } catch (IllegalArgumentException e) { + System.err.println("SQL error: " + e.getMessage()); + } catch (Exception e) { + System.err.println("Error: " + e.getMessage()); + } + } + + private void runRepl(CommandContext ctx, SqlParser parser, SqlExecutor executor) + throws Exception { + System.out.println("Paimon SQL (type 'help' for usage, 'exit' to quit)"); + System.out.println(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); + StringBuilder sqlBuf = new StringBuilder(); + + while (true) { + String prompt = sqlBuf.length() == 0 ? "paimon> " : " > "; + System.out.print(prompt); + System.out.flush(); + + String line = reader.readLine(); + if (line == null) { + break; + } + + String trimmed = line.trim(); + if (sqlBuf.length() == 0) { + if ("exit".equalsIgnoreCase(trimmed) || "quit".equalsIgnoreCase(trimmed)) { + System.out.println("Bye!"); + break; + } + if ("help".equalsIgnoreCase(trimmed)) { + printReplHelp(); + continue; + } + } + + sqlBuf.append(line).append(" "); + + if (trimmed.endsWith(";")) { + String sql = sqlBuf.toString().trim(); + if (sql.endsWith(";")) { + sql = sql.substring(0, sql.length() - 1).trim(); + } + if (!sql.isEmpty()) { + executeSql(ctx, parser, executor, sql); + } + sqlBuf.setLength(0); + System.out.println(); + } + } + } + + private void printReplHelp() { + System.out.println("Supported SQL:"); + System.out.println( + " SELECT [DISTINCT] [cols|agg(col)|*] FROM [db.]table" + + " [WHERE ...] [GROUP BY ...] [HAVING ...] [ORDER BY ...] [LIMIT n]"); + System.out.println(" SHOW DATABASES;"); + System.out.println(" SHOW TABLES [IN database];"); + System.out.println(" USE database;"); + System.out.println(); + System.out.println("Aggregate functions: COUNT, SUM, AVG, MIN, MAX"); + System.out.println(); + System.out.println("Commands:"); + System.out.println(" help - Show this help"); + System.out.println(" exit - Exit the REPL"); + System.out.println(); + System.out.println("SQL statements must end with ';'"); + } + + @Override + public String usage() { + return "Usage: paimon sql [QUERY]\n\n" + + "Execute SQL queries on Paimon tables.\n\n" + + "If QUERY is provided, execute it and exit.\n" + + "If no QUERY is provided, start an interactive REPL.\n\n" + + "Supported SQL:\n" + + " SELECT [DISTINCT] [cols|agg(col)|*] FROM [db.]table\n" + + " [WHERE ...] [GROUP BY ...] [HAVING ...] [ORDER BY ...] [LIMIT n]\n" + + " SHOW DATABASES\n" + + " SHOW TABLES [IN database]\n" + + " USE database\n\n" + + "Aggregate functions: COUNT(*), COUNT(col), SUM, AVG, MIN, MAX\n\n" + + "Examples:\n" + + " paimon sql \"SELECT * FROM mydb.users WHERE age > 18 LIMIT 10\"\n" + + " paimon sql \"SELECT city, COUNT(*) FROM mydb.users GROUP BY city\"\n" + + " paimon sql \"SELECT DISTINCT city FROM mydb.users\""; + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlExecutor.java b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlExecutor.java new file mode 100644 index 000000000000..c169c2e6ed07 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlExecutor.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.cli.CommandContext; + +import org.apache.calcite.jdbc.CalciteConnection; +import org.apache.calcite.schema.SchemaPlus; + +import javax.annotation.Nullable; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Executes SQL against Paimon tables via Apache Calcite's query planner and JDBC interface. */ +public class SqlExecutor { + + private static final int DEFAULT_LIMIT = 100; + private static final Pattern SCHEMA_REF = + Pattern.compile("(?i)(?:FROM|JOIN|INTO)\\s+([a-zA-Z_][a-zA-Z0-9_]*)\\."); + + @Nullable private String defaultDatabase; + @Nullable private Connection connection; + @Nullable private SchemaPlus rootSchema; + @Nullable private Catalog catalog; + private final Set registeredSchemas = new HashSet<>(); + + public void setDefaultDatabase(String database) { + this.defaultDatabase = database; + } + + @Nullable + public String getDefaultDatabase() { + return defaultDatabase; + } + + public void execute(CommandContext ctx, ParseResult result) throws Exception { + switch (result.type()) { + case SHOW_DATABASES: + executeShowDatabases(ctx); + break; + case SHOW_TABLES: + executeShowTables(ctx, result.database()); + break; + case USE_DATABASE: + executeUseDatabase(result.database()); + break; + case QUERY: + executeQuery(ctx, result); + break; + } + } + + public void close() { + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + // best effort + } + connection = null; + } + } + + private void executeQuery(CommandContext ctx, ParseResult result) throws Exception { + Connection conn = getOrCreateConnection(ctx); + + String sql = result.originalSql(); + + // Register schemas referenced in the SQL on demand + ensureSchemasForSql(sql); + if (defaultDatabase != null) { + ensureSchema(defaultDatabase); + conn.setSchema(defaultDatabase); + } + + if (!hasLimit(result) && !isModifyingStatement(sql)) { + sql = sql + " LIMIT " + DEFAULT_LIMIT; + } + + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + printResultSet(rs); + } + } + + private void ensureSchemasForSql(String sql) { + Matcher m = SCHEMA_REF.matcher(sql); + while (m.find()) { + ensureSchema(m.group(1)); + } + } + + private void ensureSchema(String dbName) { + if (registeredSchemas.contains(dbName)) { + return; + } + rootSchema.add(dbName, new PaimonSchema(catalog, dbName)); + registeredSchemas.add(dbName); + } + + private Connection getOrCreateConnection(CommandContext ctx) throws Exception { + if (connection != null) { + return connection; + } + + catalog = ctx.getCatalog(); + + Properties info = new Properties(); + info.setProperty("lex", "MYSQL"); + info.setProperty("caseSensitive", "false"); + info.setProperty("unquotedCasing", "UNCHANGED"); + info.setProperty("quotedCasing", "UNCHANGED"); + + Connection conn = DriverManager.getConnection("jdbc:calcite:", info); + CalciteConnection calciteConn = conn.unwrap(CalciteConnection.class); + rootSchema = calciteConn.getRootSchema(); + + connection = conn; + return conn; + } + + private static boolean hasLimit(ParseResult result) { + if (result.sqlNode() == null) { + return false; + } + String sqlUpper = result.originalSql().toUpperCase(); + return sqlUpper.contains(" LIMIT "); + } + + private static boolean isModifyingStatement(String sql) { + String upper = sql.trim().toUpperCase(); + return upper.startsWith("INSERT") + || upper.startsWith("UPDATE") + || upper.startsWith("DELETE"); + } + + private static void printResultSet(ResultSet rs) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + int cols = meta.getColumnCount(); + + StringBuilder header = new StringBuilder(); + for (int i = 1; i <= cols; i++) { + if (i > 1) { + header.append("\t"); + } + header.append(meta.getColumnLabel(i)); + } + System.out.println(header.toString()); + + int count = 0; + while (rs.next()) { + StringBuilder row = new StringBuilder(); + for (int i = 1; i <= cols; i++) { + if (i > 1) { + row.append("\t"); + } + Object val = rs.getObject(i); + row.append(val == null ? "NULL" : val); + } + System.out.println(row.toString()); + count++; + } + System.err.println("(" + count + " rows)"); + } + + private void executeShowDatabases(CommandContext ctx) throws Exception { + for (String db : ctx.getCatalog().listDatabases()) { + System.out.println(db); + } + } + + private void executeShowTables(CommandContext ctx, @Nullable String database) throws Exception { + String db = database != null ? database : defaultDatabase; + if (db == null) { + System.err.println( + "No database selected. Use 'USE ' or 'SHOW TABLES IN '."); + return; + } + for (String t : ctx.getCatalog().listTables(db)) { + System.out.println(t); + } + } + + private void executeUseDatabase(String database) { + this.defaultDatabase = database; + System.out.println("Using database '" + database + "'."); + } +} diff --git a/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlParser.java b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlParser.java new file mode 100644 index 000000000000..ac98c8244166 --- /dev/null +++ b/paimon-cli/src/main/java/org/apache/paimon/cli/sql/SqlParser.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.calcite.config.Lex; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.parser.SqlParseException; + +import javax.annotation.Nullable; + +/** Thin SQL parser wrapper around Apache Calcite. */ +public class SqlParser { + + private static final org.apache.calcite.sql.parser.SqlParser.Config PARSER_CONFIG = + org.apache.calcite.sql.parser.SqlParser.config() + .withLex(Lex.MYSQL) + .withIdentifierMaxLength(256); + + public ParseResult parse(String sql) { + String trimmed = sql.trim(); + if (trimmed.endsWith(";")) { + trimmed = trimmed.substring(0, trimmed.length() - 1).trim(); + } + + String upper = trimmed.toUpperCase(); + if (upper.startsWith("SHOW DATABASES")) { + return ParseResult.showDatabases(); + } + if (upper.startsWith("SHOW TABLES")) { + return ParseResult.showTables(extractDatabase(trimmed)); + } + if (upper.startsWith("USE ")) { + return ParseResult.useDatabase(trimmed.substring(4).trim()); + } + + try { + org.apache.calcite.sql.parser.SqlParser calciteParser = + org.apache.calcite.sql.parser.SqlParser.create(trimmed, PARSER_CONFIG); + SqlNode node = calciteParser.parseStmt(); + return ParseResult.query(node, trimmed); + } catch (SqlParseException e) { + throw new IllegalArgumentException("SQL parse error: " + e.getMessage(), e); + } + } + + @Nullable + private static String extractDatabase(String sql) { + String upper = sql.toUpperCase(); + int inIdx = upper.indexOf(" IN "); + if (inIdx > 0) { + return sql.substring(inIdx + 4).trim(); + } + return null; + } + + /** Strips backtick/double-quote quoting from Calcite's SqlNode.toString() output. */ + static String unquoteIdentifiers(String expr) { + StringBuilder sb = new StringBuilder(); + int i = 0; + while (i < expr.length()) { + char c = expr.charAt(i); + if (c == '\'') { + sb.append(c); + i++; + while (i < expr.length()) { + char ch = expr.charAt(i); + sb.append(ch); + if (ch == '\'') { + i++; + if (i < expr.length() && expr.charAt(i) == '\'') { + sb.append('\''); + i++; + } else { + break; + } + } else { + i++; + } + } + } else if (c == '`') { + i++; + while (i < expr.length() && expr.charAt(i) != '`') { + sb.append(expr.charAt(i)); + i++; + } + if (i < expr.length()) { + i++; + } + } else { + sb.append(c); + i++; + } + } + return sb.toString(); + } +} diff --git a/paimon-cli/src/main/resources/META-INF/services/org.apache.paimon.cli.Command b/paimon-cli/src/main/resources/META-INF/services/org.apache.paimon.cli.Command new file mode 100644 index 000000000000..010084df61df --- /dev/null +++ b/paimon-cli/src/main/resources/META-INF/services/org.apache.paimon.cli.Command @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.paimon.cli.commands.ListDatabasesCommand +org.apache.paimon.cli.commands.ListTablesCommand +org.apache.paimon.cli.commands.SchemaCommand +org.apache.paimon.cli.commands.ReadCommand +org.apache.paimon.cli.commands.WriteCommand +org.apache.paimon.cli.commands.CreateDatabaseCommand +org.apache.paimon.cli.commands.DropDatabaseCommand +org.apache.paimon.cli.commands.CreateTableCommand +org.apache.paimon.cli.commands.DropTableCommand +org.apache.paimon.cli.commands.ExpireSnapshotsCommand +org.apache.paimon.cli.commands.TagCommand +org.apache.paimon.cli.commands.RollbackCommand +org.apache.paimon.cli.commands.SnapshotCommand +org.apache.paimon.cli.commands.ListPartitionsCommand +org.apache.paimon.cli.commands.RenameTableCommand +org.apache.paimon.cli.commands.AlterTableCommand +org.apache.paimon.cli.commands.GetDatabaseCommand +org.apache.paimon.cli.commands.AlterDatabaseCommand +org.apache.paimon.cli.commands.ExplainCommand +org.apache.paimon.cli.commands.FullTextSearchCommand +org.apache.paimon.cli.commands.OrphanCleanCommand +org.apache.paimon.cli.commands.BranchCommand +org.apache.paimon.cli.sql.SqlCommand diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/CliArgsTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/CliArgsTest.java new file mode 100644 index 000000000000..c16f3ffe4abc --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/CliArgsTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import org.junit.jupiter.api.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +class CliArgsTest { + + @Test + void testPositionalArgs() { + CliArgs args = new CliArgs(new String[] {"mydb.mytable", "extra"}); + assertThat(args.positionalCount()).isEqualTo(2); + assertThat(args.positional(0, "table")).isEqualTo("mydb.mytable"); + assertThat(args.positional(1, "extra")).isEqualTo("extra"); + } + + @Test + void testKeyValueOptions() { + CliArgs args = new CliArgs(new String[] {"--select", "id,name", "--limit", "50"}); + assertThat(args.get("select")).isEqualTo("id,name"); + assertThat(args.get("limit")).isEqualTo("50"); + assertThat(args.positionalCount()).isEqualTo(0); + } + + @Test + void testShortAliases() { + CliArgs args = + new CliArgs( + new String[] {"-s", "id,name", "-l", "10"}, + "-s", + "--select", + "-l", + "--limit"); + assertThat(args.get("select")).isEqualTo("id,name"); + assertThat(args.get("limit")).isEqualTo("10"); + } + + @Test + void testBooleanFlags() { + Set flags = new HashSet<>(); + flags.add("cascade"); + flags.add("ignore-if-exists"); + CliArgs args = + new CliArgs( + new String[] {"mydb", "--cascade", "--ignore-if-exists"}, + flags, + "-i", + "--ignore-if-exists"); + assertThat(args.positional(0, "db")).isEqualTo("mydb"); + assertThat(args.get("cascade")).isEqualTo("true"); + assertThat(args.get("ignore-if-exists")).isEqualTo("true"); + } + + @Test + void testMixedPositionalAndOptions() { + CliArgs args = + new CliArgs( + new String[] {"mydb.table", "--format", "json", "--limit", "20"}, + "-f", + "--format", + "-l", + "--limit"); + assertThat(args.positional(0, "table")).isEqualTo("mydb.table"); + assertThat(args.get("format")).isEqualTo("json"); + assertThat(args.get("limit")).isEqualTo("20"); + } + + @Test + void testHasHelp() { + CliArgs args1 = new CliArgs(new String[] {"--help"}); + assertThat(args1.hasHelp()).isTrue(); + + CliArgs args2 = new CliArgs(new String[] {"-h"}); + assertThat(args2.hasHelp()).isTrue(); + + CliArgs args3 = new CliArgs(new String[] {"mydb"}); + assertThat(args3.hasHelp()).isFalse(); + } + + @Test + void testGetOrDefault() { + CliArgs args = new CliArgs(new String[] {"--format", "csv"}); + assertThat(args.getOrDefault("format", "tsv")).isEqualTo("csv"); + assertThat(args.getOrDefault("limit", "100")).isEqualTo("100"); + } + + @Test + void testNullForMissingOption() { + CliArgs args = new CliArgs(new String[] {"mydb"}); + assertThat(args.get("select")).isNull(); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/CliConfigTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/CliConfigTest.java new file mode 100644 index 000000000000..436809f2b801 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/CliConfigTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import org.apache.paimon.options.Options; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class CliConfigTest { + + @TempDir Path tempDir; + + @Test + void testLoadYaml() throws Exception { + Path configFile = tempDir.resolve("paimon.yaml"); + String yaml = "metastore: filesystem\nwarehouse: /tmp/warehouse\n"; + Files.write(configFile, yaml.getBytes(StandardCharsets.UTF_8)); + + Options options = CliConfig.load(configFile.toString()); + assertThat(options.get("metastore")).isEqualTo("filesystem"); + assertThat(options.get("warehouse")).isEqualTo("/tmp/warehouse"); + } + + @Test + void testLoadYamlWithNestedKeys() throws Exception { + Path configFile = tempDir.resolve("paimon.yaml"); + String yaml = + "metastore: rest\n" + "uri: http://localhost:8080\n" + "warehouse: my_catalog\n"; + Files.write(configFile, yaml.getBytes(StandardCharsets.UTF_8)); + + Options options = CliConfig.load(configFile.toString()); + assertThat(options.get("metastore")).isEqualTo("rest"); + assertThat(options.get("uri")).isEqualTo("http://localhost:8080"); + assertThat(options.get("warehouse")).isEqualTo("my_catalog"); + } + + @Test + void testFlattenedNestedMap() throws Exception { + Path configFile = tempDir.resolve("paimon.yaml"); + String yaml = + "metastore: filesystem\nwarehouse: /tmp/wh\nfs:\n oss:\n endpoint: oss-cn-hangzhou.aliyuncs.com\n"; + Files.write(configFile, yaml.getBytes(StandardCharsets.UTF_8)); + + Options options = CliConfig.load(configFile.toString()); + assertThat(options.get("fs.oss.endpoint")).isEqualTo("oss-cn-hangzhou.aliyuncs.com"); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/PaimonCliTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/PaimonCliTest.java new file mode 100644 index 000000000000..d51b93e41a5c --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/PaimonCliTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class PaimonCliTest { + + @Test + void testSpiLoadsAllBuiltinCommands() { + PaimonCli cli = new PaimonCli(); + Map commands = cli.getCommands(); + + assertThat(commands).containsKey("list-databases"); + assertThat(commands).containsKey("list-tables"); + assertThat(commands).containsKey("schema"); + assertThat(commands).containsKey("read"); + assertThat(commands).containsKey("write"); + assertThat(commands).containsKey("create-database"); + assertThat(commands).containsKey("drop-database"); + assertThat(commands).containsKey("create-table"); + assertThat(commands).containsKey("drop-table"); + assertThat(commands).containsKey("orphan-clean"); + assertThat(commands).containsKey("branch"); + } + + @Test + void testRegisterCustomCommand() { + PaimonCli cli = new PaimonCli(); + Command custom = + new Command() { + @Override + public String name() { + return "custom"; + } + + @Override + public String description() { + return "A custom command"; + } + + @Override + public void execute(CommandContext ctx, String[] args) {} + }; + cli.registerCommand(custom); + assertThat(cli.getCommands()).containsKey("custom"); + assertThat(cli.getCommands().get("custom").description()).isEqualTo("A custom command"); + } + + @Test + void testCommandDescriptionsNotEmpty() { + PaimonCli cli = new PaimonCli(); + for (Command cmd : cli.getCommands().values()) { + assertThat(cmd.name()).isNotEmpty(); + assertThat(cmd.description()).isNotEmpty(); + assertThat(cmd.usage()).isNotEmpty(); + } + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/AlterTableCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/AlterTableCommandTest.java new file mode 100644 index 000000000000..73bb986dc657 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/AlterTableCommandTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class AlterTableCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + catalog.createDatabase("testdb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .column("age", DataTypes.INT()) + .primaryKey("id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("testdb", "t1"), schema, true); + catalog.close(); + } + + @Test + void testSetOption() throws Exception { + String output = + execute( + "testdb.t1", + "set-option", + "--key", + "write-buffer-size", + "--value", + "128mb"); + assertThat(output).contains("altered successfully"); + + // Verify option was set + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + assertThat(table.options()).containsEntry("write-buffer-size", "128mb"); + catalog.close(); + } + + @Test + void testRemoveOption() throws Exception { + execute("testdb.t1", "set-option", "--key", "custom-opt", "--value", "v1"); + String output = execute("testdb.t1", "remove-option", "--key", "custom-opt"); + assertThat(output).contains("altered successfully"); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + assertThat(table.options()).doesNotContainKey("custom-opt"); + catalog.close(); + } + + @Test + void testAddColumn() throws Exception { + String output = execute("testdb.t1", "add-column", "--name", "email", "--type", "STRING"); + assertThat(output).contains("altered successfully"); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + assertThat(table.rowType().getFieldNames()).contains("email"); + catalog.close(); + } + + @Test + void testDropColumn() throws Exception { + String output = execute("testdb.t1", "drop-column", "--name", "age"); + assertThat(output).contains("altered successfully"); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + assertThat(table.rowType().getFieldNames()).doesNotContain("age"); + catalog.close(); + } + + @Test + void testRenameColumn() throws Exception { + String output = + execute("testdb.t1", "rename-column", "--name", "name", "--new-name", "username"); + assertThat(output).contains("altered successfully"); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + assertThat(table.rowType().getFieldNames()).contains("username"); + assertThat(table.rowType().getFieldNames()).doesNotContain("name"); + catalog.close(); + } + + @Test + void testRenameTable() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new RenameTableCommand().execute(ctx, new String[] {"testdb.t1", "testdb.t1_renamed"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("renamed"); + + CatalogContext cctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(cctx); + List tables = catalog.listTables("testdb"); + assertThat(tables).contains("t1_renamed"); + assertThat(tables).doesNotContain("t1"); + catalog.close(); + } + + private String execute(String... args) throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new AlterTableCommand().execute(ctx, args); + } finally { + System.setOut(originalOut); + } + return baos.toString(); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/BranchCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/BranchCommandTest.java new file mode 100644 index 000000000000..72bf8278a4cb --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/BranchCommandTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class BranchCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + catalog.createDatabase("testdb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("testdb", "t1"), schema, true); + catalog.close(); + } + + @Test + void testListBranchesEmpty() throws Exception { + String output = execute("list", "testdb.t1"); + assertThat(output).contains("No branches found"); + } + + @Test + void testCreateAndListBranch() throws Exception { + String output = execute("create", "testdb.t1", "--name", "dev"); + assertThat(output).contains("Branch 'dev' created"); + + output = execute("list", "testdb.t1"); + assertThat(output.trim()).isEqualTo("dev"); + } + + @Test + void testRenameBranch() throws Exception { + execute("create", "testdb.t1", "--name", "old-name"); + execute("rename", "testdb.t1", "--name", "old-name", "--new-name", "new-name"); + + String output = execute("list", "testdb.t1"); + assertThat(output.trim()).isEqualTo("new-name"); + } + + @Test + void testDeleteBranch() throws Exception { + execute("create", "testdb.t1", "--name", "to-delete"); + String output = execute("delete", "testdb.t1", "--name", "to-delete"); + assertThat(output).contains("deleted"); + + output = execute("list", "testdb.t1"); + assertThat(output).contains("No branches found"); + } + + @Test + void testMultipleBranches() throws Exception { + execute("create", "testdb.t1", "--name", "branch-a"); + execute("create", "testdb.t1", "--name", "branch-b"); + + String output = execute("list", "testdb.t1"); + assertThat(output).contains("branch-a"); + assertThat(output).contains("branch-b"); + } + + private String execute(String... args) throws Exception { + String[] fullArgs = new String[args.length]; + System.arraycopy(args, 0, fullArgs, 0, args.length); + + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new BranchCommand().execute(ctx, fullArgs); + } finally { + System.setOut(originalOut); + } + return baos.toString(); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/DatabaseCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/DatabaseCommandTest.java new file mode 100644 index 000000000000..58f9d9a3769b --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/DatabaseCommandTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.options.Options; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class DatabaseCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + } + + @Test + void testCreateAndListDatabase() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new CreateDatabaseCommand().execute(ctx, new String[] {"mydb"}); + new CreateDatabaseCommand().execute(ctx, new String[] {"analytics"}); + + baos.reset(); + new ListDatabasesCommand().execute(ctx, new String[] {}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("mydb"); + assertThat(output).contains("analytics"); + } + + @Test + void testCreateDatabaseIgnoreIfExists() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new CreateDatabaseCommand().execute(ctx, new String[] {"mydb"}); + // Should not throw + new CreateDatabaseCommand().execute(ctx, new String[] {"mydb", "--ignore-if-exists"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("created successfully"); + } + + @Test + void testDropDatabase() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new CreateDatabaseCommand().execute(ctx, new String[] {"tempdb"}); + + baos.reset(); + new DropDatabaseCommand().execute(ctx, new String[] {"tempdb"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("dropped successfully"); + } + + @Test + void testDropDatabaseIgnoreIfNotExists() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + // Should not throw + new DropDatabaseCommand() + .execute(ctx, new String[] {"nonexistent", "--ignore-if-not-exists"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("dropped successfully"); + } + + @Test + void testListDatabasesEmpty() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new ListDatabasesCommand().execute(ctx, new String[] {}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("No databases found"); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/ExplainCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/ExplainCommandTest.java new file mode 100644 index 000000000000..cf94bcc3cd21 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/ExplainCommandTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ExplainCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + catalog.createDatabase("testdb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .column("city", DataTypes.STRING()) + .primaryKey("id") + .option("bucket", "2") + .build(); + catalog.createTable(Identifier.create("testdb", "users"), schema, true); + + Table table = catalog.getTable(Identifier.create("testdb", "users")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + writer.write( + GenericRow.of( + 1, BinaryString.fromString("Alice"), BinaryString.fromString("Beijing"))); + writer.write( + GenericRow.of( + 2, BinaryString.fromString("Bob"), BinaryString.fromString("Shanghai"))); + List messages = writer.prepareCommit(); + writer.close(); + writeBuilder.newCommit().commit(messages); + catalog.close(); + } + + @Test + void testExplainBasic() throws Exception { + String output = execute("testdb.users"); + assertThat(output).contains("Scan Plan"); + assertThat(output).contains("Snapshot"); + } + + @Test + void testExplainWithFilter() throws Exception { + String output = execute("testdb.users", "--where", "id > 1"); + assertThat(output).contains("Scan Plan"); + } + + @Test + void testExplainWithProjection() throws Exception { + String output = execute("testdb.users", "--select", "id,name"); + assertThat(output).contains("Scan Plan"); + } + + @Test + void testListPartitionsNonPartitioned() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new ListPartitionsCommand().execute(ctx, new String[] {"testdb.users"}); + } finally { + System.setOut(originalOut); + } + + assertThat(baos.toString()).contains("not partitioned"); + } + + @Test + void testListPartitionsWithData() throws Exception { + // Create partitioned table + CatalogContext cctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(cctx); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("city", DataTypes.STRING()) + .column("value", DataTypes.INT()) + .partitionKeys("city") + .primaryKey("id", "city") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("testdb", "partitioned"), schema, true); + + Table table = catalog.getTable(Identifier.create("testdb", "partitioned")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + writer.write(GenericRow.of(1, BinaryString.fromString("Beijing"), 100)); + writer.write(GenericRow.of(2, BinaryString.fromString("Shanghai"), 200)); + List messages = writer.prepareCommit(); + writer.close(); + writeBuilder.newCommit().commit(messages); + catalog.close(); + + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new ListPartitionsCommand().execute(ctx, new String[] {"testdb.partitioned"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("Beijing"); + assertThat(output).contains("Shanghai"); + assertThat(output).contains("RecordCount"); + } + + private String execute(String... args) throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new ExplainCommand().execute(ctx, args); + } finally { + System.setOut(originalOut); + } + return baos.toString(); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/OrphanCleanCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/OrphanCleanCommandTest.java new file mode 100644 index 000000000000..6df1f01bbd28 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/OrphanCleanCommandTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class OrphanCleanCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + catalog.createDatabase("testdb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("value", DataTypes.STRING()) + .primaryKey("id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("testdb", "t1"), schema, true); + + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + writer.write(GenericRow.of(1, org.apache.paimon.data.BinaryString.fromString("a"))); + List messages = writer.prepareCommit(); + writer.close(); + writeBuilder.newCommit().commit(messages); + + catalog.close(); + } + + @Test + void testDryRunNoOrphans() throws Exception { + String output = execute("testdb.t1", "--dry-run"); + assertThat(output).contains("Dry run"); + assertThat(output).contains("0 orphan files"); + } + + @Test + void testCleanNoOrphans() throws Exception { + String output = execute("testdb.t1"); + assertThat(output).contains("Deleted 0 orphan files"); + } + + @Test + void testCleanWithOrphanFile() throws Exception { + // Create an orphan file in the data directory + Path dataDir = tempDir.resolve("testdb.db").resolve("t1").resolve("bucket-0"); + Files.createDirectories(dataDir); + Path orphan = dataDir.resolve("orphan-file-001.parquet"); + Files.write(orphan, new byte[1024]); + // Set file modification time to 2 days ago so it's older than default 1d threshold + Files.setLastModifiedTime( + orphan, + java.nio.file.attribute.FileTime.fromMillis( + System.currentTimeMillis() - 2 * 24 * 3600 * 1000L)); + + String output = execute("testdb.t1"); + assertThat(output).contains("Deleted 1 orphan files"); + } + + @Test + void testCleanAllTablesInDatabase() throws Exception { + String output = execute("testdb.*"); + assertThat(output).contains("Deleted 0 orphan files"); + } + + @Test + void testOlderThanDurationParsing() throws Exception { + // With --older-than 0d, everything is considered old + String output = execute("testdb.t1", "--older-than", "0d", "--dry-run"); + assertThat(output).contains("Dry run"); + } + + private String execute(String... args) throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new OrphanCleanCommand().execute(ctx, args); + } finally { + System.setOut(originalOut); + } + return baos.toString(); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/ReadCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/ReadCommandTest.java new file mode 100644 index 000000000000..824b5bf0850a --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/ReadCommandTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ReadCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + // Create test database and table + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + + catalog.createDatabase("testdb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .column("age", DataTypes.INT()) + .primaryKey("id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("testdb", "users"), schema, true); + + // Insert test data + Table table = catalog.getTable(Identifier.create("testdb", "users")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + + writer.write(GenericRow.of(1, BinaryString.fromString("Alice"), 25)); + writer.write(GenericRow.of(2, BinaryString.fromString("Bob"), 30)); + writer.write(GenericRow.of(3, BinaryString.fromString("Charlie"), 35)); + + List messages = writer.prepareCommit(); + writer.close(); + writeBuilder.newCommit().commit(messages); + catalog.close(); + } + + @Test + void testReadAll() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + ReadCommand cmd = new ReadCommand(); + cmd.execute(ctx, new String[] {"testdb.users"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("id"); + assertThat(output).contains("name"); + assertThat(output).contains("Alice"); + assertThat(output).contains("Bob"); + assertThat(output).contains("Charlie"); + } + + @Test + void testReadWithSelect() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + ReadCommand cmd = new ReadCommand(); + cmd.execute(ctx, new String[] {"testdb.users", "--select", "id,name"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("id"); + assertThat(output).contains("name"); + assertThat(output).contains("Alice"); + // age column should not be in output header + String headerLine = output.split("\n")[0]; + assertThat(headerLine).doesNotContain("age"); + } + + @Test + void testReadWithLimit() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + ReadCommand cmd = new ReadCommand(); + cmd.execute(ctx, new String[] {"testdb.users", "--limit", "1"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + String[] lines = output.trim().split("\n"); + // 1 header + 1 data row + assertThat(lines.length).isEqualTo(2); + } + + @Test + void testReadJsonFormat() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + ReadCommand cmd = new ReadCommand(); + cmd.execute(ctx, new String[] {"testdb.users", "--format", "json", "--limit", "1"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString().trim(); + assertThat(output).startsWith("{"); + assertThat(output).contains("\"id\":"); + assertThat(output).contains("\"name\":"); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/SchemaCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/SchemaCommandTest.java new file mode 100644 index 000000000000..19fee1b1d8b6 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/SchemaCommandTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class SchemaCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + + catalog.createDatabase("testdb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("name", DataTypes.STRING()) + .column("dt", DataTypes.STRING()) + .primaryKey("id") + .partitionKeys("dt") + .option("bucket", "4") + .build(); + catalog.createTable(Identifier.create("testdb", "events"), schema, true); + catalog.close(); + } + + @Test + void testSchemaDisplaysColumns() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + SchemaCommand cmd = new SchemaCommand(); + cmd.execute(ctx, new String[] {"testdb.events"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("Table: testdb.events"); + assertThat(output).contains("id"); + assertThat(output).contains("name"); + assertThat(output).contains("dt"); + assertThat(output).contains("BIGINT"); + assertThat(output).contains("STRING"); + } + + @Test + void testSchemaDisplaysPrimaryKeys() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + SchemaCommand cmd = new SchemaCommand(); + cmd.execute(ctx, new String[] {"testdb.events"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("Primary keys: id"); + } + + @Test + void testSchemaDisplaysPartitionKeys() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + SchemaCommand cmd = new SchemaCommand(); + cmd.execute(ctx, new String[] {"testdb.events"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("Partition keys: dt"); + } + + @Test + void testSchemaDisplaysOptions() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + SchemaCommand cmd = new SchemaCommand(); + cmd.execute(ctx, new String[] {"testdb.events"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("bucket = 4"); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/SnapshotCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/SnapshotCommandTest.java new file mode 100644 index 000000000000..a809148e3c61 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/SnapshotCommandTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.Command; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class SnapshotCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + catalog.createDatabase("testdb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .primaryKey("id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("testdb", "t1"), schema, true); + + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + writer.write(GenericRow.of(1, BinaryString.fromString("Alice"))); + List messages = writer.prepareCommit(); + writer.close(); + writeBuilder.newCommit().commit(messages); + catalog.close(); + } + + @Test + void testSnapshotShowsJson() throws Exception { + String output = execute(new SnapshotCommand(), "testdb.t1"); + assertThat(output).contains("\"id\""); + assertThat(output).contains("\"schemaId\""); + } + + @Test + void testSnapshotNoData() throws Exception { + // Create empty table + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .primaryKey("id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("testdb", "empty"), schema, true); + catalog.close(); + + String output = execute(new SnapshotCommand(), "testdb.empty"); + assertThat(output).contains("No snapshot found"); + } + + @Test + void testTagCreateAndList() throws Exception { + execute(new TagCommand(), "create", "testdb.t1", "--tag-name", "v1"); + String output = execute(new TagCommand(), "list", "testdb.t1"); + assertThat(output).contains("v1"); + } + + @Test + void testTagDelete() throws Exception { + execute(new TagCommand(), "create", "testdb.t1", "--tag-name", "v2"); + String output = execute(new TagCommand(), "delete", "testdb.t1", "--tag-name", "v2"); + assertThat(output).contains("deleted"); + + output = execute(new TagCommand(), "list", "testdb.t1"); + assertThat(output).contains("No tags found"); + } + + @Test + void testExpireSnapshots() throws Exception { + // Write a second batch to create more snapshots + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + writer.write(GenericRow.of(2, BinaryString.fromString("Bob"))); + List messages = writer.prepareCommit(); + writer.close(); + writeBuilder.newCommit().commit(messages); + catalog.close(); + + String output = execute(new ExpireSnapshotsCommand(), "testdb.t1", "--retain-max", "1"); + assertThat(output).contains("Expired"); + assertThat(output).contains("snapshots"); + } + + @Test + void testRollbackToTag() throws Exception { + execute(new TagCommand(), "create", "testdb.t1", "--tag-name", "rollback-point"); + + // Write more data + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Table table = catalog.getTable(Identifier.create("testdb", "t1")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + writer.write(GenericRow.of(2, BinaryString.fromString("Bob"))); + List messages = writer.prepareCommit(); + writer.close(); + writeBuilder.newCommit().commit(messages); + catalog.close(); + + String output = execute(new RollbackCommand(), "testdb.t1", "--tag", "rollback-point"); + assertThat(output).contains("rolled back to tag 'rollback-point'"); + } + + private String execute(Command cmd, String... args) throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + cmd.execute(ctx, args); + } finally { + System.setOut(originalOut); + } + return baos.toString(); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/TableCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/TableCommandTest.java new file mode 100644 index 000000000000..36066c1c63f0 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/TableCommandTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.options.Options; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class TableCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + catalog.createDatabase("testdb", true); + catalog.close(); + } + + @Test + void testCreateTableFromSchema() throws Exception { + Path schemaFile = tempDir.resolve("schema.json"); + String schemaJson = + "{\n" + + " \"fields\": [\n" + + " {\"name\": \"id\", \"type\": \"BIGINT\"},\n" + + " {\"name\": \"name\", \"type\": \"STRING\"},\n" + + " {\"name\": \"amount\", \"type\": \"DECIMAL(10,2)\"}\n" + + " ],\n" + + " \"primaryKeys\": [\"id\"],\n" + + " \"options\": {\"bucket\": \"2\"}\n" + + "}"; + Files.write(schemaFile, schemaJson.getBytes(StandardCharsets.UTF_8)); + + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new CreateTableCommand() + .execute( + ctx, new String[] {"testdb.orders", "--schema", schemaFile.toString()}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("created successfully"); + + // Verify table exists via list-tables + baos.reset(); + System.setOut(new PrintStream(baos)); + try (CommandContext ctx = new CommandContext(options)) { + new ListTablesCommand().execute(ctx, new String[] {"testdb"}); + } finally { + System.setOut(originalOut); + } + assertThat(baos.toString()).contains("orders"); + } + + @Test + void testCreateTableIgnoreIfExists() throws Exception { + Path schemaFile = tempDir.resolve("schema.json"); + String schemaJson = + "{\"fields\": [{\"name\": \"id\", \"type\": \"INT\"}]," + + "\"primaryKeys\": [\"id\"], \"options\": {\"bucket\": \"1\"}}"; + Files.write(schemaFile, schemaJson.getBytes(StandardCharsets.UTF_8)); + + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + CreateTableCommand cmd = new CreateTableCommand(); + cmd.execute(ctx, new String[] {"testdb.t1", "--schema", schemaFile.toString()}); + // Should not throw + cmd.execute( + ctx, + new String[] { + "testdb.t1", "--schema", schemaFile.toString(), "--ignore-if-exists" + }); + } finally { + System.setOut(originalOut); + } + + assertThat(baos.toString()).contains("created successfully"); + } + + @Test + void testDropTable() throws Exception { + Path schemaFile = tempDir.resolve("schema.json"); + String schemaJson = + "{\"fields\": [{\"name\": \"id\", \"type\": \"INT\"}]," + + "\"primaryKeys\": [\"id\"], \"options\": {\"bucket\": \"1\"}}"; + Files.write(schemaFile, schemaJson.getBytes(StandardCharsets.UTF_8)); + + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new CreateTableCommand() + .execute( + ctx, + new String[] {"testdb.to_drop", "--schema", schemaFile.toString()}); + + baos.reset(); + new DropTableCommand().execute(ctx, new String[] {"testdb.to_drop"}); + } finally { + System.setOut(originalOut); + } + + assertThat(baos.toString()).contains("dropped successfully"); + } + + @Test + void testDropTableIgnoreIfNotExists() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new DropTableCommand() + .execute(ctx, new String[] {"testdb.nonexistent", "--ignore-if-not-exists"}); + } finally { + System.setOut(originalOut); + } + + assertThat(baos.toString()).contains("dropped successfully"); + } + + @Test + void testListTablesEmpty() throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + new ListTablesCommand().execute(ctx, new String[] {"testdb"}); + } finally { + System.setOut(originalOut); + } + + assertThat(baos.toString()).contains("No tables found"); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/commands/WriteCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/WriteCommandTest.java new file mode 100644 index 000000000000..9e5856a82206 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/commands/WriteCommandTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.commands; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.options.Options; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class WriteCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + + catalog.createDatabase("testdb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .column("score", DataTypes.DOUBLE()) + .primaryKey("id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("testdb", "scores"), schema, true); + catalog.close(); + } + + @Test + void testWriteCsv() throws Exception { + Path csvFile = tempDir.resolve("data.csv"); + Files.write( + csvFile, + "id,name,score\n1,Alice,95.5\n2,Bob,88.0\n3,Charlie,72.3\n" + .getBytes(StandardCharsets.UTF_8)); + + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + WriteCommand cmd = new WriteCommand(); + cmd.execute(ctx, new String[] {csvFile.toString(), "testdb.scores"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("Successfully wrote 3 rows"); + + List rows = readAllRows("testdb", "scores"); + assertThat(rows).hasSize(3); + } + + @Test + void testWriteJson() throws Exception { + Path jsonFile = tempDir.resolve("data.json"); + String json = + "[{\"id\":1,\"name\":\"Alice\",\"score\":95.5}," + + "{\"id\":2,\"name\":\"Bob\",\"score\":88.0}]"; + Files.write(jsonFile, json.getBytes(StandardCharsets.UTF_8)); + + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + WriteCommand cmd = new WriteCommand(); + cmd.execute( + ctx, new String[] {jsonFile.toString(), "testdb.scores", "--format", "json"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("Successfully wrote 2 rows"); + + List rows = readAllRows("testdb", "scores"); + assertThat(rows).hasSize(2); + } + + @Test + void testWriteCsvWithoutHeader() throws Exception { + Path csvFile = tempDir.resolve("noheader.csv"); + Files.write(csvFile, "1,Alice,95.5\n2,Bob,88.0\n".getBytes(StandardCharsets.UTF_8)); + + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + WriteCommand cmd = new WriteCommand(); + cmd.execute(ctx, new String[] {csvFile.toString(), "testdb.scores"}); + } finally { + System.setOut(originalOut); + } + + String output = baos.toString(); + assertThat(output).contains("Successfully wrote 2 rows"); + } + + private List readAllRows(String db, String table) throws Exception { + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + Table t = catalog.getTable(Identifier.create(db, table)); + ReadBuilder readBuilder = t.newReadBuilder(); + List splits = readBuilder.newScan().plan().splits(); + List rows = new ArrayList<>(); + for (Split split : splits) { + RecordReader reader = readBuilder.newRead().createReader(split); + reader.forEachRemaining(rows::add); + } + catalog.close(); + return rows; + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/sql/SqlCommandTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/sql/SqlCommandTest.java new file mode 100644 index 000000000000..b656c632b515 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/sql/SqlCommandTest.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.CatalogFactory; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.cli.CommandContext; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.options.Options; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Integration tests for SQL command execution end-to-end. */ +class SqlCommandTest { + + @TempDir Path tempDir; + private Options options; + + @BeforeEach + void setUp() throws Exception { + options = new Options(); + options.set("metastore", "filesystem"); + options.set("warehouse", tempDir.toString()); + + CatalogContext ctx = CatalogContext.create(options); + Catalog catalog = CatalogFactory.createCatalog(ctx); + catalog.createDatabase("mydb", true); + + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .column("age", DataTypes.INT()) + .column("city", DataTypes.STRING()) + .primaryKey("id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("mydb", "users"), schema, true); + + Table table = catalog.getTable(Identifier.create("mydb", "users")); + BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder(); + BatchTableWrite writer = writeBuilder.newWrite(); + writer.write( + GenericRow.of( + 1, + BinaryString.fromString("Alice"), + 25, + BinaryString.fromString("Beijing"))); + writer.write( + GenericRow.of( + 2, + BinaryString.fromString("Bob"), + 30, + BinaryString.fromString("Shanghai"))); + writer.write( + GenericRow.of( + 3, + BinaryString.fromString("Charlie"), + 35, + BinaryString.fromString("Beijing"))); + writer.write( + GenericRow.of( + 4, + BinaryString.fromString("David"), + 28, + BinaryString.fromString("Shanghai"))); + writer.write( + GenericRow.of( + 5, BinaryString.fromString("Eve"), 22, BinaryString.fromString("Beijing"))); + List messages = writer.prepareCommit(); + writer.close(); + writeBuilder.newCommit().commit(messages); + + // Create a second table for JOIN tests + Schema orderSchema = + Schema.newBuilder() + .column("order_id", DataTypes.INT()) + .column("user_id", DataTypes.INT()) + .column("amount", DataTypes.INT()) + .primaryKey("order_id") + .option("bucket", "1") + .build(); + catalog.createTable(Identifier.create("mydb", "orders"), orderSchema, true); + + Table orderTable = catalog.getTable(Identifier.create("mydb", "orders")); + BatchWriteBuilder owb = orderTable.newBatchWriteBuilder(); + BatchTableWrite ow = owb.newWrite(); + ow.write(GenericRow.of(101, 1, 500)); + ow.write(GenericRow.of(102, 2, 300)); + ow.write(GenericRow.of(103, 1, 200)); + ow.write(GenericRow.of(104, 3, 700)); + List orderMessages = ow.prepareCommit(); + ow.close(); + owb.newCommit().commit(orderMessages); + + catalog.close(); + } + + @Test + void testSelectAll() throws Exception { + String output = executeSql("SELECT * FROM mydb.users"); + assertThat(output).contains("Alice"); + assertThat(output).contains("Bob"); + assertThat(output).contains("Charlie"); + assertThat(output).contains("David"); + assertThat(output).contains("Eve"); + } + + @Test + void testSelectWithWhere() throws Exception { + String output = executeSql("SELECT * FROM mydb.users WHERE id = 1"); + assertThat(output).contains("Alice"); + } + + @Test + void testSelectWithLimit() throws Exception { + String output = executeSql("SELECT * FROM mydb.users LIMIT 2"); + // Only 2 data rows (plus header) + String[] lines = output.trim().split("\n"); + assertThat(lines).hasSize(3); + } + + @Test + void testSelectColumns() throws Exception { + String output = executeSql("SELECT id, name FROM mydb.users LIMIT 1"); + String headerLine = output.split("\n")[0]; + assertThat(headerLine).contains("id"); + assertThat(headerLine).contains("name"); + assertThat(headerLine).doesNotContain("age"); + assertThat(headerLine).doesNotContain("city"); + } + + @Test + void testGroupByCount() throws Exception { + String output = executeSql("SELECT city, COUNT(*) FROM mydb.users GROUP BY city"); + assertThat(output).contains("Beijing"); + assertThat(output).contains("Shanghai"); + assertThat(output).contains("3"); + assertThat(output).contains("2"); + } + + @Test + void testGroupByAvg() throws Exception { + String output = executeSql("SELECT city, AVG(age) FROM mydb.users GROUP BY city"); + assertThat(output).contains("Beijing"); + assertThat(output).contains("Shanghai"); + } + + @Test + void testDistinct() throws Exception { + String output = executeSql("SELECT DISTINCT city FROM mydb.users"); + assertThat(output).contains("Beijing"); + assertThat(output).contains("Shanghai"); + // Only 2 distinct cities + header + String[] lines = output.trim().split("\n"); + assertThat(lines).hasSize(3); + } + + @Test + void testOrderByLimit() throws Exception { + String output = executeSql("SELECT * FROM mydb.users ORDER BY age DESC LIMIT 2"); + String[] lines = output.trim().split("\n"); + // header + 2 data rows + assertThat(lines).hasSize(3); + } + + @Test + void testJoin() throws Exception { + String output = + executeSql( + "SELECT u.name, o.amount FROM mydb.users u" + + " JOIN mydb.orders o ON u.id = o.user_id" + + " ORDER BY o.amount DESC LIMIT 10"); + assertThat(output).contains("Alice"); + assertThat(output).contains("Charlie"); + assertThat(output).contains("700"); + } + + @Test + void testSubquery() throws Exception { + String output = + executeSql( + "SELECT name FROM mydb.users" + + " WHERE id IN (SELECT user_id FROM mydb.orders WHERE amount > 400)"); + assertThat(output).contains("Alice"); + assertThat(output).contains("Charlie"); + } + + @Test + void testCaseWhen() throws Exception { + String output = + executeSql( + "SELECT name, CASE WHEN age >= 30 THEN 'senior' ELSE 'junior' END" + + " FROM mydb.users LIMIT 5"); + assertThat(output).contains("senior"); + assertThat(output).contains("junior"); + } + + @Test + void testUnion() throws Exception { + String output = + executeSql( + "SELECT name FROM mydb.users WHERE city = 'Beijing'" + + " UNION SELECT name FROM mydb.users WHERE age > 29"); + assertThat(output).contains("Alice"); + assertThat(output).contains("Charlie"); + assertThat(output).contains("Bob"); + } + + @Test + void testHaving() throws Exception { + String output = + executeSql( + "SELECT city, COUNT(*) FROM mydb.users" + + " GROUP BY city HAVING COUNT(*) >= 3"); + assertThat(output).contains("Beijing"); + assertThat(output).doesNotContain("Shanghai"); + } + + @Test + void testArithmeticExpression() throws Exception { + String output = executeSql("SELECT name, age * 2 FROM mydb.users WHERE id = 1"); + assertThat(output).contains("Alice"); + assertThat(output).contains("50"); + } + + @Test + void testWindowFunction() throws Exception { + String output = + executeSql( + "SELECT name, age, ROW_NUMBER() OVER (ORDER BY age DESC) as rn" + + " FROM mydb.users LIMIT 5"); + assertThat(output).contains("rn"); + assertThat(output).contains("1"); + assertThat(output).contains("Charlie"); + } + + @Test + void testShowDatabases() throws Exception { + String output = executeSql("SHOW DATABASES"); + assertThat(output).contains("mydb"); + } + + @Test + void testShowTables() throws Exception { + String output = executeSql("SHOW TABLES IN mydb"); + assertThat(output).contains("users"); + } + + private String executeSql(String sql) throws Exception { + PrintStream originalOut = System.out; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + try (CommandContext ctx = new CommandContext(options)) { + SqlCommand cmd = new SqlCommand(); + cmd.execute(ctx, new String[] {sql}); + } finally { + System.setOut(originalOut); + } + return baos.toString(); + } +} diff --git a/paimon-cli/src/test/java/org/apache/paimon/cli/sql/SqlParserTest.java b/paimon-cli/src/test/java/org/apache/paimon/cli/sql/SqlParserTest.java new file mode 100644 index 000000000000..23bfd14ab774 --- /dev/null +++ b/paimon-cli/src/test/java/org/apache/paimon/cli/sql/SqlParserTest.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.cli.sql; + +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOrderBy; +import org.apache.calcite.sql.SqlSelect; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link SqlParser}. */ +class SqlParserTest { + + private final SqlParser parser = new SqlParser(); + + @Test + void testSelectAll() { + ParseResult result = parser.parse("SELECT * FROM mydb.users"); + assertThat(result.type()).isEqualTo(ParseResult.Type.QUERY); + SqlSelect select = (SqlSelect) result.sqlNode(); + assertThat(select.getSelectList().get(0)).isInstanceOf(SqlIdentifier.class); + assertThat(((SqlIdentifier) select.getSelectList().get(0)).isStar()).isTrue(); + assertThat(extractTableName(select)).isEqualTo("mydb.users"); + } + + @Test + void testSelectColumns() { + ParseResult result = parser.parse("SELECT id, name, age FROM mydb.users"); + SqlSelect select = (SqlSelect) result.sqlNode(); + SqlNodeList cols = select.getSelectList(); + assertThat(cols).hasSize(3); + assertThat(((SqlIdentifier) cols.get(0)).getSimple()).isEqualTo("id"); + assertThat(((SqlIdentifier) cols.get(1)).getSimple()).isEqualTo("name"); + assertThat(((SqlIdentifier) cols.get(2)).getSimple()).isEqualTo("age"); + } + + @Test + void testSelectWithWhere() { + ParseResult result = + parser.parse("SELECT * FROM mydb.users WHERE age > 18 AND city = 'Beijing'"); + SqlSelect select = (SqlSelect) result.sqlNode(); + assertThat(select.getWhere()).isNotNull(); + String where = SqlParser.unquoteIdentifiers(select.getWhere().toString()); + assertThat(where).contains("age").contains("18").contains("city").contains("Beijing"); + } + + @Test + void testSelectWithLimit() { + ParseResult result = parser.parse("SELECT * FROM mydb.users LIMIT 50"); + SqlNode node = result.sqlNode(); + if (node instanceof SqlOrderBy) { + assertThat(((SqlOrderBy) node).fetch.toString()).isEqualTo("50"); + } else { + SqlSelect select = (SqlSelect) node; + assertThat(select.getFetch().toString()).isEqualTo("50"); + } + } + + @Test + void testSelectWithOrderBy() { + ParseResult result = parser.parse("SELECT * FROM mydb.users ORDER BY age DESC LIMIT 10"); + SqlOrderBy orderBy = (SqlOrderBy) result.sqlNode(); + assertThat(orderBy.orderList).hasSize(1); + SqlBasicCall descCall = (SqlBasicCall) orderBy.orderList.get(0); + assertThat(descCall.getKind()).isEqualTo(SqlKind.DESCENDING); + assertThat(orderBy.fetch.toString()).isEqualTo("10"); + } + + @Test + void testSelectWithMultipleOrderBy() { + ParseResult result = + parser.parse( + "SELECT * FROM mydb.users ORDER BY city ASC, age DESC NULLS LAST LIMIT 20"); + SqlOrderBy orderBy = (SqlOrderBy) result.sqlNode(); + assertThat(orderBy.orderList).hasSize(2); + assertThat(orderBy.fetch.toString()).isEqualTo("20"); + } + + @Test + void testSelectWithGroupBy() { + ParseResult result = parser.parse("SELECT city, COUNT(*) FROM mydb.users GROUP BY city"); + SqlSelect select = (SqlSelect) result.sqlNode(); + assertThat(select.getGroup()).isNotNull(); + assertThat(select.getGroup()).hasSize(1); + assertThat(((SqlIdentifier) select.getGroup().get(0)).getSimple()).isEqualTo("city"); + } + + @Test + void testSelectWithAggregates() { + ParseResult result = + parser.parse( + "SELECT city, SUM(age), AVG(age), MIN(age), MAX(age) " + + "FROM mydb.users GROUP BY city"); + SqlSelect select = (SqlSelect) result.sqlNode(); + SqlNodeList selectList = select.getSelectList(); + assertThat(selectList).hasSize(5); + assertThat(((SqlBasicCall) selectList.get(1)).getOperator().getName()).isEqualTo("SUM"); + assertThat(((SqlBasicCall) selectList.get(2)).getOperator().getName()).isEqualTo("AVG"); + assertThat(((SqlBasicCall) selectList.get(3)).getOperator().getName()).isEqualTo("MIN"); + assertThat(((SqlBasicCall) selectList.get(4)).getOperator().getName()).isEqualTo("MAX"); + } + + @Test + void testSelectDistinct() { + ParseResult result = parser.parse("SELECT DISTINCT city FROM mydb.users"); + SqlSelect select = (SqlSelect) result.sqlNode(); + assertThat(select.isDistinct()).isTrue(); + } + + @Test + void testSelectWithHaving() { + ParseResult result = + parser.parse( + "SELECT city, COUNT(*) FROM mydb.users GROUP BY city HAVING COUNT(*) > 5"); + SqlSelect select = (SqlSelect) result.sqlNode(); + assertThat(select.getHaving()).isNotNull(); + } + + @Test + void testShowDatabases() { + ParseResult result = parser.parse("SHOW DATABASES"); + assertThat(result.type()).isEqualTo(ParseResult.Type.SHOW_DATABASES); + } + + @Test + void testShowDatabasesWithSemicolon() { + ParseResult result = parser.parse("SHOW DATABASES;"); + assertThat(result.type()).isEqualTo(ParseResult.Type.SHOW_DATABASES); + } + + @Test + void testShowTables() { + ParseResult result = parser.parse("SHOW TABLES"); + assertThat(result.type()).isEqualTo(ParseResult.Type.SHOW_TABLES); + assertThat(result.database()).isNull(); + } + + @Test + void testShowTablesInDatabase() { + ParseResult result = parser.parse("SHOW TABLES IN mydb"); + assertThat(result.type()).isEqualTo(ParseResult.Type.SHOW_TABLES); + assertThat(result.database()).isEqualTo("mydb"); + } + + @Test + void testUseDatabase() { + ParseResult result = parser.parse("USE mydb"); + assertThat(result.type()).isEqualTo(ParseResult.Type.USE_DATABASE); + assertThat(result.database()).isEqualTo("mydb"); + } + + @Test + void testCaseInsensitive() { + ParseResult result = parser.parse("select * from mydb.users where age > 10 limit 5"); + assertThat(result.type()).isEqualTo(ParseResult.Type.QUERY); + } + + @Test + void testBacktickIdentifier() { + ParseResult result = parser.parse("SELECT `order`, `from` FROM mydb.`my-table`"); + SqlSelect select = (SqlSelect) result.sqlNode(); + SqlNodeList cols = select.getSelectList(); + assertThat(((SqlIdentifier) cols.get(0)).getSimple()).isEqualTo("order"); + assertThat(((SqlIdentifier) cols.get(1)).getSimple()).isEqualTo("from"); + } + + @Test + void testInvalidSyntax() { + assertThatThrownBy(() -> parser.parse("SELECTT * FROMM mydb.users")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("parse error"); + } + + @Test + void testUnquoteIdentifiers() { + assertThat(SqlParser.unquoteIdentifiers("`age` > 18")).isEqualTo("age > 18"); + assertThat(SqlParser.unquoteIdentifiers("`name` = 'O''Brien'")) + .isEqualTo("name = 'O''Brien'"); + assertThat(SqlParser.unquoteIdentifiers("plain text")).isEqualTo("plain text"); + } + + private static String extractTableName(SqlSelect select) { + SqlIdentifier id = (SqlIdentifier) select.getFrom(); + if (id.names.size() == 2) { + return id.names.get(0) + "." + id.names.get(1); + } + return id.getSimple(); + } +} diff --git a/pom.xml b/pom.xml index c33e22abaa03..fcfe43e4ae64 100644 --- a/pom.xml +++ b/pom.xml @@ -75,6 +75,7 @@ under the License. paimon-lumina paimon-vortex paimon-tantivy + paimon-cli