DBTOOLS-1666 added api

Author: rYamal4

README.md

@@ -9,10 +9,62 @@ A tool for easier PostgreSQL, GreenPlum, MS SQL, ClickHouse development.
 
 # Usage
 
-API is simple. Currently static getters are available:
+API is simple. Currently static methods are available:
 
 ```java
-// TODO fill
+import org.pgcodekeeper.core.DatabaseType;
+import org.pgcodekeeper.core.PgCodekeeperException;
+import org.pgcodekeeper.core.api.DatabaseFactory;
+import org.pgcodekeeper.core.api.PgCodeKeeperApi;
+import org.pgcodekeeper.core.utils.IMonitor;
+import org.pgcodekeeper.core.utils.NullMonitor;
+import org.pgcodekeeper.core.schema.AbstractDatabase;
+import org.pgcodekeeper.core.settings.CoreSettings;
+import org.pgcodekeeper.core.settings.ISettings;
+
+import java.io.IOException;
+
+public class Main {
+  public static void main(String[] args) {
+    try {
+         // Example 1: Compare two databases from JDBC connections and generate migration script
+         ISettings settings = new CoreSettings(DatabaseType.PG);
+         // settings init...
+         AbstractDatabase oldDb = DatabaseFactory.loadFromJdbc(settings, "jdbc:postgresql://localhost/old_db");
+         AbstractDatabase newDb = DatabaseFactory.loadFromJdbc(settings, "jdbc:postgresql://localhost/new_db");
+         String migrationScript = PgCodeKeeperApi.diff(settings, oldDb, newDb);
+   
+         // Example 2: Load database from project and compare with JDBC database
+         AbstractDatabase projectDb = DatabaseFactory.loadFromProject(settings, "/path/to/project");
+         AbstractDatabase liveDb = DatabaseFactory.loadFromJdbc(settings, "jdbc:postgresql://localhost/live_db");
+         String script = PgCodeKeeperApi.diff(settings, projectDb, liveDb);
+   
+         // Example 3: Compare databases with object filtering using ignore list
+         AbstractDatabase db1 = DatabaseFactory.loadFromJdbc(settings, "jdbc:postgresql://localhost/db1");
+         AbstractDatabase db2 = DatabaseFactory.loadFromProject(settings, "/path/to/project");
+         String diff = PgCodeKeeperApi.diff(settings, db1, db2, "/path/to/object_ignore_list.txt");
+   
+         // Example 4: Export database to project with filtering and progress monitoring
+         IMonitor monitor = new NullMonitor(); // or implement custom progress monitoring
+         AbstractDatabase db = DatabaseFactory.loadFromJdbc(settings, "jdbc:postgresql://localhost/db");
+         PgCodeKeeperApi.export(settings, db, "/path/to/export", "/path/to/ignore_list.txt", monitor);
+
+         // Example 5: Update project with changes from database
+         AbstractDatabase updatedDb = DatabaseFactory.loadFromJdbc(settings, "jdbc:postgresql://localhost/updated_db");
+         PgCodeKeeperApi.update(settings, updatedDb, "/path/to/project");
+
+         // Example 6: Update project with filtering and progress monitoring
+         PgCodeKeeperApi.update(settings, updatedDb, "/path/to/project", 
+                              "/path/to/object_ignore_list.txt", 
+                              "/path/to/schema_ignore_list.txt", 
+                              monitor);
+
+    } catch (PgCodekeeperException | IOException | InterruptedException e) {
+         System.err.println("Operation failed: " + e.getMessage());
+         e.printStackTrace();
+    }
+  }
+}
 ```
 
 ## Documentation
@@ -38,52 +90,34 @@ Build requires Java (JDK) 17+ and Apache Maven 3.9+.
 This project was derived from [apgdiff](https://github.com/fordfrog/apgdiff) project. At this point it is almost fully rewritten according to our needs.  
 Primary packages of this module are:
 
-- `org.pgcodekeeper.core.schema` - contains classes that describe SQL objects. Each class contains object properties, creation SQL code generator, object comparison logic and ALTER code generator. Notable classes:
-  - `PgStatement` - all object classes must be derived from this one. Defines main API for working with objects.
-  - `PgDatabase` - root class for every loaded database schema.
-  - `GenericColumn` - generic reference to any object (used to be a column reference).
+
+- `org.pgcodekeeper.core.api` - high-level API classes for database operations.
+- `org.pgcodekeeper.core.schema` - contains classes that describe SQL objects. Each class contains object properties, creation SQL code generator, object comparison logic and ALTER code generator.
 - `org.pgcodekeeper.core.schema.meta` - simplified SQL object classes for database structure serialization.
 - `org.pgcodekeeper.core.schema.meta` (resources) - serialized descriptions of PostgreSQL's system objects (built using JdbcSystemLoader).
 - `org.pgcodekeeper.core.loader` - database schema loaders: project, SQL file, JDBC, library.
-  - `ProjectLoader` - loads a database from project structure. **(!)** Keep in sync with `UIProjectLoader`.
-  - `PgDumpLoader` - loads objects into a database structure from a file. **(!)** Keep in sync with `PgUIDumpLoader`.
 - `org.pgcodekeeper.core.loader` (resources) - SQL queries for JdbcLoaders, used by Java code via `JdbcQueries`.
 - `org.pgcodekeeper.core.loader.jdbc` - classes that create schema objects based on JDBC ResultSets.
-- `org.pgcodekeeper.core.parsers.antlr` - generated ANTLR4 parser code, plus utility parser classes. Notable classes:
-  - `AntlrParser` - factory methods for parser creation and parallel parser operations.
-  - `QNameParser` - utility class for parsing and splitting qualified names.
-  - `ScriptParser` - simple parser that splits statements for later execution.
-  - `Custom(T)SQLParserListener` - main entry point for parsing SQLs into schema objects.
-- `org.pgcodekeeper.core.parsers.antlr.statements` - processor classes that create schema objects based on parser-read data. Notable classes:
-  - `ParserAbstract` - common base class for all processors.
+- `org.pgcodekeeper.core.parsers.antlr` - generated ANTLR4 parser code, plus utility parser classes.
+- `org.pgcodekeeper.core.parsers.antlr.statements` - processor classes that create schema objects based on parser-read data.
 - `org.pgcodekeeper.core.parsers.antlr.expr` - expression analysis classes: find dependencies in expressions and infer expression types for overloaded function call resolution.
 - `org.pgcodekeeper.core.parsers.antlr.expr.launcher` - support for parallel expression analysis.
 - `org.pgcodekeeper.core.formatter` - SQL and pl/pgsql code formatters.
-- `org.pgcodekeeper.core.model.difftree` - classes representing and creating an object diff tree. Notable classes:
-  - `DbObjType` - enum of all SQL object types supported by the program.
-  - `TreeElement` - a node in an object diff tree.
-  - `DiffTree` - object diff tree factory.
-  - `TreeFlattener` - filters and flattens the tree into a list of nodes according to requested parameters.
-- `org.pgcodekeeper.core.model.graph` - object dependency graph classes, built using JGraphT library. Notable classes:
-  - `DepcyGraph` - the dependency graph and its builder.
-  - `DepcyResolver` - main script building algorithm. Creates a list of `StatementAction`s according to requested object changes and object dependencies.
-  - `DepcyTreeExtender` - extends an object diff tree according to object dependencies.
-  - `DepcyWriter` - prints a dependency tree for an object.
-- `org.pgcodekeeper.core.model.exporter` - classes that save and update the project files on disk. Notable classes:
-  - `ModelExporter`, `MsModelExporter`, `AbstractModelExporter` - exporters for PG and MS projects and their common abstract part.
-  - `OverridesModelExporter` - exports project structure containing only permission overrides.
+- `org.pgcodekeeper.core.model.difftree` - classes representing and creating an object diff tree.
+- `org.pgcodekeeper.core.model.graph` - object dependency graph classes, built using JGraphT library.
+- `org.pgcodekeeper.core.model.exporter` - classes that save and update the project files on disk.
 - `org.pgcodekeeper.core.sql` - a categorized list of all PostgreSQL keywords. Generated from PostgreSQL source.
 - `org.pgcodekeeper.core.ignoreparser` - builder for `IgnoreList`s.
-- `org.pgcodekeeper.core` - main package containing general stuff: e.g. string constants, utils and general-purpose classes. Notable classes:
-  - `PgDiff` - entry point for database schema diff operations.
+- `org.pgcodekeeper.core` - main package containing general stuff: e.g. string constants, utils and general-purpose classes.
 - `src.main.antlr4.org.pgcodekeeper.core.parsers.antlr.generated` - sources for ANTLR4 parsers. We maintain parsers for PostgreSQL, pl/pgsql, T-SQL, and also our custom Ignore Lists and PostgreSQL ACLs syntax.  
 These need to be built using your preferred ANTLR4 builder into `ru.taximaxim.codekeeper.core.parsers.antlr` package.
 
-Majority of tests are here. 
+Majority of tests are here.
 
-- `org.pgcodekeeper.core` - `PgDiffTest`, `MsDiffTest` - these test cases load old and new database schemas, generate a migration script and compare it to the expected diff file.
+- `org.pgcodekeeper.core.api` - integration tests for the high-level API operations (diff, export, update) using real test databases and projects.
+- `org.pgcodekeeper.core` - these test cases load old and new database schemas, generate a migration script and compare it to the expected diff file.
 - `org.pgcodekeeper.core.depcies` - tests here work similarly to simple diff tests above, except the concept of "user selection" is added. Migration script is built only with objects in `usr` files as starting points, other objects must be added to the script via dependency mechanism.
-- `org.pgcodekeeper.core.loader` - `PgAntlrLoaderTest`, `MsAntlrLoaderTest` - these tests load simple schemas from files and compare loaded PgDatabase objects with predefined ones.
+- `org.pgcodekeeper.core.loader` - these tests load simple schemas from files and compare loaded AbstractDatabase objects with predefined ones.
 - `org.pgcodekeeper.core.parsers` - these tests simply parse test pieces of code (taken from PostgreSQL source) to verify parser validity.
 - `org.pgcodekeeper.core.parsers.antlr.expr` - these tests verify expression analysis and type inference mechanism.
 - `org.pgcodekeeper.core.model.exporter` - these test update exported project directories and verify which files have actually been changed.
@@ -92,11 +126,11 @@ Majority of tests are here.
 
 General program lifecycle goes as follows:
 1. `ISettings` object is filled with operation parameters.
-2. `PgDatabase`s are loaded from requested sources, including their libraries and privilege overrides. Ignored schemas are skipped at this step.
+2. `AbstractDatabase`s are loaded from requested sources, including their libraries and privilege overrides. Ignored schemas are skipped at this step.
    1. During the load dependencies of each object are found and recorded. Expressions are also analyzed to extract their dependencies including overloaded function calls.
    2. All parser and expression analysis operations are run in parallel using `AntlrParser.ANTLR_POOL` thread pool to speed up the process. Parallel operations are serialized by calling `finishLoaders` at the end of each loading process.
-3. The diff tree (represented by root `TreeElement`) is created by comparing two `PgDatabase`s. In GUI this is then shown to the user to assess the situation and choose objects to work with.
+3. The diff tree (represented by root `TreeElement`) is created by comparing two `AbstractDatabase`s.
 4. The diff tree, now containing "user selection", is used to selectively update project files on disk, or to generate a migration script.
 5. In latter case, each "selected" TreeElement is passed to `DepcyResolver` to generate script actions fulfilling the requested change, including actions on dependent objects. To do this, JGraphT object dependency graphs are built using dependency information found at the loading stage.
 6. Generated actions are now converted into SQL code with some last-moment post-processing and filtering.
-7. Generated SQL script is written to a file/stdout or shown in the GUI for user to review and run on their database.
+7. Generated SQL script is returned as a String for user to review and run on their database.

src/main/java/org/pgcodekeeper/core/PgDiffUtils.java

@@ -19,9 +19,9 @@
  *******************************************************************************/
 package org.pgcodekeeper.core;
 
-import org.eclipse.core.runtime.IProgressMonitor;
 import org.pgcodekeeper.core.sql.Keyword;
 import org.pgcodekeeper.core.sql.Keyword.KeywordCategory;
+import org.pgcodekeeper.core.utils.IMonitor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -132,7 +132,6 @@ public static boolean isValidIdChar(char c, boolean allowCaps, boolean allowDigi
      * quoted.
      *
      * @param name name
-     *
      * @return quoted string if needed, otherwise not quoted string
      */
     public static String getQuotedName(final String name) {
@@ -163,6 +162,7 @@ public static String quoteString(String s) {
      * Quotes string using dollar-quoting ($$) syntax.
      * <p>
      * Function equivalent to appendStringLiteralDQ in pgdump's dumputils.c
+     *
      * @param contents the string to quote
      * @return dollar-quoted string
      */
@@ -222,7 +222,7 @@ public static byte[] getHash(String s, String instance) throws NoSuchAlgorithmEx
      * @param instance the hash algorithm to use
      * @return hexadecimal hash string
      */
-    public static String hash (String s, String instance) {
+    public static String hash(String s, String instance) {
         try {
             byte[] hash = getHash(s, instance);
             StringBuilder sb = new StringBuilder(2 * hash.length);
@@ -233,12 +233,13 @@ public static String hash (String s, String instance) {
             return sb.toString();
         } catch (NoSuchAlgorithmException e) {
             LOG.error(e.getLocalizedMessage(), e);
-            return instance +"_ERROR_" + RANDOM.nextInt();
+            return instance + "_ERROR_" + RANDOM.nextInt();
         }
     }
 
     /**
      * Returns MD5 hash of string as hexadecimal.
+     *
      * @param s the string to hash
      * @return lowercase hex MD5 for UTF-8 representation of given string
      */
@@ -248,6 +249,7 @@ public static String md5(String s) {
 
     /**
      * Returns SHA-256 hash of string as hexadecimal.
+     *
      * @param s the string to hash
      * @return lowercase hex SHA-256 for UTF-8 representation of given string
      */
@@ -297,7 +299,7 @@ public static String getErrorSubstr(String s, int pos, int len) {
      * @param monitor the progress monitor to check
      * @throws InterruptedException if monitor is cancelled
      */
-    public static void checkCancelled(IProgressMonitor monitor)
+    public static void checkCancelled(IMonitor monitor)
             throws InterruptedException {
         if (monitor != null && monitor.isCanceled()) {
             throw new InterruptedException();
@@ -311,7 +313,7 @@ public static void checkCancelled(IProgressMonitor monitor)
      * <code>List.containsAll()</code> O(N^2) call here. In general, duplicate elimination is an undesired side-effect
      * of comparison using {@link Set}s, so this solution is overall better and only *slightly* slower.<br>
      * <br>
-     *
+     * <p>
      * Performance: best case O(1), worst case O(N) + new {@link HashMap} (in case N1 == N2), assuming size() takes
      * constant time.
      *
@@ -331,7 +333,7 @@ public static boolean setlikeEquals(Collection<?> c1, Collection<?> c2) {
         // mimic HashSet(Collection) constructor
         final float loadFactor = 0.75f;
         final Map<Object, Integer> map =
-                new HashMap<>(Math.max((int) (s1/loadFactor) + 1, 16), loadFactor);
+                new HashMap<>(Math.max((int) (s1 / loadFactor) + 1, 16), loadFactor);
         for (Object el1 : c1) {
             map.compute(el1, (k, i) -> i == null ? 1 : (i + 1));
         }
@@ -414,17 +416,17 @@ public static void appendSqlWrappedInDo(StringBuilder sbResult, StringBuilder sb
         String body = sbSQL.toString().replace("\n", "\n\t");
 
         sbResult
-        .append("DO $$")
-        .append("\nBEGIN")
-        .append("\n\t").append(body)
-        .append("\nEXCEPTION WHEN OTHERS THEN")
-        .append("\n\tIF (SQLSTATE = ").append(expectedErrCode).append(") THEN")
-        .append("\n\t\tRAISE NOTICE '%, skip', SQLERRM;")
-        .append("\n\tELSE")
-        .append("\n\t\tRAISE;")
-        .append("\n\tEND IF;")
-        .append("\nEND; $$")
-        .append("\nLANGUAGE 'plpgsql'");
+                .append("DO $$")
+                .append("\nBEGIN")
+                .append("\n\t").append(body)
+                .append("\nEXCEPTION WHEN OTHERS THEN")
+                .append("\n\tIF (SQLSTATE = ").append(expectedErrCode).append(") THEN")
+                .append("\n\t\tRAISE NOTICE '%, skip', SQLERRM;")
+                .append("\n\tELSE")
+                .append("\n\t\tRAISE;")
+                .append("\n\tEND IF;")
+                .append("\nEND; $$")
+                .append("\nLANGUAGE 'plpgsql'");
     }
 
     private PgDiffUtils() {