Skip to content

Commit 00da240

Browse files
AXEPOHAXEPOH
committed
Merge branch 'DBTOOLS-1769-fix-greenplum-external-tables' into 'master'
DBTOOLS-1769 fixed greenplum external tables See merge request codekeeper/pgcodekeeper-core!90
2 parents cd03090 + baa4bc6 commit 00da240

File tree

3 files changed

+104
-64
lines changed

3 files changed

+104
-64
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
2727
- Fixed a false difference between NOT NULL constraints with the default name in PostgreSQL.
2828
- Fixed adding ONLY for columns in partitioned tables in PostgreSQL.
2929
- Fixed broken ClickHouse tests.
30+
- Fixed reading of EXTERNAL TABLES in Greenplum 7.
3031

3132
## [12.0.0] - 2025-10-14
3233

‎CHANGELOG.ru.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
- Исправлено ложное различие ограничений NOT NULL с именем по умолчанию в PostgreSQL.
2828
- Исправлено добавление ONLY для колонок в партиционных таблицах в PostgreSQL.
2929
- Исправлены неработающие тесты ClickHouse.
30+
- Исправлено чтение EXTERNAL TABLES в Greenplum 7.
3031

3132
## [12.0.0] - 2025-10-14
3233

‎src/main/java/org/pgcodekeeper/core/loader/jdbc/pg/TablesReader.java‎

Lines changed: 102 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535

3636
import java.sql.ResultSet;
3737
import java.sql.SQLException;
38+
import java.util.ArrayList;
39+
import java.util.List;
3840

3941
/**
4042
* Reader for PostgreSQL tables.
@@ -43,6 +45,9 @@
4345
public final class TablesReader extends JdbcReader {
4446

4547
private static final String CREATE_TABLE = "CREATE TABLE noname () ";
48+
private static final String GP_SYSTEM_OPTIONS = "'format', 'formatter', 'delimiter', 'null', 'escape', 'quote'," +
49+
" 'header', 'newline', 'fill_missing_fields', 'force_not_null', 'is_writable', 'location_uris'," +
50+
" 'execute_on', 'log_errors', 'reject_limit', 'reject_limit_type', 'encoding'";
4651

4752
/**
4853
* Creates a new TablesReader.
@@ -58,45 +63,9 @@ protected void processResult(ResultSet res, AbstractSchema schema) throws SQLExc
5863
String schemaName = schema.getName();
5964
String tableName = res.getString("relname");
6065
loader.setCurrentObject(new GenericColumn(schemaName, tableName, DbObjType.TABLE));
61-
String partitionBound = null;
62-
String partitionGpBound = null;
63-
String partitionGpTemplate = null;
6466

65-
if (SupportedPgVersion.GP_VERSION_7.isLE(loader.getVersion()) &&
66-
res.getBoolean("relispartition")) {
67-
partitionBound = res.getString("partition_bound");
68-
checkObjectValidity(partitionBound, DbObjType.TABLE, tableName);
69-
}
70-
if (loader.isGreenplumDb() && !SupportedPgVersion.GP_VERSION_7.isLE(loader.getVersion())) {
71-
partitionGpBound = res.getString("partclause");
72-
partitionGpTemplate = res.getString("parttemplate");
73-
}
74-
75-
AbstractPgTable t;
76-
String serverName = res.getString("server_name");
7767
long ofTypeOid = res.getLong("of_type");
78-
if (serverName != null) {
79-
if (partitionBound == null) {
80-
t = new SimpleForeignPgTable(tableName, serverName);
81-
} else {
82-
t = new PartitionForeignPgTable(tableName, serverName, partitionBound);
83-
}
84-
t.addDep(new GenericColumn(serverName, DbObjType.SERVER));
85-
} else if (ofTypeOid != 0) {
86-
JdbcType jdbcOfType = loader.getCachedTypeByOid(ofTypeOid);
87-
String ofType = jdbcOfType.getFullName();
88-
t = new TypedPgTable(tableName, ofType);
89-
jdbcOfType.addTypeDepcy(t);
90-
} else if (partitionBound != null) {
91-
t = new PartitionPgTable(tableName, partitionBound);
92-
} else if (partitionGpBound != null) {
93-
t = createGpParttionTable(tableName, partitionGpBound, partitionGpTemplate);
94-
} else if (loader.isGreenplumDb() && res.getString("exloc") != null) {
95-
t = new GpExternalTable(tableName);
96-
readExternalTable((GpExternalTable) t, res);
97-
} else {
98-
t = new SimplePgTable(tableName);
99-
}
68+
AbstractPgTable t = getTable(res, tableName, ofTypeOid);
10069

10170
if (SupportedPgVersion.GP_VERSION_7.isLE(loader.getVersion()) && t instanceof AbstractRegularTable regTable) {
10271
String accessMethod = res.getString("access_method");
@@ -105,11 +74,6 @@ protected void processResult(ResultSet res, AbstractSchema schema) throws SQLExc
10574
}
10675
}
10776

108-
String[] foptions = getColArray(res, "ftoptions", true);
109-
if (foptions != null) {
110-
ParserAbstract.fillOptionParams(foptions, t::addOption, false, true, false);
111-
}
112-
11377
// PRIVILEGES, OWNER
11478
loader.setOwner(t, res.getLong("relowner"));
11579
loader.setPrivileges(t, res.getString("aclarray"), schemaName);
@@ -151,8 +115,57 @@ protected void processResult(ResultSet res, AbstractSchema schema) throws SQLExc
151115
schema.addTable(t);
152116
}
153117

154-
private AbstractPgTable createGpParttionTable(String tableName, String partitionGpBound,
155-
String partitionGpTemplate) {
118+
private AbstractPgTable getTable(ResultSet res, String tableName, long ofTypeOid) throws SQLException {
119+
if (loader.isGreenplumDb() && res.getString("exloc") != null) {
120+
return createExternalTable(tableName, res);
121+
}
122+
123+
String partitionBound = null;
124+
String partitionGpBound = null;
125+
String partitionGpTemplate = null;
126+
127+
if (SupportedPgVersion.GP_VERSION_7.isLE(loader.getVersion())) {
128+
if (res.getBoolean("relispartition")) {
129+
partitionBound = res.getString("partition_bound");
130+
checkObjectValidity(partitionBound, DbObjType.TABLE, tableName);
131+
}
132+
} else {
133+
partitionGpBound = res.getString("partclause");
134+
partitionGpTemplate = res.getString("parttemplate");
135+
}
136+
137+
AbstractPgTable t;
138+
String serverName = res.getString("server_name");
139+
if (serverName != null) {
140+
if (partitionBound == null) {
141+
t = new SimpleForeignPgTable(tableName, serverName);
142+
} else {
143+
t = new PartitionForeignPgTable(tableName, serverName, partitionBound);
144+
}
145+
t.addDep(new GenericColumn(serverName, DbObjType.SERVER));
146+
} else if (ofTypeOid != 0) {
147+
JdbcType jdbcOfType = loader.getCachedTypeByOid(ofTypeOid);
148+
String ofType = jdbcOfType.getFullName();
149+
t = new TypedPgTable(tableName, ofType);
150+
jdbcOfType.addTypeDepcy(t);
151+
} else if (partitionBound != null) {
152+
t = new PartitionPgTable(tableName, partitionBound);
153+
} else if (partitionGpBound != null) {
154+
t = createGpPartitionTable(tableName, partitionGpBound, partitionGpTemplate);
155+
} else {
156+
t = new SimplePgTable(tableName);
157+
}
158+
159+
String[] foptions = getColArray(res, "ftoptions", true);
160+
if (foptions != null) {
161+
ParserAbstract.fillOptionParams(foptions, t::addOption, false, true, false);
162+
}
163+
164+
return t;
165+
}
166+
167+
private AbstractPgTable createGpPartitionTable(String tableName, String partitionGpBound,
168+
String partitionGpTemplate) {
156169
PartitionGpTable table = new PartitionGpTable(tableName);
157170

158171
loader.submitAntlrTask(CREATE_TABLE + partitionGpBound + ';',
@@ -350,11 +363,7 @@ private void readColumns(ResultSet res, AbstractPgTable t, long ofTypeOid,
350363
}
351364

352365
if (colGenerated != null && !colGenerated[i].isEmpty()) {
353-
if ("s".equals(colGenerated[i])) {
354-
column.setGenerationOption("STORED");
355-
} else {
356-
column.setGenerationOption("VIRTUAL");
357-
}
366+
column.setGenerationOption("s".equals(colGenerated[i]) ? "STORED" : "VIRTUAL");
358367
}
359368

360369
if (colCompression != null) {
@@ -401,7 +410,9 @@ private void readColumns(ResultSet res, AbstractPgTable t, long ofTypeOid,
401410
}
402411
}
403412

404-
private void readExternalTable(GpExternalTable extTable, ResultSet res) throws SQLException {
413+
private GpExternalTable createExternalTable(String tableName, ResultSet res) throws SQLException {
414+
var extTable = new GpExternalTable(tableName);
415+
405416
String rowUrLoc = res.getString("urloc");
406417
if (rowUrLoc != null) {
407418
if (rowUrLoc.startsWith("{http")) {
@@ -446,27 +457,49 @@ private void readExternalTable(GpExternalTable extTable, ResultSet res) throws S
446457
break;
447458
}
448459

449-
String options = PgDiffUtils.unquoteQuotedString(res.getString("options"), 1);
450-
if (!options.isBlank()) {
451-
ParserAbstract.fillOptionParams(options.split(","), extTable::addOption, false, true, false);
452-
}
460+
String formatOptions = res.getString("fmtopts");
461+
if (SupportedPgVersion.GP_VERSION_7.isLE(loader.getVersion())) {
462+
String[] ftoptionNames = getColArray(res, "ftoption_names");
463+
String[] ftoptionValues = getColArray(res, "ftoption_values");
464+
465+
int splitIndex = ftoptionNames.length != 0 ? formatOptions.indexOf(ftoptionNames[0]) : formatOptions.length();
466+
// format options in fmtopts go before options defined in OPTIONS block
467+
String formatOptionsPart = formatOptions.substring(0, splitIndex).trim();
468+
extTable.setFormatOptions(appendFormatOptions(formatOptionsPart));
469+
470+
List<String> options = new ArrayList<>();
471+
for (int i = 0; i < ftoptionNames.length; i++) {
472+
String optionName = ftoptionNames[i];
473+
options.add(optionName + "=" + ftoptionValues[i]);
474+
}
453475

454-
extTable.setFormatOptions(appendFormatOptions(res));
476+
ParserAbstract.fillOptionParams(options.toArray(String[]::new), extTable::addOption,
477+
false, true, false);
478+
} else {
479+
String options = PgDiffUtils.unquoteQuotedString(res.getString("options"), 1);
480+
if (!options.isBlank()) {
481+
ParserAbstract.fillOptionParams(options.split(","), extTable::addOption, false, true, false);
482+
}
483+
extTable.setFormatOptions(appendFormatOptions(formatOptions));
484+
}
455485

456-
extTable.setRejectLimit(res.getInt("rejlim"));
457-
extTable.setIsLogErrors(res.getBoolean("logerrors"));
458-
extTable.setRowReject(!"p".equals(res.getString("rejtyp")));
486+
var rejtype = res.getString("rejtype");
487+
if (rejtype != null) {
488+
extTable.setRejectLimit(res.getInt("rejlim"));
489+
extTable.setIsLogErrors(res.getBoolean("logerrors"));
490+
extTable.setRowReject(!"p".equals(rejtype));
491+
}
459492
extTable.setEncoding(PgDiffUtils.quoteString(res.getString("enc")));
460493
extTable.setWritable(res.getBoolean("writable"));
461494

462495
String distribution = res.getString("distribution");
463496
if (distribution != null && !distribution.isBlank()) {
464497
extTable.setDistribution(distribution);
465498
}
499+
return extTable;
466500
}
467501

468-
private String appendFormatOptions(ResultSet res) throws SQLException {
469-
String formatOptions = res.getString("fmtopts");
502+
private String appendFormatOptions(String formatOptions) {
470503
if (formatOptions.contains("formatter")) {
471504
return formatOptions.trim().replace(" ", "=");
472505
}
@@ -521,8 +554,6 @@ protected void fillQueryBuilder(QueryBuilder builder) {
521554
.column("res.relispartition")
522555
.column("pg_catalog.pg_get_partkeydef(res.oid) AS partition_by")
523556
.column("pg_catalog.pg_get_expr(res.relpartbound, res.oid) AS partition_bound");
524-
} else {
525-
builder.column("res.relhasoids AS has_oids");
526557
}
527558

528559
if (loader.isGreenplumDb()) {
@@ -534,15 +565,22 @@ protected void fillQueryBuilder(QueryBuilder builder) {
534565
.column("x.fmtopts")
535566
.column("x.command")
536567
.column("x.rejectlimit AS rejlim")
537-
.column("x.rejectlimittype AS rejtyp")
568+
.column("x.rejectlimittype AS rejtype")
538569
.column("x.logerrors")
539570
.column("x.options")
540571
.column("pg_catalog.pg_encoding_to_char(x.encoding) AS enc")
541572
.column("x.writable")
542573
.join("LEFT JOIN pg_exttable x ON res.oid = x.reloid");
543574

544-
if (!SupportedPgVersion.GP_VERSION_7.isLE(loader.getVersion())) {
575+
if (SupportedPgVersion.GP_VERSION_7.isLE(loader.getVersion())) {
576+
builder.column(("ARRAY(SELECT option_name FROM pg_options_to_table(ftbl.ftoptions) " +
577+
"WHERE option_name NOT IN (%s)) AS ftoption_names").formatted(GP_SYSTEM_OPTIONS));
578+
builder.column(("ARRAY(SELECT option_value FROM pg_options_to_table(ftbl.ftoptions) " +
579+
"WHERE option_name NOT IN (%s)) AS ftoption_values").formatted(GP_SYSTEM_OPTIONS));
580+
} else {
545581
builder
582+
.column("res.relhasoids AS has_oids")
583+
.column("x.options")
546584
.column("CASE WHEN pl.parlevel = 0 THEN (SELECT pg_get_partition_def(res.oid, true, false)) END AS partclause")
547585
.column("CASE WHEN pl.parlevel = 0 THEN (SELECT pg_get_partition_template_def(res.oid, true, false)) END as parttemplate")
548586
.join("LEFT JOIN pg_partitions ps on (res.relname = ps.partitiontablename)")

0 commit comments

Comments
 (0)