fix schema.c: filter out internal partition FK clones when listing FK constraints

.github/workflows/run-tests-tiered.yml

@@ -158,6 +158,7 @@ jobs:
           - follow-defer-indexes
           - fk-not-valid
           - defer-validate-fks
+          - fk-partition-clone
           - follow-defer-validate-fks
           - follow-sequence-reset
     steps:

src/bin/pgcopydb/copydb_schema.c

@@ -994,7 +994,8 @@ copydb_prepare_index_specs(CopyDataSpec *specs, PGSQL *pgsql)
 	 * schema_list_all_indexes. pgcopydb handles FK constraint creation
 	 * directly instead of delegating to pg_restore.
 	 */
-	if (!schema_list_fk_constraints(pgsql, &(specs->filters), sourceDB))
+	if (!schema_list_fk_constraints(pgsql, &(specs->filters), sourceDB,
+									specs->deferValidateFKs))
 	{
 		/* errors have already been logged */
 		return false;

src/bin/pgcopydb/schema.c

@@ -3204,6 +3204,10 @@ static void getFKConstraintArray(void *ctx, PGresult *result);
  * Base SELECT columns and FROM/WHERE clauses shared across all FK constraint
  * filter variants. The filter-specific JOIN clauses are inserted between the
  * FROM block and the WHERE block via the array below.
+ *
+ * The conparentid partition-clone filter is added at runtime by
+ * schema_list_fk_constraints (it matches the literal " WHERE c.contype = 'f' "
+ * below, so keep that text verbatim).
  */
 #define FK_SELECT_COLS \
 	"SELECT c.oid, " \
@@ -3345,7 +3349,8 @@ struct FilteringQueries listSourceFKConstraintsSQL[] = {
 bool
 schema_list_fk_constraints(PGSQL *pgsql,
 						   SourceFilters *filters,
-						   DatabaseCatalog *catalog)
+						   DatabaseCatalog *catalog,
+						   bool deferValidateFKs)
 {
 	SourceFKConstraintContext context = { catalog, false };
 
@@ -3372,9 +3377,99 @@ schema_list_fk_constraints(PGSQL *pgsql,
 		return false;
 	}
 
-	bool ok = pgsql_execute_with_params(pgsql, sql, 0, NULL, NULL,
+	/*
+	 * Drop Postgres's internal per-partition FK clones (conparentid <> 0): the
+	 * top-level constraint re-cascades on the target, so the clones are
+	 * redundant and collide. Gated because conparentid is PG 11+ only, and
+	 * --defer-validate-fks needs the clones (PG < 18 can't create a NOT VALID
+	 * FK on a partitioned parent, so the per-leaf clones are the enforcement).
+	 */
+	char *gatedSql = NULL;
+
+	if (pgsql->pgversion_num == 0)
+	{
+		if (!pgsql_server_version(pgsql))
+		{
+			/* errors have already been logged */
+			if (filters->ctePreamble != NULL)
+			{
+				free(sql);
+			}
+			return false;
+		}
+	}
+
+	if (pgsql->pgversion_num >= 110000 && !deferValidateFKs)
+	{
+		const char *anchor = " WHERE c.contype = 'f' ";
+		const char *clause = "  AND c.conparentid = 0 ";
+		char *pos = strstr(sql, anchor);
+
+		if (pos == NULL)
+		{
+			log_error("BUG: FK constraints query is missing its \"%s\" clause; "
+					  "cannot filter out partition FK clones", anchor);
+			if (filters->ctePreamble != NULL)
+			{
+				free(sql);
+			}
+			return false;
+		}
+
+		PQExpBuffer buf = createPQExpBuffer();
+
+		if (buf == NULL)
+		{
+			log_error("Failed to allocate FK constraints query buffer");
+			if (filters->ctePreamble != NULL)
+			{
+				free(sql);
+			}
+			return false;
+		}
+
+		/* splice after the anchor; append (not printf) as the query has %I */
+		size_t prefixLen = (size_t) (pos - sql) + strlen(anchor);
+
+		appendBinaryPQExpBuffer(buf, sql, prefixLen);
+		appendPQExpBufferStr(buf, clause);
+		appendPQExpBufferStr(buf, sql + prefixLen);
+
+		if (PQExpBufferBroken(buf))
+		{
+			log_error("Failed to build FK constraints query: out of memory");
+			(void) destroyPQExpBuffer(buf);
+			if (filters->ctePreamble != NULL)
+			{
+				free(sql);
+			}
+			return false;
+		}
+
+		gatedSql = strdup(buf->data);
+		(void) destroyPQExpBuffer(buf);
+
+		if (gatedSql == NULL)
+		{
+			log_error("Failed to build FK constraints query: out of memory");
+			if (filters->ctePreamble != NULL)
+			{
+				free(sql);
+			}
+			return false;
+		}
+	}
+
+	bool ok = pgsql_execute_with_params(pgsql,
+										gatedSql != NULL ? gatedSql : sql,
+										0, NULL, NULL,
 										&context, &getFKConstraintArray);
 
+	if (gatedSql != NULL)
+	{
+		free(gatedSql);
+	}
+
 	if (filters->ctePreamble != NULL && sql != NULL)
 	{
 		free(sql);

src/bin/pgcopydb/schema.h

@@ -533,7 +533,8 @@ bool schema_list_pg_depend(PGSQL *pgsql,
 
 bool schema_list_fk_constraints(PGSQL *pgsql,
 								SourceFilters *filters,
-								DatabaseCatalog *catalog);
+								DatabaseCatalog *catalog,
+								bool deferValidateFKs);
 
 bool schema_send_table_checksum(PGSQL *pgsql, SourceTable *table);
 bool schema_fetch_table_checksum(PGSQL *pgsql, TableChecksum *sum, bool *done);

tests/Makefile

@@ -15,7 +15,7 @@ all: pagila pagila-multi-steps blobs unit filtering filtering-standby extensions
 	 follow-wal2json follow-standby follow-9.6 follow-data-only follow-target-reconnect \
 	 endpos-in-multi-wal-txn exclude-extension \
 	 blob-snapshot-release follow-defer-indexes fk-not-valid defer-validate-fks \
-	 follow-defer-validate-fks follow-sequence-reset;
+	 fk-partition-clone follow-defer-validate-fks follow-sequence-reset;
 
 pagila: build
 	$(MAKE) -C $@
@@ -86,6 +86,9 @@ fk-not-valid: build
 defer-validate-fks: build
 	$(MAKE) -C $@
 
+fk-partition-clone: build
+	$(MAKE) -C $@
+
 follow-defer-validate-fks: build
 	$(MAKE) -C $@
 
@@ -105,4 +108,4 @@ build:
 .PHONY: follow-wal2json follow-standby follow-9.6 follow-target-reconnect
 .PHONY: endpos-in-multi-wal-txn exclude-extension
 .PHONY: blob-snapshot-release follow-defer-indexes fk-not-valid defer-validate-fks
-.PHONY: follow-defer-validate-fks follow-sequence-reset
+.PHONY: fk-partition-clone follow-defer-validate-fks follow-sequence-reset

tests/fk-partition-clone/Dockerfile

@@ -0,0 +1,9 @@
+FROM pagila
+
+COPY --from=pgcopydb /usr/local/bin/pgcopydb /usr/local/bin
+
+WORKDIR /usr/src/pgcopydb
+COPY copydb.sh copydb.sh
+
+USER docker
+CMD ["/usr/src/pgcopydb/copydb.sh"]

tests/fk-partition-clone/Makefile

@@ -0,0 +1,15 @@
+# Copyright (c) 2021 The PostgreSQL Global Development Group.
+# Licensed under the PostgreSQL License.
+
+test: down run down ;
+
+run: build
+	$(DOCKER) compose run test
+
+down:
+	$(DOCKER) compose down
+
+build:
+	$(DOCKER) compose build
+
+.PHONY: down build test

tests/fk-partition-clone/compose.yaml

@@ -0,0 +1,42 @@
+services:
+  source:
+    image: postgres:${PGVERSION:-16}
+    expose:
+      - 5432
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: h4ckm3
+      POSTGRES_HOST_AUTH_METHOD: trust
+    command: >
+      -c ssl=on
+      -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+      -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
+  target:
+    image: postgres:${PGVERSION:-16}
+    expose:
+      - 5432
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: h4ckm3
+      POSTGRES_HOST_AUTH_METHOD: trust
+    command: >
+      -c ssl=on
+      -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
+      -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
+  test:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    cap_add:
+      - SYS_ADMIN
+      - SYS_PTRACE
+    environment:
+      PGSSLMODE: "require"
+      PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres
+      PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres
+      PGCOPYDB_TABLE_JOBS: 4
+      PGCOPYDB_INDEX_JOBS: 4
+      PGCOPYDB_FAIL_FAST: "true"
+    depends_on:
+      - source
+      - target