db2_fdw 18.2.0 Enable push down functions

LICENSE

@@ -1,9 +1,9 @@
 The PostgreSQL License
 
 Copyright (c) 2023, Ing Wolfgang Brandl.
-Copyright (c) 2025, Thomas Muenz, Living-Mainframe GmbH.
+Copyright (c) 2025, Thomas Muenz, PG_FDW Project.
 
-Wolfgang Brandl, Thomas Muenz and Living-Mainframe Gmbh, furthermore called "the developers".
+Wolfgang Brandl, Thomas Muenz, furthermore called "the developers".
 
 Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies.
 

META.json

@@ -2,7 +2,7 @@
     "name": "db2_fdw",
     "abstract": "PostgreSQL Data Wrappper to DB2 databases",
     "description": "With the Data Wrapper you can acces DB2 Tabels. Not supported for all Data Types (BLOB over 2 GByte)",
-    "version": "18.1.2",
+    "version": "18.2.0",
     "maintainer": [
       "Thomas Muenz <thomas.muenz@pg-fdw.de>"
    ],
@@ -12,7 +12,7 @@
             "abstract": "PostgreSQL Data Wrappper to DB2 databases",
             "file": "sql/db2_fdw.sql",
             "docfile": "doc/db2_fdw.md",
-            "version": "18.1.2"
+            "version": "18.2.0"
         }
     },
     "resources": {

Makefile

@@ -2,8 +2,11 @@ EXTENSION    = db2_fdw
 EXTVERSION   = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
 MODULE_big   = db2_fdw
 OBJS         = source/db2_fdw.o\
+               source/db2_deparse.o\
+               source/db2_de_serialize.o\
                source/db2GetForeignPlan.o\
                source/db2GetForeignPaths.o\
+               source/db2GetForeignUpperPaths.o\
                source/db2GetForeignJoinPaths.o\
                source/db2AnalyzeForeignTable.o\
                source/db2ExplainForeignScan.o\
@@ -26,12 +29,12 @@ OBJS         = source/db2_fdw.o\
                source/db2ExplainForeignModify.o\
                source/db2IsForeignRelUpdatable.o\
                source/db2ImportForeignSchema.o\
+               source/db2ImportForeignSchemaData.o\
                source/db2GetFdwState.o\
                source/db2GetForeignRelSize.o\
                source/db2ReAllocFree.o\
                source/db2SetHandlers.o\
                source/db2Callbacks.o\
-               source/db2GetOptions.o\
                source/db2Debug.o\
                source/db2ServerVersion.o\
                source/db2ClientVersion.o\
@@ -44,7 +47,6 @@ OBJS         = source/db2_fdw.o\
                source/db2FreeStmtHdl.o\
                source/db2GetSession.o\
                source/db2Describe.o\
-               source/db2GetImportColumn.o\
                source/db2PrepareQuery.o\
                source/db2BindParameter.o\
                source/db2ExecuteQuery.o\
@@ -64,6 +66,10 @@ OBJS         = source/db2_fdw.o\
                source/db2Shutdown.o\
                source/db2CopyText.o\
                source/db2IsStatementOpen.o\
+               source/db2PlanDirectModify.o\
+               source/db2BeginDirectModify.o\
+               source/db2IterateDirectModify.o\
+               source/db2EndDirectModify.o\
                source/db2_utils.o
 RELEASE      = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
 DATA         = $(wildcard sql/*--*.sql)
@@ -76,7 +82,7 @@ REGRESS_OPTS = --inputdir=test
 # to your extention.
 #
 #MODULES         = $(patsubst %.c,%,$(wildcard src/*.c))
-PG_CPPFLAGS     = -g -fPIC -I$(DB2_HOME)/include -I./include
+PG_CPPFLAGS     = -fPIC -I$(DB2_HOME)/include -I./include
 SHLIB_LINK      = -fPIC -L$(DB2_HOME)/lib64 -L$(DB2_HOME)/bin  -ldb2
 PG_CONFIG      ?= pg_config
 PGXS           := $(shell $(PG_CONFIG) --pgxs)

README.md

@@ -170,20 +170,33 @@ Foreign server options
   This can be in any of the forms that DB2 supports as long as your
   DB2 client is configured accordingly.
 
+- **batch_size** (optional, default 100)
+
+  In future enhancements batch_size will be used to operate cunks of rows of that size in batch mode.
+  Usage in LOAD and INSERT BATCH scenarios is envisaged.
+
+- **no_encoding_error** (optional, "ON", "OFF", "YES", "NO", "TRUE", "FALSE")
+
+  If this option is set to ON, YES, TRUE errors are generated on encoding problems.
+  By default these errors are suppressed.
+
 User mapping options
 --------------------
 
-- **user** (required)
+- **user** (optional - mutual exclusive to jwt_password, one use is required)
 
   The DB2 user name for the session.
   Set this to an empty string for *external authentication* if you don't
   want to store DB2 credentials in the PostgreSQL database (one simple way
   is to use an *external password store*).
 
-- **password** (required)
+- **password** (optional - mutual exclusive to jwt_password, one use is required)
 
   The password for the DB2 user.
 
+- **jwt_token** (optional - mutual exclusive to user&password, one use is required)
+
+  The password for the DB2 user.
 Foreign table options
 ---------------------
 
@@ -209,15 +222,12 @@ Foreign table options
   occurs in DB2's system catalog, so normally consist of uppercase letters
   only.
 
-- **max_long** (optional, defaults to "32767")
+- **max_long** (optional, now deprecated - no longer used)
 
   The maximal length of any LONG or LONG RAW columns in the DB2 table.
   Possible values are integers between 1 and 1073741823 (the maximal size of a
   `bytea` in PostgreSQL).  This amount of memory will be allocated at least
   twice, so large values will consume a lot of memory.
-  If **max_long** is less than the length of the longest value retrieved,
-  you will receive the error message `ORA-01406: fetched column value was
-  truncated`.
 
 - **readonly** (optional, defaults to "false")
 
@@ -228,7 +238,7 @@ Foreign table options
   on tables that you do not wish to be changed, to be prepared for an upgrade
   to PostgreSQL 9.3 or later.
 
-- **sample_percent** (optional, defaults to "100")
+- **sample_percent** (optional, defaults to "100.0")
 
   This option only influences ANALYZE processing and can be useful to
   ANALYZE very large tables in a reasonable time.
@@ -241,16 +251,35 @@ Foreign table options
   ANALYZE will fail with ORA-00933 for tables defined with DB2 queries and
   may fail with ORA-01446 for tables defined with complex DB2 views.
 
-- **prefetch** (optional, defaults to "200")
+- **prefetch** (aka SQL_ATTR_PREFETCH_NROWS optional, defaults to "100")
 
   Sets the number of rows that will be fetched with a single round-trip between
   PostgreSQL and DB2 during a foreign table scan.  This is implemented using
-  DB2 row prefetching.  The value must be between 0 and 10240, where a value
+  DB2 row prefetching.  The value must be between 0 and 1024, where a value
   of zero disables prefetching.
 
   Higher values can speed up performance, but will use more memory on the
   PostgreSQL server.
 
+- **fetch_size** (optional, fix 1)
+
+  In future enhancements fetch_size will be used to enable fetching of that number of rows at once.
+  Currently any number is ignored and 1 is used as a fix value, until the logic to cope with a larger result that one is implemnted.
+
+- **batch_size** (optional, default 100)
+
+  In future enhancements batch_size will be used to operate cunks of rows of that size in batch mode.
+  Usage in LOAD and INSERT BATCH scenarios is envisaged.
+  A value defined on a table will override the value provided on the server, just for that table.
+
+- **no_encoding_error** (optional, "ON", "OFF", "YES", "NO", "TRUE", "FALSE")
+
+  If this option is set to ON, YES, TRUE errors are generated on encoding problems.
+  By default these errors are suppressed.
+  Providing this on a table level sets all columns to this value, diffenent from the system wide setting given
+  explicitly or implicitly on the server.
+
+
 Column options (from PostgreSQL 9.2 on)
 ---------------------------------------
 
@@ -261,6 +290,48 @@ Column options (from PostgreSQL 9.2 on)
   For UPDATE and DELETE to work, you must set this option on all columns
   that belong to the table's primary key.
 
+- **db2type** (required)
+
+  On import the SQL value of the datatype is stored, identifying the DB2 datatype to db2_fdw.
+  It is required to re-construct the table description internally for db2_fdw.
+
+- **db2size** (required)
+
+  On import the length of the column as defined in the DB2 catalog is stored.
+  It is required to re-construct the table description internally for db2_fdw.
+
+- **db2bytes** (required)
+
+  On import the number of bytes derived from the datatype, encoding as provided by DB2 is stored.
+  It is required to re-construct the table description internally for db2_fdw.
+
+- **db2chars** (required)
+
+  On import the number of characters derived from the datatype, encoding as provided by DB2 is stored.
+  It is required to re-construct the table description internally for db2_fdw.
+
+- **db2scale** (required)
+
+  On import the scale of a decimal value as provided by DB2 is stored.
+  It is required to re-construct the table description internally for db2_fdw.
+
+- **db2null** (required)
+
+  The nullable characteristic (0 = NULLABLE, 1 = NOT NULL) as provided by DB2 is stored.
+  It is required to re-construct the table description internally for db2_fdw.
+
+- **db2ccsid** (required)
+
+  On import the ccsid of the encoding as provided by DB2 is stored.
+  It is required to re-construct the table description internally for db2_fdw.
+
+- **no_encoding_error** (optional, "ON", "OFF", "YES", "NO", "TRUE", "FALSE")
+
+  If this option is set to ON, YES, TRUE errors are generated on encoding problems.
+  By default these errors are suppressed.
+  Providing this on a column level sets this columns to this value, diffenent from the system wide setting given
+  explicitly or implicitly on the server, or the table wide settings given on the table level.
+
 4 Usage
 =======
 
@@ -301,6 +372,13 @@ If you want to UPDATE or DELETE, make sure that the `key` option is set on all
 columns that belong to the table's primary key.  Failure to do so will result
 in errors.
 
+If the remote table does not have a key defined you may alter the foreign table
+manually for defining a column key:
+
+    ALTER FOREIGN TABLE <schema>.<tablename>
+      ALTER COLUMN <columnname> OPTIONS (ADD key 'true');
+
+
 Data types
 ----------
 

db2_fdw.control

@@ -1,5 +1,5 @@
 # db2_fdw extension
 comment = 'foreign data wrapper for DB2 access'
-default_version = '18.1.2'
+default_version = '18.2.0'
 module_pathname = '$libdir/db2_fdw'
 relocatable = true

include/DB2Column.h

@@ -7,27 +7,24 @@
  *  @since  1.0
  */
 typedef struct db2Column {
-  char*               colName;       // column name in DB2
-  short               colType;       // column data type in DB2
-  size_t              colSize;       // column size
-  short               colScale;      // column scale of size describing digits right of decimal point
-  short               colNulls;      // column is nullable
-  size_t              colChars;      // numer of characters fit in column size, it is less if UTF8, 16 or DBCS
-  size_t              colBytes;      // number of bytes representing colSize
-  int                 colPrimKeyPart;// 1 if column is part of the primary key - only relevant for UPDATE or DELETE
-  int                 colCodepage;   // codepage set for this column (only set on char columns), if 0 the content is binary
-  char*               pgname;        // PG column name
-  int                 pgattnum;      // PG attribute number
-  Oid                 pgtype;        // PG data type
-  int                 pgtypmod;      // PG type modifier
-  int                 used;          // is the column used in the query?
-  int                 pkey;          // nonzero for primary keys, later set to the resjunk attribute number
-  char*               val;           // buffer for DB2 to return results in (LOB locator for LOBs)
-  size_t              val_size;      // allocated size in val
-  size_t              val_len;       // actual length of val
-  int                 val_null;      // indicator for NULL value
-  int                 varno;         // range table index of this column's relation
-  db2NoEncErrType     noencerr;      // no encoding error produced
+  char*           colName;        // column name in DB2
+  short           colType;        // column data type in DB2
+  size_t          colSize;        // column size
+  short           colScale;       // column scale of size describing digits right of decimal point
+  short           colNulls;       // column is nullable
+  size_t          colChars;       // numer of characters fit in column size, it is less if UTF8, 16 or DBCS
+  size_t          colBytes;       // number of bytes representing colSize
+  int             colPrimKeyPart; // 1 if column is part of the primary key - only relevant for UPDATE or DELETE
+  int             colCodepage;    // codepage set for this column (only set on char columns), if 0 the content is binary
+  int             pgrelid;        // range table index of this column's relation
+  char*           pgname;         // PG column name
+  int             pgattnum;       // PG attribute number
+  Oid             pgtype;         // PG data type
+  int             pgtypmod;       // PG type modifier
+  int             used;           // is the column used in the query?
+  int             pkey;           // nonzero for primary keys, later set to the resjunk attribute number
+  size_t          val_size;       // allocated size in val
+  db2NoEncErrType noencerr;       // no encoding error produced
 } DB2Column;
 
 #endif

include/DB2FdwDirectModifyState.h

@@ -0,0 +1,68 @@
+#ifndef DB2FDWDIRECTMODIFYSTATE_H
+#define DB2FDWDIRECTMODIFYSTATE_H
+
+#include <foreign/foreign.h>
+#include <foreign/fdwapi.h>
+#include <funcapi.h>
+#include "ParamDesc.h"
+#include "DB2ResultColumn.h"
+
+// Execution state of a foreign scan that modifies a foreign table directly. 
+typedef struct db2FdwDirectModifyState {
+  // DB2 Section
+  char*               dbserver;               // DB2 connect string
+  char*               user;                   // DB2 username
+  char*               password;               // DB2 password
+  char*               jwt_token;              // JWT token for authentication (alternative to user/password)
+  char*               nls_lang;               // DB2 locale information
+  DB2Session*         session;                // encapsulates the active DB2 session
+  ParamDesc*          paramList;              // description of parameters needed for the query
+  DB2ResultColumn*    resultList;             // list of result columns for the query
+  DB2Table*           db2Table;               // description of the remote DB2 table
+  unsigned long       prefetch;               // number of rows to prefetch (SQL_ATTR_PREFETCH_NROWS 0-1024)
+  int                 fetch_size;             // fetch size for this remote table (SQL_ATTR_ROW_ARRAY_SIZE 1 - 32767)
+  Relation	          rel;		  	            // relcache entry for the foreign table 
+  AttInMetadata*      attinmeta;      	      // attribute datatype conversion metadata 
+  // extracted fdw_private data 
+  char*               query;			            // text of UPDATE/DELETE command 
+  bool		            has_returning;	        // is there a RETURNING clause? 
+  List*               retrieved_attrs;	      // attr numbers retrieved by RETURNING 
+  bool		            set_processed;	        // do we set the command es_processed? 
+  // for remote query execution 
+  int			            numParams;		          // number of parameters passed to query 
+  FmgrInfo*           param_flinfo;	          // output conversion functions for them 
+  List*               param_exprs;	          // executable expressions for param values 
+  const char**        param_values;	          // textual values of query parameters 
+  // for storing result tuples 
+  //  PGresult*       result;			              // result for query 
+  int			            num_tuples;		          // # of result tuples 
+  int			            next_tuple;		          // index of next one to return 
+  Relation            resultRel;		          // relcache entry for the target relation 
+  AttrNumber*         attnoMap;		            // array of attnums of input user columns 
+  AttrNumber	        ctidAttno;		          // attnum of input ctid column 
+  AttrNumber	        oidAttno;		            // attnum of input oid column 
+  bool		            hasSystemCols;	        // are there system columns of resultRel? 
+  // working memory context 
+  MemoryContext       temp_cxt;		            // context for per-tuple temporary data 
+} DB2FdwDirectModifyState;
+
+/* Similarly, this enum describes what's kept in the fdw_private list for
+ * a ForeignScan node that modifies a foreign table directly.  We store:
+ *
+ * 1) UPDATE/DELETE statement text to be sent to the remote server
+ * 2) Boolean flag showing if the remote query has a RETURNING clause
+ * 3) Integer list of attribute numbers retrieved by RETURNING, if any
+ * 4) Boolean flag showing if we set the command es_processed
+ */
+enum FdwDirectModifyPrivateIndex {
+  // SQL statement to execute remotely (as a String node) 
+  FdwDirectModifyPrivateUpdateSql,
+  // has-returning flag (as a Boolean node) 
+  FdwDirectModifyPrivateHasReturning,
+  // Integer list of attribute numbers retrieved by RETURNING 
+  FdwDirectModifyPrivateRetrievedAttrs,
+  // set-processed flag (as a Boolean node) 
+  FdwDirectModifyPrivateSetProcessed,
+};
+
+#endif

include/DB2FdwPathExtraData.h

@@ -0,0 +1,21 @@
+#ifndef DB2FDWPATHEXTRADATA_H
+#define DB2FDWPATHEXTRADATA_H
+
+#include <nodes/pathnodes.h>
+
+/** Struct for extra information passed to estimate_path_cost_size()
+ * 
+ *  @author Thomas Muenz
+ *  @since  18.1.1
+ */
+
+typedef struct DB2FdwPathExtraData {
+  PathTarget* target;
+  bool        has_final_sort;
+  bool        has_limit;
+  double      limit_tuples;
+  int64       count_est;
+  int64       offset_est;
+} DB2FdwPathExtraData;
+
+#endif