DB2 fdw 18.1.2

META.json

@@ -2,7 +2,7 @@
     "name": "db2_fdw",
     "abstract": "PostgreSQL Data Wrappper to DB2 databases",
     "description": "With the Data Wrapper you can acces DB2 Tabels. Not supported for all Data Types (BLOB over 2 GByte)",
-    "version": "18.1.1",
+    "version": "18.1.2",
     "maintainer": [
       "Thomas Muenz <thomas.muenz@pg-fdw.de>"
    ],
@@ -12,7 +12,7 @@
             "abstract": "PostgreSQL Data Wrappper to DB2 databases",
             "file": "sql/db2_fdw.sql",
             "docfile": "doc/db2_fdw.md",
-            "version": "18.1.1"
+            "version": "18.1.2"
         }
     },
     "resources": {

db2_fdw.control

@@ -1,5 +1,5 @@
 # db2_fdw extension
 comment = 'foreign data wrapper for DB2 access'
-default_version = '18.1.1'
+default_version = '18.1.2'
 module_pathname = '$libdir/db2_fdw'
 relocatable = true

include/db2_fdw.h

@@ -23,9 +23,9 @@
 #endif
 
 /* defined in backend/commands/analyze.c */
-#ifndef WIDTH_THRESHOLD
-#define WIDTH_THRESHOLD 1024
-#endif /* WIDTH_THRESHOLD */
+//#ifndef WIDTH_THRESHOLD
+//#define WIDTH_THRESHOLD 1024
+//#endif /* WIDTH_THRESHOLD */
 
 /* array_create_iterator has a new signature from 9.5 on */
 #define array_create_iterator(arr, slice_ndim) array_create_iterator(arr, slice_ndim, NULL)
@@ -62,7 +62,7 @@
 */
 
 /* db2_fdw version */
-#define DB2_FDW_VERSION "18.1.1"
+#define DB2_FDW_VERSION "18.1.2"
 /* number of bytes to read per LOB chunk */
 #define LOB_CHUNK_SIZE    8192
 #define ERRBUFSIZE        2000
@@ -187,4 +187,8 @@ typedef enum { CASE_KEEP, CASE_LOWER, CASE_SMART } fold_t;
 #define serializeInt(x)                makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum((int32)(x)), false, true)
 #define serializeOid(x)                makeConst(OIDOID, -1, InvalidOid, 4, ObjectIdGetDatum(x), false, true)
 
+#ifndef MIN
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
 #endif

source/db2AnalyzeForeignTable.c

@@ -16,7 +16,7 @@ extern int          db2ExecuteQuery           (DB2Session* session, const DB2Tab
 extern int          db2FetchNext              (DB2Session* session);
 extern void         checkDataType             (short db2type, int scale, Oid pgtype, const char* tablename, const char* colname);
 extern short        c2dbType                  (short fcType);
-extern void         convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls, bool trunc_lob) ;
+extern void         convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls) ;
 extern void         db2Debug1                 (const char* message, ...);
 extern void         db2Debug2                 (const char* message, ...);
 extern void         db2Debug3                 (const char* message, ...);
@@ -40,7 +40,6 @@ bool db2AnalyzeForeignTable (Relation relation, AcquireSampleRowsFunc* func, Blo
 
 /** acquireSampleRowsFunc
  *   Perform a sequential scan on the DB2 table and return a sampe of rows.
- *   All LOB values are truncated to WIDTH_THRESHOLD+1 because anything
  *   exceeding this is not used by compute_scalar_stats().
  */
 int acquireSampleRowsFunc (Relation relation, int elevel, HeapTuple * rows, int targrows, double *totalrows, double *totaldeadrows) {
@@ -140,7 +139,7 @@ int acquireSampleRowsFunc (Relation relation, int elevel, HeapTuple * rows, int
       /* the first "targrows" rows are added as samples */
       /* use a temporary memory context during convertTuple */
       old_cxt = MemoryContextSwitchTo (tmp_cxt);
-      convertTuple (fdw_state, values, nulls, true);
+      convertTuple (fdw_state, values, nulls);
       MemoryContextSwitchTo (old_cxt);
       rows[collected_rows++] = heap_form_tuple (tupDesc, values, nulls);
       MemoryContextReset (tmp_cxt);
@@ -157,7 +156,7 @@ int acquireSampleRowsFunc (Relation relation, int elevel, HeapTuple * rows, int
         heap_freetuple (rows[k]);
         /* use a temporary memory context during convertTuple */
         old_cxt = MemoryContextSwitchTo (tmp_cxt);
-        convertTuple (fdw_state, values, nulls, true);
+        convertTuple (fdw_state, values, nulls);
         MemoryContextSwitchTo (old_cxt);
         rows[k] = heap_form_tuple (tupDesc, values, nulls);
         MemoryContextReset (tmp_cxt);

source/db2BeginForeignModify.c

@@ -171,7 +171,7 @@ DB2FdwState* deserializePlanData (List* list) {
     db2Debug2("  state->db2Table->cols[%d]->noencerr: %d",i,state->db2Table->cols[i]->noencerr);
     cell = list_next (list,cell);
     /* allocate memory for the result value only when the column is used in query */
-    state->db2Table->cols[i]->val      = (state->db2Table->cols[i]->used == 1) ? (char*) db2alloc ("state->db2Table->cols[i]->val", state->db2Table->cols[i]->val_size + 1) : NULL;
+    state->db2Table->cols[i]->val      = (state->db2Table->cols[i]->used == 1) ? (char*) db2alloc ("state->db2Table->cols[i]->val", MIN(state->db2Table->cols[i]->val_size, 1073741823)) : NULL;
     db2Debug2("  state->db2Table->cols[%d]->val: %x",i,state->db2Table->cols[i]->val);
     state->db2Table->cols[i]->val_len  = 0;
     db2Debug2("  state->db2Table->cols[%d]->val_len: %d",i,state->db2Table->cols[i]->val_len);

source/db2ExecForeignDelete.c

@@ -14,7 +14,7 @@ extern regproc* output_funcs;
 extern int             db2ExecuteQuery           (DB2Session* session, const DB2Table* db2Table, ParamDesc* paramList);
 extern void            db2Debug1                 (const char* message, ...);
 extern void            db2Debug2                 (const char* message, ...);
-extern void            convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls, bool trunc_lob) ;
+extern void            convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls) ;
 extern char*           deparseDate               (Datum datum);
 extern char*           deparseTimestamp          (Datum datum, bool hasTimezone);
 extern void*           db2alloc                  (const char* type, size_t size);
@@ -61,7 +61,7 @@ TupleTableSlot* db2ExecForeignDelete (EState* estate, ResultRelInfo* rinfo, Tupl
   ExecClearTuple (slot);
 
   /* convert result for RETURNING to arrays of values and null indicators */
-  convertTuple (fdw_state, slot->tts_values, slot->tts_isnull, false);
+  convertTuple (fdw_state, slot->tts_values, slot->tts_isnull);
 
   /* store the virtual tuple */
   ExecStoreVirtualTuple (slot);

source/db2ExecForeignInsert.c

@@ -14,7 +14,7 @@ extern bool dml_in_transaction;
 extern int             db2ExecuteInsert          (DB2Session* session, const DB2Table* db2Table, ParamDesc* paramList);
 extern void            db2Debug1                 (const char* message, ...);
 extern void            setModifyParameters       (ParamDesc* paramList, TupleTableSlot* newslot, TupleTableSlot* oldslot, DB2Table* db2Table, DB2Session* session);
-extern void            convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls, bool trunc_lob) ;
+extern void            convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls) ;
 
 /** local prototypes */
 TupleTableSlot* db2ExecForeignInsert(EState* estate, ResultRelInfo* rinfo, TupleTableSlot* slot, TupleTableSlot* planSlot);
@@ -52,7 +52,7 @@ TupleTableSlot* db2ExecForeignInsert (EState* estate, ResultRelInfo* rinfo, Tupl
   ExecClearTuple (slot);
 
   /* convert result for RETURNING to arrays of values and null indicators */
-  convertTuple (fdw_state, slot->tts_values, slot->tts_isnull, false);
+  convertTuple (fdw_state, slot->tts_values, slot->tts_isnull);
 
   /* store the virtual tuple */
   ExecStoreVirtualTuple (slot);

source/db2ExecForeignUpdate.c

@@ -15,7 +15,7 @@ extern int             db2ExecuteQuery           (DB2Session* session, const DB2
 extern void            db2Debug1                 (const char* message, ...);
 extern void            db2Debug2                 (const char* message, ...);
 extern void            setModifyParameters       (ParamDesc* paramList, TupleTableSlot* newslot, TupleTableSlot* oldslot, DB2Table* db2Table, DB2Session* session);
-extern void            convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls, bool trunc_lob) ;
+extern void            convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls) ;
 
 /** local prototypes */
 TupleTableSlot* db2ExecForeignUpdate      (EState* estate, ResultRelInfo* rinfo, TupleTableSlot* slot, TupleTableSlot* planSlot);
@@ -58,7 +58,7 @@ TupleTableSlot* db2ExecForeignUpdate (EState* estate, ResultRelInfo* rinfo, Tupl
   ExecClearTuple (slot);
 
   /* convert result for RETURNING to arrays of values and null indicators */
-  convertTuple (fdw_state, slot->tts_values, slot->tts_isnull, false);
+  convertTuple (fdw_state, slot->tts_values, slot->tts_isnull);
 
   /* store the virtual tuple */
   ExecStoreVirtualTuple (slot);

source/db2GetImportColumn.c

@@ -130,7 +130,7 @@ int db2GetImportColumn(DB2Session* session, char* schema, char* table_list, int
       case 0: {   /* FDW_IMPORT_SCHEMA_ALL      */
         char* query_str = "SELECT T.TABNAME, C.COLNAME, C.TYPENAME, C.LENGTH, C.SCALE, C.NULLS, COALESCE(C.KEYSEQ, 0) AS KEY, C.CODEPAGE"
                           " FROM SYSCAT.TABLES T JOIN SYSCAT.COLUMNS C ON T.TABSCHEMA = C.TABSCHEMA AND T.TABNAME   = C.TABNAME"
-                          " WHERE T.TABSCHEMA = ? AND T.TYPE IN ('T','V') ORDER BY T.TABNAME, C.COLNO";
+                          " WHERE UPPER(T.TABSCHEMA) = UPPER(?) AND T.TYPE IN ('T','V') AND COALESCE(C.HIDDEN,'') = '' ORDER BY T.TABNAME, C.COLNO";
         int   s_len     = strlen(query_str)+1;
         column_query = db2alloc("column_query",s_len);
         strncpy(column_query,query_str,s_len);
@@ -139,7 +139,7 @@ int db2GetImportColumn(DB2Session* session, char* schema, char* table_list, int
       case 1: {   /* FDW_IMPORT_SCHEMA_LIMIT_TO */
         char* query_str = "SELECT T.TABNAME, C.COLNAME, C.TYPENAME, C.LENGTH, C.SCALE, C.NULLS, COALESCE(C.KEYSEQ, 0) AS KEY, C.CODEPAGE"
                           " FROM SYSCAT.TABLES T JOIN SYSCAT.COLUMNS C ON T.TABSCHEMA = C.TABSCHEMA AND T.TABNAME   = C.TABNAME"
-                          " WHERE T.TABSCHEMA = ? AND T.TYPE IN ('T','V') AND T.TABNAME IN (%s) ORDER BY T.TABNAME, C.COLNO";
+                          " WHERE UPPER(T.TABSCHEMA) = UPPER(?) AND T.TYPE IN ('T','V') AND UPPER(T.TABNAME) IN (%s) AND COALESCE(C.HIDDEN,'') = '' ORDER BY T.TABNAME, C.COLNO";
         int   s_len     = strlen(query_str) + strlen(table_list) + 1;
         column_query = db2alloc("column_query",s_len);
         snprintf(column_query,s_len,query_str,table_list);
@@ -148,7 +148,7 @@ int db2GetImportColumn(DB2Session* session, char* schema, char* table_list, int
       case 2: {   /* FDW_IMPORT_SCHEMA_EXCEPT   */
         char* query_str = "SELECT T.TABNAME, C.COLNAME, C.TYPENAME, C.LENGTH, C.SCALE, C.NULLS, COALESCE(C.KEYSEQ, 0) AS KEY, C.CODEPAGE"
                           " FROM SYSCAT.TABLES T JOIN SYSCAT.COLUMNS C ON T.TABSCHEMA = C.TABSCHEMA AND T.TABNAME   = C.TABNAME"
-                          " WHERE T.TABSCHEMA = ? AND T.TYPE IN ('T','V') AND T.TABNAME NOT IN (%s) ORDER BY T.TABNAME, C.COLNO";
+                          " WHERE UPPER(T.TABSCHEMA) = UPPER(?) AND T.TYPE IN ('T','V') AND UPPER(T.TABNAME) NOT IN (%s) AND COALESCE(C.HIDDEN,'') = '' ORDER BY T.TABNAME, C.COLNO";
         int   s_len     = strlen(query_str) + strlen(table_list) + 1;
         column_query = db2alloc("column_query",s_len);
         snprintf(column_query,s_len,query_str,table_list);
@@ -172,7 +172,7 @@ int db2GetImportColumn(DB2Session* session, char* schema, char* table_list, int
     }
 
     /* bind the parameter */
-    result = SQLBindParameter(session->stmtp->hsql, SQL_PARAM_INPUT, 1, SQL_C_CHAR, SQL_VARCHAR, 128, 0, schema, sizeof(schema), &ind_s);
+    result = SQLBindParameter(session->stmtp->hsql, 1, SQL_PARAM_INPUT, SQL_C_CHAR, SQL_VARCHAR, 128, 0, schema, sizeof(schema), &ind_s);
     db2Debug2("  SQLBindParameter table_schema = '%s' rc : %d",schema, result);
     result = db2CheckErr(result, session->stmtp->hsql, session->stmtp->type, __LINE__, __FILE__);
     if (result != SQL_SUCCESS) {

source/db2GetLob.c

@@ -18,27 +18,29 @@ extern SQLRETURN    db2CheckErr          (SQLRETURN status, SQLHANDLE handle, SQ
 extern void         db2Error_d           (db2error sqlstate, const char* message, const char* detail, ...);
 
 /** internal prototypes */
-void                db2GetLob            (DB2Session* session, DB2Column* column, int cidx, char** value, long* value_len, unsigned long trunc);
+void                db2GetLob            (DB2Session* session, DB2Column* column, int cidx, char** value, long* value_len);
 
 /** db2GetLob
  *   Get the LOB contents and store them in *value and *value_len.
  *   If "trunc" is nonzero, it contains the number of bytes or characters to get.
  */
-void db2GetLob (DB2Session* session, DB2Column* column, int cidx, char** value, long* value_len, unsigned long trunc) {
+void db2GetLob (DB2Session* session, DB2Column* column, int cidx, char** value, long* value_len) {
   SQLRETURN      rc  = SQL_SUCCESS;
   SQLLEN         ind = 0;
   SQLCHAR        buf[LOB_CHUNK_SIZE+1];
+  SQLSMALLINT    fcType = (column->colType == DB2_CLOB) ? SQL_C_CHAR : SQL_C_BINARY;
   int            extend = 0;
   db2Debug1("> db2GetLob");
-  db2Debug2("  column->colName: '%s'",column->colName);
-  db2Debug2("  cidx           :  %d ",cidx);
+  db2Debug2("  column->colName  : '%s'",column->colName);
+  db2Debug2("  cidx             :  %d ",cidx);
+  db2Debug2("  column->pgattnum :  %d ",column->pgattnum);
   /* initialize result buffer length */
   *value_len = 0;
   /* read the LOB in chunks */
   do {
     db2Debug2("  value_len: %ld",*value_len);
     db2Debug2("  reading %d byte chunck of data",sizeof(buf));
-    rc = SQLGetData(session->stmtp->hsql, cidx, SQL_C_CHAR, buf, sizeof(buf), &ind);
+    rc = SQLGetData(session->stmtp->hsql, cidx, fcType, buf, sizeof(buf), &ind);
     rc = db2CheckErr(rc,session->stmtp->hsql, session->stmtp->type, __LINE__, __FILE__);
     if (rc == SQL_ERROR) {
       db2Error_d ( FDW_UNABLE_TO_CREATE_EXECUTION, "error fetching result: SQLGetData failed to read LOB chunk", db2Message);

source/db2ImportForeignSchema.c

@@ -102,9 +102,9 @@ List* db2ImportForeignSchema (ImportForeignSchemaStmt* stmt, Oid serverOid) {
       continue;
     } else if (strcmp (def->defname, "readonly") == 0) {
       char *s = STRVAL(def->arg);
-      if (pg_strcasecmp (s, "on") != 0 || pg_strcasecmp (s, "yes") != 0 || pg_strcasecmp (s, "true") != 0)
+      if (pg_strcasecmp (s, "on") == 0 || pg_strcasecmp (s, "yes") == 0 || pg_strcasecmp (s, "true") == 0)
         readonly = true;
-      else if (pg_strcasecmp (s, "off") != 0 || pg_strcasecmp (s, "no") != 0 || pg_strcasecmp (s, "false") != 0)
+      else if (pg_strcasecmp (s, "off") == 0 || pg_strcasecmp (s, "no") == 0 || pg_strcasecmp (s, "false") == 0)
         readonly = false;
       else
         ereport (ERROR, (errcode (ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), errmsg ("invalid value for option \"%s\"", def->defname)));
@@ -131,18 +131,34 @@ List* db2ImportForeignSchema (ImportForeignSchemaStmt* stmt, Oid serverOid) {
   if (stmt->list_type != FDW_IMPORT_SCHEMA_ALL) {
     foreach (cell, stmt->table_list) {
       RangeVar* rVar = lfirst(cell);
+      char* uppername;
+      char* folded;
       db2Debug2("  rVar             :  %x ", rVar);
-      if (rVar != NULL) {
-        db2Debug2("  rVar->type       :  %d ", rVar->type);
-        db2Debug2("  rVar->catalogname: '%s'", rVar->catalogname);
-        db2Debug2("  rVar->schemaname : '%s'", rVar->schemaname);
-        db2Debug2("  rVar->relname    : '%s'", rVar->relname);
-        if (tblist.len != 0) {
-          appendStringInfo(&tblist,",'%s'",rVar->relname);
-        } else {
-          appendStringInfo(&tblist,"'%s'",rVar->relname);
-        }
+      if (rVar == NULL || rVar->relname == NULL)
+        continue;
+
+      /*
+       * IMPORTANT: the table list in LIMIT TO / EXCEPT is compared against the
+       * names that will be created in PostgreSQL after case folding.
+       *
+       * So we normalize rVar->relname in-place using fold_case() so that
+       * PostgreSQL's LIMIT/EXCEPT filtering and our generated CREATE FOREIGN
+       * TABLE statements agree.
+       *
+       * For the DB2-side query filter we use an uppercased version of the name
+       * and the DB2 query itself compares with UPPER(T.TABNAME), making the
+       * matching case-insensitive.
+       */
+      uppername = str_toupper (rVar->relname, strlen (rVar->relname), DEFAULT_COLLATION_OID);
+      if (tblist.len != 0) {
+        appendStringInfo(&tblist,",'%s'", uppername);
+      } else {
+        appendStringInfo(&tblist,"'%s'", uppername);
       }
+      db2free (uppername);
+
+      folded = fold_case (rVar->relname, foldcase);
+      rVar->relname = folded;
     }
   }
   db2Debug2("  import table_list: '%s'",tblist.data);
@@ -169,6 +185,7 @@ List* db2ImportForeignSchema (ImportForeignSchemaStmt* stmt, Oid serverOid) {
       appendStringInfo (&buf, ")");
       db2Debug2 ("  pg fdw table ddl: '%s'",buf.data);
       result = lappend (result, db2strdup (buf.data));
+      db2Debug2("result(%d):%x",list_length(result), result);
     }
 
     if (rc == 1 && (oldtabname[0] == '\0' || strcmp (tabname, oldtabname))) {
@@ -274,7 +291,8 @@ List* db2ImportForeignSchema (ImportForeignSchemaStmt* stmt, Oid serverOid) {
         appendStringInfo (&buf, " NOT NULL");
     }
   } while (rc == 1);
-  db2Debug1("< db2ImportForeignSchema");
+  db2Debug2("result(%d):%x",list_length(result), result);
+  db2Debug1("< db2ImportForeignSchema : result(%d):%x",list_length(result), result);
   return result;
 }
 

source/db2IterateForeignScan.c

@@ -18,7 +18,7 @@ extern void         db2CloseStatement         (DB2Session* session);
 extern void         db2Debug1                 (const char* message, ...);
 extern void         db2Debug2                 (const char* message, ...);
 extern void         db2Debug3                 (const char* message, ...);
-extern void         convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls, bool trunc_lob) ;
+extern void         convertTuple              (DB2FdwState* fdw_state, Datum* values, bool* nulls) ;
 extern char*        deparseDate               (Datum datum);
 extern char*        deparseTimestamp          (Datum datum, bool hasTimezone);
 
@@ -60,7 +60,7 @@ TupleTableSlot* db2IterateForeignScan (ForeignScanState* node) {
     /* increase row count */
     ++fdw_state->rowcount;
     /* convert result to arrays of values and null indicators */
-    convertTuple (fdw_state, slot->tts_values, slot->tts_isnull, false);
+    convertTuple (fdw_state, slot->tts_values, slot->tts_isnull);
     /* store the virtual tuple */
     ExecStoreVirtualTuple (slot);
   } else {