This update handles large tables in BulkXMLLoad better by processing …

…records in batches. It also renames the built-in tables Keywords.dbf, ExprMap.dbf, and TypeMap.dbf to have an underscore prefix to avoid conflict with tables using those names in the database to be upsized.

.gitignore

@@ -1,4 +1,12 @@
 *.FXP
 *.ERR
 *.BAK
+*.log
 *.zip
+Upsize/*
+Solution.xml
+UPSIZINGWIZARD_MetaData.*
+matt_upsize_engine.*
+ThorUpdater/*.ps1
+ThorUpdater/Project.txt
+ThorUpdater/Version.txt

LIB/upswiz.vc2

@@ -1713,15 +1713,15 @@ DEFINE CLASS cfieldtypemap AS basecontainer OF "basecontrols.vcx"
 			lcRmtType=RTRIM(&lcMappingTable..RmtType)
 			lcLocalType=RTRIM(&lcMappingTable..DataType)
 			lnOldArea=SELECT()
-			SELECT TypeMap	
+			SELECT _TypeMap	
 			LOCATE FOR RTRIM(RemoteType)==lcRmtType ;
 				AND RTRIM(LocalType)==lcLocalType ;
 				AND Server=RTRIM(Thisform.oEngine.ServerType)
-			this.grcRmtLength.Enabled=TypeMap.VarLength
-			this.grcRmtPrec.Enabled=TypeMap.HasPrec
+			this.grcRmtLength.Enabled=_TypeMap.VarLength
+			this.grcRmtPrec.Enabled=_TypeMap.HasPrec
 
-			this.grcRmtLength.Readonly=!TypeMap.VarLength
-			this.grcRmtPrec.Readonly=!TypeMap.HasPrec
+			this.grcRmtLength.Readonly=!_TypeMap.VarLength
+			this.grcRmtPrec.Readonly=!_TypeMap.HasPrec
 			SELECT(lnOldArea)
 		ENDIF
 

PROGRAM/bulkxmlload.prg

@@ -2,7 +2,7 @@
 * Function:			BulkXMLLoad
 * Purpose:			Performs a SQL Server bulk XML load
 * Author:			Doug Hennig
-* Last revision:	02/22/2017
+* Last revision:	01/24/2019
 * Parameters:		tcAlias      - the alias of the cursor to export
 *					tcTable      - the name of the table to import into
 *					ttBlank      - the value to use for blank DateTime values
@@ -22,7 +22,7 @@
 *						loading on a local server, the temporary file path must
 *						be a UNC path (such as \\servername\sharename). If
 *						specified, transactional processing will be enabled for
-*						the XML bulk load.
+*						the XML bulk load
 * Returns:			an empty string if the bulk load succeeded or the text of
 *						an error message if it failed
 * Environment in:	the alias specified in tcAlias must be open
@@ -53,6 +53,10 @@
 *					The reason I added that is because when I was testing, the
 *					XML bulk load choked on a table where a CHR(2) had somehow
 *					gotten into a field of user-entered data.
+*
+*					Change made 2019-01-24: to handle large tables, XML output
+*					is now done in batches of up to 500 MB of records at a
+*					time.
 *==============================================================================
 
 lparameters tcAlias, ;
@@ -76,11 +80,13 @@ local lnSelect, ;
 	lcReplaceCommand, ;
 	lcSchema, ;
 	lcData, ;
+	lnRecords, ;
+	lnRecsProcessed, ;
 	lcReturn, ;
 	loException as Exception, ;
 	lcXSD, ;
 	loBulkLoad
-#include Include\AllDefs.H
+#include ..\Include\AllDefs.H
 
 * These characters are illegal according to the XML 1.0 specification. CHR(0)
 * is also illegal, but we're leaving that alone because they may be situations
@@ -151,84 +157,101 @@ if ascan(laFields, 'C', -1, -1, 2, 7) > 0 or ;
 	next lnI
 endif ascan(laFields ...
 
+* We don't want the XML files to exceed 500 MB (the files can be up to 2 GB but
+* XML is verbose so we have to account for that) so we may have to process the
+* table in batches.
+
+lnRecords       = ceiling(500000000/recsize())
+lnRecsProcessed = 0
+
 * Create the XML data and schema files.
 
 lcSchema = forceext(tcTable, 'xsd')
 lcData   = forceext(tcTable, 'xml')
-try
-	cursortoxml(alias(), lcData, 1, 512 + 8, 0, lcSchema)
-	lcReturn = ''
-catch to loException
-	lcReturn = loException.Message
-endtry
-
-* If we created a cursor, close it.
-
-if llClose
-	use
-endif llClose
+lcReturn = ''
+go top
+do while lnRecsProcessed < reccount()
+	try
+		cursortoxml(alias(), lcData, 1, 512 + 8, lnRecords, lcSchema)
+	catch to loException
+		lcReturn = loException.Message
+	endtry
 
 * Convert the XSD into a format acceptable by SQL Server. Add the SQL
 * namespace, convert the <xsd:choice> start and end tags to <xsd:sequence>,
 * use the sql:datatype attribute for DateTime fields, and specify the table
-* imported into with the sql:relation attribute.
-
-if empty(lcReturn)
-	lcXSD = filetostr(lcSchema)
-	lcXSD = strtran(lcXSD, ':xml-msdata">', ;
-		':xml-msdata" xmlns:sql="urn:schemas-microsoft-com:mapping-schema">')
-	lcXSD = strtran(lcXSD, 'IsDataSet="true">', ;
-		'IsDataSet="true" sql:is-constant="1">')
-	lcXSD = strtran(lcXSD, '<xsd:choice maxOccurs="unbounded">', ;
-		'<xsd:sequence>')
-	lcXSD = strtran(lcXSD, '</xsd:choice>', ;
-		'</xsd:sequence>')
-	lcXSD = strtran(lcXSD, 'type="xsd:dateTime"', ;
-		'type="xsd:dateTime" sql:datatype="dateTime"')
-	lcXSD = strtran(lcXSD, 'minOccurs="0"', ;
-		'sql:relation="' + lower(tcTable) + '" minOccurs="0"')
-	strtofile(lcXSD, lcSchema)
+* imported into with the sql:relation attribute. Note: although the XSD content
+* doesn't change, the file has to be recreated on each pass or the bulk XML
+* load won't import any records on the second and subsequent passes.
+
+	if empty(lcReturn)
+		lcXSD = filetostr(lcSchema)
+		lcXSD = strtran(lcXSD, ':xml-msdata">', ;
+			':xml-msdata" xmlns:sql="urn:schemas-microsoft-com:mapping-schema">')
+		lcXSD = strtran(lcXSD, 'IsDataSet="true">', ;
+			'IsDataSet="true" sql:is-constant="1">')
+		lcXSD = strtran(lcXSD, '<xsd:choice maxOccurs="unbounded">', ;
+			'<xsd:sequence>')
+		lcXSD = strtran(lcXSD, '</xsd:choice>', ;
+			'</xsd:sequence>')
+		lcXSD = strtran(lcXSD, 'type="xsd:dateTime"', ;
+			'type="xsd:dateTime" sql:datatype="dateTime"')
+		lcXSD = strtran(lcXSD, 'minOccurs="0"', ;
+			'sql:relation="' + lower(tcTable) + '" minOccurs="0"')
+		strtofile(lcXSD, lcSchema)
 
 * Instantiate the SQLXMLBulkLoad object and set its ConnectionString and other
 * properties. Note: we can set the ErrorLogFile property to the name of a file
 * to write import errors to; that isn't done here.
 
-	try
-		loBulkLoad   = createobject('SQLXMLBulkLoad.SQLXMLBulkload.4.0')
-		lcConnString = 'Provider=SQLOLEDB.1;Initial Catalog=' + tcDatabase + ;
-			';Data Source=' + tcServer + ';Persist Security Info=False;'
-		if empty(tcUserName)
-			lcConnString = lcConnString + 'Integrated Security=SSPI'
-		else
-			lcConnString = lcConnString + 'User ID=' + tcUserName + ;
-				';Password=' + tcPassword
-		endif empty(tcUserName)
-		loBulkLoad.ConnectionString = lcConnString
-		loBulkLoad.KeepNulls        = .T.
-
-* If a temp folder was specified, turn on transaction processing for the bulk
-* load. This allows the XML bulk load to succeed when you have blank string
-* fields and do not allow NULL values. I have not been able to find any
-* explanation WHY transaction processing allows blank strings when
-* non-transaction processing forces NULL values to be saved.
-
-		if not empty(tcTempFolder)
-			loBulkLoad.Transaction	= .T.
-			loBulkLoad.TempFilePath = tcTempFolder
-		endif not empty(tcTempFolder)
+		try
+			loBulkLoad   = createobject('SQLXMLBulkLoad.SQLXMLBulkload.4.0')
+			lcConnString = 'Provider=SQLOLEDB.1;Initial Catalog=' + tcDatabase + ;
+				';Data Source=' + tcServer + ';Persist Security Info=False;'
+			if empty(tcUserName)
+				lcConnString = lcConnString + 'Integrated Security=SSPI'
+			else
+				lcConnString = lcConnString + 'User ID=' + tcUserName + ;
+					';Password=' + tcPassword
+			endif empty(tcUserName)
+			loBulkLoad.ConnectionString = lcConnString
+			loBulkLoad.KeepNulls        = .T.
+			loBulkLoad.ForceTableLock   = .T.
+
+* Turn on transaction processing for the bulk load. This allows the XML bulk
+* load to succeed when you have blank string fields and do not allow NULL
+* values. I have not been able to find any explanation WHY transaction
+* processing allows blank strings when non-transaction processing forces NULL
+* values to be saved. If a temporary folder was specified, use it; otherwise
+* the location specified in the TEMP environment variable is used.
+
+			loBulkLoad.Transaction = .T.
+			if not empty(tcTempFolder)
+				loBulkLoad.TempFilePath = tcTempFolder
+			endif not empty(tcTempFolder)
 
 * Call Execute to perform the bulk import.
 
-		loBulkLoad.Execute(lcSchema, lcData)
-		lcReturn = ''
-	catch to loException
-		lcReturn = loException.Message
-	endtry
+			loBulkLoad.Execute(lcSchema, lcData)
+		catch to loException
+			lcReturn = loException.Message
+		endtry
+	endif empty(lcReturn)
+	lnRecsProcessed = lnRecsProcessed + lnRecords
+	if not eof()
+		skip
+	endif not eof()
+	if not empty(lcReturn)
+		exit
+	endif not empty(lcReturn)
+enddo while lnRecsProcessed < reccount()
 
-* Clean up.
+* Clean up. If we created a cursor, close it.
 
-	erase (lcSchema)
-	erase (lcData)
-endif empty(lcReturn)
+if llClose
+	use
+endif llClose
+erase (lcSchema)
+erase (lcData)
 select (lnSelect)
 return lcReturn

PROGRAM/wizusz.prg

@@ -502,9 +502,10 @@ DEFINE CLASS UpsizeEngine AS WizEngineAll of WZEngine.prg
         lcAction=IIF(THIS.DoReport AND THIS.ProcessingOutput,"Close","Delete")
 
         *Deal with tables that are part of the upsizing wizard project (that don't need to be deleted)
-        THIS.DisposeTable("Keywords","Close")
-        THIS.DisposeTable("ExprMap","Close")
-        THIS.DisposeTable("TypeMap","Close")
+*** 2019-01-24: prefix built-in tables with "_" to avoid conflict with tables with same names in DBC to be upsized
+        THIS.DisposeTable("_Keywords","Close")
+        THIS.DisposeTable("_ExprMap","Close")
+        THIS.DisposeTable("_TypeMap","Close")
 
         *Close this cursor (created in SQL 95 case)
         THIS.DisposeTable(THIS.OraNames,"Close")
@@ -4502,25 +4503,26 @@ DEFINE CLASS UpsizeEngine AS WizEngineAll of WZEngine.prg
     ENDPROC
 
 
+*** 2019-01-24: prefix built-in tables with "_" to avoid conflict with tables with same names in DBC to be upsized
     PROCEDURE GetDefaultMapping
         PARAMETERS aPassedArray
         LOCAL lnOldArea, lcServerConstraint
 
 *** DH 2015-09-14: handle TypeMap not existing
-		if not file('TypeMap.dbf')
+		if not file('_TypeMap.dbf')
 			This.HadError = .T.
-			This.ErrorMessage = 'TypeMap.dbf cannot be found'
+			This.ErrorMessage = '_TypeMap.dbf cannot be found'
 			This.Die()
-		endif not file('TypeMap.dbf')
+		endif not file('_TypeMap.dbf')
 *** DH 2015-09-14: end of new code
         lnOldArea=SELECT()
-        IF NOT USED("TypeMap")
+        IF NOT USED("_TypeMap")
             SELECT 0
 *** DH 09/05/2013: don't open exclusively
-***            USE TypeMap EXCLUSIVE
-            USE TypeMap
+***            USE _TypeMap EXCLUSIVE
+            USE _TypeMap
         ELSE
-            SELECT TypeMap
+            SELECT _TypeMap
         ENDIF
 
         *Didn't foresee a problem, thus this cheezy snippet
@@ -4536,8 +4538,8 @@ DEFINE CLASS UpsizeEngine AS WizEngineAll of WZEngine.prg
 ***            ENDIF
         ENDIF
 
-        SELECT LocalType, RemoteType, VarLength, FullLocal FROM TypeMap ;
-            WHERE  TypeMap.DEFAULT=.T. AND TypeMap.SERVER=lcServerConstraint ;
+        SELECT LocalType, RemoteType, VarLength, FullLocal FROM _TypeMap ;
+            WHERE _TypeMap.DEFAULT=.T. AND _TypeMap.SERVER=lcServerConstraint ;
             INTO ARRAY aPassedArray
 
         SELECT(lnOldArea)
@@ -5283,16 +5285,17 @@ DEFINE CLASS UpsizeEngine AS WizEngineAll of WZEngine.prg
         lcSetTalk = SET('TALK')
         SET TALK OFF
 
-        IF !USED("ExprMap") THEN
+*** 2019-01-24: prefix built-in tables with "_" to avoid conflict with tables with same names in DBC to be upsized
+        IF !USED("_ExprMap") THEN
             SELECT 0
-            USE ExprMap EXCLUSIVE
+            USE _ExprMap EXCLUSIVE
             IF THIS.ServerType = "Oracle" THEN
-                SET FILTER TO !EMPTY(ExprMap.ORACLE)
+                SET FILTER TO !EMPTY(_ExprMap.ORACLE)
             ELSE
-                SET FILTER TO !EMPTY(ExprMap.SQLServer)
+                SET FILTER TO !EMPTY(_ExprMap.SQLServer)
             ENDIF
         ELSE
-            SELECT ExprMap
+            SELECT _ExprMap
         ENDIF
 
         lcRemoteExpression = ''
@@ -5365,12 +5368,13 @@ DEFINE CLASS UpsizeEngine AS WizEngineAll of WZEngine.prg
 
         lcExpression = CHR(1)+lcLocalExpression
         SCAN
-            IF !ExprMap.PAD
-                lcServerSQL = RTRIM(ExprMap.&lcMapField.)
+*** 2019-01-24: prefix built-in tables with "_" to avoid conflict with tables with same names in DBC to be upsized
+            IF !_ExprMap.PAD
+                lcServerSQL = RTRIM(_ExprMap.&lcMapField.)
             ELSE
-                lcServerSQL = " "+ RTRIM(ExprMap.&lcMapField.) + " "
+                lcServerSQL = " "+ RTRIM(_ExprMap.&lcMapField.) + " "
             ENDIF
-            lcXbase = LOWER(RTRIM(ExprMap.FoxExpr))
+            lcXbase = LOWER(RTRIM(_ExprMap.FoxExpr))
             lcExpression = STRTRAN(lcExpression, lcXbase, LOWER(lcServerSQL))
         ENDSCAN
         lcExpression = SUBSTR(lcExpression,2)
@@ -8448,12 +8452,13 @@ DEFINE CLASS UpsizeEngine AS WizEngineAll of WZEngine.prg
         lnOldArea=SELECT()
 
         * Check keyword table
-        IF !USED("Keywords")
+*** 2019-01-24: prefix built-in tables with "_" to avoid conflict with tables with same names in DBC to be upsized
+        IF !USED("_Keywords")
             SELECT 0
-            USE Keywords
+            USE _Keywords
             SET ORDER TO Keyword
         ELSE
-            SELECT Keywords
+            SELECT _Keywords
         ENDIF
 *** DH 12/15/2014: use only SQL Server rather than variants of it
 ***        IF RTRIM(THIS.ServerType)=="SQL Server95" THEN
@@ -8577,6 +8582,7 @@ DEFINE CLASS UpsizeEngine AS WizEngineAll of WZEngine.prg
     ENDFUNC
 
 
+*** 2019-01-24: prefix built-in tables with "_" to avoid conflict with tables with same names in DBC to be upsized
     PROCEDURE CreateTypeArrays
         *Creates an array for each FoxPro datatype; each array of possible remote datatypes
         *has the same name as the local FoxPro datatype
@@ -8596,12 +8602,12 @@ DEFINE CLASS UpsizeEngine AS WizEngineAll of WZEngine.prg
         ENDIF
 
         *Find all the local types
-        SELECT LocalType,"this." + LocalType FROM TypeMap ;
+        SELECT LocalType,"this." + LocalType FROM _TypeMap ;
             WHERE DEFAULT=.T. AND SERVER=lcServerConstraint ;
             INTO ARRAY aArrays
 
         FOR I=1 TO ALEN(aArrays,1)
-            SELECT RemoteType FROM TypeMap WHERE LocalType=aArrays[i,1] AND SERVER=lcServerConstraint ;
+            SELECT RemoteType FROM _TypeMap WHERE LocalType=aArrays[i,1] AND SERVER=lcServerConstraint ;
                 INTO ARRAY &aArrays[i,2]
         NEXT