From 72bd6a5e1dffae1ce668c4573c9f5253a2e8099b Mon Sep 17 00:00:00 2001 From: Hans Van Akelyen Date: Fri, 7 Feb 2025 11:17:45 +0100 Subject: [PATCH] add HopMetadataProperty to sql file output, fixes #1935 remove clone --- .../sqlfileoutput/SQLFileOutput.java | 59 +-- .../sqlfileoutput/SQLFileOutputDialog.java | 54 +- .../sqlfileoutput/SQLFileOutputMeta.java | 469 ++++-------------- .../sqlfileoutput/SQLFileOutputMetaTest.java | 102 ++-- .../resources/sql-file-output-transform.xml | 56 +++ 5 files changed, 238 insertions(+), 502 deletions(-) create mode 100644 plugins/transforms/sqlfileoutput/src/test/resources/sql-file-output-transform.xml diff --git a/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutput.java b/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutput.java index 33c2ade367b..03e0ad525ca 100644 --- a/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutput.java +++ b/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutput.java @@ -66,12 +66,10 @@ public boolean processRow() throws HopException { meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); data.insertRowMeta = getInputRowMeta().clone(); - if (meta.isDoNotOpenNewFileInit()) { - if (!openNewFile()) { - logError("Couldn't open file [" + buildFilename() + "]"); - setErrors(1); - return false; - } + if (meta.getFile().isDoNotOpenNewFileInit() && !openNewFile()) { + logError("Couldn't open file [" + buildFilename() + "]"); + setErrors(1); + return false; } } @@ -80,26 +78,24 @@ public boolean processRow() throws HopException { if (r != null && getLinesOutput() > 0 - && meta.getSplitEvery() > 0 - && ((getLinesOutput() + 1) % meta.getSplitEvery()) == 0) { + && meta.getFile().getSplitEvery() > 0 + && ((getLinesOutput() + 1) % meta.getFile().getSplitEvery()) == 0) { // Done with this part or with everything. closeFile(); // Not finished: open another file... - if (r != null) { - if (!openNewFile()) { - logError("Unable to open new file (split #" + data.splitnr + "..."); - setErrors(1); - return false; - } + if (r != null && !openNewFile()) { + logError("Unable to open new file (split #" + data.splitnr + "..."); + setErrors(1); + return false; } } try { if (getLinesOutput() == 0) { // Add creation table once to the top - if (meta.createTable()) { + if (meta.isCreateTable()) { String crTable = data.db.getDDLCreationTable(schemaTable, data.insertRowMeta); if (isRowLevel()) { @@ -110,7 +106,7 @@ && getLinesOutput() > 0 } // Truncate table - if (meta.truncateTable()) { + if (meta.isTruncateTable()) { // Write to file String truncatetable = data.db.getDDLTruncateTable(schemaName, tableName + ";" + Const.CR + Const.CR); @@ -128,7 +124,7 @@ && getLinesOutput() > 0 + ";"; // Do we start a new line for this statement ? - if (meta.StartNewLine()) { + if (meta.isStartNewLine()) { sql = sql + Const.CR; } @@ -146,10 +142,8 @@ && getLinesOutput() > 0 putRow(data.outputRowMeta, r); // in case we want it go further... incrementLinesOutput(); - if (checkFeedback(getLinesRead())) { - if (isBasic()) { - logBasic("linenr " + getLinesRead()); - } + if (checkFeedback(getLinesRead()) && isBasic()) { + logBasic("linenr " + getLinesRead()); } } catch (HopException e) { @@ -176,7 +170,7 @@ && getLinesOutput() > 0 } public String buildFilename() { - return meta.buildFilename(this, resolve(meta.getFileName()), getCopy(), data.splitnr); + return meta.buildFilename(this, resolve(meta.getFile().getFileName()), getCopy(), data.splitnr); } public boolean openNewFile() { @@ -186,7 +180,7 @@ public boolean openNewFile() { try { String filename = buildFilename(); - if (meta.AddToResult()) { + if (meta.isAddToResult()) { // Add this to the result file names... ResultFile resultFile = new ResultFile( @@ -202,7 +196,8 @@ public boolean openNewFile() { if (isDetailed()) { logDetailed("Opening output stream in nocompress mode"); } - OutputStream fos = HopVfs.getOutputStream(filename, meta.isFileAppended(), variables); + OutputStream fos = + HopVfs.getOutputStream(filename, meta.getFile().isFileAppended(), variables); outputStream = fos; if (isDetailed()) { @@ -291,12 +286,12 @@ public boolean init() { logBasic("Connected to database [" + meta.getConnection() + "]"); - if (meta.isCreateParentFolder()) { + if (meta.getFile().isCreateParentFolder()) { // Check for parent folder FileObject parentfolder = null; try { // Get parent folder - String filename = resolve(meta.getFileName()); + String filename = resolve(meta.getFile().getFileName()); parentfolder = HopVfs.getFileObject(filename, variables).getParent(); if (!parentfolder.exists()) { logBasic( @@ -319,15 +314,13 @@ public boolean init() { } } - if (!meta.isDoNotOpenNewFileInit()) { - if (!openNewFile()) { - logError("Couldn't open file [" + buildFilename() + "]"); - setErrors(1L); - stopAll(); - } + if (!meta.getFile().isDoNotOpenNewFileInit() && !openNewFile()) { + logError("Couldn't open file [" + buildFilename() + "]"); + setErrors(1L); + stopAll(); } - tableName = resolve(meta.getTablename()); + tableName = resolve(meta.getTableName()); schemaName = resolve(meta.getSchemaName()); if (Utils.isEmpty(tableName)) { diff --git a/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputDialog.java b/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputDialog.java index 1ef23c3621a..56fdcba2933 100644 --- a/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputDialog.java +++ b/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputDialog.java @@ -824,19 +824,19 @@ public void getData() { if (input.getSchemaName() != null) { wSchema.setText(input.getSchemaName()); } - if (input.getTablename() != null) { - wTable.setText(input.getTablename()); + if (input.getTableName() != null) { + wTable.setText(input.getTableName()); } if (input.getConnection() != null) { wConnection.setText(input.getConnection()); } - if (input.getFileName() != null) { - wFilename.setText(input.getFileName()); + if (input.getFile().getFileName() != null) { + wFilename.setText(input.getFile().getFileName()); } - wCreateParentFolder.setSelection(input.isCreateParentFolder()); - if (input.getExtension() != null) { - wExtension.setText(input.getExtension()); + wCreateParentFolder.setSelection(input.getFile().isCreateParentFolder()); + if (input.getFile().getExtension() != null) { + wExtension.setText(input.getFile().getExtension()); } else { wExtension.setText("sql"); } @@ -845,21 +845,21 @@ public void getData() { wFormat.setText(input.getDateFormat()); } - wSplitEvery.setText("" + input.getSplitEvery()); - wAddDate.setSelection(input.isDateInFilename()); - wAddTime.setSelection(input.isTimeInFilename()); - wAppend.setSelection(input.isFileAppended()); - wAddTransformNr.setSelection(input.isTransformNrInFilename()); + wSplitEvery.setText("" + input.getFile().getSplitEvery()); + wAddDate.setSelection(input.getFile().isDateInFilename()); + wAddTime.setSelection(input.getFile().isTimeInFilename()); + wAppend.setSelection(input.getFile().isFileAppended()); + wAddTransformNr.setSelection(input.getFile().isTransformNrInFilename()); - wTruncate.setSelection(input.truncateTable()); - wAddCreate.setSelection(input.createTable()); + wTruncate.setSelection(input.isTruncateTable()); + wAddCreate.setSelection(input.isCreateTable()); if (input.getEncoding() != null) { wEncoding.setText(input.getEncoding()); } - wAddToResult.setSelection(input.AddToResult()); - wStartNewLine.setSelection(input.StartNewLine()); - wDoNotOpenNewFileInit.setSelection(input.isDoNotOpenNewFileInit()); + wAddToResult.setSelection(input.isAddToResult()); + wStartNewLine.setSelection(input.isStartNewLine()); + wDoNotOpenNewFileInit.setSelection(input.getFile().isDoNotOpenNewFileInit()); wTransformName.selectAll(); wTransformName.setFocus(); @@ -873,26 +873,26 @@ private void cancel() { private void getInfo(SQLFileOutputMeta info) { info.setSchemaName(wSchema.getText()); - info.setTablename(wTable.getText()); + info.setTableName(wTable.getText()); info.setConnection(wConnection.getText()); info.setTruncateTable(wTruncate.getSelection()); - info.setCreateParentFolder(wCreateParentFolder.getSelection()); + info.getFile().setCreateParentFolder(wCreateParentFolder.getSelection()); info.setCreateTable(wAddCreate.getSelection()); - info.setFileName(wFilename.getText()); - info.setExtension(wExtension.getText()); + info.getFile().setFileName(wFilename.getText()); + info.getFile().setExtension(wExtension.getText()); info.setDateFormat(wFormat.getText()); - info.setSplitEvery(Const.toInt(wSplitEvery.getText(), 0)); - info.setFileAppended(wAppend.getSelection()); - info.setTransformNrInFilename(wAddTransformNr.getSelection()); - info.setDateInFilename(wAddDate.getSelection()); - info.setTimeInFilename(wAddTime.getSelection()); + info.getFile().setSplitEvery(Const.toInt(wSplitEvery.getText(), 0)); + info.getFile().setFileAppended(wAppend.getSelection()); + info.getFile().setTransformNrInFilename(wAddTransformNr.getSelection()); + info.getFile().setDateInFilename(wAddDate.getSelection()); + info.getFile().setTimeInFilename(wAddTime.getSelection()); info.setEncoding(wEncoding.getText()); info.setAddToResult(wAddToResult.getSelection()); info.setStartNewLine(wStartNewLine.getSelection()); - info.setDoNotOpenNewFileInit(wDoNotOpenNewFileInit.getSelection()); + info.getFile().setDoNotOpenNewFileInit(wDoNotOpenNewFileInit.getSelection()); } private void ok() { diff --git a/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMeta.java b/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMeta.java index 1be9cbfef09..7b43fdf9f78 100644 --- a/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMeta.java +++ b/plugins/transforms/sqlfileoutput/src/main/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMeta.java @@ -21,6 +21,8 @@ import java.util.Date; import java.util.List; import java.util.Map; +import lombok.Getter; +import lombok.Setter; import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.CheckResult; import org.apache.hop.core.Const; @@ -40,6 +42,7 @@ import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.core.xml.XmlHandler; import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; import org.apache.hop.metadata.api.IHopMetadataProvider; import org.apache.hop.pipeline.DatabaseImpact; import org.apache.hop.pipeline.PipelineMeta; @@ -57,308 +60,64 @@ categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Output", keywords = "i18n::SQLFileOutputMeta.keyword", documentationUrl = "/pipeline/transforms/sqlfileoutput.html") +@Getter +@Setter public class SQLFileOutputMeta extends BaseTransformMeta { private static final Class PKG = SQLFileOutputMeta.class; private static final String CONST_SPACE = " "; private static final String CONST_SPACE_SHORT = " "; + @HopMetadataProperty(key = "connection") private String connection; + + @HopMetadataProperty(key = "schema") private String schemaName; + + @HopMetadataProperty(key = "table") private String tableName; + + @HopMetadataProperty(key = "truncate") private boolean truncateTable; + @HopMetadataProperty(key = "AddToResult") private boolean addToResult; + @HopMetadataProperty(key = "create") private boolean createTable; - /** The base name of the output file */ - private String fileName; - - /** The file extention in case of a generated filename */ - private String extension; - - /** - * if this value is larger then 0, the text file is split up into parts of this number of lines - */ - private int splitEvery; - - /** Flag to indicate the we want to append to the end of an existing file (if it exists) */ - private boolean fileAppended; - - /** Flag: add the transformnr in the filename */ - private boolean transformNrInFilename; - - /** Flag: add the partition number in the filename */ - private boolean partNrInFilename; - - /** Flag: add the date in the filename */ - private boolean dateInFilename; - - /** Flag: add the time in the filename */ - private boolean timeInFilename; + @HopMetadataProperty(key = "file") + private SqlFile file; /** The encoding to use for reading: null or empty string means system default encoding */ + @HopMetadataProperty(key = "encoding") private String encoding; /** The date format */ - private String dateformat; + @HopMetadataProperty(key = "dateformat") + private String dateFormat; /** Start New line for each statement */ + @HopMetadataProperty(key = "StartNewLine") private boolean startNewLine; - /** Flag: create parent folder if needed */ - private boolean createparentfolder; - - private boolean doNotOpenNewFileInit; - + /** + * @deprecated keep for backwards compatibility + * @param transformNode the XML of the transform node + * @param metadataProvider the metadata provider + * @throws HopXmlException when unable to parse the XML + */ @Override + @Deprecated(since = "2.13") public void loadXml(Node transformNode, IHopMetadataProvider metadataProvider) throws HopXmlException { - readData(transformNode, metadataProvider); - } - - @Override - public Object clone() { - - return (SQLFileOutputMeta) super.clone(); - } - - /** - * Get connection name - * - * @return - */ - public String getConnection() { - return connection; - } - - /** - * Set connection name - * - * @param connection - */ - public void setConnection(String connection) { - this.connection = connection; - } - - /** - * @return Returns the extension. - */ - public String getExtension() { - return extension; - } - - /** - * @param extension The extension to set. - */ - public void setExtension(String extension) { - this.extension = extension; - } - - /** - * @return Returns the fileAppended. - */ - public boolean isFileAppended() { - return fileAppended; - } - - /** - * @param fileAppended The fileAppended to set. - */ - public void setFileAppended(boolean fileAppended) { - this.fileAppended = fileAppended; - } - - /** - * @return Returns the fileName. - */ - public String getFileName() { - return fileName; - } - - /** - * @return Returns the splitEvery. - */ - public int getSplitEvery() { - return splitEvery; - } - - /** - * @param splitEvery The splitEvery to set. - */ - public void setSplitEvery(int splitEvery) { - this.splitEvery = splitEvery; - } - - /** - * @return Returns the transformNrInFilename. - */ - public boolean isTransformNrInFilename() { - return transformNrInFilename; - } - - /** - * @param transformNrInFilename The transformNrInFilename to set. - */ - public void setTransformNrInFilename(boolean transformNrInFilename) { - this.transformNrInFilename = transformNrInFilename; - } - - /** - * @return Returns the timeInFilename. - */ - public boolean isTimeInFilename() { - return timeInFilename; - } - - /** - * @return Returns the dateInFilename. - */ - public boolean isDateInFilename() { - return dateInFilename; - } - - /** - * @param dateInFilename The dateInFilename to set. - */ - public void setDateInFilename(boolean dateInFilename) { - this.dateInFilename = dateInFilename; - } - - /** - * @param timeInFilename The timeInFilename to set. - */ - public void setTimeInFilename(boolean timeInFilename) { - this.timeInFilename = timeInFilename; - } - - /** - * @param fileName The fileName to set. - */ - public void setFileName(String fileName) { - this.fileName = fileName; - } - - /** - * @return The desired encoding of output file, null or empty if the default system encoding needs - * to be used. - */ - public String getEncoding() { - return encoding; - } - - /** - * @return The desired date format. - */ - public String getDateFormat() { - return dateformat; - } - - /** - * @param encoding The desired encoding of output file, null or empty if the default system - * encoding needs to be used. - */ - public void setEncoding(String encoding) { - this.encoding = encoding; - } - - /** - * @param dateFormat The desired date format of output field date used. - */ - public void setDateFormat(String dateFormat) { - this.dateformat = dateFormat; - } - - /** - * @return Returns the table name. - */ - public String getTablename() { - return tableName; - } - - /** - * @param tableName The table name to set. - */ - public void setTablename(String tableName) { - this.tableName = tableName; - } - - /** - * @return Returns the truncate table flag. - */ - public boolean truncateTable() { - return truncateTable; - } - - /** - * @return Returns the Add to result filesname flag. - */ - public boolean AddToResult() { - return addToResult; - } - - /** - * @return Returns the Start new line flag. - */ - public boolean StartNewLine() { - return startNewLine; - } - - public boolean isDoNotOpenNewFileInit() { - return doNotOpenNewFileInit; - } - - public void setDoNotOpenNewFileInit(boolean doNotOpenNewFileInit) { - this.doNotOpenNewFileInit = doNotOpenNewFileInit; - } - - /** - * @return Returns the create table flag. - */ - public boolean createTable() { - return createTable; - } - - /** - * @param truncateTable The truncate table flag to set. - */ - public void setTruncateTable(boolean truncateTable) { - this.truncateTable = truncateTable; - } - - /** - * @param addToResult The Add file to result to set. - */ - public void setAddToResult(boolean addToResult) { - this.addToResult = addToResult; - } - - /** - * @param startNewLine The Start NEw Line to set. - */ - public void setStartNewLine(boolean startNewLine) { - this.startNewLine = startNewLine; - } - - /** - * @param createTable The create table flag to set. - */ - public void setCreateTable(boolean createTable) { - this.createTable = createTable; - } - - /** - * @return Returns the create parent folder flag. - */ - public boolean isCreateParentFolder() { - return createparentfolder; - } + try { + super.loadXml(transformNode, metadataProvider); - /** - * @param createparentfolder The create parent folder flag to set. - */ - public void setCreateParentFolder(boolean createparentfolder) { - this.createparentfolder = createparentfolder; + // Rename from extention to extension + file.extension = XmlHandler.getTagValue(transformNode, "file", "extention"); + } catch (Exception e) { + throw new HopXmlException("Unable to load transform info from XML", e); + } } public String[] getFiles(IVariables variables, String fileName) { @@ -366,15 +125,15 @@ public String[] getFiles(IVariables variables, String fileName) { int splits = 1; int parts = 1; - if (transformNrInFilename) { + if (file.transformNrInFilename) { copies = 3; } - if (partNrInFilename) { + if (file.partNrInFilename) { parts = 3; } - if (splitEvery != 0) { + if (file.splitEvery != 0) { splits = 3; } @@ -409,111 +168,37 @@ public String buildFilename(IVariables variables, String fileName, int transform Date now = new Date(); - if (dateInFilename) { + if (file.dateInFilename) { daf.applyPattern("yyyMMdd"); String d = daf.format(now); retval += "_" + d; } - if (timeInFilename) { + if (file.timeInFilename) { daf.applyPattern("HHmmss"); String t = daf.format(now); retval += "_" + t; } - if (transformNrInFilename) { + if (file.transformNrInFilename) { retval += "_" + transformnr; } - if (splitEvery > 0) { + if (file.splitEvery > 0) { retval += "_" + splitnr; } - if (extension != null && extension.length() != 0) { - retval += "." + variables.resolve(extension); + if (file.extension != null && !file.extension.isEmpty()) { + retval += "." + variables.resolve(file.extension); } return retval; } - private void readData(Node transformNode, IHopMetadataProvider metadataProvider) - throws HopXmlException { - try { - - connection = XmlHandler.getTagValue(transformNode, "connection"); - schemaName = XmlHandler.getTagValue(transformNode, "schema"); - tableName = XmlHandler.getTagValue(transformNode, "table"); - truncateTable = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "truncate")); - createTable = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "create")); - encoding = XmlHandler.getTagValue(transformNode, "encoding"); - dateformat = XmlHandler.getTagValue(transformNode, "dateformat"); - addToResult = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "AddToResult")); - - startNewLine = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "StartNewLine")); - - fileName = XmlHandler.getTagValue(transformNode, "file", "name"); - createparentfolder = - "Y" - .equalsIgnoreCase( - XmlHandler.getTagValue(transformNode, "file", "create_parent_folder")); - extension = XmlHandler.getTagValue(transformNode, "file", "extention"); - fileAppended = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "file", "append")); - transformNrInFilename = - "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "file", "split")); - partNrInFilename = - "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "file", "haspartno")); - dateInFilename = - "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "file", "add_date")); - timeInFilename = - "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode, "file", "add_time")); - splitEvery = Const.toInt(XmlHandler.getTagValue(transformNode, "file", "splitevery"), 0); - doNotOpenNewFileInit = - "Y" - .equalsIgnoreCase( - XmlHandler.getTagValue(transformNode, "file", "DoNotOpenNewFileInit")); - - } catch (Exception e) { - throw new HopXmlException("Unable to load transform info from XML", e); - } - } - @Override public void setDefault() { connection = ""; tableName = ""; - createparentfolder = false; - doNotOpenNewFileInit = false; - } - - @Override - public String getXml() { - StringBuilder retval = new StringBuilder(); - - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("connection", connection)); - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("schema", schemaName)); - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("table", tableName)); - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("truncate", truncateTable)); - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("create", createTable)); - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("encoding", encoding)); - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("dateformat", dateformat)); - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("addtoresult", addToResult)); - - retval.append(CONST_SPACE_SHORT + XmlHandler.addTagValue("startnewline", startNewLine)); - - retval.append(" " + Const.CR); - retval.append(CONST_SPACE + XmlHandler.addTagValue("name", fileName)); - retval.append(CONST_SPACE + XmlHandler.addTagValue("extention", extension)); - retval.append(CONST_SPACE + XmlHandler.addTagValue("append", fileAppended)); - retval.append(CONST_SPACE + XmlHandler.addTagValue("split", transformNrInFilename)); - retval.append(CONST_SPACE + XmlHandler.addTagValue("haspartno", partNrInFilename)); - retval.append(CONST_SPACE + XmlHandler.addTagValue("add_date", dateInFilename)); - retval.append(CONST_SPACE + XmlHandler.addTagValue("add_time", timeInFilename)); - retval.append(CONST_SPACE + XmlHandler.addTagValue("splitevery", splitEvery)); - retval.append(CONST_SPACE + XmlHandler.addTagValue("create_parent_folder", createparentfolder)); - retval.append( - CONST_SPACE + XmlHandler.addTagValue("DoNotOpenNewFileInit", doNotOpenNewFileInit)); - - retval.append(" " + Const.CR); - - return retval.toString(); + file.createParentFolder = false; + file.doNotOpenNewFileInit = false; } @Override @@ -814,7 +499,7 @@ public SqlStatement getSqlStatements( String crTable = db.getDDL(schemaTable, prev); // Empty string means: nothing to do: set it to null... - if (crTable == null || crTable.length() == 0) { + if (crTable == null || crTable.isEmpty()) { crTable = null; } @@ -880,20 +565,6 @@ public IRowMeta getRequiredFields(IVariables variables) throws HopException { } } - /** - * @return the schemaName - */ - public String getSchemaName() { - return schemaName; - } - - /** - * @param schemaName the schemaName to set - */ - public void setSchemaName(String schemaName) { - this.schemaName = schemaName; - } - @Override public boolean supportsErrorHandling() { return true; @@ -925,20 +596,62 @@ public String exportResources( // From : ${Internal.Pipeline.Filename.Directory}/../foo/bar.data // To : /home/matt/test/files/foo/bar.data // - FileObject fileObject = HopVfs.getFileObject(variables.resolve(fileName), variables); + FileObject fileObject = HopVfs.getFileObject(variables.resolve(file.fileName), variables); // If the file doesn't exist, forget about this effort too! // if (fileObject.exists()) { // Convert to an absolute path... // - fileName = iResourceNaming.nameResource(fileObject, variables, true); + file.fileName = iResourceNaming.nameResource(fileObject, variables, true); - return fileName; + return file.fileName; } return null; } catch (Exception e) { throw new HopException(e); } } + + @Getter + @Setter + public static final class SqlFile { + @HopMetadataProperty(key = "name") + private String fileName; + + @HopMetadataProperty(key = "create_parent_folder") + private boolean createParentFolder; + + @HopMetadataProperty(key = "DoNotOpenNewFileInit") + private boolean doNotOpenNewFileInit; + + @HopMetadataProperty(key = "extension") + private String extension; + + /** + * if this value is larger then 0, the text file is split up into parts of this number of lines + */ + @HopMetadataProperty(key = "splitevery") + private int splitEvery; + + /** Flag to indicate the we want to append to the end of an existing file (if it exists) */ + @HopMetadataProperty(key = "append") + private boolean fileAppended; + + /** Flag: add the transformnr in the filename */ + @HopMetadataProperty(key = "split") + private boolean transformNrInFilename; + + /** Flag: add the partition number in the filename */ + @HopMetadataProperty(key = "haspartno") + private boolean partNrInFilename; + + /** Flag: add the date in the filename */ + @HopMetadataProperty(key = "add_date") + private boolean dateInFilename; + + /** Flag: add the time in the filename */ + @HopMetadataProperty(key = "add_time") + private boolean timeInFilename; + } } diff --git a/plugins/transforms/sqlfileoutput/src/test/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMetaTest.java b/plugins/transforms/sqlfileoutput/src/test/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMetaTest.java index 5f2976300f6..1095c9486aa 100644 --- a/plugins/transforms/sqlfileoutput/src/test/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMetaTest.java +++ b/plugins/transforms/sqlfileoutput/src/test/java/org/apache/hop/pipeline/transforms/sqlfileoutput/SQLFileOutputMetaTest.java @@ -16,76 +16,50 @@ */ package org.apache.hop.pipeline.transforms.sqlfileoutput; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.hop.core.HopEnvironment; -import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.plugins.PluginRegistry; -import org.apache.hop.junit.rules.RestoreHopEngineEnvironment; -import org.apache.hop.pipeline.transforms.loadsave.LoadSaveTester; -import org.apache.hop.pipeline.transforms.loadsave.validator.IFieldLoadSaveValidator; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class SQLFileOutputMetaTest { - LoadSaveTester loadSaveTester; - Class testMetaClass = SQLFileOutputMeta.class; - @ClassRule public static RestoreHopEngineEnvironment env = new RestoreHopEngineEnvironment(); +import org.apache.hop.pipeline.transform.TransformSerializationTestUtil; +import org.junit.jupiter.api.Test; - @Before - public void setUpLoadSave() throws Exception { - HopEnvironment.init(); - PluginRegistry.init(); - List attributes = - Arrays.asList( - "connection", - "schemaName", - "tablename", - "truncateTable", - "AddToResult", - "createTable", - "fileName", - "extension", - "splitEvery", - "fileAppended", - "transformNrInFilename", - "dateInFilename", - "timeInFilename", - "encoding", - "dateFormat", - "StartNewLine", - "createParentFolder", - "DoNotOpenNewFileInit"); - - // Note - "partNrInFilename" is used in serialization/deserialization, but there is no - // getter/setter for it and it's - // not present in the dialog. Looks like a copy/paste thing, and the value itself will end up - // serialized/deserialized - // as false. - Map getterMap = - new HashMap() { - { - put("truncateTable", "truncateTable"); - put("AddToResult", "AddToResult"); - put("createTable", "createTable"); - put("StartNewLine", "StartNewLine"); - } - }; - Map setterMap = new HashMap<>(); +class SQLFileOutputMetaTest { + @Test + void testSerialization() throws Exception { + SQLFileOutputMeta meta = + TransformSerializationTestUtil.testSerialization( + "/sql-file-output-transform.xml", SQLFileOutputMeta.class); - Map> attrValidatorMap = new HashMap<>(); - Map> typeValidatorMap = new HashMap<>(); + assertEquals("testtable", meta.getTableName()); + assertEquals("${DATABASE_NAME}", meta.getConnection()); + assertEquals("public", meta.getSchemaName()); + assertFalse(meta.isTruncateTable()); + assertTrue(meta.isStartNewLine()); - loadSaveTester = - new LoadSaveTester( - testMetaClass, attributes, getterMap, setterMap, attrValidatorMap, typeValidatorMap); + assertEquals("${PROJECT_HOME}/output/filename", meta.getFile().getFileName()); + assertFalse(meta.getFile().isFileAppended()); + assertFalse(meta.getFile().isTransformNrInFilename()); + assertEquals(0, meta.getFile().getSplitEvery()); } @Test - public void testSerialization() throws HopException { - loadSaveTester.testSerialization(); + void testClone() throws Exception { + SQLFileOutputMeta meta = + TransformSerializationTestUtil.testSerialization( + "/sql-file-output-transform.xml", SQLFileOutputMeta.class); + + SQLFileOutputMeta clone = (SQLFileOutputMeta) meta.clone(); + + assertEquals(clone.getTableName(), meta.getTableName()); + assertEquals(clone.getConnection(), meta.getConnection()); + assertEquals(clone.getSchemaName(), meta.getSchemaName()); + assertEquals(clone.isTruncateTable(), meta.isTruncateTable()); + assertEquals(clone.isStartNewLine(), meta.isStartNewLine()); + + assertEquals(clone.getFile().getFileName(), meta.getFile().getFileName()); + assertEquals(clone.getFile().isFileAppended(), meta.getFile().isFileAppended()); + assertEquals( + clone.getFile().isTransformNrInFilename(), meta.getFile().isTransformNrInFilename()); + assertEquals(clone.getFile().getSplitEvery(), meta.getFile().getSplitEvery()); } } diff --git a/plugins/transforms/sqlfileoutput/src/test/resources/sql-file-output-transform.xml b/plugins/transforms/sqlfileoutput/src/test/resources/sql-file-output-transform.xml new file mode 100644 index 00000000000..2d77283359f --- /dev/null +++ b/plugins/transforms/sqlfileoutput/src/test/resources/sql-file-output-transform.xml @@ -0,0 +1,56 @@ + + + + SQL file output + SQLFileOutput + + Y + + 1 + + none + + + ${DATABASE_NAME} + public + testtable
+ N + Y + UTF-8 + + N + Y + + ${PROJECT_HOME}/output/filename + sql + N + N + N + N + N + 0 + Y + Y + + + + 432 + 112 + +