diff --git a/docker/integration-tests/integration-tests-mysql.yaml b/docker/integration-tests/integration-tests-mysql.yaml new file mode 100644 index 00000000000..36bfb4dabb7 --- /dev/null +++ b/docker/integration-tests/integration-tests-mysql.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + integration_test_database: + extends: + file: integration-tests-base.yaml + service: integration_test + depends_on: + mysql: + condition: service_healthy + links: + - mysql + + mysql: + image: mysql:9.1.0 + env_file: + - mysql.env + ports: + - "3306" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-p${MYSQL_ROOT_PASSWORD}"] + interval: 20s + timeout: 10s + retries: 6 + start_period: 120s + volumes: + - ./resource/mysql/config/my.cnf:/etc/mysql/my.cnf + - ../../integration-tests/mysql/import:/var/lib/mysql-files + diff --git a/docker/integration-tests/mysql.env b/docker/integration-tests/mysql.env new file mode 100644 index 00000000000..6f5f0f2c1af --- /dev/null +++ b/docker/integration-tests/mysql.env @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +MYSQL_USER=hop_user +MYSQL_PASSWORD=hop_password +MYSQL_DATABASE=hop_database +MYSQL_ROOT_PASSWORD=hop_root_password diff --git a/docker/integration-tests/resource/mysql/config/my.cnf b/docker/integration-tests/resource/mysql/config/my.cnf new file mode 100644 index 00000000000..977d073a2a1 --- /dev/null +++ b/docker/integration-tests/resource/mysql/config/my.cnf @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +[mysqld] +local-infile = 1 diff --git a/integration-tests/mysql/dev-env-config.json b/integration-tests/mysql/dev-env-config.json new file mode 100644 index 00000000000..60e8888dac6 --- /dev/null +++ b/integration-tests/mysql/dev-env-config.json @@ -0,0 +1,23 @@ +{ + "variables" : [ { + "name" : "MYSQL_HOSTNAME", + "value" : "mysql", + "description" : "The MySQL hostname" + }, { + "name" : "MYSQL_PORT", + "value" : "3306", + "description" : "The data port " + }, { + "name" : "MYSQL_USERNAME", + "value" : "root", + "description" : "Username: none" + }, { + "name" : "MYSQL_PASSWORD", + "value" : "hop_root_password", + "description" : "Password: none" + }, { + "name" : "MYSQL_DATABASE", + "value" : "hop_database", + "description" : "" + } ] +} \ No newline at end of file diff --git a/integration-tests/mysql/disabled.txt b/integration-tests/mysql/disabled.txt new file mode 100644 index 00000000000..7560e5b3bce --- /dev/null +++ b/integration-tests/mysql/disabled.txt @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# \ No newline at end of file diff --git a/integration-tests/mysql/hop-config.json b/integration-tests/mysql/hop-config.json new file mode 100644 index 00000000000..102ac981d58 --- /dev/null +++ b/integration-tests/mysql/hop-config.json @@ -0,0 +1,290 @@ +{ + "variables": [ + { + "name": "HOP_LENIENT_STRING_TO_NUMBER_CONVERSION", + "value": "N", + "description": "System wide flag to allow lenient string to number conversion for backward compatibility. If this setting is set to \"Y\", an string starting with digits will be converted successfully into a number. (example: 192.168.1.1 will be converted into 192 or 192.168 or 192168 depending on the decimal and grouping symbol). The default (N) will be to throw an error if non-numeric symbols are found in the string." + }, + { + "name": "HOP_COMPATIBILITY_DB_IGNORE_TIMEZONE", + "value": "N", + "description": "System wide flag to ignore timezone while writing date/timestamp value to the database." + }, + { + "name": "HOP_LOG_SIZE_LIMIT", + "value": "0", + "description": "The log size limit for all pipelines and workflows that don't have the \"log size limit\" property set in their respective properties." + }, + { + "name": "HOP_EMPTY_STRING_DIFFERS_FROM_NULL", + "value": "N", + "description": "NULL vs Empty String. If this setting is set to Y, an empty string and null are different. Otherwise they are not." + }, + { + "name": "HOP_MAX_LOG_SIZE_IN_LINES", + "value": "0", + "description": "The maximum number of log lines that are kept internally by Hop. Set to 0 to keep all rows (default)" + }, + { + "name": "HOP_MAX_LOG_TIMEOUT_IN_MINUTES", + "value": "1440", + "description": "The maximum age (in minutes) of a log line while being kept internally by Hop. Set to 0 to keep all rows indefinitely (default)" + }, + { + "name": "HOP_MAX_WORKFLOW_TRACKER_SIZE", + "value": "5000", + "description": "The maximum number of workflow trackers kept in memory" + }, + { + "name": "HOP_MAX_ACTIONS_LOGGED", + "value": "5000", + "description": "The maximum number of action results kept in memory for logging purposes." + }, + { + "name": "HOP_MAX_LOGGING_REGISTRY_SIZE", + "value": "10000", + "description": "The maximum number of logging registry entries kept in memory for logging purposes." + }, + { + "name": "HOP_LOG_TAB_REFRESH_DELAY", + "value": "1000", + "description": "The hop log tab refresh delay." + }, + { + "name": "HOP_LOG_TAB_REFRESH_PERIOD", + "value": "1000", + "description": "The hop log tab refresh period." + }, + { + "name": "HOP_PLUGIN_CLASSES", + "value": null, + "description": "A comma delimited list of classes to scan for plugin annotations" + }, + { + "name": "HOP_PLUGIN_PACKAGES", + "value": null, + "description": "A comma delimited list of packages to scan for plugin annotations (warning: slow!!)" + }, + { + "name": "HOP_TRANSFORM_PERFORMANCE_SNAPSHOT_LIMIT", + "value": "0", + "description": "The maximum number of transform performance snapshots to keep in memory. Set to 0 to keep all snapshots indefinitely (default)" + }, + { + "name": "HOP_ROWSET_GET_TIMEOUT", + "value": "50", + "description": "The name of the variable that optionally contains an alternative rowset get timeout (in ms). This only makes a difference for extremely short lived pipelines." + }, + { + "name": "HOP_ROWSET_PUT_TIMEOUT", + "value": "50", + "description": "The name of the variable that optionally contains an alternative rowset put timeout (in ms). This only makes a difference for extremely short lived pipelines." + }, + { + "name": "HOP_CORE_TRANSFORMS_FILE", + "value": null, + "description": "The name of the project variable that will contain the alternative location of the hop-transforms.xml file. You can use this to customize the list of available internal transforms outside of the codebase." + }, + { + "name": "HOP_CORE_WORKFLOW_ACTIONS_FILE", + "value": null, + "description": "The name of the project variable that will contain the alternative location of the hop-workflow-actions.xml file." + }, + { + "name": "HOP_SERVER_OBJECT_TIMEOUT_MINUTES", + "value": "1440", + "description": "This project variable will set a time-out after which waiting, completed or stopped pipelines and workflows will be automatically cleaned up. The default value is 1440 (one day)." + }, + { + "name": "HOP_PIPELINE_PAN_JVM_EXIT_CODE", + "value": null, + "description": "Set this variable to an integer that will be returned as the Pan JVM exit code." + }, + { + "name": "HOP_DISABLE_CONSOLE_LOGGING", + "value": "N", + "description": "Set this variable to Y to disable standard Hop logging to the console. (stdout)" + }, + { + "name": "HOP_REDIRECT_STDERR", + "value": "N", + "description": "Set this variable to Y to redirect stderr to Hop logging." + }, + { + "name": "HOP_REDIRECT_STDOUT", + "value": "N", + "description": "Set this variable to Y to redirect stdout to Hop logging." + }, + { + "name": "HOP_DEFAULT_NUMBER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default number format" + }, + { + "name": "HOP_DEFAULT_BIGNUMBER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default bignumber format" + }, + { + "name": "HOP_DEFAULT_INTEGER_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default integer format" + }, + { + "name": "HOP_DEFAULT_DATE_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default date format" + }, + { + "name": "HOP_DEFAULT_TIMESTAMP_FORMAT", + "value": null, + "description": "The name of the variable containing an alternative default timestamp format" + }, + { + "name": "HOP_DEFAULT_SERVLET_ENCODING", + "value": null, + "description": "Defines the default encoding for servlets, leave it empty to use Java default encoding" + }, + { + "name": "HOP_FAIL_ON_LOGGING_ERROR", + "value": "N", + "description": "Set this variable to Y when you want the workflow/pipeline fail with an error when the related logging process (e.g. to a database) fails." + }, + { + "name": "HOP_AGGREGATION_MIN_NULL_IS_VALUED", + "value": "N", + "description": "Set this variable to Y to set the minimum to NULL if NULL is within an aggregate. Otherwise by default NULL is ignored by the MIN aggregate and MIN is set to the minimum value that is not NULL. See also the variable HOP_AGGREGATION_ALL_NULLS_ARE_ZERO." + }, + { + "name": "HOP_AGGREGATION_ALL_NULLS_ARE_ZERO", + "value": "N", + "description": "Set this variable to Y to return 0 when all values within an aggregate are NULL. Otherwise by default a NULL is returned when all values are NULL." + }, + { + "name": "HOP_COMPATIBILITY_TEXT_FILE_OUTPUT_APPEND_NO_HEADER", + "value": "N", + "description": "Set this variable to Y for backward compatibility for the Text File Output transform. Setting this to Ywill add no header row at all when the append option is enabled, regardless if the file is existing or not." + }, + { + "name": "HOP_PASSWORD_ENCODER_PLUGIN", + "value": "Hop", + "description": "Specifies the password encoder plugin to use by ID (Hop is the default)." + }, + { + "name": "HOP_SYSTEM_HOSTNAME", + "value": null, + "description": "You can use this variable to speed up hostname lookup. Hostname lookup is performed by Hop so that it is capable of logging the server on which a workflow or pipeline is executed." + }, + { + "name": "HOP_SERVER_JETTY_ACCEPTORS", + "value": null, + "description": "A variable to configure jetty option: acceptors for Carte" + }, + { + "name": "HOP_SERVER_JETTY_ACCEPT_QUEUE_SIZE", + "value": null, + "description": "A variable to configure jetty option: acceptQueueSize for Carte" + }, + { + "name": "HOP_SERVER_JETTY_RES_MAX_IDLE_TIME", + "value": null, + "description": "A variable to configure jetty option: lowResourcesMaxIdleTime for Carte" + }, + { + "name": "HOP_COMPATIBILITY_MERGE_ROWS_USE_REFERENCE_STREAM_WHEN_IDENTICAL", + "value": "N", + "description": "Set this variable to Y for backward compatibility for the Merge Rows (diff) transform. Setting this to Y will use the data from the reference stream (instead of the comparison stream) in case the compared rows are identical." + }, + { + "name": "HOP_SPLIT_FIELDS_REMOVE_ENCLOSURE", + "value": "false", + "description": "Set this variable to false to preserve enclosure symbol after splitting the string in the Split fields transform. Changing it to true will remove first and last enclosure symbol from the resulting string chunks." + }, + { + "name": "HOP_ALLOW_EMPTY_FIELD_NAMES_AND_TYPES", + "value": "false", + "description": "Set this variable to TRUE to allow your pipeline to pass 'null' fields and/or empty types." + }, + { + "name": "HOP_GLOBAL_LOG_VARIABLES_CLEAR_ON_EXPORT", + "value": "false", + "description": "Set this variable to false to preserve global log variables defined in pipeline / workflow Properties -> Log panel. Changing it to true will clear it when export pipeline / workflow." + }, + { + "name": "HOP_FILE_OUTPUT_MAX_STREAM_COUNT", + "value": "1024", + "description": "This project variable is used by the Text File Output transform. It defines the max number of simultaneously open files within the transform. The transform will close/reopen files as necessary to insure the max is not exceeded" + }, + { + "name": "HOP_FILE_OUTPUT_MAX_STREAM_LIFE", + "value": "0", + "description": "This project variable is used by the Text File Output transform. It defines the max number of milliseconds between flushes of files opened by the transform." + }, + { + "name": "HOP_USE_NATIVE_FILE_DIALOG", + "value": "N", + "description": "Set this value to Y if you want to use the system file open/save dialog when browsing files" + }, + { + "name": "HOP_AUTO_CREATE_CONFIG", + "value": "Y", + "description": "Set this value to N if you don't want to automatically create a hop configuration file (hop-config.json) when it's missing" + } + ], + "LocaleDefault": "en_BE", + "guiProperties": { + "FontFixedSize": "13", + "MaxUndo": "100", + "DarkMode": "Y", + "FontNoteSize": "13", + "ShowOSLook": "Y", + "FontFixedStyle": "0", + "FontNoteName": ".AppleSystemUIFont", + "FontFixedName": "Monospaced", + "FontGraphStyle": "0", + "FontDefaultSize": "13", + "GraphColorR": "255", + "FontGraphSize": "13", + "IconSize": "32", + "BackgroundColorB": "255", + "FontNoteStyle": "0", + "FontGraphName": ".AppleSystemUIFont", + "FontDefaultName": ".AppleSystemUIFont", + "GraphColorG": "255", + "UseGlobalFileBookmarks": "Y", + "FontDefaultStyle": "0", + "GraphColorB": "255", + "BackgroundColorR": "255", + "BackgroundColorG": "255", + "WorkflowDialogStyle": "RESIZE,MAX,MIN", + "LineWidth": "1", + "ContextDialogShowCategories": "Y" + }, + "projectsConfig": { + "enabled": true, + "projectMandatory": true, + "environmentMandatory": true, + "defaultProject": "default", + "defaultEnvironment": null, + "standardParentProject": "default", + "standardProjectsFolder": null, + "projectConfigurations": [ + { + "projectName": "default", + "projectHome": "${HOP_CONFIG_FOLDER}", + "configFilename": "project-config.json" + } + ], + "lifecycleEnvironments": [ + { + "name": "dev", + "purpose": "Testing", + "projectName": "default", + "configurationFiles": [ + "${PROJECT_HOME}/dev-env-config.json" + ] + } + ], + "projectLifecycles": [] + } +} \ No newline at end of file diff --git a/integration-tests/mysql/import/movies.csv b/integration-tests/mysql/import/movies.csv new file mode 100644 index 00000000000..5ed206dcfec --- /dev/null +++ b/integration-tests/mysql/import/movies.csv @@ -0,0 +1,4 @@ +Der Andere,1913,48,5.5,114,Max Mack +Atlantis,1913,121,6.5,480,August Blom +From Dusk to Dawn,1913,90,5.9,113,Frank E. Wolfe +Germinal,1913,150,6.9,173,Albert Capellani diff --git a/integration-tests/mysql/main-0001-bulk-ops.hwf b/integration-tests/mysql/main-0001-bulk-ops.hwf new file mode 100644 index 00000000000..6eed127f20f --- /dev/null +++ b/integration-tests/mysql/main-0001-bulk-ops.hwf @@ -0,0 +1,355 @@ + + + + main-0001-bulk-ops + Y + + + + - + 2024/12/04 14:44:21.948 + - + 2024/12/04 14:44:21.948 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 32 + 128 + + + + Success + + SUCCESS + + N + 1408 + 64 + + + + Bulk load from CSV to MySQL + + MYSQL_BULK_LOAD + + + movie + /var/lib/mysql-files/movies.csv + , + " + + + + N + + + N + 0 + N + MySQL + N + 640 + 64 + + + + Create table movies + + SQL + + MySQL + N + create table movie +( + title text null, + released int null, + length int null, + rating double null, + rating_count int null, + directors text null +); + N + N + N + 224 + 64 + + + + Drop tables + + SQL + + MySQL + N + drop table if exists movie; +drop table if exists wine; + N + N + N + 128 + 64 + + + + Bulk load from MySQL into file + + MYSQL_BULK_FILE + + + wine + /var/lib/mysql-files/wine.csv + , + " + Y + \n + 0 + + N + 0 + 1 + N + MySQL + N + 912 + 64 + + + + Insert Wines + + SQL + + MySQL + N + -- auto-generated definition +create table wine +( + country text null, + description text null, + designation text null, + points int null, + price double null, + province text null, + region text null, + region_2 text null, + taster_name text null, + taster_twitter_handle text null, + title text null, + variety text null, + winery text null +); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('Italy', 'Aromas include tropical fruit, broom, brimstone and dried herb. The palate isn\'t overly expressive, offering unripened apple, citrus and dried sage alongside brisk acidity.', 'Vulkà Bianco', 87, null, 'Sicily & Sardinia', 'Etna', null, 'Kerin O’Keefe', '@kerinokeefe', 'Nicosia 2013 Vulkà Bianco (Etna)', 'White Blend', 'Nicosia'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('Portugal', 'This is ripe and fruity, a wine that is smooth while still structured. Firm tannins are filled out with juicy red berry fruits and freshened with acidity. It\'s already drinkable, although it will certainly be better from 2016.', 'Avidagos', 87, 15, 'Douro', null, null, 'Roger Voss', '@vossroger', 'Quinta dos Avidagos 2011 Avidagos Red (Douro)', 'Portuguese Red', 'Quinta dos Avidagos'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('US', 'Tart and snappy, the flavors of lime flesh and rind dominate. Some green pineapple pokes through, with crisp acidity underscoring the flavors. The wine was all stainless-steel fermented.', null, 87, 14, 'Oregon', 'Willamette Valley', 'Willamette Valley', 'Paul Gregutt', '@paulgwine ', 'Rainstorm 2013 Pinot Gris (Willamette Valley)', 'Pinot Gris', 'Rainstorm'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('US', 'Pineapple rind, lemon pith and orange blossom start off the aromas. The palate is a bit more opulent, with notes of honey-drizzled guava and mango giving way to a slightly astringent, semidry finish.', 'Reserve Late Harvest', 87, 13, 'Michigan', 'Lake Michigan Shore', null, 'Alexander Peartree', null, 'St. Julian 2013 Reserve Late Harvest Riesling (Lake Michigan Shore)', 'Riesling', 'St. Julian'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('US', 'Much like the regular bottling from 2012, this comes across as rather rough and tannic, with rustic, earthy, herbal characteristics. Nonetheless, if you think of it as a pleasantly unfussy country wine, it\'s a good companion to a hearty winter stew.', 'Vintner\'s Reserve Wild Child Block', 87, 65, 'Oregon', 'Willamette Valley', 'Willamette Valley', 'Paul Gregutt', '@paulgwine ', 'Sweet Cheeks 2012 Vintner\'s Reserve Wild Child Block Pinot Noir (Willamette Valley)', 'Pinot Noir', 'Sweet Cheeks'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('Spain', 'Blackberry and raspberry aromas show a typical Navarran whiff of green herbs and, in this case, horseradish. In the mouth, this is fairly full bodied, with tomatoey acidity. Spicy, herbal flavors complement dark plum fruit, while the finish is fresh but grabby.', 'Ars In Vitro', 87, 15, 'Northern Spain', 'Navarra', null, 'Michael Schachner', '@wineschach', 'Tandem 2011 Ars In Vitro Tempranillo-Merlot (Navarra)', 'Tempranillo-Merlot', 'Tandem'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('Italy', 'Here\'s a bright, informal red that opens with aromas of candied berry, white pepper and savory herb that carry over to the palate. It\'s balanced with fresh acidity and soft tannins.', 'Belsito', 87, 16, 'Sicily & Sardinia', 'Vittoria', null, 'Kerin O’Keefe', '@kerinokeefe', 'Terre di Giurfo 2013 Belsito Frappato (Vittoria)', 'Frappato', 'Terre di Giurfo'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('France', 'This dry and restrained wine offers spice in profusion. Balanced with acidity and a firm texture, it\'s very much for food.', null, 87, 24, 'Alsace', 'Alsace', null, 'Roger Voss', '@vossroger', 'Trimbach 2012 Gewurztraminer (Alsace)', 'Gewürztraminer', 'Trimbach'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('Germany', 'Savory dried thyme notes accent sunnier flavors of preserved peach in this brisk, off-dry wine. It\'s fruity and fresh, with an elegant, sprightly footprint.', 'Shine', 87, 12, 'Rheinhessen', null, null, 'Anna Lee C. Iijima', null, 'Heinz Eifel 2013 Shine Gewürztraminer (Rheinhessen)', 'Gewürztraminer', 'Heinz Eifel'); +INSERT INTO wine (country, description, designation, points, price, province, region, region_2, taster_name, taster_twitter_handle, title, variety, winery) VALUES ('France', 'This has great depth of flavor with its fresh apple and pear fruits and touch of spice. It\'s off dry while balanced with acidity and a crisp texture. Drink now.', 'Les Natures', 87, 27, 'Alsace', 'Alsace', null, 'Roger Voss', '@vossroger', 'Jean-Baptiste Adam 2012 Les Natures Pinot Gris (Alsace)', 'Pinot Gris', 'Jean-Baptiste Adam'); + + + + + N + N + N + 432 + 64 + + + + Abort workflow + + ABORT + + N + N + 544 + 416 + + + + Dummy + + DUMMY + + N + 544 + 304 + + + + Delete files + + DELETE_FILES + + N + N + + + ${PROJECT_HOME}/import/wine.csv + + + + N + 128 + 176 + + + + File exists + + FILE_EXISTS + + ${PROJECT_HOME}/import/wine.csv + N + 1168 + 64 + + + + + + Bulk load from CSV to MySQL + Bulk load from MySQL into file + Y + Y + N + + + Drop tables + Create table movies + Y + Y + N + + + Create table movies + Insert Wines + Y + Y + N + + + Insert Wines + Bulk load from CSV to MySQL + Y + Y + N + + + Drop tables + Dummy + Y + N + N + + + Create table movies + Dummy + Y + N + N + + + Insert Wines + Dummy + Y + N + N + + + Bulk load from CSV to MySQL + Dummy + Y + N + N + + + Bulk load from MySQL into file + Dummy + Y + N + N + + + Dummy + Abort workflow + Y + Y + Y + + + Start + Delete files + Y + Y + Y + + + Delete files + Drop tables + Y + Y + N + + + Bulk load from MySQL into file + File exists + Y + Y + N + + + File exists + Success + Y + Y + N + + + File exists + Dummy + Y + N + N + + + + + + diff --git a/integration-tests/mysql/metadata/pipeline-run-configuration/local.json b/integration-tests/mysql/metadata/pipeline-run-configuration/local.json new file mode 100644 index 00000000000..d6b7ca4eede --- /dev/null +++ b/integration-tests/mysql/metadata/pipeline-run-configuration/local.json @@ -0,0 +1,20 @@ +{ + "engineRunConfiguration": { + "Local": { + "feedback_size": "50000", + "sample_size": "100", + "sample_type_in_gui": "Last", + "wait_time": "20", + "rowset_size": "10000", + "safe_mode": false, + "show_feedback": false, + "topo_sort": false, + "gather_metrics": false, + "transactional": false + } + }, + "defaultSelection": true, + "configurationVariables": [], + "name": "local", + "description": "Runs your pipelines locally with the standard local Hop pipeline engine" +} \ No newline at end of file diff --git a/integration-tests/mysql/metadata/rdbms/MySQL.json b/integration-tests/mysql/metadata/rdbms/MySQL.json new file mode 100644 index 00000000000..789bb0a4c7e --- /dev/null +++ b/integration-tests/mysql/metadata/rdbms/MySQL.json @@ -0,0 +1,31 @@ +{ + "rdbms": { + "MYSQL": { + "databaseName": "${MYSQL_DATABASE}", + "pluginId": "MYSQL", + "accessType": 0, + "hostname": "${MYSQL_HOSTNAME}", + "password": "${MYSQL_PASSWORD}", + "pluginName": "MySQL", + "port": "${MYSQL_PORT}", + "driverClassName": "Mysql 8+", + "attributes": { + "SUPPORTS_TIMESTAMP_DATA_TYPE": "Y", + "STREAM_RESULTS": "N", + "EXTRA_OPTION_MYSQL.useCursorFetch": "true", + "EXTRA_OPTION_MYSQL.defaultFetchSize": "500", + "FORCE_IDENTIFIERS_TO_LOWERCASE": "N", + "PRESERVE_RESERVED_WORD_CASE": "Y", + "FORCE_IDENTIFIERS_TO_UPPERCASE": "N", + "QUOTE_ALL_FIELDS": "N", + "EXTRA_OPTION_MYSQL.zeroDateTimeBehaviorValue": "CONVERT_TO_NULL", + "SUPPORTS_BOOLEAN_DATA_TYPE": "Y", + "SQL_CONNECT": "", + "PREFERRED_SCHEMA_NAME": "" + }, + "manualUrl": "", + "username": "${MYSQL_USERNAME}" + } + }, + "name": "MySQL" +} \ No newline at end of file diff --git a/integration-tests/mysql/metadata/workflow-run-configuration/local.json b/integration-tests/mysql/metadata/workflow-run-configuration/local.json new file mode 100644 index 00000000000..1d0cf74baec --- /dev/null +++ b/integration-tests/mysql/metadata/workflow-run-configuration/local.json @@ -0,0 +1,11 @@ +{ + "engineRunConfiguration": { + "Local": { + "safe_mode": false, + "transactional": false + } + }, + "defaultSelection": true, + "name": "local", + "description": "Runs your workflows locally with the standard local Hop workflow engine" +} \ No newline at end of file diff --git a/integration-tests/mysql/project-config.json b/integration-tests/mysql/project-config.json new file mode 100644 index 00000000000..df9d54f0243 --- /dev/null +++ b/integration-tests/mysql/project-config.json @@ -0,0 +1,16 @@ +{ + "metadataBaseFolder": "${PROJECT_HOME}/metadata", + "unitTestsBasePath": "${PROJECT_HOME}", + "dataSetsCsvFolder": "${PROJECT_HOME}/datasets", + "enforcingExecutionInHome": true, + "parentProjectName": "base-project", + "config": { + "variables": [ + { + "name": "HOP_LICENSE_HEADER_FILE", + "value": "${PROJECT_HOME}/../asf-header.txt", + "description": "This will automatically serialize the ASF license header into pipelines and workflows in the integration test projects" + } + ] + } +} \ No newline at end of file diff --git a/plugins/transforms/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoaderDialog.java b/plugins/transforms/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoaderDialog.java index 5dc34c5c6be..06aa72a9864 100644 --- a/plugins/transforms/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoaderDialog.java +++ b/plugins/transforms/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoaderDialog.java @@ -1530,7 +1530,7 @@ private void getTableName() { // New class: SelectTableDialog int connr = wConnection.getSelectionIndex(); if (connr >= 0) { - DatabaseMeta inf = pipelineMeta.getDatabases().get(connr); + DatabaseMeta inf = pipelineMeta.findDatabase(wConnection.getText(), variables); if (log.isDebug()) { logDebug(