diff --git a/README.md b/README.md index dc2533a6..e54b776f 100644 --- a/README.md +++ b/README.md @@ -149,10 +149,10 @@ This profile allows you to launch only the technical services : postgres, elasti |---------------|------------|------------| | Postgres | 14.9 | | | RabbitMQ | 4.0.4 | management | -| Elasticsearch | 8.15.5 | | +| Elasticsearch | 8.19.10 | | | Grafana | 11.3.0 | | | Prometheus | v3.8.1 | | -| Minio | 2023-09-27 | | +| Minio | 2025-09-07 | | It is used for k8s deployment with Minikube. diff --git a/docker-compose/technical/docker-compose.technical.yml b/docker-compose/technical/docker-compose.technical.yml index 91f77605..b9aec8de 100644 --- a/docker-compose/technical/docker-compose.technical.yml +++ b/docker-compose/technical/docker-compose.technical.yml @@ -157,7 +157,7 @@ services: restart: unless-stopped s3-storage: - image: minio/minio:RELEASE.2023-09-27T15-22-50Z + image: minio/minio:RELEASE.2025-09-07T16-13-09Z # need to override entrypoint to create the bucket, is there a simpler way ? entrypoint: sh command: -c 'mkdir -p /data/ws-bucket && /usr/bin/docker-entrypoint.sh server /data --console-address ":19090"' diff --git a/k8s/resources/state-estimation-orchestrator-server-config.yml b/k8s/resources/state-estimation-orchestrator-server-config.yml new file mode 100644 index 00000000..50291f2e --- /dev/null +++ b/k8s/resources/state-estimation-orchestrator-server-config.yml @@ -0,0 +1,12 @@ +estim: + homeDir: /estim + debug: false + +#Workaround iidm in-memory impl present on classpath (needed by cvg-extension) +network: + default-impl-name: NetworkStore + +# Nb of estim concurrent processes - should be in-sync with stateestimation.run.stateestimationGroup concurrency +computation-local: + available-core: 2 + diff --git a/k8s/resources/state-estimation-server-config.yml b/k8s/resources/state-estimation-server-config.yml new file mode 100644 index 00000000..50291f2e --- /dev/null +++ b/k8s/resources/state-estimation-server-config.yml @@ -0,0 +1,12 @@ +estim: + homeDir: /estim + debug: false + +#Workaround iidm in-memory impl present on classpath (needed by cvg-extension) +network: + default-impl-name: NetworkStore + +# Nb of estim concurrent processes - should be in-sync with stateestimation.run.stateestimationGroup concurrency +computation-local: + available-core: 2 + diff --git a/migration/migrate_parameters.log b/migration/migrate_parameters.log new file mode 100644 index 00000000..9e0abac2 --- /dev/null +++ b/migration/migrate_parameters.log @@ -0,0 +1,3106 @@ +********* QUERY ********** +search_path=public +************************** + +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --host=host_name --port=5432 --username=user_name --dbname=study -csearch_path=public --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * multi-schema case : $ psql --host=host_name --port=5432 --username=user_name --dbname=database_name -csearch_path=study --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}', + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study. to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +********* QUERY ********** +search_path=public +************************** + +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --host=host_name --port=5432 --username=user_name --dbname=study -csearch_path=public --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * multi-schema case : $ psql --host=host_name --port=5432 --username=user_name --dbname=database_name -csearch_path=study --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}', + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +********* QUERY ********** +search_path=public +************************** + +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --host=host_name --port=5432 --username=user_name --dbname=study -csearch_path=public --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * multi-schema case : $ psql --host=host_name --port=5432 --username=user_name --dbname=database_name -csearch_path=study --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}', + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +********* QUERY ********** +search_path=public +************************** + +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --host=host_name --port=5432 --username=user_name --dbname=study -csearch_path=public --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * multi-schema case : $ psql --host=host_name --port=5432 --username=user_name --dbname=database_name -csearch_path=study --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}', + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +********* QUERY ********** +search_path=public +************************** + +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --host=host_name --port=5432 --username=user_name --dbname=study -csearch_path=public --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * multi-schema case : $ psql --host=host_name --port=5432 --username=user_name --dbname=database_name -csearch_path=study --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}', + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --host=host_name --port=5432 --username=user_name --dbname=study -csearch_path=public --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * multi-schema case : $ psql --host=host_name --port=5432 --username=user_name --dbname=database_name -csearch_path=study --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}', + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --host=host_name --port=5432 --username=user_name --dbname=study -csearch_path=public --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * multi-schema case : $ psql --host=host_name --port=5432 --username=user_name --dbname=database_name -csearch_path=study --echo-errors --expanded=auto --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=file.sql --log-file=migrate_parameters.log + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}' + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}', + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}' + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}', + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}' + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}', + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}' + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + else + study_name := concat(study_name, '.', study_name); --study server use same name for schema and table name + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}', + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}' + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}', + '{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}' + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + '{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + --'{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}' + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + '{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + --'{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}' + --'{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + --'{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}', + '{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO +********* QUERY ********** +/* + * Run command: + * multi-database case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=study port=5432 options=-csearch_path=public" + * multi-schema case : $ psql --expanded --echo-errors --single-transaction --command='\conninfo' --command='\encoding utf-8' --file=migrate_parameters_from_study.sql --log-file=migrate_parameters.log "host=host_name user=user_name dbname=database_name port=5432 options=-csearch_path=study" + * + * How to rollback in case of error? + * Normally this script run itself inside a transaction, so if an exception is raised, the transaction is rollback. + * There is multiples migrations done by this script (see "migrate" variable), + * so only rollback the migration who failed. + * If it failed at the end of the script, check before if there wasn't non-rollback-able update/delete! + * If you're not sure, ask GridSuite devs or read this script in detail. + * To rollback a single migration who failed up to around half way: + * $ truncate table . + * $ update study.study set =coalesce(, ), =null + */ +/* Dev notes: + * Because json functions to extract list of elements from array or object return a setof, which isn't usable from for & foreach loops, + * we use tricks by casting into text and treating it and re-casting it to json array. + */ +DO +$body$ +<> +DECLARE + study_name text := quote_ident('study'); + migrate constant jsonb[] := array[ + /* Config format: + * - from_table (string): source table name + * - to_schema (string): destination database/schema (depends if multi-database/schema structure) + * - to_table (string): destination table name + * - from_old_id (string): source table old entity ID column name + * - from_new_uuid (string): source table new UUID column name + * - additional_tables (array[object]): additional tables to copy + * - from_table (string): source table name + * - to_table (string): destination table name + */ + --'{"from_table": "short_circuit_parameters", "from_old_id": "short_circuit_parameters_entity_id", "from_new_uuid": "short_circuit_parameters_uuid", "to_schema": "shortcircuit", "to_table": "analysis_parameters", "additional_tables": []}', + --'{"from_table": "load_flow_parameters", "from_old_id": "load_flow_parameters_entity_id", "from_new_uuid": "load_flow_parameters_uuid", "to_schema": "loadflow", "to_table": "load_flow_parameters", "additional_tables": [{"from_table": "load_flow_parameters_entity_countries_to_balance", "to_table": "load_flow_parameters_entity_countries_to_balance"}, {"from_table": "load_flow_specific_parameters", "to_table": "load_flow_specific_parameters"}]}' + --'{"from_table": "security_analysis_parameters", "from_old_id": "security_analysis_parameters_entity_id", "from_new_uuid": "security_analysis_parameters_uuid", "to_schema": "sa", "to_table": "security_analysis_parameters", "additional_tables": []}', + '{"from_table": "sensitivity_analysis_parameters", "from_old_id": "sensitivity_analysis_parameters_entity_id", "from_new_uuid": "sensitivity_analysis_parameters_uuid", "to_schema": "sensitivityanalysis", "to_table": "sensitivity_analysis_parameters", "additional_tables": [{"from_table": "contingencies", "to_table": "contingencies"}, {"from_table": "injections", "to_table": "injections"}, {"from_table": "monitored_branch", "to_table": "monitored_branch"}, {"from_table": "sensitivity_factor_for_injection_entity", "to_table": "sensitivity_factor_for_injection_entity"}, {"from_table": "sensitivity_factor_for_node_entity", "to_table": "sensitivity_factor_for_node_entity"}, {"from_table": "sensitivity_factor_with_distrib_type_entity", "to_table": "sensitivity_factor_with_distrib_type_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity", "to_table": "sensitivity_factor_with_sensi_type_for_hvdc_entity"}, {"from_table": "sensitivity_factor_with_sensi_type_for_pst_entity", "to_table": "sensitivity_factor_with_sensi_type_for_pst_entity"}]}' + --TODO dynaflow + ]; + params jsonb; + additional_table jsonb; + additional_first bool; + path_from text; + path_to text; + remote_databases bool; + insert_columns text; + rows_affected integer; + err_returned_sqlstate text; + err_column_name text; + err_constraint_name text; + err_pg_datatype_name text; + err_message_text text; + err_table_name text; + err_schema_name text; + err_pg_exception_detail text; + err_pg_exception_hint text; + err_pg_exception_context text; +BEGIN + --lock table study in exclusive mode; + raise log 'Script % starting at %', 'v1.7', now(); + raise log 'user=%, db=%, schema=%', current_user, current_database(), current_schema(); + + /* Case OPF: copy from db.study.
to db..
(easy, no problem to migrate) + * Case localdev & Azure: copy from study.public.
to .public.
(need to copy between databases...) + * [0A000] cross-database references are not implemented: "ref.to.remote.table" + */ + if exists(SELECT nspname FROM pg_catalog.pg_namespace where nspname = fn.study_name) then + raise notice 'multi schemas structure detected'; + if current_schema() != study_name then + raise exception 'Invalid current schema "%"', current_schema() using hint='Assuming script launch at "'||study_name||'.*"', errcode='invalid_schema_name'; + end if; + remote_databases := false; + elsif exists(SELECT datname FROM pg_catalog.pg_database WHERE datistemplate = false and datname = fn.study_name) then + raise notice 'separate databases structure detected'; + if current_database() != study_name then + raise exception 'Invalid current database "%"', current_database() using hint='Assuming script launch at "'||study_name||'.*.*"', errcode='invalid_database_definition'; + end if; + remote_databases := true; + create extension if not exists postgres_fdw; + else + raise exception 'Can''t detect type of database' using + hint='Is it the good database?', + errcode='invalid_database_definition', + detail='Can''t find schema nor database "study" from current session, use to determine the database structure.'; + end if; + + foreach params in array migrate loop + raise debug 'migration data = %', params; + /*Note: quote_indent() ⇔ %I ≠ quote_literal() ⇔ %L */ + begin + + if remote_databases then + execute format('create server if not exists %s foreign data wrapper postgres_fdw options (dbname %L)', concat('lnk_', params->>'to_schema'), params->>'to_schema'); + execute format('create user mapping if not exists for %s server %s options (user %L)', current_user, concat('lnk_', params->>'to_schema'), current_user); + end if; + + + additional_first := true; + foreach additional_table in array array_prepend(format('{"from_table": %s, "to_table": %s}', params->'from_table', params->'to_table'), + array_remove(string_to_array(replace(replace(trim(both '[]' from (params->'additional_tables')::text), '}, {', '}\n{'), '},{', '}\n{'), '\n', ''), null) + )::jsonb[] loop + raise debug 'debug table input = %', additional_table; + + if remote_databases then + -- rename for potential conflict with foreign table name + execute format('alter table if exists %I rename to %I', additional_table->>'to_table', concat(additional_table->>'to_table', '_old')); + + execute format('import foreign schema %I limit to (%I) from server %s into %I', 'public', additional_table->>'to_table', concat('lnk_', params->>'to_schema'), 'public'); + path_from := concat(quote_ident('public'), '.', quote_ident(concat(additional_table->>'from_table', '_old'))); + path_to := concat('public.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attnum >=1 and attrelid = (select ft.ftrelid' || + ' from pg_foreign_table ft left join pg_foreign_server fs on ft.ftserver=fs.oid left join pg_foreign_data_wrapper fdw on fs.srvfdw = fdw.oid left join pg_roles on pg_roles.oid=fdw.fdwowner' || + ' where pg_roles.rolname=current_user and fdw.fdwname=%L and fs.srvname=%L and %L=any(ft.ftoptions))', + 'postgres_fdw', concat('lnk_', params->>'to_schema'), concat('table_name=', additional_table->>'to_table')) --and ftoptions like '%tablename=...%' + into insert_columns; --[...] ; there isn't oid cast for foreign tables + --the create&import commands don't raise an exception if something isn't right, so insert_column result can be null + else + path_from := concat(study_name, '.', quote_ident(additional_table->>'from_table')); + path_to := concat(quote_ident(params->>'to_schema'), '.', quote_ident(additional_table->>'to_table')); + execute format('select string_agg(attname, '','') from pg_attribute where attrelid = %L::regclass and attnum >=1', path_to) + into insert_columns; --columns order may be different between src & dst tables + end if; + raise notice 'insert_columns=%', insert_columns; + if insert_columns is null then + raise exception 'A silent problem seem to happen during the connection to the remote database' using errcode = 'fdw_error', hint='Check if the server is created and the table imported'; + end if; + raise debug 'table locations: study=% src="%" dst="%"', study_name, path_from, path_to; + + raise notice 'copy data from % to %', path_from, path_to; + execute format('insert into %s(%s) select %s from %s', path_to, insert_columns, insert_columns, path_from); --... on conflict do nothing/update + get current diagnostics rows_affected = row_count; + raise info 'Copied % items from % to %', rows_affected, path_from, path_to; + + if additional_first then + additional_first := false; + execute format('update %s set %I=%I, %I=null', study_name, params->>'from_new_uuid', params->>'from_old_id', params->>'from_old_id'); + get current diagnostics rows_affected = row_count; + raise info 'Moved % IDs in % from % to %', rows_affected, study_name, params->>'from_old_id', params->>'from_new_uuid'; + end if; + + if remote_databases then + execute format('drop foreign table if exists public.%I cascade', additional_table->>'to_table'); + execute format('alter table if exists %I rename to %I', concat(additional_table->>'to_table', '_old'), additional_table->>'to_table'); --restore original name + end if; + end loop; + + /*execute format('truncate table %s', path_from); + raise info 'Emptied old table %', path_from;*/ + + if remote_databases then + /*use "drop ... if exist ...", so not need "if remote_databases then ..."*/ + --execute format('drop foreign table if exists %I.%I' cascade, 'public', params[i]->>'to_schema'); + --execute format('drop user mapping if exists for current_user server %s', 'lnk_'||params[i]->>'to_schema'); + execute format('drop server if exists %s cascade', concat('lnk_', params->>'to_schema')); + end if; + + exception when others then --we don't block the script but alert the admin + get stacked diagnostics + err_returned_sqlstate = returned_sqlstate, + err_column_name = column_name, + err_constraint_name = constraint_name, + err_pg_datatype_name = pg_datatype_name, + err_message_text = message_text, + err_table_name = table_name, + err_schema_name = schema_name, + err_pg_exception_detail = pg_exception_detail, + err_pg_exception_hint = pg_exception_hint, + err_pg_exception_context = pg_exception_context; + raise warning using + message = err_message_text, + detail = err_pg_exception_detail, + errcode = err_returned_sqlstate, + column = err_column_name, + constraint = err_constraint_name, + datatype = err_pg_datatype_name, + hint = err_pg_exception_hint, + schema = err_schema_name, + table = err_table_name; + raise debug E'--- Call Stack ---\n%', err_pg_exception_context; + end; + end loop; + raise log 'Script finish at %', now(); +END; +--anonymous function, so return void +$body$ +LANGUAGE plpgsql; +************************** + +DO