From 76e6062e06f20894a36645739869be0fb9a9259c Mon Sep 17 00:00:00 2001 From: Brian Moses Hall Date: Mon, 20 Jan 2025 14:39:05 -0500 Subject: [PATCH] DEV-1472 post-zephir processing: get database config from env vars - Move all Perl DB connection code to a single module under `perl_lib` - Remove config/database.yml - Add necessary `MARIADB_HT...` and `MARIADB_RIGHTS...` vars to docker compose. --- bin/grin_gfv.pl | 30 ++++------------------- bin/populate_rights_data.pl | 29 +++------------------- config/database.yml | 4 --- docker-compose.yml | 15 ++++++------ lib/services.rb | 9 +++---- perl_lib/Database.pm | 49 +++++++++++++++++++++++++++++++++++++ rightsDB.pm | 26 ++++---------------- t/db.t | 26 +++++--------------- 8 files changed, 81 insertions(+), 107 deletions(-) delete mode 100644 config/database.yml create mode 100644 perl_lib/Database.pm diff --git a/bin/grin_gfv.pl b/bin/grin_gfv.pl index a85a873..43d6e14 100644 --- a/bin/grin_gfv.pl +++ b/bin/grin_gfv.pl @@ -19,13 +19,16 @@ use strict; use warnings; +use lib "$ENV{ROOTDIR}/perl_lib"; + use Date::Manip qw(ParseDate UnixDate); -use DBI; use Getopt::Long; use Mail::Mailer; use ProgressTracker; use YAML::XS; +use Database; + my $noop = undef; # set with --noop my $mailer = undef; my $email_body = ""; # holds the current email body @@ -40,10 +43,9 @@ # config my $config_dir = $ENV{CONFIG_DIR} || '/usr/src/app/config'; my $config_yaml = "$config_dir/rights.yml"; -my $db_yaml = "$config_dir/database.yml"; my $config = YAML::XS::LoadFile($config_yaml); my $rights_dir = $config->{rights}->{rights_dir}; -my $dbh = get_dbh(); +my $dbh = Database::get_rights_rw_dbh(); GetOptions( # skip update queries, emails, log file & tracker @@ -185,25 +187,3 @@ sub new_mailer { $mailer->open($email); } - -# Inspired by HTFeed::DBTools::_init -sub get_dbh { - my $db_conf = YAML::XS::LoadFile($db_yaml); - my $dbname = $db_conf->{dbname}; - my $hostname = $db_conf->{hostname}; - my $dbuser = $db_conf->{user}; - my $passwd = $db_conf->{password}; - - my $extra_params = { - 'RaiseError' => 1, - }; - - my $dbh = DBI->connect( - "DBI:MariaDB:$dbname:$hostname", - $dbuser, - $passwd, - $extra_params - ); - - return $dbh; -} diff --git a/bin/populate_rights_data.pl b/bin/populate_rights_data.pl index aa80880..edf8076 100755 --- a/bin/populate_rights_data.pl +++ b/bin/populate_rights_data.pl @@ -3,8 +3,7 @@ use strict; use warnings; -use FindBin; -use lib "$FindBin::Bin/../lib"; +use lib "$ENV{ROOTDIR}/perl_lib"; use Date::Manip; use DBI; @@ -13,6 +12,8 @@ use ProgressTracker; use YAML::XS; +use Database; + # Set up precedence. New rights with >= precedence are allowed to override. # PRIORITY_MAN is special - a note must be set. my $PRIORITY_BIB = 1; @@ -59,7 +60,6 @@ # read config from yaml my $config_dir = $ENV{CONFIG_DIR} || '/usr/src/app/config'; my $config_yaml = "$config_dir/rights.yml"; -my $db_yaml = "$config_dir/database.yml"; my $config = YAML::XS::LoadFile($config_yaml); my $archive = $config->{rights}->{archive}; my $rights_dir = $config->{rights}->{rights_dir}; @@ -129,7 +129,7 @@ } if (@rights_files) { - $dbh = get_dbh(); + $dbh = Database::get_rights_rw_dbh(); prepare_statements(); foreach my $file (@rights_files) { @@ -574,27 +574,6 @@ sub export_barcodes { } } -sub get_dbh { - my $db_conf = YAML::XS::LoadFile($db_yaml); - my $dbname = $db_conf->{dbname}; - my $hostname = $db_conf->{hostname}; - my $user = $db_conf->{user}; - my $passwd = $db_conf->{password}; - - my $extra_params = { - 'RaiseError' => 1, - }; - - my $dbh = DBI->connect( - "DBI:MariaDB:$dbname:$hostname", - $user, - $passwd, - $extra_params - ); - - return $dbh; -} - =head1 NAME diff --git a/config/database.yml b/config/database.yml deleted file mode 100644 index 9f886f7..0000000 --- a/config/database.yml +++ /dev/null @@ -1,4 +0,0 @@ -user: ht_rights -password: ht_rights -hostname: mariadb -dbname: ht diff --git a/docker-compose.yml b/docker-compose.yml index 4726fcc..f3922ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,6 @@ services: pushgateway: *healthy environment: DATA_ROOT: "/usr/src/app/data" - DB_CONNECTION_STRING: "mysql2://ht_rights:ht_rights@mariadb/ht" POST_ZEPHIR_PROCESSING_LOGGER_LEVEL: "1" PUSHGATEWAY: "http://pushgateway:9091" command: @@ -36,14 +35,16 @@ services: pushgateway: *healthy environment: - DATA_ROOT=/usr/src/app/data - - DB_CONNECTION_STRING="mysql2://ht_rights:ht_rights@mariadb/ht" - POST_ZEPHIR_PROCESSING_LOGGER_LEVEL=1 - PUSHGATEWAY="http://pushgateway:9091" - - DB_HT_RO_USER=ht_rights - - DB_HT_RO_PASSWORD=ht_rights - - DB_HT_RO_HOST=mariadb - - DB_HT_RO_PORT=3306 - - DB_HT_RO_DATABASE=ht + - MARIADB_HT_RO_USERNAME=ht_rights + - MARIADB_HT_RO_PASSWORD=ht_rights + - MARIADB_HT_RO_HOST=mariadb + - MARIADB_HT_RO_DATABASE=ht + - MARIADB_RIGHTS_RW_USERNAME=ht_rights + - MARIADB_RIGHTS_RW_PASSWORD=ht_rights + - MARIADB_RIGHTS_RW_HOST=mariadb + - MARIADB_RIGHTS_RW_DATABASE=ht # pass through info needed by coveralls uploader - GITHUB_TOKEN - GITHUB_RUN_ID diff --git a/lib/services.rb b/lib/services.rb index b8e5d20..7a0ce44 100644 --- a/lib/services.rb +++ b/lib/services.rb @@ -16,11 +16,10 @@ module PostZephirProcessing Services.register(:database) do Sequel.connect( adapter: "mysql2", - user: ENV["DB_HT_RO_USER"], - password: ENV["DB_HT_RO_PASSWORD"], - host: ENV["DB_HT_RO_HOST"], - port: ENV["DB_HT_RO_PORT"], - database: ENV["DB_HT_RO_DATABASE"], + user: ENV["MARIADB_HT_RO_USERNAME"], + password: ENV["MARIADB_HT_RO_PASSWORD"], + host: ENV["MARIADB_HT_RO_HOST"], + database: ENV["MARIADB_HT_RO_DATABASE"], encoding: "utf8mb4" ) end diff --git a/perl_lib/Database.pm b/perl_lib/Database.pm new file mode 100644 index 0000000..4f5c4d6 --- /dev/null +++ b/perl_lib/Database.pm @@ -0,0 +1,49 @@ +package Database; + +# Shared utility for read-only connection to Rights DB (via views in ht database) +# or, for populate_rights_data.pl, read-write access to the Rights DB directly. + +use DBI; + +my $DB_ENV = { + 'ht_ro' => [ + 'MARIADB_HT_RO_USERNAME', + 'MARIADB_HT_RO_PASSWORD', + 'MARIADB_HT_RO_DATABASE', + 'MARIADB_HT_RO_HOST' + ], + 'rights_rw' => [ + 'MARIADB_RIGHTS_RW_USERNAME', + 'MARIADB_RIGHTS_RW_PASSWORD', + 'MARIADB_RIGHTS_RW_DATABASE', + 'MARIADB_RIGHTS_RW_HOST' + ], +}; + +sub get_ht_ro_dbh { + return _get_dbh('ht_ro'); +} + +sub get_rights_rw_dbh { + return _get_dbh('rights_rw'); +} + +sub _get_dbh { + my $database_option = shift; + + my ($dbuser, $passwd, $dbname, $hostname) = map { $ENV{$_}; } @{$DB_ENV->{$database_option}}; + my $extra_params = { + 'RaiseError' => 1, + }; + + my $dbh = DBI->connect( + "DBI:MariaDB:$dbname:$hostname", + $dbuser, + $passwd, + $extra_params + ); + + return $dbh; +} + +1; diff --git a/rightsDB.pm b/rightsDB.pm index fbb9acd..80a9375 100755 --- a/rightsDB.pm +++ b/rightsDB.pm @@ -3,11 +3,12 @@ package rightsDB; use strict; no strict 'refs'; no strict 'subs'; -#use Sys::Hostname; -use DBI; -#use Data::Dumper; + +use lib "$ENV{ROOTDIR}/perl_lib"; + use File::Basename; use YAML; +use Database; sub new { my $class = shift; @@ -15,11 +16,8 @@ sub new { my $self; - # config - my $config_path = dirname(__FILE__) . '/config/database.yml'; - my %config = %{ YAML::LoadFile($config_path) }; # globals - my $sdr_dbh = ConnectToSdrDb($config{user}, $config{password}, $config{hostname}, $config{dbname}); + my $sdr_dbh = Database::get_ht_ro_dbh; my $sdr_sth = InitSdrSth($sdr_dbh); $self->{sdr_dbh} = $sdr_dbh; $self->{sdr_sth} = $sdr_sth; @@ -33,20 +31,6 @@ sub new { return bless $self, $class; } -sub ConnectToSdrDb -{ - my $db_user = shift; - my $db_passwd = shift; - my $db_server = shift; - my $db_name = shift; - - my $sdr_dbh; - $sdr_dbh = DBI->connect( "DBI:MariaDB:$db_name:$db_server", $db_user, $db_passwd, - { RaiseError => 0, AutoCommit => 1 } ) || die "Cannot connect: $DBI::errstr"; - - return $sdr_dbh; -} - sub InitSdrSth { my $dbh = shift; diff --git a/t/db.t b/t/db.t index 09ba49a..8871731 100644 --- a/t/db.t +++ b/t/db.t @@ -3,38 +3,24 @@ use strict; use warnings; use utf8; + +use lib "$ENV{ROOTDIR}/perl_lib"; + use DBI; use Encode qw(encode); use Test::More; use YAML; +use Database; + # This test is here as an example to help characterize the behavior of the # database connection and ensure that we can process UTF-8 at least from the # perl side. YMMV in production as it might be a different version of mariadb # with different settings, etc. -sub get_dbh { - my $db_conf = YAML::LoadFile("/usr/src/app/config/database.yml"); - my $dbname = $db_conf->{dbname}; - my $hostname = $db_conf->{hostname}; - my $user = $db_conf->{user}; - my $passwd = $db_conf->{password}; - - my $extra_params = { - 'RaiseError' => 1, - }; - - my $dbh = DBI->connect( - "DBI:MariaDB:$dbname:$hostname", - $user, - $passwd, - $extra_params - ); - return $dbh; -} -my $dbh = get_dbh(); +my $dbh = Database::get_rights_rw_dbh; subtest "UTF-8 support for ht_rights" => sub { my @tables = ('rights_current', 'rights_log');