Skip to content

DEV-1472 post-zephir processing: get database config from env vars #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aelkiss merged 1 commit into from
Jan 24, 2025
Merged

DEV-1472 post-zephir processing: get database config from env vars #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 5 additions & 25 deletions bin/grin_gfv.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,16 @@
use strict;
use warnings;

use lib "$ENV{ROOTDIR}/perl_lib";

use Date::Manip qw(ParseDate UnixDate);
use DBI;
use Getopt::Long;
use Mail::Mailer;
use ProgressTracker;
use YAML::XS;

use Database;

my $noop = undef; # set with --noop
my $mailer = undef;
my $email_body = ""; # holds the current email body
Expand All @@ -40,10 +43,9 @@
# config
my $config_dir = $ENV{CONFIG_DIR} || '/usr/src/app/config';
my $config_yaml = "$config_dir/rights.yml";
my $db_yaml = "$config_dir/database.yml";
my $config = YAML::XS::LoadFile($config_yaml);
my $rights_dir = $config->{rights}->{rights_dir};
my $dbh = get_dbh();
my $dbh = Database::get_rights_rw_dbh();

GetOptions(
# skip update queries, emails, log file & tracker
Expand Down Expand Up @@ -185,25 +187,3 @@ sub new_mailer {

$mailer->open($email);
}

# Inspired by HTFeed::DBTools::_init
sub get_dbh {
my $db_conf = YAML::XS::LoadFile($db_yaml);
my $dbname = $db_conf->{dbname};
my $hostname = $db_conf->{hostname};
my $dbuser = $db_conf->{user};
my $passwd = $db_conf->{password};

my $extra_params = {
'RaiseError' => 1,
};

my $dbh = DBI->connect(
"DBI:MariaDB:$dbname:$hostname",
$dbuser,
$passwd,
$extra_params
);

return $dbh;
}
29 changes: 4 additions & 25 deletions bin/populate_rights_data.pl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@
use strict;
use warnings;

use FindBin;
use lib "$FindBin::Bin/../lib";
use lib "$ENV{ROOTDIR}/perl_lib";

use Date::Manip;
use DBI;
Expand All @@ -13,6 +12,8 @@
use ProgressTracker;
use YAML::XS;

use Database;

# Set up precedence. New rights with >= precedence are allowed to override.
# PRIORITY_MAN is special - a note must be set.
my $PRIORITY_BIB = 1;
Expand Down Expand Up @@ -59,7 +60,6 @@
# read config from yaml
my $config_dir = $ENV{CONFIG_DIR} || '/usr/src/app/config';
my $config_yaml = "$config_dir/rights.yml";
my $db_yaml = "$config_dir/database.yml";
my $config = YAML::XS::LoadFile($config_yaml);
my $archive = $config->{rights}->{archive};
my $rights_dir = $config->{rights}->{rights_dir};
Expand Down Expand Up @@ -129,7 +129,7 @@
}

if (@rights_files) {
$dbh = get_dbh();
$dbh = Database::get_rights_rw_dbh();
prepare_statements();

foreach my $file (@rights_files) {
Expand Down Expand Up @@ -574,27 +574,6 @@ sub export_barcodes {
}
}

sub get_dbh {
my $db_conf = YAML::XS::LoadFile($db_yaml);
my $dbname = $db_conf->{dbname};
my $hostname = $db_conf->{hostname};
my $user = $db_conf->{user};
my $passwd = $db_conf->{password};

my $extra_params = {
'RaiseError' => 1,
};

my $dbh = DBI->connect(
"DBI:MariaDB:$dbname:$hostname",
$user,
$passwd,
$extra_params
);

return $dbh;
}


=head1 NAME

Expand Down
4 changes: 0 additions & 4 deletions config/database.yml
View file
Open in desktop

This file was deleted.

15 changes: 8 additions & 7 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ services:
pushgateway: *healthy
environment:
DATA_ROOT: "/usr/src/app/data"
DB_CONNECTION_STRING: "mysql2://ht_rights:ht_rights@mariadb/ht"
POST_ZEPHIR_PROCESSING_LOGGER_LEVEL: "1"
PUSHGATEWAY: "http://pushgateway:9091"
command:
Expand All @@ -36,14 +35,16 @@ services:
pushgateway: *healthy
environment:
- DATA_ROOT=/usr/src/app/data
- DB_CONNECTION_STRING="mysql2://ht_rights:ht_rights@mariadb/ht"
- POST_ZEPHIR_PROCESSING_LOGGER_LEVEL=1
- PUSHGATEWAY="http://pushgateway:9091"
- DB_HT_RO_USER=ht_rights
- DB_HT_RO_PASSWORD=ht_rights
- DB_HT_RO_HOST=mariadb
- DB_HT_RO_PORT=3306
- DB_HT_RO_DATABASE=ht
- MARIADB_HT_RO_USERNAME=ht_rights
- MARIADB_HT_RO_PASSWORD=ht_rights
- MARIADB_HT_RO_HOST=mariadb
- MARIADB_HT_RO_DATABASE=ht
- MARIADB_RIGHTS_RW_USERNAME=ht_rights
- MARIADB_RIGHTS_RW_PASSWORD=ht_rights
- MARIADB_RIGHTS_RW_HOST=mariadb
- MARIADB_RIGHTS_RW_DATABASE=ht
# pass through info needed by coveralls uploader
- GITHUB_TOKEN
- GITHUB_RUN_ID
Expand Down
9 changes: 4 additions & 5 deletions lib/services.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,10 @@ module PostZephirProcessing
Services.register(:database) do
Sequel.connect(
adapter: "mysql2",
user: ENV["DB_HT_RO_USER"],
password: ENV["DB_HT_RO_PASSWORD"],
host: ENV["DB_HT_RO_HOST"],
port: ENV["DB_HT_RO_PORT"],
database: ENV["DB_HT_RO_DATABASE"],
user: ENV["MARIADB_HT_RO_USERNAME"],
password: ENV["MARIADB_HT_RO_PASSWORD"],
host: ENV["MARIADB_HT_RO_HOST"],
database: ENV["MARIADB_HT_RO_DATABASE"],
encoding: "utf8mb4"
)
end
Expand Down
49 changes: 49 additions & 0 deletions perl_lib/Database.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package Database;

# Shared utility for read-only connection to Rights DB (via views in ht database)
# or, for populate_rights_data.pl, read-write access to the Rights DB directly.

use DBI;

my $DB_ENV = {
'ht_ro' => [
'MARIADB_HT_RO_USERNAME',
'MARIADB_HT_RO_PASSWORD',
'MARIADB_HT_RO_DATABASE',
'MARIADB_HT_RO_HOST'
],
'rights_rw' => [
'MARIADB_RIGHTS_RW_USERNAME',
'MARIADB_RIGHTS_RW_PASSWORD',
'MARIADB_RIGHTS_RW_DATABASE',
'MARIADB_RIGHTS_RW_HOST'
],
};

sub get_ht_ro_dbh {
return _get_dbh('ht_ro');
}

sub get_rights_rw_dbh {
return _get_dbh('rights_rw');
}

sub _get_dbh {
my $database_option = shift;

my ($dbuser, $passwd, $dbname, $hostname) = map { $ENV{$_}; } @{$DB_ENV->{$database_option}};
my $extra_params = {
'RaiseError' => 1,
};

my $dbh = DBI->connect(
"DBI:MariaDB:$dbname:$hostname",
$dbuser,
$passwd,
$extra_params
);

return $dbh;
}

1;
26 changes: 5 additions & 21 deletions rightsDB.pm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,21 @@ package rightsDB;
use strict;
no strict 'refs';
no strict 'subs';
#use Sys::Hostname;
use DBI;
#use Data::Dumper;

use lib "$ENV{ROOTDIR}/perl_lib";

use File::Basename;
use YAML;
use Database;

sub new {
my $class = shift;
$class = ref($class) || $class; # Handle cloning

my $self;

# config
my $config_path = dirname(__FILE__) . '/config/database.yml';
my %config = %{ YAML::LoadFile($config_path) };
# globals
my $sdr_dbh = ConnectToSdrDb($config{user}, $config{password}, $config{hostname}, $config{dbname});
my $sdr_dbh = Database::get_ht_ro_dbh;
my $sdr_sth = InitSdrSth($sdr_dbh);
$self->{sdr_dbh} = $sdr_dbh;
$self->{sdr_sth} = $sdr_sth;
Expand All @@ -33,20 +31,6 @@ sub new {
return bless $self, $class;
}

sub ConnectToSdrDb
{
my $db_user = shift;
my $db_passwd = shift;
my $db_server = shift;
my $db_name = shift;

my $sdr_dbh;
$sdr_dbh = DBI->connect( "DBI:MariaDB:$db_name:$db_server", $db_user, $db_passwd,
{ RaiseError => 0, AutoCommit => 1 } ) || die "Cannot connect: $DBI::errstr";
Copy link
Member

@aelkiss aelkiss Jan 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just noting that the move removes the AutoCommit => 1, but that should be fine as this code is only using the read-only connection (and we weren't previously using autocommit for the read-write stuff)


return $sdr_dbh;
}

sub InitSdrSth {
my $dbh = shift;

Expand Down
26 changes: 6 additions & 20 deletions t/db.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,24 @@
use strict;
use warnings;
use utf8;

use lib "$ENV{ROOTDIR}/perl_lib";

use DBI;
use Encode qw(encode);
use Test::More;
use YAML;

use Database;

# This test is here as an example to help characterize the behavior of the
# database connection and ensure that we can process UTF-8 at least from the
# perl side. YMMV in production as it might be a different version of mariadb
# with different settings, etc.

sub get_dbh {
my $db_conf = YAML::LoadFile("/usr/src/app/config/database.yml");
my $dbname = $db_conf->{dbname};
my $hostname = $db_conf->{hostname};
my $user = $db_conf->{user};
my $passwd = $db_conf->{password};

my $extra_params = {
'RaiseError' => 1,
};

my $dbh = DBI->connect(
"DBI:MariaDB:$dbname:$hostname",
$user,
$passwd,
$extra_params
);

return $dbh;
}

my $dbh = get_dbh();
my $dbh = Database::get_rights_rw_dbh;

subtest "UTF-8 support for ht_rights" => sub {
my @tables = ('rights_current', 'rights_log');
Expand Down
Loading