Skip to content

Commit

Permalink
* Added a check that throws an error if the striker.conf points to th…
Browse files Browse the repository at this point in the history 
…e same database target more than once.

* Fixed a bug in 'AN::Remote->remote_call()' where the same ssh filehandle is cached when two nodes have the same IP but different ports (as is the case in most remote connections via port forwards).

Signed-off-by: Digimer <[email protected]>
  • Loading branch information
@digimer
digimer committed Dec 5, 2016
1 parent 0068df5 commit 0aa87ef
Show file tree
Hide file tree
Showing 10 changed files with 119 additions and 49 deletions.
4 changes: 4 additions & 0 deletions AN/Tools/Alert.pod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -982,6 +982,10 @@ The 'AN::Tools::Get' module's 'ram_used_by_program()' method was called without

The 'AN::Tools::ScanCore' module's 'check_ram_usage()' method was called without a program name to check.

=head2 C<193>

The 'AN::Tools::DB' module's 'connect_to_databases()' method tried to connect to the same database twice.


=head2 C<999>

Expand Down
19 changes: 19 additions & 0 deletions AN/Tools/DB.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ sub connect_to_databases
name1 => "sys::host_uuid", value1 => $an->data->{sys}{host_uuid},
}, file => $THIS_FILE, line => __LINE__});

my $seen_connections = [];
my $connections = 0;
my $failed_connections = [];
my $successful_connections = [];
Expand All @@ -207,6 +208,24 @@ sub connect_to_databases
my $postgres_password = $an->data->{scancore}{db}{$id}{postgres_password} ? $an->data->{scancore}{db}{$id}{postgres_password} : "";
my $initialize = $an->data->{scancore}{db}{$id}{initialize} ? $an->data->{scancore}{db}{$id}{initialize} : 0;

# Make sure the user didn't specify the same target twice.
my $target_host = "$host:$port";
my $duplicate = 0;
foreach my $existing_host (sort {$a cmp $b} @{$seen_connections})
{
if ($existing_host eq $target_host)
{
# User is connecting to the same target twice.
$an->Alert->error({title_key => "error_title_0005", message_key => "error_message_0193", message_variables => { target => $target_host}, code => 193, file => $THIS_FILE, line => __LINE__});
$duplicate = 1;
}
}
if (not $duplicate)
{
push @{$seen_connections}, $target_host;
}
next if $duplicate;

# Log what we're doing.
$an->Log->entry({log_level => 3, title_key => "an_alert_title_0001", message_key => "tools_log_0007", message_variables => {
id => $id,
Expand Down
20 changes: 17 additions & 3 deletions AN/Tools/Get.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4511,6 +4511,14 @@ sub target_details
my $target = $parameter->{target} ? $parameter->{target} : $an->hostname;
my $port = $parameter->{port} ? $parameter->{port} : 22;
my $password = $parameter->{password} ? $parameter->{password} : "";
$an->Log->entry({log_level => 2, message_key => "an_variables_0002", message_variables => {
name1 => "target", value1 => $target,
name2 => "port", value2 => $port,
}, file => $THIS_FILE, line => __LINE__});
$an->Log->entry({log_level => 4, message_key => "an_variables_0001", message_variables => {
name1 => "password", value1 => $password,
}, file => $THIS_FILE, line => __LINE__});

my $return = {
anvil_name => "",
network => {
Expand All @@ -4535,9 +4543,10 @@ sub target_details
{
### Remote calls
# UUID
$an->Log->entry({log_level => 2, message_key => "an_variables_0002", message_variables => {
$an->Log->entry({log_level => 2, message_key => "an_variables_0003", message_variables => {
name1 => "uuid_shell_call", value1 => $uuid_shell_call,
name2 => "target", value2 => $target,
name3 => "port", value3 => $port,
}, file => $THIS_FILE, line => __LINE__});
(my $error, my $ssh_fh, $uuid_return) = $an->Remote->remote_call({
target => $target,
Expand All @@ -4548,9 +4557,10 @@ sub target_details
});

# IP info
$an->Log->entry({log_level => 3, message_key => "an_variables_0002", message_variables => {
$an->Log->entry({log_level => 2, message_key => "an_variables_0003", message_variables => {
name1 => "ip_shell_call", value1 => $ip_shell_call,
name2 => "target", value2 => $target,
name3 => "port", value3 => $port,
}, file => $THIS_FILE, line => __LINE__});
($error, $ssh_fh, $ip_return) = $an->Remote->remote_call({
target => $target,
Expand Down Expand Up @@ -4581,12 +4591,16 @@ sub target_details
{
chomp;
my $line = $_;
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
name1 => "line", value1 => $line,
}, file => $THIS_FILE, line => __LINE__});

push @{$uuid_return}, $line;
}
close $file_handle;

# IP Info
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
name1 => "ip_shell_call", value1 => $ip_shell_call,
}, file => $THIS_FILE, line => __LINE__});
open ($file_handle, "$ip_shell_call 2>&1 |") or $an->Alert->error({title_key => "error_title_0020", message_key => "error_message_0022", message_variables => { shell_call => $ip_shell_call, error => $! }, code => 30, file => $THIS_FILE, line => __LINE__});
Expand Down
28 changes: 15 additions & 13 deletions AN/Tools/InstallManifest.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11894,12 +11894,14 @@ sub map_network_on_node
my $an = $self->parent;
$an->Log->entry({log_level => 2, title_key => "tools_log_0001", title_variables => { function => "map_network_on_node" }, message_key => "tools_log_0002", file => $THIS_FILE, line => __LINE__});

my $remap = $parameter->{remap} ? $parameter->{remap} : "";
my $say_node = $parameter->{say_node} ? $parameter->{say_node} : "";
my $node = $parameter->{node} ? $parameter->{node} : "";
my $target = $parameter->{target} ? $parameter->{target} : "";
my $port = $parameter->{port} ? $parameter->{port} : "";
my $password = $parameter->{password} ? $parameter->{password} : "";
### TODO: Why are we not using $an->Remote->remote_call() ?
my $remap = $parameter->{remap} ? $parameter->{remap} : "";
my $say_node = $parameter->{say_node} ? $parameter->{say_node} : "";
my $node = $parameter->{node} ? $parameter->{node} : "";
my $target = $parameter->{target} ? $parameter->{target} : "";
my $port = $parameter->{port} ? $parameter->{port} : 22;
my $ssh_fh_key = $target.":".$port;
my $password = $parameter->{password} ? $parameter->{password} : "";
$an->Log->entry({log_level => 2, message_key => "an_variables_0005", message_variables => {
name1 => "remap", value1 => $remap,
name2 => "say_node", value2 => $say_node,
Expand Down Expand Up @@ -11979,10 +11981,10 @@ fi";
}
}
$an->Log->entry({log_level => 2, message_key => "an_variables_0004", message_variables => {
name1 => "proceed", value1 => $proceed,
name2 => "return_code", value2 => $return_code,
name3 => "ssh_fh", value3 => $ssh_fh,
name4 => "node::${target}::ssh_fh", value4 => $an->data->{target}{$target}{ssh_fh},
name1 => "proceed", value1 => $proceed,
name2 => "return_code", value2 => $return_code,
name3 => "ssh_fh", value3 => $ssh_fh,
name4 => "node::${ssh_fh_key}::ssh_fh", value4 => $an->data->{target}{$ssh_fh_key}{ssh_fh},
}, file => $THIS_FILE, line => __LINE__});

my $nics_seen = 0;
Expand All @@ -11993,20 +11995,20 @@ fi";
print $an->String->get({key => "message_0378"});
}
}
elsif ($an->data->{target}{$target}{ssh_fh} !~ /^Net::SSH2/)
elsif ($an->data->{target}{$ssh_fh_key}{ssh_fh} !~ /^Net::SSH2/)
{
# Invalid or broken SSH handle.
$an->Log->entry({log_level => 1, message_key => "log_0186", message_variables => {
node => $node,
ssh_fh => $an->data->{target}{$target}{ssh_fh},
ssh_fh => $an->data->{target}{$ssh_fh_key}{ssh_fh},
}, file => $THIS_FILE, line => __LINE__});
$return_code = 8;
}
else
{
### WARNING: Don't use 'remote_call()'! We need input from the user, so we need to call the
### target directly
my $ssh_fh = $an->data->{target}{$target}{ssh_fh};
my $ssh_fh = $an->data->{target}{$ssh_fh_key}{ssh_fh};
my $close = 0;

### Build the shell call
Expand Down
48 changes: 26 additions & 22 deletions AN/Tools/Remote.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,26 +354,29 @@ sub remote_call
my $parameter = shift;
my $an = $self->parent;

# Get the target
my $target = $parameter->{target};
# Get the target and port so that we can create the ssh_fh key
my $target = $parameter->{target};
my $port = $parameter->{port} ? $parameter->{port} : 22;
my $ssh_fh_key = $target.":".$port;

# This will store the SSH file handle for the given target after the initial connection.
$an->data->{target}{$target}{ssh_fh} = defined $an->data->{target}{$target}{ssh_fh} ? $an->data->{target}{$target}{ssh_fh} : "";
$an->data->{target}{$ssh_fh_key}{ssh_fh} = defined $an->data->{target}{$ssh_fh_key}{ssh_fh} ? $an->data->{target}{$ssh_fh_key}{ssh_fh} : "";

# Now pick up the rest of the variables.
my $port = $parameter->{port} ? $parameter->{port} : 22;
my $user = $parameter->{user} ? $parameter->{user} : "root";
my $password = $parameter->{password} ? $parameter->{password} : $an->data->{sys}{root_password};
my $ssh_fh = $parameter->{ssh_fh} ? $parameter->{ssh_fh} : $an->data->{target}{$target}{ssh_fh};
my $ssh_fh = $parameter->{ssh_fh} ? $parameter->{ssh_fh} : $an->data->{target}{$ssh_fh_key}{ssh_fh};
my $close = defined $parameter->{'close'} ? $parameter->{'close'} : 0;
my $shell_call = $parameter->{shell_call};
$an->Log->entry({log_level => 3, message_key => "an_variables_0006", message_variables => {
name1 => "time", value1 => time,
my $start_time = time;
$an->Log->entry({log_level => 3, message_key => "an_variables_0007", message_variables => {
name1 => "start_time", value1 => $start_time,
name2 => "target", value2 => $target,
name3 => "port", value3 => $port,
name4 => "user", value4 => $user,
name5 => "ssh_fh", value5 => $ssh_fh,
name6 => "close", value6 => $close,
name5 => "ssh_fh_key", value5 => $ssh_fh_key,
name6 => "ssh_fh", value6 => $ssh_fh,
name7 => "close", value7 => $close,
}, file => $THIS_FILE, line => __LINE__});
# Shell calls can expose passwords, which is why it is down here.
$an->Log->entry({log_level => 4, message_key => "an_variables_0002", message_variables => {
Expand Down Expand Up @@ -431,13 +434,13 @@ sub remote_call
if (not $an->Validate->is_ipv4({ip => $target}))
{
my $new_target = $an->Get->ip_from_hostname({host_name => $target});
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "new_target", value1 => $new_target,
}, file => $THIS_FILE, line => __LINE__});
if ($new_target)
{
$target = $new_target;
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "target", value1 => $target,
}, file => $THIS_FILE, line => __LINE__});
}
Expand Down Expand Up @@ -541,29 +544,30 @@ sub remote_call
if ($ssh_fh->auth_publickey($user, $public_key, $private_key))
{
# We're in! Record the file handle for this target.
$an->data->{target}{$target}{ssh_fh} = $ssh_fh;
$an->data->{target}{$ssh_fh_key}{ssh_fh} = $ssh_fh;
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "target::${target}::ssh_fh", value1 => $an->data->{target}{$target}{ssh_fh},
name1 => "target::${ssh_fh_key}::ssh_fh", value1 => $an->data->{target}{$ssh_fh_key}{ssh_fh},
}, file => $THIS_FILE, line => __LINE__});

$an->Log->entry({log_level => 3, message_key => "notice_message_0014", message_variables => { target => $target }, file => $THIS_FILE, line => __LINE__});
$an->Log->entry({log_level => 2, message_key => "notice_message_0014", message_variables => { target => $ssh_fh_key }, file => $THIS_FILE, line => __LINE__});
}
else
{
# This is for the user
$error = $an->String->get({key => "error_message_0032", variables => { target => $target }});
$error = $an->String->get({key => "error_message_0032", variables => { target => $ssh_fh_key }});
# This is for our logs
$an->Log->entry({log_level => 1, message_key => "error_message_0032", message_variables => { target => $target }, file => $THIS_FILE, line => __LINE__});
$an->Log->entry({log_level => 1, message_key => "error_message_0032", message_variables => { target => $ssh_fh_key }, file => $THIS_FILE, line => __LINE__});
}
}
else
{
# We're in! Record the file handle for this target.
$an->data->{target}{$target}{ssh_fh} = $ssh_fh;
$an->data->{target}{$ssh_fh_key}{ssh_fh} = $ssh_fh;
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "target::${target}::ssh_fh", value1 => $an->data->{target}{$target}{ssh_fh},
name1 => "target::${ssh_fh_key}::ssh_fh", value1 => $an->data->{target}{$ssh_fh_key}{ssh_fh},
}, file => $THIS_FILE, line => __LINE__});
$an->Log->entry({log_level => 3, message_key => "notice_message_0004", message_variables => { target => $target }, file => $THIS_FILE, line => __LINE__});

$an->Log->entry({log_level => 2, message_key => "notice_message_0004", message_variables => { target => $ssh_fh_key }, file => $THIS_FILE, line => __LINE__});
}
}
}
Expand Down Expand Up @@ -682,14 +686,14 @@ sub remote_call
# Close the connection if requested.
if ($close)
{
$an->Log->entry({log_level => 2, message_key => "notice_message_0005", message_variables => { target => $target }, file => $THIS_FILE, line => __LINE__});
$an->Log->entry({log_level => 2, message_key => "notice_message_0005", message_variables => { target => $ssh_fh_key }, file => $THIS_FILE, line => __LINE__});
$ssh_fh->disconnect() if $ssh_fh;

# For good measure, blank both variables.
$an->data->{target}{$target}{ssh_fh} = "";
$an->data->{target}{$ssh_fh_key}{ssh_fh} = "";
$ssh_fh = "";
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "target::${target}::ssh_fh", value1 => $an->data->{target}{$target}{ssh_fh},
name1 => "target::${ssh_fh_key}::ssh_fh", value1 => $an->data->{target}{$ssh_fh_key}{ssh_fh},
}, file => $THIS_FILE, line => __LINE__});
}

Expand Down
12 changes: 6 additions & 6 deletions AN/Tools/ScanCore.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2042,7 +2042,7 @@ sub insert_or_update_nodes
my $node_sn = $parameter->{node_sn} ? $parameter->{node_sn} : "NULL";
my $node_ifn = $parameter->{node_ifn} ? $parameter->{node_ifn} : "NULL";
my $node_password = $parameter->{node_password} ? $parameter->{node_password} : "NULL";
$an->Log->entry({log_level => 3, message_key => "an_variables_0009", message_variables => {
$an->Log->entry({log_level => 2, message_key => "an_variables_0009", message_variables => {
name1 => "node_uuid", value1 => $node_uuid,
name2 => "node_anvil_uuid", value2 => $node_anvil_uuid,
name3 => "node_host_uuid", value3 => $node_host_uuid,
Expand Down Expand Up @@ -2285,7 +2285,7 @@ AND
AND
node_cache_name = ".$an->data->{sys}{use_db_fh}->quote($node_cache_name)."
;";
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "query", value1 => $query,
}, file => $THIS_FILE, line => __LINE__});

Expand All @@ -2298,7 +2298,7 @@ AND
foreach my $row (@{$results})
{
$node_cache_uuid = $row->[0];
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "node_cache_uuid", value1 => $node_cache_uuid,
}, file => $THIS_FILE, line => __LINE__});
}
Expand Down Expand Up @@ -2330,7 +2330,7 @@ INSERT INTO
);
";
$query =~ s/'NULL'/NULL/g;
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "query", value1 => $query,
}, file => $THIS_FILE, line => __LINE__});
$an->DB->do_db_write({query => $query, source => $THIS_FILE, line => __LINE__});
Expand All @@ -2351,7 +2351,7 @@ FROM
WHERE
node_cache_uuid = ".$an->data->{sys}{use_db_fh}->quote($node_cache_uuid)."
;";
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "query", value1 => $query,
}, file => $THIS_FILE, line => __LINE__});

Expand Down Expand Up @@ -2403,7 +2403,7 @@ WHERE
node_cache_uuid = ".$an->data->{sys}{use_db_fh}->quote($node_cache_uuid)."
";
$query =~ s/'NULL'/NULL/g;
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
$an->Log->entry({log_level => 4, message_key => "an_variables_0001", message_variables => {
name1 => "query", value1 => $query,
}, file => $THIS_FILE, line => __LINE__});
$an->DB->do_db_write({query => $query, source => $THIS_FILE, line => __LINE__});
Expand Down
2 changes: 1 addition & 1 deletion AN/Tools/Striker.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7264,7 +7264,7 @@ sub _gather_node_details
my $target = $an->data->{sys}{anvil}{$node_key}{use_ip};
my $port = $an->data->{sys}{anvil}{$node_key}{use_port};
my $password = $an->data->{sys}{anvil}{$node_key}{password};
$an->Log->entry({log_level => 2, message_key => "an_variables_0007", message_variables => {
$an->Log->entry({log_level => 3, message_key => "an_variables_0007", message_variables => {
name1 => "node_uuid", value1 => $node_uuid,
name2 => "node_key", value2 => $node_key,
name3 => "anvil_uuid", value3 => $anvil_uuid,
Expand Down
1 change: 1 addition & 0 deletions AN/tools.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,7 @@ Failed to send an alert because this host is not yet in the database. This can h
<key name="error_message_0190"><![CDATA[A request to shut down: [#!variable!node_name!#] appears to have failed.<br />In some cases, this can be caused by something holding open the shared storage, resulting in the node getting fenced.<br />Please go back to the main page, wait a few minutes and see if the node comes back online.<br />Once things stabilize, please try shutting down the system again.]]></key>
<key name="error_message_0191"><![CDATA[The 'AN::Tools::Get' module's 'ram_used_by_program()' method was called without a program name to check.]]></key>
<key name="error_message_0192"><![CDATA[The 'AN::Tools::ScanCore' module's 'check_ram_usage()' method was called without a program name to check.]]></key>
<key name="error_message_0193"><![CDATA[The 'AN::Tools::DB' module's 'connect_to_databases()' method tried to connect to the same database twice: [#!variable!target!#].]]></key>

<!-- Cleared warning/error messages -->
<key name="cleared_message_0001">Connection to the database: [#!variable!name!#] on host: [#!variable!host!#:#!variable!port!#] has been re-established.</key>
Expand Down
Loading

0 comments on commit 0aa87ef

Please sign in to comment.