-
Notifications
You must be signed in to change notification settings - Fork 79
/
Copy pathcapacity-checker
executable file
·155 lines (138 loc) · 6.27 KB
/
capacity-checker
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
#!/usr/bin/env ruby
# For use with OpenShift Enterprise 1.0 and 1.1
#
# This is an EXAMPLE script to show how you could monitor
# OpenShift capacity using oo-stats and then alert
# a separate system to spin up new nodes as needed.
# It would need a lot of work to actually do that in any
# specific system.
#
# Installation: Place this script in the same directory
# as the oo-stats script on a broker.
#
# Usage Example for new nodes when 50% profile capacity reached:
# capacity-checker -p 50
#
# Keep in mind two major kinds of capacity:
# 1) Active capacity - the number of active gears node hosts can run
# 2) District capacity - the number of gears (in any state) that the
# district can allocate.
#--
# Copyright 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#++
load 'oo-stats'
require 'optparse'
require 'ostruct'
options = OpenStruct.new :dist_t => 90.0, :compact_t => 50.0, :profile_t => 80.0, :wait => 2
opts = OptionParser.new do |opts|
opts.banner = "Usage: capacity-checker [options]"
opts.on("-d", "--district PERCENTAGE", Float,
"Active usage threshold before adding nodes to a district, as a percentage (default 90)") do |threshold|
options.dist_t = threshold
end
opts.on("-c", "--compact PERCENTAGE", Float,
"When to consider compacting a full district: available UIDs as percentage of node available active capacity (default 50)") do |threshold|
options.compact_t = threshold
end
opts.on("-p", "--profile PERCENTAGE", Float,
"Active usage threshold before adding nodes to a profile, as a percentage (default 80)") do |threshold|
options.profile_t = threshold
end
opts.on('-w','--wait SECONDS', Float, <<WAIT) { |wait| options.wait = wait }
Seconds for broker to wait for node responses (default 2).
\tIf nodes are not responding in time, increase this as needed.
WAIT
opts.on("-h", "--help", "Print usage and exit") {|h| puts opts; exit 0 }
end
opts.parse!
o = OOStats.new :wait => options.wait
o.gather_statistics
MAX_DISTRICT_CAPACITY = 6000 # for now = the cap on district capacity
node_additions = 0
# May want to consult the districts first to see if any need adjustment
o.results[:district_summaries].each do |district|
# TODO: fill in with your own automation instead of just messages.
# Ignore NONE districts (i.e. non-districted nodes) for now
next if district[:uuid].start_with? 'NONE ' # not a real district
unless district[:missing_nodes].empty?
puts "Warning: these nodes from district '#{district[:name]}' are not responding:"
puts district[:missing_nodes].join ', '
end
# For districts with more available active gears on the nodes than district capacity,
if district[:dist_avail_capacity] < district[:available_active_gears]
# consider simply adding capacity if possible
if district[:district_capacity] < MAX_DISTRICT_CAPACITY
puts <<CAPACITY
District '#{district[:name]}' has unusable node capacity.
Consider adding district capacity with this broker command (customize as needed):
oo-admin-ctl-district -c add-capacity -n '#{district[:name]}' -s #{district[:available_active_gears]-district[:dist_avail_capacity]}
CAPACITY
# or if up against the 6000 limit, consider compacting by removing a node.
# -c specifies a threshold where this becomes desirable, e.g. when
# dist_avail_capacity is <50% of available_active_gears.
elsif district[:dist_avail_capacity] * options.compact_t < district[:available_active_gears] * 100.0
puts <<COMPACT
District '#{district[:name]}' has unusable node capacity.
Consider compacting the district by removing a node.
COMPACT
end
else
# For districts with very high active usage but not lacking capacity,
# consider adding a node to that district.
# TODO: also establish a threshold for how much capacity the district should have.
usage_perct = 100.0 * district[:gears_active_count] /
(district[:available_active_gears] + district[:gears_active_count])
if usage_perct > options.dist_t
puts <<ADD
District '#{district[:name]}' has active usage #{ usage_perct }.
To give it another node, create a node with profile '#{district[:profile]}'
and add it to the district with:
oo-admin-ctl-district -c add-node -n '#{district[:name]}' -i <node.host.name>
ADD
end
end
end
# Calculate need for nodes by seeing if any of the profiles
# are over the specified usage
o.results[:profile_summaries].each do |profile|
usage_perct = 100.0 * profile[:gears_active_count].to_f / (profile[:available_active_gears] + profile[:gears_active_count])
puts "Current '#{profile[:profile]}' active gear usage = #{usage_perct}%"
if usage_perct >= options.profile_t
unless profile[:missing_nodes].empty?
puts "Warning: these nodes from profile #{profile[:profile]} are not responding:"
puts profile[:missing_nodes].join ', '
puts "Take these recommendations with a grain of salt."
end
puts "Profile '#{profile[:profile]}' has active usage #{ usage_perct }."
if node_additions > 0
puts "Consider adding the #{node_additions} nodes discussed above first, then:"
end
puts <<ADD
To add nodes to this profile, create the nodes and add them to either existing
districts or a new node.
You can create a new district:
oo-admin-ctl-district -c create -n <district name> -p '#{profile[:profile]}'
oo-admin-ctl-district -c add-node -n <district name> -i <node.host.name>
Or choose from the following districts sorted by available capacity:
ADD
o.results[:district_summaries].
select {|d| d[:profile] == profile[:profile]}.
sort_by {|x| x[:dist_avail_capacity]}.
reverse.each_with_index do |district, i|
break if i > 5 # really need more than that to choose from?
puts "district '#{district[:name]}' has available capacity #{district[:dist_avail_capacity]}"
end
end
end