Skip to content

Commit

Permalink
Support a new metric option that might be useful
Browse files Browse the repository at this point in the history 
Signed-off-by: Łukasz Gryglicki <[email protected]>
  • Loading branch information
@lukaszgryglicki
lukaszgryglicki committed Sep 28, 2024
1 parent 8a47718 commit a79343a
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 53 deletions.
5 changes: 3 additions & 2 deletions calc_metric.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,6 @@ cd ../devstats || exit 3
# GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_QOUT=1 GHA2DB_DEBUG=2 GHA2DB_SQLDEBUG=1 GHA2DB_LOCAL=1 GHA2DB_ST=1 GHA2DB_NCPUS=1 ../devstatscode/calc_metric events_hll ./events_hll.sql '2024-01-01 0' '2024-10-01 0' w 'skip_past,hll'
# GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_QOUT=1 GHA2DB_DEBUG=2 GHA2DB_SQLDEBUG=1 GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries.sql '2024-01-01 0' '2024-01-01 0' q 'multivalue,hll,merge_series:prjcntr,drop:sprjcntr' > ../devstatscode/out @>&1
# GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_QOUT=1 GHA2DB_DEBUG=2 GHA2DB_SQLDEBUG=1 GHA2DB_LOCAL=1 GHA2DB_ST=1 GHA2DB_NCPUS=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries.sql '2024-01-01 0' '2024-10-01 0' q 'multivalue,hll,merge_series:prjcntr,drop:sprjcntr'
GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries.sql '2024-01-01 0' '2024-10-01 0' m 'multivalue,hll,merge_series:prjcntr,drop:sprjcntr'
PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries_commiters.sql '2024-01-01 0' '2024-10-01 0' m 'multivalue,hll,merge_series:prjcntr,drop:sprjcntr'
# GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries.sql '2024-01-01 0' '2024-10-01 0' m 'multivalue,hll,merge_series:prjcntr,drop:sprjcntr'
# GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries_commiters.sql '2014-01-01 0' '2024-10-01 0' m 'hll,merge_series:prjcntr,drop:sprjcntr,skip_escape_series_name'
GHA2DB_ENABLE_METRICS_DROP=1 PG_DB=tuf GHA2DB_PROJECT=tuf GHA2DB_LOCAL=1 ../devstatscode/calc_metric multi_row_multi_column ./metrics/shared/project_countries_commiters.sql '2014-01-01 0' '2024-10-01 0' m 'multivalue,hll,skip_escape_series_name,merge_series:prjcntr,drop:sprjcntr'
76 changes: 50 additions & 26 deletions cmd/calc_metric/calc_metric.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,19 @@ import (

// calcMetricData structure to hold metric calculation data
type calcMetricData struct {
hist bool
multivalue bool
escapeValueName bool
annotationsRanges bool
skipPast bool
desc string
mergeSeries string
customData bool
seriesNameMap map[string]string
drop []string
projectScale string
hll bool
hist bool
multivalue bool
escapeValueName bool
skipEscapeSeriesName bool
annotationsRanges bool
skipPast bool
desc string
mergeSeries string
customData bool
seriesNameMap map[string]string
drop []string
projectScale string
hll bool
}

// Global start date & command line to be used to insert data into `gha_last_computed` table.
Expand Down Expand Up @@ -92,7 +93,7 @@ func valueDescription(descFunc string, value float64) (result string) {
// if multivalue is true then rowName is not used for generating series name
// Series name is independent from rowName, and metric returns "series_name;rowName"
// Multivalue series can even have partialy multivalue row: "this_comes_to_multivalues`this_comes_to_series_name", separator is `
func multiRowMultiColumn(cfg *calcMetricData, expr string, multivalue, escapeValueName bool) (result []string) {
func multiRowMultiColumn(cfg *calcMetricData, expr string, multivalue, escapeValueName, skipEscapeSeriesName bool) (result []string) {
ary := strings.Split(expr, ";")
pref := ary[0]
if pref == "" {
Expand All @@ -108,7 +109,12 @@ func multiRowMultiColumn(cfg *calcMetricData, expr string, multivalue, escapeVal
}
rowName = mapName(cfg, rowName)
if len(rowNameAry) > 1 {
rowNameNonMulti := lib.NormalizeName(rowNameAry[1])
var rowNameNonMulti string
if skipEscapeSeriesName {
rowNameNonMulti = rowNameAry[1]
} else {
rowNameNonMulti = lib.NormalizeName(rowNameAry[1])
}
for _, series := range splitColumns {
result = append(result, fmt.Sprintf("%s%s%s;%s", pref, rowNameNonMulti, series, rowName))
}
Expand All @@ -119,7 +125,12 @@ func multiRowMultiColumn(cfg *calcMetricData, expr string, multivalue, escapeVal
}
return
}
rowName := lib.NormalizeName(ary[1])
var rowName string
if skipEscapeSeriesName {
rowName = ary[1]
} else {
rowName = lib.NormalizeName(ary[1])
}
if rowName == "" {
lib.Printf("multiRowMultiColumn: Info: rowName '%v' (%+v) maps to empty string, skipping\n", ary[1], ary)
return
Expand All @@ -137,7 +148,7 @@ func multiRowMultiColumn(cfg *calcMetricData, expr string, multivalue, escapeVal
// if multivalue is true then rowName is not used for generating series name
// Series name is independent from rowName, and metric returns "series_name;rowName"
// Multivalue series can even have partialy multivalue row: "this_comes_to_multivalues`this_comes_to_series_name", separator is `
func multiRowSingleColumn(cfg *calcMetricData, col string, multivalue, escapeValueName bool) (result []string) {
func multiRowSingleColumn(cfg *calcMetricData, col string, multivalue, escapeValueName, skipEscapeSeriesName bool) (result []string) {
ary := strings.Split(col, ",")
pref := ary[0]
if pref == "" {
Expand All @@ -152,12 +163,22 @@ func multiRowSingleColumn(cfg *calcMetricData, col string, multivalue, escapeVal
}
rowName = mapName(cfg, rowName)
if len(rowNameAry) > 1 {
rowNameNonMulti := lib.NormalizeName(rowNameAry[1])
var rowNameNonMulti string
if skipEscapeSeriesName {
rowNameNonMulti = rowNameAry[1]
} else {
rowNameNonMulti = lib.NormalizeName(rowNameAry[1])
}
return []string{fmt.Sprintf("%s%s;%s", pref, rowNameNonMulti, rowName)}
}
return []string{fmt.Sprintf("%s;%s", pref, rowName)}
}
rowName := lib.NormalizeName(ary[1])
var rowName string
if skipEscapeSeriesName {
rowName = ary[1]
} else {
rowName = lib.NormalizeName(ary[1])
}
if rowName == "" {
lib.Printf("multiRowSingleColumn: Info: rowName '%v' (%+v) maps to empty string, skipping\n", ary[1], ary)
return
Expand All @@ -167,14 +188,14 @@ func multiRowSingleColumn(cfg *calcMetricData, col string, multivalue, escapeVal
}

// Generate name for given series row and period
func nameForMetricsRow(cfg *calcMetricData, metric, name string, multivalue, escapeValueName bool) []string {
func nameForMetricsRow(cfg *calcMetricData, metric, name string, multivalue, escapeValueName, skipEscapeSeriesName bool) []string {
switch metric {
case "single_row_multi_column":
return strings.Split(name, ",")
case "multi_row_single_column":
return multiRowSingleColumn(cfg, name, multivalue, escapeValueName)
return multiRowSingleColumn(cfg, name, multivalue, escapeValueName, skipEscapeSeriesName)
case "multi_row_multi_column":
return multiRowMultiColumn(cfg, name, multivalue, escapeValueName)
return multiRowMultiColumn(cfg, name, multivalue, escapeValueName, skipEscapeSeriesName)
default:
lib.Printf("Error\nUnknown metric '%v'\n", metric)
fmt.Fprintf(os.Stdout, "Error\nUnknown metric '%v'\n", metric)
Expand Down Expand Up @@ -293,7 +314,7 @@ func calcSingleHLLRange(
// Get first column name, and using it all series names
// First column should contain nColumns - 1 names separated by ","
name := string(*pValues[0].(*[]uint8))
names := nameForMetricsRow(cfg, seriesNameOrFunc, name, cfg.multivalue, cfg.escapeValueName)
names := nameForMetricsRow(cfg, seriesNameOrFunc, name, cfg.multivalue, cfg.escapeValueName, cfg.skipEscapeSeriesName)
if ctx.Debug > 0 {
lib.Printf("nameForMetricsRow: %s -> %v\n", name, names)
}
Expand Down Expand Up @@ -481,7 +502,7 @@ func calcSingleNumericRange(
// Get first column name, and using it all series names
// First column should contain nColumns - 1 names separated by ","
name := string(*pValues[0].(*sql.RawBytes))
names := nameForMetricsRow(cfg, seriesNameOrFunc, name, cfg.multivalue, cfg.escapeValueName)
names := nameForMetricsRow(cfg, seriesNameOrFunc, name, cfg.multivalue, cfg.escapeValueName, cfg.skipEscapeSeriesName)
if ctx.Debug > 0 {
lib.Printf("nameForMetricsRow: %s -> %v\n", name, names)
}
Expand Down Expand Up @@ -953,7 +974,7 @@ func calcHistogram(ctx *lib.Ctx, seriesNameOrFunc, sqlFile, sqlQuery, excludeBot
// Get row values
lib.FatalOnError(rows.Scan(pValues...))
name := string(*pValues[0].(*sql.RawBytes))
names := nameForMetricsRow(cfg, seriesNameOrFunc, name, cfg.multivalue, false)
names := nameForMetricsRow(cfg, seriesNameOrFunc, name, cfg.multivalue, false, false)
if ctx.Debug > 0 {
lib.Printf("nameForMetricsRow: %s -> %v\n", name, names)
}
Expand Down Expand Up @@ -1227,8 +1248,8 @@ func calcMetric(seriesNameOrFunc, sqlFile, from, to, intervalAbbr string, cfg *c

// Run
lib.Printf(
"calc_metric.go: %s: Running (on %d CPUs): %v - %v with interval %d %s, descriptions '%s', multivalue: %v, escape_value_name: %v, custom_data: %v\n",
sqlFile, thrN, dFrom, dTo, nIntervals, interval, cfg.desc, cfg.multivalue, cfg.escapeValueName, cfg.customData,
"calc_metric.go: %s: Running (on %d CPUs): %v - %v with interval %d %s, descriptions '%s', multivalue: %v, escape_value_name: %v, skip_escape_series_name: %v, custom_data: %v\n",
sqlFile, thrN, dFrom, dTo, nIntervals, interval, cfg.desc, cfg.multivalue, cfg.escapeValueName, cfg.skipEscapeSeriesName, cfg.customData,
)

dt := dFrom
Expand Down Expand Up @@ -1366,6 +1387,9 @@ func main() {
if _, ok := optMap["escape_value_name"]; ok {
cfg.escapeValueName = true
}
if _, ok := optMap["skip_escape_series_name"]; ok {
cfg.skipEscapeSeriesName = true
}
if _, ok := optMap["annotations_ranges"]; ok {
cfg.annotationsRanges = true
}
Expand Down
54 changes: 29 additions & 25 deletions cmd/gha2db_sync/gha2db_sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,31 +20,32 @@ type metrics struct {
// metric contain each metric data
// some metrics can be allowed to fail
type metric struct {
Name string `yaml:"name"`
Periods string `yaml:"periods"`
SeriesNameOrFunc string `yaml:"series_name_or_func"`
MetricSQL string `yaml:"sql"`
MetricSQLs *[]string `yaml:"sqls"`
AddPeriodToName bool `yaml:"add_period_to_name"`
Histogram bool `yaml:"histogram"`
Aggregate string `yaml:"aggregate"`
Skip string `yaml:"skip"`
Desc string `yaml:"desc"`
MultiValue bool `yaml:"multi_value"`
EscapeValueName bool `yaml:"escape_value_name"`
AnnotationsRanges bool `yaml:"annotations_ranges"`
MergeSeries string `yaml:"merge_series"`
CustomData bool `yaml:"custom_data"`
StartFrom *time.Time `yaml:"start_from"`
LastHours int `yaml:"last_hours"`
SeriesNameMap map[string]string `yaml:"series_name_map"`
EnvMap map[string]string `yaml:"env"`
Disabled bool `yaml:"disabled"`
Drop string `yaml:"drop"`
Project string `yaml:"project"`
AllowFail bool `yaml:"allow_fail"`
WaitAfterFail int `yaml:"wait_after_fail"`
HLL bool `yaml:"hll"`
Name string `yaml:"name"`
Periods string `yaml:"periods"`
SeriesNameOrFunc string `yaml:"series_name_or_func"`
MetricSQL string `yaml:"sql"`
MetricSQLs *[]string `yaml:"sqls"`
AddPeriodToName bool `yaml:"add_period_to_name"`
Histogram bool `yaml:"histogram"`
Aggregate string `yaml:"aggregate"`
Skip string `yaml:"skip"`
Desc string `yaml:"desc"`
MultiValue bool `yaml:"multi_value"`
EscapeValueName bool `yaml:"escape_value_name"`
SkipEscapeSeriesName bool `yaml:"skip_escape_series_name"`
AnnotationsRanges bool `yaml:"annotations_ranges"`
MergeSeries string `yaml:"merge_series"`
CustomData bool `yaml:"custom_data"`
StartFrom *time.Time `yaml:"start_from"`
LastHours int `yaml:"last_hours"`
SeriesNameMap map[string]string `yaml:"series_name_map"`
EnvMap map[string]string `yaml:"env"`
Disabled bool `yaml:"disabled"`
Drop string `yaml:"drop"`
Project string `yaml:"project"`
AllowFail bool `yaml:"allow_fail"`
WaitAfterFail int `yaml:"wait_after_fail"`
HLL bool `yaml:"hll"`
}

// randomize - shufflues array of metrics to calculate, making sure that ctx.LastSeries is still last
Expand Down Expand Up @@ -552,6 +553,9 @@ func sync(ctx *lib.Ctx, args []string) {
if metric.EscapeValueName {
extraParams = append(extraParams, "escape_value_name")
}
if metric.SkipEscapeSeriesName {
extraParams = append(extraParams, "skip_escape_series_name")
}
if metric.Desc != "" {
extraParams = append(extraParams, "desc:"+metric.Desc)
}
Expand Down

0 comments on commit a79343a

Please sign in to comment.