From 343bb06084a2a9e96d48b57d16892440055b5723 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 3 Aug 2015 18:04:37 +0200 Subject: [PATCH 01/80] experiments to get an SQLite'ish create schema --- sql/benchmark.sqlite | 165 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 sql/benchmark.sqlite diff --git a/sql/benchmark.sqlite b/sql/benchmark.sqlite new file mode 100644 index 0000000..9fb3579 --- /dev/null +++ b/sql/benchmark.sqlite @@ -0,0 +1,165 @@ +DROP TABLE IF EXISTS `bench_backup_additional_relations`; +DROP TABLE IF EXISTS `bench_backup_values`; +DROP TABLE IF EXISTS `bench_additional_relations`; +DROP TABLE IF EXISTS `bench_additional_type_relations`; +DROP TABLE IF EXISTS `bench_additional_values`; +DROP TABLE IF EXISTS `bench_additional_types`; +DROP TABLE IF EXISTS `bench_values`; +DROP TABLE IF EXISTS `benchs`; +DROP TABLE IF EXISTS `bench_units`; +DROP TABLE IF EXISTS `bench_subsume_types`; + +CREATE TABLE `bench_subsume_types` ( + `bench_subsume_type_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_subsume_type` VARCHAR(32) NOT NULL , -- COMMENT 'unique string identifier', + `bench_subsume_type_rank` TINYINT(3) NOT NULL , -- COMMENT 'subsume type order', + `datetime_strftime_pattern` VARCHAR(32) NULL , -- COMMENT 'format pattern for per DateTime->strftime for grouping', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' +) ; -- COMMENT 'types of subsume values'; + +CREATE TABLE `bench_units` ( + `bench_unit_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_unit` VARCHAR(64) NOT NULL , -- COMMENT 'unique string identifier', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date', +) ; -- COMMENT 'units for benchmark data points'; + +CREATE TABLE `benchs` ( + `bench_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_unit_id` TINYINT(3) NULL, + `bench` VARCHAR(64) NOT NULL , -- COMMENT 'unique string identifier', + `active` TINYINT(3) NOT NULL , -- COMMENT 'is entry still active (1=yes,0=no)', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + CONSTRAINT `fk_benchs_01` + FOREIGN KEY (`bench_unit_id`) + REFERENCES `bench_units` (`bench_unit_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'containg benchmark head data'; + +CREATE TABLE `bench_values` ( + `bench_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_id` INT(10) NOT NULL , -- COMMENT 'FK to benchs', + `bench_subsume_type_id` TINYINT(3) NOT NULL , -- COMMENT 'FK to bench_subsume_types', + `bench_value` FLOAT NULL , -- COMMENT 'value for bench data point', + `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + CONSTRAINT `fk_bench_values_01` + FOREIGN KEY (`bench_id`) + REFERENCES `benchs` (`bench_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_values_02` + FOREIGN KEY (`bench_subsume_type_id`) + REFERENCES `bench_subsume_types` (`bench_subsume_type_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'containing data points for benchmark'; + +CREATE TABLE `bench_additional_types` ( + `bench_additional_type_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_additional_type` VARCHAR(32) NOT NULL , -- COMMENT 'unique string identifier', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' +) ; -- COMMENT 'types of additional values for benchmark data points'; + +CREATE TABLE `bench_additional_values` ( + `bench_additional_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_additional_type_id` SMALLINT(5) NOT NULL , -- COMMENT 'FK to bench_additional_types', + `bench_additional_value` VARCHAR(128) NOT NULL , -- COMMENT 'additional value', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + CONSTRAINT `fk_bench_additional_values_01` + FOREIGN KEY (`bench_additional_type_id`) + REFERENCES `bench_additional_types` (`bench_additional_type_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'additional values for benchmark data point'; + +CREATE TABLE `bench_additional_type_relations` ( + `bench_id` INT(10) NOT NULL , -- COMMENT 'FK to benchs (PK)', + `bench_additional_type_id` SMALLINT(5) NOT NULL , -- COMMENT 'FK to bench_additional_types (PK)', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + PRIMARY KEY (`bench_id`,`bench_additional_type_id`), + CONSTRAINT `fk_bench_additional_type_relations_01` + FOREIGN KEY (`bench_id`) + REFERENCES `benchs` (`bench_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_additional_type_relations_02` + FOREIGN KEY (`bench_additional_type_id`) + REFERENCES `bench_additional_types` (`bench_additional_type_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'additional values for benchmark data point'; + +CREATE TABLE `bench_additional_relations` ( + `bench_value_id` INT(10) NOT NULL , -- COMMENT 'FK to bench_values', + `bench_additional_value_id` INT(10) NOT NULL , -- COMMENT 'FK to bench_additional_values', + `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + PRIMARY KEY (`bench_value_id`,`bench_additional_value_id`), + CONSTRAINT `fk_bench_additional_relations_01` + FOREIGN KEY (`bench_value_id`) + REFERENCES `bench_values` (`bench_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_additional_relations_02` + FOREIGN KEY (`bench_additional_value_id`) + REFERENCES `bench_additional_values` (`bench_additional_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'add additional values to benchmarks'; + +CREATE TABLE `bench_backup_values` ( + `bench_backup_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_value_id` int(10) NOT NULL , -- COMMENT 'FK to bench_values', + `bench_id` int(10) NOT NULL , -- COMMENT 'FK to benchs', + `bench_subsume_type_id` tinyint(3) NOT NULL , -- COMMENT 'FK to bench_subsume_types', + `bench_value` float DEFAULT NULL , -- COMMENT 'value for bench data point', + `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + CONSTRAINT `fk_bench_backup_values_01` + FOREIGN KEY (`bench_id`) + REFERENCES `benchs` (`bench_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_backup_values_02` + FOREIGN KEY (`bench_subsume_type_id`) + REFERENCES `bench_subsume_types` (`bench_subsume_type_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_backup_values_03` + FOREIGN KEY (`bench_value_id`) + REFERENCES `bench_values` (`bench_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT='backup table for data points for benchmark'; + +CREATE TABLE `bench_backup_additional_relations` ( + `bench_backup_value_id` int(10) NOT NULL , -- COMMENT 'FK to bench_backup_values', + `bench_additional_value_id` int(10) NOT NULL , -- COMMENT 'FK to bench_additional_values', + `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + PRIMARY KEY (`bench_backup_value_id`,`bench_additional_value_id`), + CONSTRAINT `fk_bench_backup_additional_relations_01` + FOREIGN KEY (`bench_backup_value_id`) + REFERENCES `bench_backup_values` (`bench_backup_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_backup_additional_relations_02` + FOREIGN KEY (`bench_additional_value_id`) + REFERENCES `bench_additional_values` (`bench_additional_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT='add additional values to benchmarks'; + +INSERT INTO bench_subsume_types + ( bench_subsume_type, bench_subsume_type_rank, datetime_strftime_pattern, created_at ) +VALUES + ( 'atomic' , 1, NULL , datetime() ), + ( 'second' , 2, '%Y%m%d%H%M%S' , datetime() ), + ( 'minute' , 3, '%Y%m%d%H%M' , datetime() ), + ( 'hour' , 4, '%Y%m%d%H' , datetime() ), + ( 'day' , 5, '%Y%m%d' , datetime() ), + ( 'week' , 6, '%Y%W' , datetime() ), + ( 'month' , 7, '%Y%m' , datetime() ), + ( 'year' , 8, '%Y' , datetime() ) +; \ No newline at end of file From 842ae5a75870880ec2516f91efcb2d7b431c0efe Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 4 Aug 2015 10:17:37 +0200 Subject: [PATCH 02/80] dist.ini: explicit github location --- dist.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dist.ini b/dist.ini index cd4c36e..5343493 100644 --- a/dist.ini +++ b/dist.ini @@ -3,6 +3,8 @@ author = Roberto Schaefer license = FreeBSD copyright_holder = Amazon.com, Inc. or its affiliates -[@TAPPER] +[@SCHWIGON] dist = Tapper-Benchmark repository_at = github +repository = git://github.com/benchmarkanything/Tapper-Benchmark.git +repository_web = http://github.com/benchmarkanything/Tapper-Benchmark From a436903d7816bcd07a7e5465180c6681aba3905a Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 4 Aug 2015 10:17:48 +0200 Subject: [PATCH 03/80] .gitignore --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..28a94fb --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.* +!.gitignore +Tapper-Benchmark-* +Debian_CPANTS.txt +nytprof* From 8814e4c154579f716723e5f3b9d9193e5390e5e4 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 11 Aug 2015 10:12:31 +0200 Subject: [PATCH 04/80] dzil: use @TAPPER bundle ...to correctly handle explicit LICENSE file. --- dist.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist.ini b/dist.ini index 5343493..4e60455 100644 --- a/dist.ini +++ b/dist.ini @@ -3,7 +3,7 @@ author = Roberto Schaefer license = FreeBSD copyright_holder = Amazon.com, Inc. or its affiliates -[@SCHWIGON] +[@TAPPER] dist = Tapper-Benchmark repository_at = github repository = git://github.com/benchmarkanything/Tapper-Benchmark.git From b8a4cad70de29dc3232056599a6598bf085976a8 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 11 Aug 2015 10:13:35 +0200 Subject: [PATCH 05/80] docs: fix POD glitches Basically all =over's were unclosed, missing =back. Fixed that. --- lib/Tapper/Benchmark.pm | 47 ++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 2d5fbb0..34ccefc 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -680,7 +680,7 @@ way to the the database. A search function with complexe filters already exists. =head3 new -=over +=over 4 =item @@ -692,7 +692,7 @@ Create a new B object. config => YAML::Syck::LoadFile('~/conf/tapper_benchmark.conf'), }); -=over +=over 4 =item dbh @@ -710,9 +710,12 @@ written to STDOUT. The default is 0. =back +=back + + =head3 add_single_benchmark -=over +=over 4 =item @@ -739,7 +742,7 @@ Add one or more data points to a single benchmark to the database. force => 1 }); -=over +=over 4 =item 1. Parameter Hash => NAME @@ -761,9 +764,11 @@ Ignore forgivable errors while writing. =back +=back + =head3 add_multi_benchmark -=over +=over 4 =item @@ -791,7 +796,7 @@ Add one or more data points for multiple benchmarks to the database. force => 1 }); -=over +=over 4 =item 1. Parameter Array of Hashes => NAME @@ -815,9 +820,11 @@ Ignore forgivable errors while writing. =back +=back + =head3 search -=over +=over 4 =item @@ -847,7 +854,7 @@ Statement Handle. ], }); -=over +=over 4 =item select [optional] @@ -973,9 +980,11 @@ An integer value which determine the number of omitted benchmark data points. =back +=back + =head3 search_array -=over +=over 4 =item @@ -1002,7 +1011,7 @@ Returning all benchmark data points as Array of Hashes. =head3 search_hash -=over +=over 4 =item @@ -1033,9 +1042,11 @@ Every "key" create a new nested hash. ], }); +=back + =head3 subsume -=over +=over 4 =item @@ -1054,7 +1065,7 @@ It is highly recommended to do this periodically for better search performance. backup => 0, }); -=over +=over 4 =item subsume_type @@ -1088,15 +1099,17 @@ isn't desired a false value must be passed. =back +=back + =head1 Configuration -=over +=over 4 =item The following elements are required in configuration: -=over +=over 4 =item default_aggregation @@ -1122,4 +1135,8 @@ Containing the names of the tables used bei B =item select_cache [optional] -In case of a true value the module cache some select results \ No newline at end of file +In case of a true value the module cache some select results + +=back + +=back From e6d83f80a6d0cf55abfd8529a423ba0eb4650db8 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 11 Aug 2015 10:22:50 +0200 Subject: [PATCH 06/80] docs: fix confusing enumeration The items seemed to look like a numbered list but in fact meant the two hashes to be provided. Reowrd this to use '1st hash' and '2nd hash' making it less confusing. --- lib/Tapper/Benchmark.pm | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 34ccefc..734085d 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -744,21 +744,21 @@ Add one or more data points to a single benchmark to the database. =over 4 -=item 1. Parameter Hash => NAME +=item 1st Parameter Hash => NAME The name of the benchmark for grouping benchmark data points. -=item 1. Parameter Hash => data +=item 1st Parameter Hash => data This parameter contains the benchmark data points. It's an array of hashes. The element C is the only required element in this hashes. The C is the benchmark data point value. -=item 1. Parameter Hash => UNIT [optional] +=item 1st Parameter Hash => UNIT [optional] Containing a unit for benchmark data point values. -=item 2. Parameter Hash => force [optional] +=item 2nd Parameter Hash => force [optional] Ignore forgivable errors while writing. @@ -798,23 +798,23 @@ Add one or more data points for multiple benchmarks to the database. =over 4 -=item 1. Parameter Array of Hashes => NAME +=item 1st Parameter Array of Hashes => NAME The name of the benchmark for grouping benchmark data points. -=item 1. Parameter Hash => VALUE +=item 1st Parameter Hash => VALUE The value is the benchmark data point value. -=item 1. Parameter Hash => UNIT [optional] +=item 1st Parameter Hash => UNIT [optional] Containing a unit for benchmark data point values. -=item 1. Parameter Hash => all others +=item 1st Parameter Hash => all others All other elements in the hashes are additional values added to this data point. -=item 2. Parameter Hash => force [optional] +=item 2nd Parameter Hash => force [optional] Ignore forgivable errors while writing. From 87757d919eb9ec0f5137f2b44508daff15be88ad Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 11 Aug 2015 10:50:21 +0200 Subject: [PATCH 07/80] docs: provide podweaver ABSTRACTs --- lib/Tapper/Benchmark.pm | 1 + lib/Tapper/Benchmark/Query.pm | 9 +-------- lib/Tapper/Benchmark/Query/mysql.pm | 9 +-------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 734085d..9f89008 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -1,4 +1,5 @@ package Tapper::Benchmark; +# ABSTRACT: Autonomous SQL backend to store benchmarks use strict; use warnings; diff --git a/lib/Tapper/Benchmark/Query.pm b/lib/Tapper/Benchmark/Query.pm index a3f99ba..ca745b8 100644 --- a/lib/Tapper/Benchmark/Query.pm +++ b/lib/Tapper/Benchmark/Query.pm @@ -1,4 +1,5 @@ package Tapper::Benchmark::Query; +# ABSTRACT: Tapper::Benchmark - qerying - base class use strict; use warnings; @@ -106,11 +107,3 @@ sub finish_transaction { } 1; - -__END__ - -=pod - -=head1 NAME - -Tapper::Benchmark::Query - Base class for the database work used by Tapper::Benchmark \ No newline at end of file diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index c00c690..c12f6a0 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -1,4 +1,5 @@ package Tapper::Benchmark::Query::mysql; +# ABSTRACT: Tapper::Benchmark - qerying - MySQL backend use strict; use warnings; @@ -830,11 +831,3 @@ sub delete_benchmark_value { } 1; - -__END__ - -=pod - -=head1 NAME - -Tapper::Benchmark::Query::mysql - Base class for the database work used by Tapper::Benchmark when MySQL is used \ No newline at end of file From 1ef1bccad2296c158ed3205a18b6ff802541cbd8 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 11 Aug 2015 12:00:30 +0200 Subject: [PATCH 08/80] rename DB files according to DBD driver ...so we can automatically correlate the correct file. --- sql/{benchmark.sqlite => benchmark.SQLite} | 0 sql/{benchmark.sql => benchmark.mysql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename sql/{benchmark.sqlite => benchmark.SQLite} (100%) rename sql/{benchmark.sql => benchmark.mysql} (100%) diff --git a/sql/benchmark.sqlite b/sql/benchmark.SQLite similarity index 100% rename from sql/benchmark.sqlite rename to sql/benchmark.SQLite diff --git a/sql/benchmark.sql b/sql/benchmark.mysql similarity index 100% rename from sql/benchmark.sql rename to sql/benchmark.mysql From 0582ccd79124ba3b30c908dfcbe5cc709aae0997 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 11 Aug 2015 15:11:59 +0200 Subject: [PATCH 09/80] mv schema DB files to share/ ...so they get available using File::ShareDir, like for creating the DB with cmdline tools. --- .../tapper-benchmark-create-schema.SQLite | 0 sql/benchmark.mysql => share/tapper-benchmark-create-schema.mysql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename sql/benchmark.SQLite => share/tapper-benchmark-create-schema.SQLite (100%) rename sql/benchmark.mysql => share/tapper-benchmark-create-schema.mysql (100%) diff --git a/sql/benchmark.SQLite b/share/tapper-benchmark-create-schema.SQLite similarity index 100% rename from sql/benchmark.SQLite rename to share/tapper-benchmark-create-schema.SQLite diff --git a/sql/benchmark.mysql b/share/tapper-benchmark-create-schema.mysql similarity index 100% rename from sql/benchmark.mysql rename to share/tapper-benchmark-create-schema.mysql From d647693b17b9575eb85b0a75cb0f518b4b6c5b0f Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 11 Aug 2015 15:12:51 +0200 Subject: [PATCH 10/80] dzil: disable POD coverage tests --- dist.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/dist.ini b/dist.ini index 4e60455..dbda691 100644 --- a/dist.ini +++ b/dist.ini @@ -8,3 +8,4 @@ dist = Tapper-Benchmark repository_at = github repository = git://github.com/benchmarkanything/Tapper-Benchmark.git repository_web = http://github.com/benchmarkanything/Tapper-Benchmark +disable_pod_coverage_tests = 1 From 2255b9b88c37b725d0fcbd8581f6d2739daffa96 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 11 Aug 2015 15:16:10 +0200 Subject: [PATCH 11/80] query backend: SQLite Basically copy'n'pasted from ::Query::mysql, and replaced NOW() with CURRENT_TIMESTAMP. More changes are expected to come while reverse engineering how SQLite differs from MySQL... :-) --- lib/Tapper/Benchmark/Query/SQLite.pm | 836 +++++++++++++++++++++++++++ 1 file changed, 836 insertions(+) create mode 100644 lib/Tapper/Benchmark/Query/SQLite.pm diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm new file mode 100644 index 0000000..c16c53d --- /dev/null +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -0,0 +1,836 @@ +package Tapper::Benchmark::Query::SQLite; +# ABSTRACT: Tapper::Benchmark - qerying - SQLite backend + +use strict; +use warnings; +use feature 'switch'; +use base 'Tapper::Benchmark::Query'; + +use List::MoreUtils qw( any ); + +my %h_used_selects; +my %h_default_columns = ( + 'NAME' => 'b.bench', + 'UNIT' => 'bu.bench_unit', + 'VALUE' => 'bv.bench_value', + 'VALUE_ID' => 'bv.bench_value_id', + 'CREATED' => 'bv.created_at', +); + +sub default_columns { + return %h_default_columns; +} + +sub benchmark_operators { + return ( '=', '!=', 'like', 'not like', '<', '>', '<=', '>=' ); +} + +sub create_where_clause { + + my ( $s_column_name, $ar_value ) = @_; + + my $s_where_clause = q##; + if ( $ar_value->[0] eq 'not like' ) { + $s_where_clause = "$s_column_name NOT LIKE ?"; + } + elsif ( $ar_value->[0] eq 'like' ) { + $s_where_clause = "$s_column_name LIKE ?"; + } + elsif ( + $ar_value->[0] eq '<' + || $ar_value->[0] eq '>' + || $ar_value->[0] eq '<=' + || $ar_value->[0] eq '>=' + ) { + $s_where_clause = "$s_column_name $ar_value->[0] ?"; + } + elsif ( $ar_value->[0] eq '=' ) { + if ( $#{$ar_value} > 1 ) { + $s_where_clause = "$s_column_name IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; + } + else { + $s_where_clause = "$s_column_name = ?"; + } + } + elsif ( $ar_value->[0] eq '!=' ) { + if ( $#{$ar_value} > 1 ) { + $s_where_clause = "$s_column_name NOT IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; + } + else { + $s_where_clause = "$s_column_name != ?"; + } + } + else { + require Carp; + Carp::confess("unknown operator '$ar_value->[0]'"); + return; + } + + return $s_where_clause; + +} + +sub create_select_column { + + my ( $or_self, $ar_select, $i_counter, $b_aggregate_all ) = @_; + + my $s_aggr_func = q##; + my ( $s_aggr, $s_column ) = @{$ar_select}; + my $s_return_select = q##; + + AGGR: { + given( $s_aggr ) { + when ( q## ) { + # aggregate all columns if a single column is aggregated + if ( $b_aggregate_all ) { + $s_aggr = $or_self->{config}{default_aggregation}; + redo AGGR; + } + $s_return_select = '${COLUMN}'; + } + when ( 'min' ) { + $s_return_select = 'MIN( ${COLUMN} )'; + } + when ( 'max' ) { + $s_return_select = 'MAX( ${COLUMN} )'; + } + when ( 'avg' ) { + $s_return_select = 'AVG( ${COLUMN} )'; + } + # Geometric Mean, unsupported in SQLite due to lack of EXP(), + # see http://stackoverflow.com/questions/13190064/how-to-find-power-of-a-number-in-sqlite + # + # when ( 'gem' ) { + # $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; + # } + when ( 'sum' ) { + $s_return_select = 'SUM( ${COLUMN} )'; + } + when ( 'cnt' ) { + $s_return_select = 'COUNT( ${COLUMN} )'; + } + when ( 'cnd' ) { + $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; + } + default { + require Carp; + Carp::confess("unknown aggregate function '$s_aggr'"); + return; + } + } + } # AGGR + + my ( $s_return_column ); + my $s_replace_as = $s_aggr ? $s_aggr . "_$s_column" : $s_column; + + if ( $h_used_selects{$or_self}{$s_replace_as} ) { + return; + } + if ( any { $s_column eq $_ } keys %h_default_columns ) { + $h_used_selects{$or_self}{$s_replace_as} = $h_default_columns{$s_column}; + } + else { + $s_return_column = $s_column; + $h_used_selects{$or_self}{$s_replace_as} = "bav$i_counter.bench_additional_value"; + } + + $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; + + return ( $s_return_column, "$s_return_select AS $s_replace_as", ); + +} + +sub create_period_check { + + my ( $s_column, $dt_from, $dt_to ) = @_; + + my @a_vals; + my $s_where; + if ( $dt_from ) { + if ( my ( $s_date, $s_time ) = $dt_from =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { + $s_where .= "\nAND $s_column > ?"; + push @a_vals, $s_date . ( $s_time || ' 00:00:00' ); + } + else { + require Carp; + Carp::confess(q#unknown date format for 'date_from'#); + return; + } + } + if ( $dt_to ) { + if ( my ( $s_date, $s_time ) = $dt_to =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { + $s_where .= "\nAND $s_column < ?"; + push @a_vals, $s_date . ( $s_time || ' 23:59:59' ); + } + else { + require Carp; + Carp::confess(q#unknown date format for 'date_to'#); + return; + } + } + + return { + vals => \@a_vals, + where => $s_where, + }; + +} + +sub select_benchmark_values { + + my ( $or_self, $hr_search ) = @_; + + # clear selected columns + $h_used_selects{$or_self} = {}; + + # deep copy hash + require JSON::XS; + $hr_search = JSON::XS::decode_json( + JSON::XS::encode_json( $hr_search ) + ); + + my ( + $s_limit, + $s_offset, + $s_order_by, + @a_select, + @a_from, + @a_from_vals, + @a_where, + @a_where_vals, + ) = ( + q##, + q##, + q##, + ); + + # limit clause + if ( $hr_search->{limit} ) { + if ( $hr_search->{limit} =~ /^\d+$/ ) { + $s_limit = "LIMIT $hr_search->{limit}"; + } + else { + require Carp; + Carp::confess("invalid limit value '$hr_search->{limit}'"); + return; + } + } + + # offset clause + if ( $hr_search->{offset} ) { + if ( $hr_search->{offset} =~ /^\d+$/ ) { + $s_offset = "OFFSET $hr_search->{offset}"; + } + else { + require Carp; + Carp::confess("invalid offset value '$hr_search->{offset}'"); + return; + } + } + + # where clause + my $i_counter = 0; + if ( $hr_search->{where} ) { + + for my $ar_where ( @{$hr_search->{where}} ) { + if ( any { $ar_where->[1] eq $_ } keys %h_default_columns ) { + my $s_column = splice( @{$ar_where}, 1, 1 ); + push @a_where, create_where_clause( $h_default_columns{$s_column}, $ar_where ); + push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; + } + else { + my $hr_additional_type = $or_self + ->select_addtype_by_name( splice( @{$ar_where}, 1, 1 ) ) + ->fetchrow_hashref() + ; + if ( !$hr_additional_type || !$hr_additional_type->{bench_additional_type_id} ) { + require Carp; + Carp::confess("benchmark additional value '$ar_where->[1]' not exists"); + return; + } + push @a_from, " + JOIN ( + $or_self->{config}{tables}{additional_relation_table} bar$i_counter + JOIN $or_self->{config}{tables}{additional_value_table} bav$i_counter + ON ( bav$i_counter.bench_additional_value_id = bar$i_counter.bench_additional_value_id ) + ) + ON ( + bar$i_counter.bench_value_id = bv.bench_value_id + AND bav$i_counter.bench_additional_type_id = ? + ) + "; + push @a_from_vals, $hr_additional_type->{bench_additional_type_id}; + push @a_where, create_where_clause( "bav$i_counter.bench_additional_value", $ar_where ); + push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; + $i_counter++; + } + } + } + + # select clause + my $b_aggregate_all = 0; + if ( $hr_search->{select} ) { + for my $i_counter ( 0..$#{$hr_search->{select}} ) { + if ( ref $hr_search->{select}[$i_counter] ne 'ARRAY' ) { + $hr_search->{select}[$i_counter] = ['',$hr_search->{select}[$i_counter]]; + } + elsif ( !$b_aggregate_all && $hr_search->{select}[$i_counter][0] ne q## ) { + $b_aggregate_all = 1; + for my $s_clause (qw/ order_by limit offset /) { + if ( $hr_search->{$s_clause} ) { + require Carp; + Carp::confess("cannot use '$s_clause' with aggregation"); + } + } + } + } + } + push @{$hr_search->{select} ||= []}, map {['',$_]} keys %h_default_columns; + + for my $ar_select ( @{$hr_search->{select}} ) { + + my ( $s_column, $s_select ) = $or_self->create_select_column( + $ar_select, $i_counter, $b_aggregate_all, + ); + + if ( $s_select ) { + + push @a_select, $s_select; + + if ( $s_column ) { + + my $hr_additional_type = $or_self + ->select_addtype_by_name( $s_column ) + ->fetchrow_hashref() + ; + if ( !$hr_additional_type || !$hr_additional_type->{bench_additional_type_id} ) { + require Carp; + Carp::confess("benchmark additional value '$s_column' not exists"); + return; + } + + push @a_from_vals, $hr_additional_type->{bench_additional_type_id}; + push @a_from, " + LEFT JOIN ( + $or_self->{config}{tables}{additional_relation_table} bar$i_counter + JOIN $or_self->{config}{tables}{additional_value_table} bav$i_counter + ON ( bav$i_counter.bench_additional_value_id = bar$i_counter.bench_additional_value_id ) + ) + ON ( + bar$i_counter.bench_value_id = bv.bench_value_id + AND bav$i_counter.bench_additional_type_id = ? + ) + "; + $i_counter++; + } + } + + } + + # order_by clause + if ( $hr_search->{order_by} ) { + my @a_order_by_possible = keys %h_default_columns; + my @a_order_by_direction = qw/ ASC DESC /; + if ( $hr_search->{select} ) { + push @a_order_by_possible, map { $_->[1] } @{$hr_search->{select}}; + } + my @a_order_by; + for my $order_column ( @{$hr_search->{order_by}} ) { + if ( ref $order_column ) { + if ( any { $order_column->[0] eq $_ } @a_order_by_possible ) { + if ( any { $order_column->[1] eq $_ } @a_order_by_direction ) { + my $s_numeric_cast = q##; + if ( $order_column->[2] && $order_column->[2]{numeric} ) { + $s_numeric_cast = '0 + '; + } + if ( any { $order_column->[0] eq $_ } keys %h_default_columns ) { + push @a_order_by, "$s_numeric_cast$h_default_columns{$order_column->[0]} $order_column->[1]"; + } + else { + push @a_order_by, "$s_numeric_cast$order_column->[0] $order_column->[1]"; + } + } + else { + require Carp; + Carp::confess("unknown order by direction '$order_column->[1]'"); + return; + } + } + else { + require Carp; + Carp::confess("unknown order by column '$order_column->[0]'"); + return; + } + } + else { + if ( any { $order_column eq $_ } @a_order_by_possible ) { + if ( any { $order_column eq $_ } keys %h_default_columns ) { + push @a_order_by, "$h_default_columns{$order_column} ASC"; + } + else { + push @a_order_by, "$order_column ASC"; + } + } + else { + require Carp; + Carp::confess("unknown order by column '$order_column'"); + return; + } + } + } + $s_order_by = 'ORDER BY ' . (join ', ', @a_order_by) + } + + # replace placeholders inside of raw sql where clause + my $s_raw_where = $hr_search->{where_sql}; + if ( $s_raw_where ) { + $s_raw_where =~ s/ + \${(.+?)} + / + $h_used_selects{$or_self}{$1} + ? $h_used_selects{$or_self}{$1} + : die "column '$1' not exists in SELECT clause" + /gex; + } + + return $or_self->execute_query( + " + SELECT + " . ( join ",\n", map {"$_"} @a_select ) . " + FROM + $or_self->{config}{tables}{benchmark_table} b + JOIN $or_self->{config}{tables}{benchmark_value_table} bv + ON ( bv.bench_id = b.bench_id ) + LEFT JOIN $or_self->{config}{tables}{unit_table} bu + ON ( bu.bench_unit_id = b.bench_unit_id ) + " . ( join "\n", @a_from ) . " + WHERE + b.active = 1 + AND bv.active = 1 + " . + ( @a_where ? join "\n", map { "AND $_" } @a_where : q## ) . + ( $s_raw_where ? " $s_raw_where" : q## ) . + " + $s_order_by + $s_limit + $s_offset + ", + @a_from_vals, + @a_where_vals, + ); + +} + +sub select_addtype_by_name { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_type_id + FROM $or_self->{config}{tables}{additional_type_table} + WHERE bench_additional_type = ? + ", @a_vals ); + +} + +sub select_min_subsume_type { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_subsume_type_id + FROM $or_self->{config}{tables}{subsume_type_table} + ORDER BY bench_subsume_type_rank ASC + LIMIT 1 + " ); + +} + +sub select_subsume_type { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT + bench_subsume_type_id, + bench_subsume_type_rank, + datetime_strftime_pattern + FROM + $or_self->{config}{tables}{subsume_type_table} + WHERE + bench_subsume_type = ? + ", @a_vals ); + +} + +sub select_check_subsumed_values { + + my ( $or_self, $hr_vals ) = @_; + + if (! $hr_vals->{subsume_type_id} ) { + require Carp; + Carp::confess(q#required parameter 'subsume_type_id' is missing#); + return; + } + + my $hr_period_check = create_period_check( + 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} + ); + + return $or_self->execute_query( + " + SELECT + bv.bench_value_id + FROM + bench_values bv + JOIN bench_subsume_types bet + ON ( bv.bench_subsume_type_id = bet.bench_subsume_type_id ) + WHERE + bet.bench_subsume_type_rank > ( + SELECT beti.bench_subsume_type_rank + FROM bench_subsume_types beti + WHERE bench_subsume_type_id = ? + ) + $hr_period_check->{where} + LIMIT + 1 + ", + $hr_vals->{subsume_type_id}, + @{$hr_period_check->{vals}}, + ); + +} + +sub select_data_values_for_subsume { + + my ( $or_self, $hr_vals ) = @_; + + my $hr_period_check = create_period_check( + 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} + ); + + my @a_addexclude_vals; + my $s_addexclude_where = q##; + if ( $hr_vals->{exclude_additionals} && @{$hr_vals->{exclude_additionals}} ) { + $s_addexclude_where = 'AND bav.bench_additional_type_id NOT IN (' . (join ',', map {'?'} @{$hr_vals->{exclude_additionals}}) . ')'; + push @a_addexclude_vals, @{$hr_vals->{exclude_additionals}}; + } + + return $or_self->execute_query( + " + SELECT + b.bench_id, + bv.bench_value_id, + bv.created_at, + bv.bench_value, + bet.bench_subsume_type_rank, + GROUP_CONCAT( + bav.bench_additional_type_id, + '|', + bav.bench_additional_value_id + ORDER BY + bav.bench_additional_type_id, + bav.bench_additional_value_id + SEPARATOR + '-' + ) AS additionals + FROM + benchs b + JOIN bench_values bv + ON ( bv.bench_id = b.bench_id ) + JOIN bench_subsume_types bet + ON ( bet.bench_subsume_type_id = bv.bench_subsume_type_id ) + LEFT JOIN ( + bench_additional_relations bar + JOIN bench_additional_values bav + ON ( bav.bench_additional_value_id = bar.bench_additional_value_id ) + ) + ON ( + bar.active = 1 + AND bar.bench_value_id = bv.bench_value_id + $s_addexclude_where + ) + WHERE + b.active = 1 + AND bv.active = 1 + $hr_period_check->{where} + GROUP BY + bet.bench_subsume_type_rank, + b.bench_id, + bv.created_at, + bv.bench_value, + bv.bench_value_id + ORDER BY + b.bench_id, + additionals, + bv.created_at + ", + @a_addexclude_vals, + @{$hr_period_check->{vals}}, + ); +} + +sub select_benchmark { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_id + FROM $or_self->{config}{tables}{benchmark_table} + WHERE bench = ? + ", @a_vals ); + +} + +sub select_unit { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_unit_id + FROM $or_self->{config}{tables}{unit_table} + WHERE bench_unit = ? + ", @a_vals ); + +} + +sub select_addtype { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_type_id + FROM $or_self->{config}{tables}{additional_type_table} + WHERE bench_additional_type = ? + ", @a_vals ); + +} + +sub select_addvalue { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_value_id + FROM $or_self->{config}{tables}{additional_value_table} + WHERE bench_additional_type_id = ? AND bench_additional_value = ? + ", @a_vals ); + +} + +sub select_addtyperelation { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_id, bench_additional_type_id, created_at + FROM $or_self->{config}{tables}{additional_type_relation_table} + WHERE bench_id = ? AND bench_additional_type_id = ? + ", @a_vals ); + +} + +sub insert_addtyperelation { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{additional_type_relation_table} + ( bench_id, bench_additional_type_id, created_at ) + VALUES + ( ?, ?, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_unit { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{unit_table} + ( bench_unit, created_at ) + VALUES + ( ?, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_benchmark { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{benchmark_table} + ( bench, bench_unit_id, active, created_at ) + VALUES + ( ?, ?, 1, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_benchmark_value { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{benchmark_value_table} + ( bench_id, bench_subsume_type_id, bench_value, active, created_at ) + VALUES + ( ?, ?, ?, 1, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub copy_additional_values { + + my ( $or_self, $hr_vals ) = @_; + + for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { + if (! $hr_vals->{$s_param} ) { + require Carp; + Carp::confess("missing parameter '$s_param'"); + return; + } + } + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{additional_relation_table} + ( bench_value_id, bench_additional_value_id, active, created_at ) + SELECT + ?, bench_additional_value_id, 1, CURRENT_TIMESTAMP + FROM + $or_self->{config}{tables}{additional_relation_table} + WHERE + bench_value_id = ? + ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); + +} + +sub copy_benchmark_backup_value { + + my ( $or_self, $hr_vals ) = @_; + + for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { + if (! $hr_vals->{$s_param} ) { + require Carp; + Carp::confess("missing parameter '$s_param'"); + return; + } + } + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{benchmark_backup_value_table} + ( bench_value_id, bench_id, bench_subsume_type_id, bench_value, active, created_at ) + SELECT + ?, bench_id, bench_subsume_type_id, bench_value, active, created_at + FROM + $or_self->{config}{tables}{benchmark_value_table} + WHERE + bench_value_id = ? + ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); + +} + +sub copy_benchmark_backup_additional_relations { + + my ( $or_self, $hr_vals ) = @_; + + for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { + if (! $hr_vals->{$s_param} ) { + require Carp; + Carp::confess("missing parameter '$s_param'"); + return; + } + } + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{backup_additional_relation_table} + ( bench_backup_value_id, bench_additional_value_id, active, created_at ) + SELECT + ?, bench_additional_value_id, active, created_at + FROM + $or_self->{config}{tables}{additional_relation_table} + WHERE + bench_value_id = ? + ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); + +} + +sub insert_addtype { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{additional_type_table} + ( bench_additional_type, created_at ) + VALUES + ( ?, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_addvalue { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{additional_value_table} + ( bench_additional_type_id, bench_additional_value, created_at ) + VALUES + ( ?, ?, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_addvaluerelation { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{additional_relation_table} + ( bench_value_id, bench_additional_value_id, active, created_at ) + VALUES + ( ?, ?, 1, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub update_benchmark_backup_value { + + my ( $or_self, $hr_vals ) = @_; + + return $or_self->execute_query( " + UPDATE $or_self->{config}{tables}{benchmark_backup_value_table} + SET bench_value_id = ? + WHERE bench_value_id = ? + ", @{$hr_vals}{qw/ + new_bench_value_id + old_bench_value_id + /} ); + +} + +sub delete_benchmark_additional_relations { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM $or_self->{config}{tables}{additional_relation_table} + WHERE bench_value_id = ? + ", @a_vals ); + +} + +sub delete_benchmark_value { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM $or_self->{config}{tables}{benchmark_value_table} + WHERE bench_value_id = ? + ", @a_vals ); + +} + +1; From 6bef415e1a82ce9e87f57be6b2167f52c296db2f Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Thu, 13 Aug 2015 15:06:42 +0200 Subject: [PATCH 12/80] docs: fix order_by examples (argument order) --- lib/Tapper/Benchmark.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 9f89008..d1714c0 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -655,7 +655,7 @@ Tapper::Benchmark - Save and search benchmark points by database ], order_by => [ 'machine', - ['ASC','testrun_id',{ numeric => 1 }] + ['testrun_id','ASC',{ numeric => 1 }] ], limit => 2, offset => 1, @@ -851,7 +851,7 @@ Statement Handle. offset => 1, order_by => [ 'machine', - ['ASC','testrun_id'] + ['testrun_id','ASC'] ], }); @@ -1004,7 +1004,7 @@ Returning all benchmark data points as Array of Hashes. offset => 1, order_by => [ 'machine', - ['ASC','testrun_id'] + ['testrun_id','ASC'] ], }); @@ -1039,7 +1039,7 @@ Every "key" create a new nested hash. offset => 1, order_by => [ 'machine', - ['ASC','testrun_id'] + ['testrun_id','ASC'] ], }); From 24a0f684fee88339d9615ae5292c36279b787804 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 16 Aug 2015 11:07:38 +0200 Subject: [PATCH 13/80] fix: experiment with quoting ...in order to find the bug on SQLite: no such column: bav2.bench_additional_value --- lib/Tapper/Benchmark/Query/SQLite.pm | 2 +- lib/Tapper/Benchmark/Query/mysql.pm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index c16c53d..f61bfdb 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -136,7 +136,7 @@ sub create_select_column { $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; - return ( $s_return_column, "$s_return_select AS $s_replace_as", ); + return ( $s_return_column, "$s_return_select AS '$s_replace_as'", ); } diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index c12f6a0..e93a7a2 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -133,7 +133,7 @@ sub create_select_column { $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; - return ( $s_return_column, "$s_return_select AS $s_replace_as", ); + return ( $s_return_column, "$s_return_select AS '$s_replace_as'", ); } From 6b60ebbe1ec586a8c6471b64deaa1c3c9d5d748a Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 16 Aug 2015 11:13:26 +0200 Subject: [PATCH 14/80] changelog++ --- Changes | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Changes diff --git a/Changes b/Changes new file mode 100644 index 0000000..22cc830 --- /dev/null +++ b/Changes @@ -0,0 +1,11 @@ +Revision history for {{$dist->name}} + +{{$NEXT}} + - Initial release - CPAN DAY - 2015-08-16 + - based on the released code from early 2014 + - This is a library to provide a schema for + storing benchmark values in an SQL database + - Primarily targets mysql, SQLite support + is basically so far by cutting away known + trouble maker SQL statement parts + - Read more about the schema on http://benchmarkanything.org/ From 3a71c89ee7d1293cd400cc648d201b010dc11b3f Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 16 Aug 2015 11:15:11 +0200 Subject: [PATCH 15/80] v0.001 - Initial release - CPAN DAY - 2015-08-16 - based on the released code from early 2014 - This is a library to provide a schema for storing benchmark values in an SQL database - Primarily targets mysql, SQLite support is basically so far by cutting away known trouble maker SQL statement parts - Read more about the schema on http://benchmarkanything.org/ --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 22cc830..e6a9ffa 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.001 2015-08-16 - Initial release - CPAN DAY - 2015-08-16 - based on the released code from early 2014 - This is a library to provide a schema for From 0e041574c1c642962663d056b24b1f2c969ac29a Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 16 Aug 2015 12:58:15 +0200 Subject: [PATCH 16/80] store bench_values as VARCHAR ...to allow ultimate precision. --- share/tapper-benchmark-create-schema.SQLite | 2 +- share/tapper-benchmark-create-schema.mysql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/share/tapper-benchmark-create-schema.SQLite b/share/tapper-benchmark-create-schema.SQLite index 9fb3579..f55276e 100644 --- a/share/tapper-benchmark-create-schema.SQLite +++ b/share/tapper-benchmark-create-schema.SQLite @@ -40,7 +40,7 @@ CREATE TABLE `bench_values` ( `bench_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', `bench_id` INT(10) NOT NULL , -- COMMENT 'FK to benchs', `bench_subsume_type_id` TINYINT(3) NOT NULL , -- COMMENT 'FK to bench_subsume_types', - `bench_value` FLOAT NULL , -- COMMENT 'value for bench data point', + `bench_value` VARCHAR(255) NULL , -- COMMENT 'value for bench data point', `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', CONSTRAINT `fk_bench_values_01` diff --git a/share/tapper-benchmark-create-schema.mysql b/share/tapper-benchmark-create-schema.mysql index 7c3c09c..343657f 100644 --- a/share/tapper-benchmark-create-schema.mysql +++ b/share/tapper-benchmark-create-schema.mysql @@ -51,7 +51,7 @@ CREATE TABLE `bench_values` ( `bench_value_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_id` INT(10) UNSIGNED NOT NULL COMMENT 'FK to benchs', `bench_subsume_type_id` TINYINT(3) UNSIGNED NOT NULL COMMENT 'FK to bench_subsume_types', - `bench_value` FLOAT NULL COMMENT 'value for bench data point', + `bench_value` VARCHAR(255) NULL COMMENT 'value for bench data point', `active` tinyint(3) unsigned NOT NULL COMMENT 'is entry still active (0=no,1=yes)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_value_id`), From 0803605f510ba8d170724dfd54958a2ddb3e87d6 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 16 Aug 2015 16:54:50 +0200 Subject: [PATCH 17/80] dzil: +Test::Compile --- dist.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dist.ini b/dist.ini index dbda691..59233ca 100644 --- a/dist.ini +++ b/dist.ini @@ -9,3 +9,5 @@ repository_at = github repository = git://github.com/benchmarkanything/Tapper-Benchmark.git repository_web = http://github.com/benchmarkanything/Tapper-Benchmark disable_pod_coverage_tests = 1 + +[Test::Compile] From bdc813448e7e504e8a474ca9e250437c917d9d64 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 16 Aug 2015 16:55:19 +0200 Subject: [PATCH 18/80] changelog++ --- Changes | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes b/Changes index e6a9ffa..128739c 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for {{$dist->name}} {{$NEXT}} + - store values as VARCHAR for max precision 0.001 2015-08-16 - Initial release - CPAN DAY - 2015-08-16 From e3b89b7c810118cd42db3775e7cad384088b2ae5 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 17 Aug 2015 13:51:02 +0200 Subject: [PATCH 19/80] deps: DBD::SQLite 1.48 This avoids the error DBD::SQLite::db prepare failed: no such column: bav2.bench_additional_value at .../Tapper/Benchmark/Query.pm probably by correctly using the SQLite pragmas full_column_names short_column_names which at least can be configured to reproduce the error in plain sqlite3 (though I don't have proof that this is the ultimate solution to that problem). --- dist.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dist.ini b/dist.ini index 59233ca..d4236cf 100644 --- a/dist.ini +++ b/dist.ini @@ -11,3 +11,6 @@ repository_web = http://github.com/benchmarkanything/Tapper-Benchmark disable_pod_coverage_tests = 1 [Test::Compile] + +[Prereqs] +DBD::SQLite = 1.48 From 0fb0718111ecc046a899add9645e1395d1ce6635 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 17 Aug 2015 13:55:02 +0200 Subject: [PATCH 20/80] improve error message --- lib/Tapper/Benchmark/Query/SQLite.pm | 5 +++-- lib/Tapper/Benchmark/Query/mysql.pm | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index f61bfdb..82d8eb3 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -239,13 +239,14 @@ sub select_benchmark_values { push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; } else { + my $s_additional_type = splice( @{$ar_where}, 1, 1 ); my $hr_additional_type = $or_self - ->select_addtype_by_name( splice( @{$ar_where}, 1, 1 ) ) + ->select_addtype_by_name( $s_additional_type ) ->fetchrow_hashref() ; if ( !$hr_additional_type || !$hr_additional_type->{bench_additional_type_id} ) { require Carp; - Carp::confess("benchmark additional value '$ar_where->[1]' not exists"); + Carp::confess("benchmark additional value '$s_additional_type' not exists"); return; } push @a_from, " diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index e93a7a2..42121f2 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -236,13 +236,14 @@ sub select_benchmark_values { push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; } else { + my $s_additional_type = splice( @{$ar_where}, 1, 1 ); my $hr_additional_type = $or_self - ->select_addtype_by_name( splice( @{$ar_where}, 1, 1 ) ) + ->select_addtype_by_name( $s_additional_type ) ->fetchrow_hashref() ; if ( !$hr_additional_type || !$hr_additional_type->{bench_additional_type_id} ) { require Carp; - Carp::confess("benchmark additional value '$ar_where->[1]' not exists"); + Carp::confess("benchmark additional value '$s_additional_type' not exists"); return; } push @a_from, " From a501dabf94c84c28b8d99109ee13b136de1c44a8 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 17 Aug 2015 13:58:03 +0200 Subject: [PATCH 21/80] changelog++ --- Changes | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Changes b/Changes index 128739c..94789a6 100644 --- a/Changes +++ b/Changes @@ -1,7 +1,10 @@ Revision history for {{$dist->name}} {{$NEXT}} - - store values as VARCHAR for max precision + - store benchmark VALUEs as VARCHAR to avoid rounding + and allow non-number values en passant + - dependency++ to DBD::SQLite 1.48 to avoid error + 'no such column: bav2.bench_additional_value" 0.001 2015-08-16 - Initial release - CPAN DAY - 2015-08-16 From 5a34d32316b64e1f355eb756d3bd637ca8b1e20b Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 17 Aug 2015 13:59:27 +0200 Subject: [PATCH 22/80] v0.002 - store benchmark VALUEs as VARCHAR to avoid rounding and allow non-number values en passant - dependency++ to DBD::SQLite 1.48 to avoid error 'no such column: bav2.bench_additional_value" --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 94789a6..61ab352 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.002 2015-08-17 - store benchmark VALUEs as VARCHAR to avoid rounding and allow non-number values en passant - dependency++ to DBD::SQLite 1.48 to avoid error From 47de8336529b9ad6e35abfef1ecbeb024d4978ba Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 18 Aug 2015 15:19:31 +0200 Subject: [PATCH 23/80] skip undefined values For key=value pairs where the key is undefined we skip the whole pair. I am not sure it makes sense but I need to avoid db errors resulting from that. --- lib/Tapper/Benchmark.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index d1714c0..dc77584 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -239,6 +239,7 @@ sub add_single_benchmark { ADDITIONAL: for my $s_key ( keys %{$hr_point} ) { next ADDITIONAL if $s_key eq 'VALUE'; + next ADDITIONAL if not defined $hr_point->{$s_key}; # additional type my $i_addtype_id; From 229ce790d6b7e204915b61f9fd3768e6caa745cb Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 18 Aug 2015 15:21:20 +0200 Subject: [PATCH 24/80] ignore duplicate inserts When we insert similar entries it used to insert similar fields and failed. Trying to fix this with simple INSERT IGNORE statements, for mysql and SQLite respectively. --- lib/Tapper/Benchmark/Query/SQLite.pm | 6 +++--- lib/Tapper/Benchmark/Query/mysql.pm | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index 82d8eb3..4491b90 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -636,7 +636,7 @@ sub insert_addtyperelation { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{additional_type_relation_table} + INSERT OR IGNORE INTO $or_self->{config}{tables}{additional_type_relation_table} ( bench_id, bench_additional_type_id, created_at ) VALUES ( ?, ?, CURRENT_TIMESTAMP ) @@ -675,7 +675,7 @@ sub insert_benchmark_value { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{benchmark_value_table} + INSERT OR IGNORE INTO $or_self->{config}{tables}{benchmark_value_table} ( bench_id, bench_subsume_type_id, bench_value, active, created_at ) VALUES ( ?, ?, ?, 1, CURRENT_TIMESTAMP ) @@ -789,7 +789,7 @@ sub insert_addvaluerelation { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{additional_relation_table} + INSERT OR IGNORE INTO $or_self->{config}{tables}{additional_relation_table} ( bench_value_id, bench_additional_value_id, active, created_at ) VALUES ( ?, ?, 1, CURRENT_TIMESTAMP ) diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index 42121f2..746d386 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -633,7 +633,7 @@ sub insert_addtyperelation { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{additional_type_relation_table} + INSERT IGNORE INTO $or_self->{config}{tables}{additional_type_relation_table} ( bench_id, bench_additional_type_id, created_at ) VALUES ( ?, ?, NOW() ) @@ -672,7 +672,7 @@ sub insert_benchmark_value { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{benchmark_value_table} + INSERT IGNORE INTO $or_self->{config}{tables}{benchmark_value_table} ( bench_id, bench_subsume_type_id, bench_value, active, created_at ) VALUES ( ?, ?, ?, 1, NOW() ) @@ -760,7 +760,7 @@ sub insert_addtype { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{additional_type_table} + INSERT IGNORE INTO $or_self->{config}{tables}{additional_type_table} ( bench_additional_type, created_at ) VALUES ( ?, NOW() ) @@ -786,7 +786,7 @@ sub insert_addvaluerelation { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{additional_relation_table} + INSERT IGNORE INTO $or_self->{config}{tables}{additional_relation_table} ( bench_value_id, bench_additional_value_id, active, created_at ) VALUES ( ?, ?, 1, NOW() ) From 5695a61ca31c2e6c769a1939a15cd902c1e98620 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 18 Aug 2015 15:22:43 +0200 Subject: [PATCH 25/80] more duplicate handling For some reason it seems to Just Work(tm) in SQLite. Let's see if explicitely updating the last inserted id in mysql does the Right Thing. --- lib/Tapper/Benchmark/Query/mysql.pm | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index 746d386..df0c736 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -650,6 +650,8 @@ sub insert_unit { ( bench_unit, created_at ) VALUES ( ?, NOW() ) + ON DUPLICATE KEY + UPDATE bench_unit_id=LAST_INSERT_ID(bench_unit_id) ", @a_vals ); } @@ -663,6 +665,8 @@ sub insert_benchmark { ( bench, bench_unit_id, active, created_at ) VALUES ( ?, ?, 1, NOW() ) + ON DUPLICATE KEY + UPDATE bench_id=LAST_INSERT_ID(bench_id) ", @a_vals ); } @@ -764,6 +768,8 @@ sub insert_addtype { ( bench_additional_type, created_at ) VALUES ( ?, NOW() ) + ON DUPLICATE KEY + UPDATE bench_additional_type_id=LAST_INSERT_ID(bench_additional_type_id) ", @a_vals ); } @@ -777,6 +783,8 @@ sub insert_addvalue { ( bench_additional_type_id, bench_additional_value, created_at ) VALUES ( ?, ?, NOW() ) + ON DUPLICATE KEY + UPDATE bench_additional_value_id=LAST_INSERT_ID(bench_additional_value_id) ", @a_vals ); } From 3940af0e1bd05dc5029e135810badee474f47e4f Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 18 Aug 2015 15:25:25 +0200 Subject: [PATCH 26/80] changelog++ --- Changes | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes b/Changes index 61ab352..90ec15c 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for {{$dist->name}} {{$NEXT}} + - handling of duplicates and undefined values 0.002 2015-08-17 - store benchmark VALUEs as VARCHAR to avoid rounding From 3bf9ece89752738be620bcc930574c071aecc2d8 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 18 Aug 2015 15:26:03 +0200 Subject: [PATCH 27/80] v0.003 - handling of duplicates and undefined values --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 90ec15c..e9cb70b 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.003 2015-08-18 - handling of duplicates and undefined values 0.002 2015-08-17 From e181cec6f3187eaffcc823ad7287045e0315cfe6 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 19 Aug 2015 23:32:10 +0200 Subject: [PATCH 28/80] get list of benchmark NAMEs Quite some code duplication but we are in stupid extension mode, refactoring later this week. --- lib/Tapper/Benchmark.pm | 28 ++++++++++++++++++++++++++++ lib/Tapper/Benchmark/Query/SQLite.pm | 13 +++++++++++++ lib/Tapper/Benchmark/Query/mysql.pm | 13 +++++++++++++ 3 files changed, 54 insertions(+) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index dc77584..35692cb 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -396,6 +396,34 @@ sub search { } +sub list_benchmark_names { + + my ( $or_self, $s_pattern ) = @_; + + my $ar_pattern = defined($s_pattern) ? [$s_pattern] : []; + + my $s_key; + if ( $or_self->{cache} ) { + require JSON::XS; + $s_key = JSON::XS::encode_json($ar_pattern); + if ( my $ar_search_data = $or_self->{cache}->get("list_benchmark_names||$s_key") ) { + return $ar_search_data; + } + } + + my $ar_result = $or_self->{query} + ->select_benchmark_names( @$ar_pattern ) + ->fetchall_arrayref([0]); + my $ar_benchmark_names = [ map { $_->[0] } @$ar_result ]; + + if ( $or_self->{cache} ) { + $or_self->{cache}->set( "list_benchmark_names||$s_key" => $ar_benchmark_names ); + } + + return $ar_benchmark_names; + +} + sub search_array { my ( $or_self, $hr_search ) = @_; diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index 4491b90..ea2a29b 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -583,6 +583,19 @@ sub select_benchmark { } +sub select_benchmark_names { + + my ( $or_self, @a_vals ) = @_; + + my $query = " + SELECT DISTINCT bench + FROM $or_self->{config}{tables}{benchmark_table}"; + $query .= " + WHERE bench LIKE ? " if @a_vals; + return $or_self->execute_query( $query, @a_vals ); + +} + sub select_unit { my ( $or_self, @a_vals ) = @_; diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index df0c736..7dbb732 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -580,6 +580,19 @@ sub select_benchmark { } +sub select_benchmark_names { + + my ( $or_self, @a_vals ) = @_; + + my $query = " + SELECT DISTINCT bench + FROM $or_self->{config}{tables}{benchmark_table}"; + $query .= " + WHERE bench LIKE ? " if @a_vals; + return $or_self->execute_query( $query, @a_vals ); + +} + sub select_unit { my ( $or_self, @a_vals ) = @_; From e9d62993dc837f87d6088a3fc2cceaff4b6717fa Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Thu, 20 Aug 2015 06:54:28 +0200 Subject: [PATCH 29/80] changelog++ --- Changes | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes b/Changes index e9cb70b..efbe5ad 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for {{$dist->name}} {{$NEXT}} + - get list of benchmark NAMEs 0.003 2015-08-18 - handling of duplicates and undefined values From 0ab0a176153ea6d28beba9c2be4bc6ef3588744b Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Thu, 20 Aug 2015 06:54:58 +0200 Subject: [PATCH 30/80] v0.004 - get list of benchmark NAMEs --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index efbe5ad..ed5a62a 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.004 2015-08-20 - get list of benchmark NAMEs 0.003 2015-08-18 From 166d4c41fa55a71ebe2a4577563907f67c3da86b Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Fri, 21 Aug 2015 18:11:05 +0200 Subject: [PATCH 31/80] all fields and values VARCHAR++ Aargh, data loss due to too short columns and MySQL not complaining by default. Every couple of years I learn the same bitter lesson again. Postgres, I'm coming. --- share/tapper-benchmark-create-schema.mysql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/share/tapper-benchmark-create-schema.mysql b/share/tapper-benchmark-create-schema.mysql index 343657f..a250482 100644 --- a/share/tapper-benchmark-create-schema.mysql +++ b/share/tapper-benchmark-create-schema.mysql @@ -25,7 +25,7 @@ CREATE TABLE `bench_subsume_types` ( CREATE TABLE `bench_units` ( `bench_unit_id` TINYINT(3) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', - `bench_unit` VARCHAR(64) NOT NULL COMMENT 'unique string identifier', + `bench_unit` VARCHAR(255) NOT NULL COMMENT 'unique string identifier', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_unit_id`), UNIQUE INDEX `ux_bench_units_01` (`bench_unit` ASC) @@ -34,7 +34,7 @@ CREATE TABLE `bench_units` ( CREATE TABLE `benchs` ( `bench_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_unit_id` TINYINT(3) UNSIGNED NULL, - `bench` VARCHAR(64) NOT NULL COMMENT 'unique string identifier', + `bench` VARCHAR(2048) NOT NULL COMMENT 'unique string identifier', `active` TINYINT(3) UNSIGNED NOT NULL COMMENT 'is entry still active (1=yes,0=no)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_id`), @@ -51,7 +51,7 @@ CREATE TABLE `bench_values` ( `bench_value_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_id` INT(10) UNSIGNED NOT NULL COMMENT 'FK to benchs', `bench_subsume_type_id` TINYINT(3) UNSIGNED NOT NULL COMMENT 'FK to bench_subsume_types', - `bench_value` VARCHAR(255) NULL COMMENT 'value for bench data point', + `bench_value` VARCHAR(2048) NULL COMMENT 'value for bench data point', `active` tinyint(3) unsigned NOT NULL COMMENT 'is entry still active (0=no,1=yes)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_value_id`), @@ -71,7 +71,7 @@ CREATE TABLE `bench_values` ( CREATE TABLE `bench_additional_types` ( `bench_additional_type_id` SMALLINT(5) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', - `bench_additional_type` VARCHAR(32) NOT NULL COMMENT 'unique string identifier', + `bench_additional_type` VARCHAR(2048) NOT NULL COMMENT 'unique string identifier', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_additional_type_id`), UNIQUE INDEX `ux_bench_additional_types_01` (`bench_additional_type` ASC) @@ -80,7 +80,7 @@ CREATE TABLE `bench_additional_types` ( CREATE TABLE `bench_additional_values` ( `bench_additional_value_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_additional_type_id` SMALLINT(5) UNSIGNED NOT NULL COMMENT 'FK to bench_additional_types', - `bench_additional_value` VARCHAR(128) NOT NULL COMMENT 'additional value', + `bench_additional_value` VARCHAR(2048) NOT NULL COMMENT 'additional value', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_additional_value_id`), INDEX `fk_bench_additional_values_01` (`bench_additional_type_id` ASC), From f5537b86d32bb05d6b516327f46668fe74b25225 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 24 Aug 2015 13:23:29 +0200 Subject: [PATCH 32/80] fix: key columns are size restricted Too bad that key columns can't be longer than 767 in mysql. Downsizing the affected ones. --- share/tapper-benchmark-create-schema.mysql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/share/tapper-benchmark-create-schema.mysql b/share/tapper-benchmark-create-schema.mysql index a250482..f240ee4 100644 --- a/share/tapper-benchmark-create-schema.mysql +++ b/share/tapper-benchmark-create-schema.mysql @@ -25,7 +25,7 @@ CREATE TABLE `bench_subsume_types` ( CREATE TABLE `bench_units` ( `bench_unit_id` TINYINT(3) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', - `bench_unit` VARCHAR(255) NOT NULL COMMENT 'unique string identifier', + `bench_unit` VARCHAR(767) NOT NULL COMMENT 'unique string identifier', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_unit_id`), UNIQUE INDEX `ux_bench_units_01` (`bench_unit` ASC) @@ -34,7 +34,7 @@ CREATE TABLE `bench_units` ( CREATE TABLE `benchs` ( `bench_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_unit_id` TINYINT(3) UNSIGNED NULL, - `bench` VARCHAR(2048) NOT NULL COMMENT 'unique string identifier', + `bench` VARCHAR(767) NOT NULL COMMENT 'unique string identifier', `active` TINYINT(3) UNSIGNED NOT NULL COMMENT 'is entry still active (1=yes,0=no)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_id`), @@ -51,7 +51,7 @@ CREATE TABLE `bench_values` ( `bench_value_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_id` INT(10) UNSIGNED NOT NULL COMMENT 'FK to benchs', `bench_subsume_type_id` TINYINT(3) UNSIGNED NOT NULL COMMENT 'FK to bench_subsume_types', - `bench_value` VARCHAR(2048) NULL COMMENT 'value for bench data point', + `bench_value` VARCHAR(767) NULL COMMENT 'value for bench data point', `active` tinyint(3) unsigned NOT NULL COMMENT 'is entry still active (0=no,1=yes)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_value_id`), @@ -71,7 +71,7 @@ CREATE TABLE `bench_values` ( CREATE TABLE `bench_additional_types` ( `bench_additional_type_id` SMALLINT(5) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', - `bench_additional_type` VARCHAR(2048) NOT NULL COMMENT 'unique string identifier', + `bench_additional_type` VARCHAR(767) NOT NULL COMMENT 'unique string identifier', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_additional_type_id`), UNIQUE INDEX `ux_bench_additional_types_01` (`bench_additional_type` ASC) @@ -80,7 +80,7 @@ CREATE TABLE `bench_additional_types` ( CREATE TABLE `bench_additional_values` ( `bench_additional_value_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_additional_type_id` SMALLINT(5) UNSIGNED NOT NULL COMMENT 'FK to bench_additional_types', - `bench_additional_value` VARCHAR(2048) NOT NULL COMMENT 'additional value', + `bench_additional_value` VARCHAR(767) NOT NULL COMMENT 'additional value', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_additional_value_id`), INDEX `fk_bench_additional_values_01` (`bench_additional_type_id` ASC), From 7e78058ec40744ab10f2740ab10fb614221bf264 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 24 Aug 2015 13:24:14 +0200 Subject: [PATCH 33/80] sync SQLite with mysql schema SQLite doesn't really care for field size but we keep the files in sync as much as possible. --- share/tapper-benchmark-create-schema.SQLite | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/share/tapper-benchmark-create-schema.SQLite b/share/tapper-benchmark-create-schema.SQLite index f55276e..e1014ce 100644 --- a/share/tapper-benchmark-create-schema.SQLite +++ b/share/tapper-benchmark-create-schema.SQLite @@ -19,14 +19,14 @@ CREATE TABLE `bench_subsume_types` ( CREATE TABLE `bench_units` ( `bench_unit_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', - `bench_unit` VARCHAR(64) NOT NULL , -- COMMENT 'unique string identifier', + `bench_unit` VARCHAR(767) NOT NULL , -- COMMENT 'unique string identifier', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date', ) ; -- COMMENT 'units for benchmark data points'; CREATE TABLE `benchs` ( `bench_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', `bench_unit_id` TINYINT(3) NULL, - `bench` VARCHAR(64) NOT NULL , -- COMMENT 'unique string identifier', + `bench` VARCHAR(767) NOT NULL , -- COMMENT 'unique string identifier', `active` TINYINT(3) NOT NULL , -- COMMENT 'is entry still active (1=yes,0=no)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', CONSTRAINT `fk_benchs_01` @@ -40,7 +40,7 @@ CREATE TABLE `bench_values` ( `bench_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', `bench_id` INT(10) NOT NULL , -- COMMENT 'FK to benchs', `bench_subsume_type_id` TINYINT(3) NOT NULL , -- COMMENT 'FK to bench_subsume_types', - `bench_value` VARCHAR(255) NULL , -- COMMENT 'value for bench data point', + `bench_value` VARCHAR(767) NULL , -- COMMENT 'value for bench data point', `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', CONSTRAINT `fk_bench_values_01` @@ -57,14 +57,14 @@ CREATE TABLE `bench_values` ( CREATE TABLE `bench_additional_types` ( `bench_additional_type_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', - `bench_additional_type` VARCHAR(32) NOT NULL , -- COMMENT 'unique string identifier', + `bench_additional_type` VARCHAR(767) NOT NULL , -- COMMENT 'unique string identifier', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' ) ; -- COMMENT 'types of additional values for benchmark data points'; CREATE TABLE `bench_additional_values` ( `bench_additional_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', `bench_additional_type_id` SMALLINT(5) NOT NULL , -- COMMENT 'FK to bench_additional_types', - `bench_additional_value` VARCHAR(128) NOT NULL , -- COMMENT 'additional value', + `bench_additional_value` VARCHAR(767) NOT NULL , -- COMMENT 'additional value', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', CONSTRAINT `fk_bench_additional_values_01` FOREIGN KEY (`bench_additional_type_id`) From 17ff40bbad233ccd17b6ad17a707ac44f86ce1d0 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 24 Aug 2015 14:52:58 +0200 Subject: [PATCH 34/80] +select_benchmark_point_essentials() Get NAME, VALUE, UNIT of a single benchmark point. --- lib/Tapper/Benchmark/Query/SQLite.pm | 27 +++++++++++++++++++++++++++ lib/Tapper/Benchmark/Query/mysql.pm | 27 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index ea2a29b..d91d832 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -422,6 +422,33 @@ sub select_benchmark_values { } +sub select_benchmark_point_essentials { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT + b.bench, + bv.bench_value, + bu.bench_unit + FROM + $or_self->{config}{tables}{benchmark_table} b + JOIN + $or_self->{config}{tables}{benchmark_value_table} bv + ON + b.bench_id = bv.bench_id + LEFT JOIN + $or_self->{config}{tables}{unit_table} bu + ON + b.bench_unit_id = bu.bench_unit_id + WHERE + bv.bench_value_id = ? + ; + ", @a_vals ); + +} + + sub select_addtype_by_name { my ( $or_self, @a_vals ) = @_; diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index 7dbb732..5441e40 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -419,6 +419,33 @@ sub select_benchmark_values { } +sub select_benchmark_point_essentials { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT + b.bench, + bv.bench_value, + bu.bench_unit + FROM + $or_self->{config}{tables}{benchmark_table} b + JOIN + $or_self->{config}{tables}{benchmark_value_table} bv + ON + b.bench_id = bv.bench_id + LEFT JOIN + $or_self->{config}{tables}{unit_table} bu + ON + b.bench_unit_id = bu.bench_unit_id + WHERE + bv.bench_value_id = ? + ; + ", @a_vals ); + +} + + sub select_addtype_by_name { my ( $or_self, @a_vals ) = @_; From 1e369256a8c201454ca6d7d868c11615a3e92d0b Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 24 Aug 2015 14:54:10 +0200 Subject: [PATCH 35/80] +select_complete_benchmark_point() Get all additional key/value fields of a single benchmark point. This does not contain the essentials like NAME, VALUE, UNIT. --- lib/Tapper/Benchmark/Query/SQLite.pm | 37 ++++++++++++++++++++++++++++ lib/Tapper/Benchmark/Query/mysql.pm | 37 ++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index d91d832..3f24eb7 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -448,6 +448,43 @@ sub select_benchmark_point_essentials { } +sub select_complete_benchmark_point { + + my ( $or_self, @a_vals ) = @_; + + my $query = " + SELECT + bat.bench_additional_type, + bav.bench_additional_value + FROM + benchs b + JOIN + bench_values bv + ON + b.bench_id = bv.bench_id + JOIN + bench_additional_type_relations batr + ON + bv.bench_id = batr.bench_id + JOIN + bench_additional_types bat + ON + batr.bench_additional_type_id = bat.bench_additional_type_id + JOIN + bench_additional_relations bar + ON + bv.bench_value_id = bar.bench_value_id + JOIN + bench_additional_values bav + ON + bar.bench_additional_value_id = bav.bench_additional_value_id AND + bat.bench_additional_type_id = bav.bench_additional_type_id + WHERE + bv.bench_value_id = ? + ORDER BY + bat.bench_additional_type"; + return $or_self->execute_query( $query, @a_vals ); +} sub select_addtype_by_name { diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index 5441e40..f45cbdc 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -445,6 +445,43 @@ sub select_benchmark_point_essentials { } +sub select_complete_benchmark_point { + + my ( $or_self, @a_vals ) = @_; + + my $query = " + SELECT + bat.bench_additional_type, + bav.bench_additional_value + FROM + benchs b + JOIN + bench_values bv + ON + b.bench_id = bv.bench_id + JOIN + bench_additional_type_relations batr + ON + bv.bench_id = batr.bench_id + JOIN + bench_additional_types bat + ON + batr.bench_additional_type_id = bat.bench_additional_type_id + JOIN + bench_additional_relations bar + ON + bv.bench_value_id = bar.bench_value_id + JOIN + bench_additional_values bav + ON + bar.bench_additional_value_id = bav.bench_additional_value_id AND + bat.bench_additional_type_id = bav.bench_additional_type_id + WHERE + bv.bench_value_id = ? + ORDER BY + bat.bench_additional_type"; + return $or_self->execute_query( $query, @a_vals ); +} sub select_addtype_by_name { From 4e54abf4112ec21a9b42496cd8e2ff517906f871 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 24 Aug 2015 14:58:35 +0200 Subject: [PATCH 36/80] +get_single_benchmark_point() Reconstruct a single benchmark point from all essentials (NAME, VALUE, UNIT) and all additional key/value pairs. --- lib/Tapper/Benchmark.pm | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 35692cb..f646fa8 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -424,6 +424,47 @@ sub list_benchmark_names { } +sub get_single_benchmark_point { + + my ( $or_self, $i_bench_value_id ) = @_; + + return {} unless $i_bench_value_id; + + # cache? + my $s_key; + if ( $or_self->{cache} ) { + require JSON::XS; + $s_key = JSON::XS::encode_json({bench_value_id => $i_bench_value_id}); + if ( my $hr_search_data = $or_self->{cache}->get("get_single_benchmark_point||$s_key") ) { + return $hr_search_data; + } + } + + # fetch all additional key/value fields + my $ar_query_result = $or_self->{query} + ->select_complete_benchmark_point( $i_bench_value_id ) + ->fetchall_arrayref({}); + + # fetch essentials, like NAME, VALUE, UNIT + my $hr_essentials = $or_self->{query} + ->select_benchmark_point_essentials( $i_bench_value_id ) + ->fetchrow_hashref(); + + # create complete BenchmarkAnything-like key/value entry + my $hr_result; + $hr_result = { map { ($_->{bench_additional_type} => $_->{bench_additional_value} ) } @$ar_query_result }; + $hr_result->{NAME} = $hr_essentials->{bench}; + $hr_result->{VALUE} = $hr_essentials->{bench_value}; + $hr_result->{UNIT} = $hr_essentials->{bench_unit} if $hr_essentials->{bench_unit}; + + # cache! + if ( $or_self->{cache} ) { + $or_self->{cache}->set( "get_single_benchmark_point||$s_key" => $hr_result ); + } + + return $hr_result; +} + sub search_array { my ( $or_self, $hr_search ) = @_; From 7631791a908f57ad36f002b8a60388a5365587f7 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 24 Aug 2015 15:03:40 +0200 Subject: [PATCH 37/80] changelog++ --- Changes | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Changes b/Changes index ed5a62a..e106a20 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,13 @@ Revision history for {{$dist->name}} {{$NEXT}} + - larger field sizes + - select_benchmark_point_essentials() + Get NAME, VALUE, UNIT of a single benchmark point + - select_complete_benchmark_point() + Get all additional key/value fields of a single benchmark point + - get_single_benchmark_point() + Reconstruct a complete single benchmark point 0.004 2015-08-20 - get list of benchmark NAMEs From 8d50b3ef5a14a09924e5fb426612706b8c2a849e Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 24 Aug 2015 15:04:49 +0200 Subject: [PATCH 38/80] v0.005 - larger field sizes - select_benchmark_point_essentials() Get NAME, VALUE, UNIT of a single benchmark point - select_complete_benchmark_point() Get all additional key/value fields of a single benchmark point - get_single_benchmark_point() Reconstruct a complete single benchmark point --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index e106a20..b48c337 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.005 2015-08-24 - larger field sizes - select_benchmark_point_essentials() Get NAME, VALUE, UNIT of a single benchmark point From 2939f71eafee7962c0bd10f78b18cbb2ee660855 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 24 Aug 2015 15:27:47 +0200 Subject: [PATCH 39/80] docs++ --- lib/Tapper/Benchmark.pm | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index f646fa8..5fad002 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -1115,6 +1115,27 @@ Every "key" create a new nested hash. =back + +=head3 select_benchmark_point_essentials + +Get a single data points from the database including all essential +fields (NAME, VALUE, UNIT) and additional fields. + + my $point = $or_bench->get_single_benchmark_point($value_id); + +=back + + +=head3 list_benchmark_names + +Get a list of all benchmark NAMEs, optionally matching a given pattern +(SQL LIKE syntax, i.e., using C<%> as placeholder. + + $benchmarkanythingdata = $or_bench->list_benchmark_names($pattern); + +=back + + =head3 subsume =over 4 From 989965353a26b32816728924a94c0bd95f774587 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 25 Aug 2015 20:47:20 +0200 Subject: [PATCH 40/80] disabled cache Ui ui ui, the caching is strange and actually breaks things when run repeatedly within the same process. --- lib/Tapper/Benchmark.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 5fad002..45eb972 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -5,7 +5,7 @@ use strict; use warnings; my $hr_default_config = { - select_cache => 1, + select_cache => 0, default_aggregation => 'min', tables => { unit_table => 'bench_units', From 12dfe75dcd56ea5a6a7e85bd2618e4d43b00e741 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 26 Aug 2015 13:39:26 +0200 Subject: [PATCH 41/80] fix POD errors --- lib/Tapper/Benchmark.pm | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 45eb972..1c5aa75 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -837,11 +837,8 @@ Ignore forgivable errors while writing. =back -=head3 add_multi_benchmark - -=over 4 -=item +=head3 add_multi_benchmark Add one or more data points for multiple benchmarks to the database. @@ -891,14 +888,9 @@ Ignore forgivable errors while writing. =back -=back =head3 search -=over 4 - -=item - Search for benchmark data points in the database. Function returns a DBI Statement Handle. @@ -1051,14 +1043,9 @@ An integer value which determine the number of omitted benchmark data points. =back -=back =head3 search_array -=over 4 - -=item - Returning all benchmark data points as Array of Hashes. my $or_benchmark_points = $or_bench->search_array({ @@ -1078,14 +1065,9 @@ Returning all benchmark data points as Array of Hashes. ], }); -=back =head3 search_hash -=over 4 - -=item - Returning all benchmark data points as Hash of Hashes. As compared to search C this function needs the parameter C. C is an Array of Strings which determine the columns used as the keys for the nested hashes. @@ -1113,8 +1095,6 @@ Every "key" create a new nested hash. ], }); -=back - =head3 select_benchmark_point_essentials @@ -1123,8 +1103,6 @@ fields (NAME, VALUE, UNIT) and additional fields. my $point = $or_bench->get_single_benchmark_point($value_id); -=back - =head3 list_benchmark_names @@ -1133,15 +1111,9 @@ Get a list of all benchmark NAMEs, optionally matching a given pattern $benchmarkanythingdata = $or_bench->list_benchmark_names($pattern); -=back - =head3 subsume -=over 4 - -=item - This is a maintenance function for reducing the number of data points in the database. Calling this function reduces the rows in the benchmark values table by building an average value for all benchmark data points grouped by specfic @@ -1191,14 +1163,9 @@ isn't desired a false value must be passed. =back -=back =head1 Configuration -=over 4 - -=item - The following elements are required in configuration: =over 4 @@ -1230,5 +1197,3 @@ Containing the names of the tables used bei B In case of a true value the module cache some select results =back - -=back From a32373ca916cd6a871d90658b211c04122e4b285 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 26 Aug 2015 13:40:33 +0200 Subject: [PATCH 42/80] changelog++ --- Changes | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes b/Changes index b48c337..3f15d00 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for {{$dist->name}} {{$NEXT}} + - disable caching due to wrong behaviour 0.005 2015-08-24 - larger field sizes From 2b4f1fbb324cbee4129e19eb581640da499e97c5 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 26 Aug 2015 13:40:51 +0200 Subject: [PATCH 43/80] v0.006 - disable caching due to wrong behaviour --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 3f15d00..ab04cfc 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.006 2015-08-26 - disable caching due to wrong behaviour 0.005 2015-08-24 From ba5f952af0cc92a88c9f9e4f1011e3080e16366b Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 6 Sep 2015 04:19:47 +0200 Subject: [PATCH 44/80] tables to support simple queueing Just enqueue raw incoming data here, and later process/dequeue them. --- share/tapper-benchmark-create-schema.SQLite | 17 ++++++++++++++++- share/tapper-benchmark-create-schema.mysql | 17 ++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/share/tapper-benchmark-create-schema.SQLite b/share/tapper-benchmark-create-schema.SQLite index e1014ce..3d5991b 100644 --- a/share/tapper-benchmark-create-schema.SQLite +++ b/share/tapper-benchmark-create-schema.SQLite @@ -8,6 +8,7 @@ DROP TABLE IF EXISTS `bench_values`; DROP TABLE IF EXISTS `benchs`; DROP TABLE IF EXISTS `bench_units`; DROP TABLE IF EXISTS `bench_subsume_types`; +DROP TABLE IF EXISTS `raw_bench_bundles`; CREATE TABLE `bench_subsume_types` ( `bench_subsume_type_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', @@ -34,7 +35,7 @@ CREATE TABLE `benchs` ( REFERENCES `bench_units` (`bench_unit_id`) ON DELETE NO ACTION ON UPDATE NO ACTION -) ; -- COMMENT 'containg benchmark head data'; +) ; -- COMMENT 'containing benchmark head data'; CREATE TABLE `bench_values` ( `bench_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', @@ -151,6 +152,20 @@ CREATE TABLE `bench_backup_additional_relations` ( ON UPDATE NO ACTION ) ; -- COMMENT='add additional values to benchmarks'; +CREATE TABLE `raw_bench_bundles` ( + `raw_bench_bundle_id` INTEGER PRIMARY KEY NOT NULL, + `raw_bench_bundle_serialized` TEXT NOT NULL, + `processed` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' +) ; -- COMMENT='queue raw incoming data for later processing'; +-- Usage: +-- * create elements: +-- INSERT INTO raw_bench_bundles (raw_bench_bundle_serialized) VALUES ('hot content'); +-- * get next: +-- SELECT raw_bench_bundle_id, raw_bench_bundle_serialized FROM raw_bench_bundles WHERE processed=0 ORDER BY raw_bench_bundle_id ASC LIMIT 1; +-- * mark as done: +-- UPDATE raw_bench_bundles SET processed=1 WHERE raw_bench_bundle_id=1; + INSERT INTO bench_subsume_types ( bench_subsume_type, bench_subsume_type_rank, datetime_strftime_pattern, created_at ) VALUES diff --git a/share/tapper-benchmark-create-schema.mysql b/share/tapper-benchmark-create-schema.mysql index f240ee4..66c5426 100644 --- a/share/tapper-benchmark-create-schema.mysql +++ b/share/tapper-benchmark-create-schema.mysql @@ -12,6 +12,7 @@ DROP TABLE IF EXISTS `bench_values`; DROP TABLE IF EXISTS `benchs`; DROP TABLE IF EXISTS `bench_units`; DROP TABLE IF EXISTS `bench_subsume_types`; +DROP TABLE IF EXISTS `raw_bench_bundles`; CREATE TABLE `bench_subsume_types` ( `bench_subsume_type_id` TINYINT(3) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', @@ -45,7 +46,7 @@ CREATE TABLE `benchs` ( REFERENCES `bench_units` (`bench_unit_id`) ON DELETE NO ACTION ON UPDATE NO ACTION -) COMMENT 'containg benchmark head data'; +) COMMENT 'containing benchmark head data'; CREATE TABLE `bench_values` ( `bench_value_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', @@ -180,6 +181,20 @@ CREATE TABLE `bench_backup_additional_relations` ( ON UPDATE NO ACTION ) COMMENT='add additional values to benchmarks'; +CREATE TABLE `raw_bench_bundles` ( + `raw_bench_bundle_id` INTEGER PRIMARY KEY AUTO_INCREMENT NOT NULL, + `raw_bench_bundle_serialized` TEXT NOT NULL, + `processed` tinyint(3) NOT NULL DEFAULT 0 COMMENT 'is entry processed (0=no,1=yes)', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date' +) COMMENT='queue raw incoming data for later processing'; +-- Usage: +-- * create elements: +-- INSERT INTO raw_bench_bundles (raw_bench_bundle_serialized) VALUES ('hot content'); +-- * get next: +-- SELECT raw_bench_bundle_id, raw_bench_bundle_serialized FROM raw_bench_bundles WHERE processed=0 ORDER BY raw_bench_bundle_id ASC LIMIT 1; +-- * mark as done: +-- UPDATE raw_bench_bundles SET processed=1 WHERE raw_bench_bundle_id=1; + SET FOREIGN_KEY_CHECKS = 1; INSERT INTO bench_subsume_types From 5a4d83ca897e142c707cd1d36ba1dd266bd828f6 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 6 Sep 2015 04:21:24 +0200 Subject: [PATCH 45/80] DB functions to support queueing --- lib/Tapper/Benchmark/Query/SQLite.pm | 38 ++++++++++++++++++++++++++++ lib/Tapper/Benchmark/Query/mysql.pm | 38 ++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index 3f24eb7..28d35b2 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -708,6 +708,20 @@ sub select_addtyperelation { } +sub select_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT raw_bench_bundle_id, + raw_bench_bundle_serialized + FROM raw_bench_bundles + WHERE processed=0 + ORDER BY raw_bench_bundle_id ASC + LIMIT 1 + ", @a_vals ); +} + sub insert_addtyperelation { my ( $or_self, @a_vals ) = @_; @@ -760,6 +774,18 @@ sub insert_benchmark_value { } +sub insert_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO raw_bench_bundles + (raw_bench_bundle_serialized) + VALUES ( ? ) + ", @a_vals ); + +} + sub copy_additional_values { my ( $or_self, $hr_vals ) = @_; @@ -889,6 +915,18 @@ sub update_benchmark_backup_value { } +sub update_raw_bench_bundle_set_processed { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + UPDATE raw_bench_bundles + SET processed=1 + WHERE raw_bench_bundle_id = ? + ", @a_vals ); + +} + sub delete_benchmark_additional_relations { my ( $or_self, @a_vals ) = @_; diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index f45cbdc..5446f33 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -705,6 +705,20 @@ sub select_addtyperelation { } +sub select_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT raw_bench_bundle_id, + raw_bench_bundle_serialized + FROM raw_bench_bundles + WHERE processed=0 + ORDER BY raw_bench_bundle_id ASC + LIMIT 1 + ", @a_vals ); +} + sub insert_addtyperelation { my ( $or_self, @a_vals ) = @_; @@ -761,6 +775,18 @@ sub insert_benchmark_value { } +sub insert_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO raw_bench_bundles + (raw_bench_bundle_serialized) + VALUES ( ? ) + ", @a_vals ); + +} + sub copy_additional_values { my ( $or_self, $hr_vals ) = @_; @@ -894,6 +920,18 @@ sub update_benchmark_backup_value { } +sub update_raw_bench_bundle_set_processed { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + UPDATE raw_bench_bundles + SET processed=1 + WHERE raw_bench_bundle_id = ? + ", @a_vals ); + +} + sub delete_benchmark_additional_relations { my ( $or_self, @a_vals ) = @_; From d75d35ea7394d231a3255cf5c7d3ad3ebf133970 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Sun, 6 Sep 2015 04:33:41 +0200 Subject: [PATCH 46/80] functions to enqueue+process values --- lib/Tapper/Benchmark.pm | 49 +++++++++++++++++++++++++++++++++++ lib/Tapper/Benchmark/Query.pm | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 1c5aa75..66ec802 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -340,6 +340,55 @@ sub add_single_benchmark { } +sub enqueue_multi_benchmark { + + my ( $or_self, $ar_data_points, $hr_options ) = @_; + + require Sereal::Encoder; + + my $s_serialized = Sereal::Encoder::encode_sereal($ar_data_points); + $or_self->{query}->insert_raw_bench_bundle($s_serialized); + + return 1; + +} + +# deques a single bundle (can contain multiple data points) +sub process_queued_multi_benchmark { + + my ( $or_self, $hr_options ) = @_; + + require Sereal::Decoder; + + # TODO: BEGIN_transaction + # TODO: LOCK - so only one worker works here! + + my $ar_result = $or_self->{query} + ->select_raw_bench_bundle + ->fetchrow_hashref(); + + my $i_id = $ar_result->{raw_bench_bundle_id}; + my $s_serialized = $ar_result->{raw_bench_bundle_serialized}; + + return unless $i_id; + + my $ar_data_points = Sereal::Decoder::decode_sereal($s_serialized); + + # preserve order, otherwise add_multi_benchmark() would reorder to optimize insert + foreach my $chunk (@$ar_data_points) + { + $or_self->add_multi_benchmark([$chunk], $hr_options); + } + + $or_self->{query}->update_raw_bench_bundle_set_processed($i_id); + + # TODO: UNLOCK + # TODO: END_transaction + + return $i_id; + +} + sub add_multi_benchmark { my ( $or_self, $ar_data_points, $hr_options ) = @_; diff --git a/lib/Tapper/Benchmark/Query.pm b/lib/Tapper/Benchmark/Query.pm index ca745b8..a40b568 100644 --- a/lib/Tapper/Benchmark/Query.pm +++ b/lib/Tapper/Benchmark/Query.pm @@ -1,5 +1,5 @@ package Tapper::Benchmark::Query; -# ABSTRACT: Tapper::Benchmark - qerying - base class +# ABSTRACT: Tapper::Benchmark - querying - base class use strict; use warnings; From 07d04fcb7eb4b4995c9410bd1912050d4272453b Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 7 Sep 2015 18:31:48 +0200 Subject: [PATCH 47/80] schema: raw_bench_bundles track processing --- share/tapper-benchmark-create-schema.SQLite | 1 + share/tapper-benchmark-create-schema.mysql | 1 + 2 files changed, 2 insertions(+) diff --git a/share/tapper-benchmark-create-schema.SQLite b/share/tapper-benchmark-create-schema.SQLite index 3d5991b..5b07b52 100644 --- a/share/tapper-benchmark-create-schema.SQLite +++ b/share/tapper-benchmark-create-schema.SQLite @@ -155,6 +155,7 @@ CREATE TABLE `bench_backup_additional_relations` ( CREATE TABLE `raw_bench_bundles` ( `raw_bench_bundle_id` INTEGER PRIMARY KEY NOT NULL, `raw_bench_bundle_serialized` TEXT NOT NULL, + `processing` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', `processed` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' ) ; -- COMMENT='queue raw incoming data for later processing'; diff --git a/share/tapper-benchmark-create-schema.mysql b/share/tapper-benchmark-create-schema.mysql index 66c5426..55d4c45 100644 --- a/share/tapper-benchmark-create-schema.mysql +++ b/share/tapper-benchmark-create-schema.mysql @@ -184,6 +184,7 @@ CREATE TABLE `bench_backup_additional_relations` ( CREATE TABLE `raw_bench_bundles` ( `raw_bench_bundle_id` INTEGER PRIMARY KEY AUTO_INCREMENT NOT NULL, `raw_bench_bundle_serialized` TEXT NOT NULL, + `processing` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', `processed` tinyint(3) NOT NULL DEFAULT 0 COMMENT 'is entry processed (0=no,1=yes)', `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date' ) COMMENT='queue raw incoming data for later processing'; From 5174b33127331cc2ae918f6ff34cbf15c78eaf30 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 7 Sep 2015 18:33:34 +0200 Subject: [PATCH 48/80] raw_value processing split up ...into first mark the row as processing (transaction 1), then actually process it and mark it as done (transaction 2). --- lib/Tapper/Benchmark/Query/SQLite.pm | 35 ++++++++++++++++++++++---- lib/Tapper/Benchmark/Query/mysql.pm | 37 ++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index 28d35b2..ab1d219 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -708,20 +708,32 @@ sub select_addtyperelation { } -sub select_raw_bench_bundle { +sub select_raw_bench_bundle_for_lock { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - SELECT raw_bench_bundle_id, - raw_bench_bundle_serialized + SELECT raw_bench_bundle_id FROM raw_bench_bundles - WHERE processed=0 + WHERE processed=0 AND + processing=0 ORDER BY raw_bench_bundle_id ASC LIMIT 1 ", @a_vals ); } +sub select_raw_bench_bundle_for_processing { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT raw_bench_bundle_serialized + FROM raw_bench_bundles + WHERE raw_bench_bundle_id = ? + LIMIT 1 + ", @a_vals ); +} + sub insert_addtyperelation { my ( $or_self, @a_vals ) = @_; @@ -915,13 +927,26 @@ sub update_benchmark_backup_value { } +sub start_processing_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + UPDATE raw_bench_bundles + SET processing = 1 + WHERE raw_bench_bundle_id = ? + ", @a_vals ); + +} + sub update_raw_bench_bundle_set_processed { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " UPDATE raw_bench_bundles - SET processed=1 + SET processed=1, + processing=0 WHERE raw_bench_bundle_id = ? ", @a_vals ); diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index 5446f33..caa0d6d 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -705,17 +705,31 @@ sub select_addtyperelation { } -sub select_raw_bench_bundle { +sub select_raw_bench_bundle_for_lock { my ( $or_self, @a_vals ) = @_; + use Data::Dumper; print Dumper(\@a_vals); return $or_self->execute_query( " - SELECT raw_bench_bundle_id, - raw_bench_bundle_serialized + SELECT raw_bench_bundle_id FROM raw_bench_bundles - WHERE processed=0 + WHERE processed=0 AND + processing=0 ORDER BY raw_bench_bundle_id ASC LIMIT 1 + FOR UPDATE + ", @a_vals ); +} + +sub select_raw_bench_bundle_for_processing { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT raw_bench_bundle_serialized + FROM raw_bench_bundles + WHERE raw_bench_bundle_id = ? + FOR UPDATE ", @a_vals ); } @@ -920,13 +934,26 @@ sub update_benchmark_backup_value { } +sub start_processing_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + UPDATE raw_bench_bundles + SET processing = 1 + WHERE raw_bench_bundle_id = ? + ", @a_vals ); + +} + sub update_raw_bench_bundle_set_processed { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " UPDATE raw_bench_bundles - SET processed=1 + SET processed=1, + processing=0 WHERE raw_bench_bundle_id = ? ", @a_vals ); From 16b719f12ba4da95e1903674b07d91328c311203 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 7 Sep 2015 18:34:21 +0200 Subject: [PATCH 49/80] the actual orchestration of the raw_bench processing --- lib/Tapper/Benchmark.pm | 59 ++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 66ec802..81d19bb 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -358,34 +358,45 @@ sub process_queued_multi_benchmark { my ( $or_self, $hr_options ) = @_; - require Sereal::Decoder; - - # TODO: BEGIN_transaction - # TODO: LOCK - so only one worker works here! - - my $ar_result = $or_self->{query} - ->select_raw_bench_bundle - ->fetchrow_hashref(); - - my $i_id = $ar_result->{raw_bench_bundle_id}; - my $s_serialized = $ar_result->{raw_bench_bundle_serialized}; - - return unless $i_id; - - my $ar_data_points = Sereal::Decoder::decode_sereal($s_serialized); + my $i_id; + my $s_serialized; + my $ar_data_points; + my $ar_results; + my $or_result; + + # ===== exclusively pick single raw entry ===== + # Lock single row via processing=1 so that only one worker handles it! + $or_self->{query}->start_transaction; + eval { + $ar_results = $or_self->{query}->select_raw_bench_bundle_for_lock; + goto RETURN unless defined $ar_results; + $or_result = $ar_results->fetchrow_hashref; + $i_id = $or_result->{raw_bench_bundle_id}; + goto RETURN unless $i_id; + $or_self->{query}->start_processing_raw_bench_bundle($i_id); + }; + $or_self->{query}->finish_transaction( $@ ); - # preserve order, otherwise add_multi_benchmark() would reorder to optimize insert - foreach my $chunk (@$ar_data_points) - { - $or_self->add_multi_benchmark([$chunk], $hr_options); - } + # ===== process that single raw entry ===== + $or_self->{query}->start_transaction; + eval { + require Sereal::Decoder; - $or_self->{query}->update_raw_bench_bundle_set_processed($i_id); + $ar_results = $or_self->{query}->select_raw_bench_bundle_for_processing($i_id); + $s_serialized = $ar_results->fetchrow_hashref->{raw_bench_bundle_serialized}; + $ar_data_points = Sereal::Decoder::decode_sereal($s_serialized); - # TODO: UNLOCK - # TODO: END_transaction + # preserve order, otherwise add_multi_benchmark() would reorder to optimize insert + foreach my $chunk (@$ar_data_points) + { + $or_self->add_multi_benchmark([$chunk], $hr_options); + } + $or_self->{query}->update_raw_bench_bundle_set_processed($i_id); + }; + $or_self->{query}->finish_transaction( $@ ); - return $i_id; + RETURN: + return $@ ? undef : $i_id; } From 322295aef58eecd131fc8131ca081f07df8911f0 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Mon, 7 Sep 2015 18:34:48 +0200 Subject: [PATCH 50/80] Let Sereal compress content --- lib/Tapper/Benchmark.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 81d19bb..c1520eb 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -346,7 +346,7 @@ sub enqueue_multi_benchmark { require Sereal::Encoder; - my $s_serialized = Sereal::Encoder::encode_sereal($ar_data_points); + my $s_serialized = Sereal::Encoder->new({compress => 1})->encode($ar_data_points); $or_self->{query}->insert_raw_bench_bundle($s_serialized); return 1; From 503eac50e78ccaf95e652a6543caf2e87714ec5f Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 10:56:55 +0200 Subject: [PATCH 51/80] code hygiene --- lib/Tapper/Benchmark.pm | 6 +----- lib/Tapper/Benchmark/Query/SQLite.pm | 12 ++++++++++++ lib/Tapper/Benchmark/Query/mysql.pm | 13 ++++++++++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index c1520eb..cea0786 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -369,7 +369,6 @@ sub process_queued_multi_benchmark { $or_self->{query}->start_transaction; eval { $ar_results = $or_self->{query}->select_raw_bench_bundle_for_lock; - goto RETURN unless defined $ar_results; $or_result = $ar_results->fetchrow_hashref; $i_id = $or_result->{raw_bench_bundle_id}; goto RETURN unless $i_id; @@ -387,10 +386,7 @@ sub process_queued_multi_benchmark { $ar_data_points = Sereal::Decoder::decode_sereal($s_serialized); # preserve order, otherwise add_multi_benchmark() would reorder to optimize insert - foreach my $chunk (@$ar_data_points) - { - $or_self->add_multi_benchmark([$chunk], $hr_options); - } + $or_self->add_multi_benchmark([$chunk], $hr_options) foreach my $chunk @$ar_data_points; $or_self->{query}->update_raw_bench_bundle_set_processed($i_id); }; $or_self->{query}->finish_transaction( $@ ); diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index ab1d219..068f1d6 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -974,4 +974,16 @@ sub delete_benchmark_value { } +# Garbage Collection +sub delete_processed_raw_bench_bundles { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM $or_self->{config}{tables}{additional_relation_table} + WHERE bench_value_id = ? + ", @a_vals ); + +} + 1; diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index caa0d6d..19c991a 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -709,7 +709,6 @@ sub select_raw_bench_bundle_for_lock { my ( $or_self, @a_vals ) = @_; - use Data::Dumper; print Dumper(\@a_vals); return $or_self->execute_query( " SELECT raw_bench_bundle_id FROM raw_bench_bundles @@ -981,4 +980,16 @@ sub delete_benchmark_value { } +# Garbage Collection +sub delete_processed_raw_bench_bundles { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM $or_self->{config}{tables}{additional_relation_table} + WHERE bench_value_id = ? + ", @a_vals ); + +} + 1; From 5a3f467c6de41d8f9296b669c58a13bf3ea28764 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 10:58:10 +0200 Subject: [PATCH 52/80] transaction handling - restore autocommit mode I am not exactly sure if this could be nested but anyway we push/pop the old values so at least that part works nested. --- lib/Tapper/Benchmark/Query.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/Tapper/Benchmark/Query.pm b/lib/Tapper/Benchmark/Query.pm index a40b568..70f5f9c 100644 --- a/lib/Tapper/Benchmark/Query.pm +++ b/lib/Tapper/Benchmark/Query.pm @@ -67,6 +67,7 @@ sub start_transaction { local $or_self->{dbh}{RaiseError} = 1; eval { + push @{$or_self->{old_AutoCommit}}, $or_self->{dbh}{AutoCommit}; # allow nested transactions (does this make sense) $or_self->{dbh}{AutoCommit} = 0; }; if ( $@ ) { @@ -100,6 +101,9 @@ sub finish_transaction { $or_self->{dbh}->commit(); } + my $old_AutoCommits = $or_self->{old_AutoCommit}; # reference, to change it + $or_self->{dbh}{AutoCommit} = pop @$old_AutoCommits if @$old_AutoCommits; + } return 1; From 80f5cd49e8637f2d88edce0e790eee23942a6841 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 10:59:28 +0200 Subject: [PATCH 53/80] garbage collector Currently this only covers the cleanup of the queued and already processed raw_bench_bundle rows, but other cleanup can be handled here, too. --- lib/Tapper/Benchmark.pm | 8 ++++++++ lib/Tapper/Benchmark/Query/SQLite.pm | 5 +++-- lib/Tapper/Benchmark/Query/mysql.pm | 5 +++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index cea0786..0dad1c9 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -396,6 +396,14 @@ sub process_queued_multi_benchmark { } +# garbage collect - initially raw_bench_bundles, but also other stuff. +sub gc { + + my ( $or_self, $hr_options ) = @_; + + $or_self->{query}->delete_processed_raw_bench_bundles; +} + sub add_multi_benchmark { my ( $or_self, $ar_data_points, $hr_options ) = @_; diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index 068f1d6..2b0ffac 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -980,8 +980,9 @@ sub delete_processed_raw_bench_bundles { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - DELETE FROM $or_self->{config}{tables}{additional_relation_table} - WHERE bench_value_id = ? + DELETE FROM raw_bench_bundles + WHERE processed=1 AND + processing=0 ", @a_vals ); } diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index 19c991a..71656d3 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -986,8 +986,9 @@ sub delete_processed_raw_bench_bundles { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - DELETE FROM $or_self->{config}{tables}{additional_relation_table} - WHERE bench_value_id = ? + DELETE FROM raw_bench_bundles + WHERE processed=1 AND + processing=0 ", @a_vals ); } From 507cd5be3267b6f273c80108c4d40be6065211fb Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 11:07:07 +0200 Subject: [PATCH 54/80] fix syntax error --- lib/Tapper/Benchmark.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 0dad1c9..814f2f7 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -386,7 +386,7 @@ sub process_queued_multi_benchmark { $ar_data_points = Sereal::Decoder::decode_sereal($s_serialized); # preserve order, otherwise add_multi_benchmark() would reorder to optimize insert - $or_self->add_multi_benchmark([$chunk], $hr_options) foreach my $chunk @$ar_data_points; + $or_self->add_multi_benchmark([$_], $hr_options) foreach @$ar_data_points; $or_self->{query}->update_raw_bench_bundle_set_processed($i_id); }; $or_self->{query}->finish_transaction( $@ ); From c8bd6d126b28e908345af3f6e0092d3d355a2b6d Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 14:00:42 +0200 Subject: [PATCH 55/80] check POD for T::Benchmark now, but skip others --- dist.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dist.ini b/dist.ini index d4236cf..6e60cd9 100644 --- a/dist.ini +++ b/dist.ini @@ -8,7 +8,8 @@ dist = Tapper-Benchmark repository_at = github repository = git://github.com/benchmarkanything/Tapper-Benchmark.git repository_web = http://github.com/benchmarkanything/Tapper-Benchmark -disable_pod_coverage_tests = 1 +disable_pod_coverage_tests = 0 +skip_pod_modules = qr/^Tapper::Benchmark::Query/ [Test::Compile] From 833875f072db8e4e9e5203221f14fb7e850cb334 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 14:00:58 +0200 Subject: [PATCH 56/80] docs++ --- lib/Tapper/Benchmark.pm | 47 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 814f2f7..1b245d4 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -353,7 +353,7 @@ sub enqueue_multi_benchmark { } -# deques a single bundle (can contain multiple data points) +# dequeues a single bundle (can contain multiple data points) sub process_queued_multi_benchmark { my ( $or_self, $hr_options ) = @_; @@ -1160,7 +1160,7 @@ Every "key" create a new nested hash. }); -=head3 select_benchmark_point_essentials +=head3 get_single_benchmark_point Get a single data points from the database including all essential fields (NAME, VALUE, UNIT) and additional fields. @@ -1175,6 +1175,49 @@ Get a list of all benchmark NAMEs, optionally matching a given pattern $benchmarkanythingdata = $or_bench->list_benchmark_names($pattern); +=head3 enqueue_multi_benchmark + +As a low-latency alternative to directly calling +L there is a queuing functionality. + +The C function simply writes the raw incoming +data structure serialized (and compressed) into a single row and +returns. The complementary function to this is +C which takes these values over using +the real C internally. + +=head3 process_queued_multi_benchmark + +This is part 2 of the low-latency queuing alternative to directly +calling L. + +It transactionally marks a single raw entry as being processed and +then takes over its values by calling C. It +preserves the order of entries by inserting each chunk sequentially, +to not confuse the IDs to the careful reader. After the bundle is +taken over it is marked as processed. + +This function only handles one single raw entry. It is expected to +called from co-operating multiple worker tasks or multiple times from +a wrapper. + +Currently the original raw values are B deleted immediately, just +for safety reasons, until the transactional code is death-proof (and +certified by Stuntman Mike). There is a dedicated funtion L/gc> for +that cleanup. + +The function returns the ID of the processed raw entry. + +=head3 gc + +This calls garbage collection, in particular deletes raw entries +created by C and already processed by +C. + +It is separated from those processing just for safety reasons until +the transactional code in there is waterproof. + +The gc function can cleanup more stuff in the future. =head3 subsume From 62a043603a00e9aaefb8289f468eb469e434479d Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 15:54:16 +0200 Subject: [PATCH 57/80] changelog++ --- Changes | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Changes b/Changes index ab04cfc..972748d 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,11 @@ Revision history for {{$dist->name}} {{$NEXT}} + - queuing mode for high-throughput incoming results + * part 1) adds raw results into separate table + * part 2) process one of those values, + likely called in bundles or from multiple workers + * part 3) garbage collect successfully processed results 0.006 2015-08-26 - disable caching due to wrong behaviour From 010096cccd6e7a543a82d16a89d4fb27b635e4b5 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 15:54:38 +0200 Subject: [PATCH 58/80] v0.007 - queuing mode for high-throughput incoming results * part 1) adds raw results into separate table * part 2) process one of those values, likely called in bundles or from multiple workers * part 3) garbage collect successfully processed results --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 972748d..0a19f1f 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.007 2015-09-08 - queuing mode for high-throughput incoming results * part 1) adds raw results into separate table * part 2) process one of those values, From fe0c010aa40ff5556dde21966d1f6e2a93f1ac16 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 16:41:13 +0200 Subject: [PATCH 59/80] transactions for gc() When run with the same dbh and the queuing code running with transactions, it seems the garbage collection also needs transactions. --- lib/Tapper/Benchmark.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 1b245d4..57c0b7f 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -401,7 +401,9 @@ sub gc { my ( $or_self, $hr_options ) = @_; + $or_self->{query}->start_transaction; $or_self->{query}->delete_processed_raw_bench_bundles; + $or_self->{query}->finish_transaction( $@ ); } sub add_multi_benchmark { From 8dfd310fe40fe35ce9dc35adbbf416be0b8bbce9 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 16:41:54 +0200 Subject: [PATCH 60/80] changelog++ --- Changes | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes b/Changes index 0a19f1f..5469e2d 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for {{$dist->name}} {{$NEXT}} + - enable transactions for garbage collector 0.007 2015-09-08 - queuing mode for high-throughput incoming results From b57ca985e4a7208556ccb02fb454c361ec531a0a Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 8 Sep 2015 16:42:17 +0200 Subject: [PATCH 61/80] v0.008 - enable transactions for garbage collector --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 5469e2d..6366f5f 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.008 2015-09-08 - enable transactions for garbage collector 0.007 2015-09-08 From 073613f1ad75447c7602d49d6c2bca9a6de6da60 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Fri, 11 Sep 2015 13:08:16 +0200 Subject: [PATCH 62/80] simplify reset of AutoCommit mode Nested transactions don't work anyway, so the use of a queue with push and pop was over-engineered. Drop it to just setting a single old value. --- lib/Tapper/Benchmark/Query.pm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/Tapper/Benchmark/Query.pm b/lib/Tapper/Benchmark/Query.pm index 70f5f9c..f06f20f 100644 --- a/lib/Tapper/Benchmark/Query.pm +++ b/lib/Tapper/Benchmark/Query.pm @@ -67,7 +67,7 @@ sub start_transaction { local $or_self->{dbh}{RaiseError} = 1; eval { - push @{$or_self->{old_AutoCommit}}, $or_self->{dbh}{AutoCommit}; # allow nested transactions (does this make sense) + $or_self->{old_AutoCommit} = $or_self->{dbh}{AutoCommit}; $or_self->{dbh}{AutoCommit} = 0; }; if ( $@ ) { @@ -101,8 +101,7 @@ sub finish_transaction { $or_self->{dbh}->commit(); } - my $old_AutoCommits = $or_self->{old_AutoCommit}; # reference, to change it - $or_self->{dbh}{AutoCommit} = pop @$old_AutoCommits if @$old_AutoCommits; + $or_self->{dbh}{AutoCommit} = $or_self->{old_AutoCommit}; } From 0ff171074a0f4d2288f2418730c95aec140934b1 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Fri, 11 Sep 2015 14:52:45 +0200 Subject: [PATCH 63/80] db schema type tweaks Be crisp about UNSIGNED and use BLOB to never hit any column limits, just in case... --- share/tapper-benchmark-create-schema.SQLite | 4 ++-- share/tapper-benchmark-create-schema.mysql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/share/tapper-benchmark-create-schema.SQLite b/share/tapper-benchmark-create-schema.SQLite index 5b07b52..4f78bde 100644 --- a/share/tapper-benchmark-create-schema.SQLite +++ b/share/tapper-benchmark-create-schema.SQLite @@ -153,8 +153,8 @@ CREATE TABLE `bench_backup_additional_relations` ( ) ; -- COMMENT='add additional values to benchmarks'; CREATE TABLE `raw_bench_bundles` ( - `raw_bench_bundle_id` INTEGER PRIMARY KEY NOT NULL, - `raw_bench_bundle_serialized` TEXT NOT NULL, + `raw_bench_bundle_id` INT(10) PRIMARY KEY NOT NULL, + `raw_bench_bundle_serialized` BLOB NOT NULL, `processing` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', `processed` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' diff --git a/share/tapper-benchmark-create-schema.mysql b/share/tapper-benchmark-create-schema.mysql index 55d4c45..11e23d5 100644 --- a/share/tapper-benchmark-create-schema.mysql +++ b/share/tapper-benchmark-create-schema.mysql @@ -182,8 +182,8 @@ CREATE TABLE `bench_backup_additional_relations` ( ) COMMENT='add additional values to benchmarks'; CREATE TABLE `raw_bench_bundles` ( - `raw_bench_bundle_id` INTEGER PRIMARY KEY AUTO_INCREMENT NOT NULL, - `raw_bench_bundle_serialized` TEXT NOT NULL, + `raw_bench_bundle_id` INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT NOT NULL, + `raw_bench_bundle_serialized` LONGBLOB NOT NULL, `processing` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', `processed` tinyint(3) NOT NULL DEFAULT 0 COMMENT 'is entry processed (0=no,1=yes)', `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date' From 366f7ed62e1787c8126446e6cd8ec9a7172142b7 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Fri, 11 Sep 2015 14:59:40 +0200 Subject: [PATCH 64/80] fix missing finish_transaction ...when all raw_bench_bundles are processed. --- lib/Tapper/Benchmark.pm | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 57c0b7f..fee068f 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -371,7 +371,10 @@ sub process_queued_multi_benchmark { $ar_results = $or_self->{query}->select_raw_bench_bundle_for_lock; $or_result = $ar_results->fetchrow_hashref; $i_id = $or_result->{raw_bench_bundle_id}; - goto RETURN unless $i_id; + if (!$i_id) { + $or_self->{query}->finish_transaction( $@ ); + goto RETURN ; + } $or_self->{query}->start_processing_raw_bench_bundle($i_id); }; $or_self->{query}->finish_transaction( $@ ); @@ -401,9 +404,7 @@ sub gc { my ( $or_self, $hr_options ) = @_; - $or_self->{query}->start_transaction; $or_self->{query}->delete_processed_raw_bench_bundles; - $or_self->{query}->finish_transaction( $@ ); } sub add_multi_benchmark { From 98e098bb13889cf79a6fb3e3c209f27c51213ed9 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Fri, 11 Sep 2015 15:35:51 +0200 Subject: [PATCH 65/80] tuning: avoid deadlocks due to gap locking (under mysql) --- lib/Tapper/Benchmark.pm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index fee068f..1b9e499 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -142,6 +142,7 @@ sub new { else { $or_self->{query} = $s_module->new({ dbh => $hr_atts->{dbh}, + driver => $hr_atts->{dbh}{Driver}{Name}, debug => $hr_atts->{debug} || 0, config => $or_self->{config}, }); @@ -363,9 +364,11 @@ sub process_queued_multi_benchmark { my $ar_data_points; my $ar_results; my $or_result; + my $driver = $or_self->{query}{driver}; # ===== exclusively pick single raw entry ===== # Lock single row via processing=1 so that only one worker handles it! + $or_self->{query}{dbh}->do("set transaction isolation level read committed") if $driver eq "mysql"; # avoid deadlocks due to gap locking $or_self->{query}->start_transaction; eval { $ar_results = $or_self->{query}->select_raw_bench_bundle_for_lock; @@ -373,11 +376,13 @@ sub process_queued_multi_benchmark { $i_id = $or_result->{raw_bench_bundle_id}; if (!$i_id) { $or_self->{query}->finish_transaction( $@ ); + $or_self->{query}{dbh}->do("set transaction isolation level repeatable read") if $driver eq "mysql"; # reset to normal gap locking goto RETURN ; } $or_self->{query}->start_processing_raw_bench_bundle($i_id); }; $or_self->{query}->finish_transaction( $@ ); + $or_self->{query}{dbh}->do("set transaction isolation level repeatable read") if $driver eq "mysql"; # reset to normal gap locking # ===== process that single raw entry ===== $or_self->{query}->start_transaction; From 15c3ba04af08a3c823f02baa1184b9619ae95191 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Fri, 11 Sep 2015 15:36:25 +0200 Subject: [PATCH 66/80] fix missing LIMIT 1 for queue workers --- lib/Tapper/Benchmark/Query/mysql.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index 71656d3..99c8329 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -728,6 +728,7 @@ sub select_raw_bench_bundle_for_processing { SELECT raw_bench_bundle_serialized FROM raw_bench_bundles WHERE raw_bench_bundle_id = ? + LIMIT 1 FOR UPDATE ", @a_vals ); } From a69edb6fc78285dc0c4309c8d1d474b35196250e Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Fri, 11 Sep 2015 15:42:00 +0200 Subject: [PATCH 67/80] changelog++ --- Changes | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Changes b/Changes index 6366f5f..a1f7590 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,11 @@ Revision history for {{$dist->name}} {{$NEXT}} + - DB tuning/tweaking: + * avoid deadlocks due to gap locking + * fix missing finish_transaction + * be crisp about UNSIGNED and use BLOB for raw queue + * simplify resetting of AutoCommit mode 0.008 2015-09-08 - enable transactions for garbage collector From 1a56241ab945f31f5185f70fc44d5e6bdf921d14 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Fri, 11 Sep 2015 15:42:17 +0200 Subject: [PATCH 68/80] v0.009 - DB tuning/tweaking: * avoid deadlocks due to gap locking * fix missing finish_transaction * be crisp about UNSIGNED and use BLOB for raw queue * simplify resetting of AutoCommit mode --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index a1f7590..dc3a594 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.009 2015-09-11 - DB tuning/tweaking: * avoid deadlocks due to gap locking * fix missing finish_transaction From d083df098085cd88c2bd68717933390ab10c5642 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 15 Sep 2015 14:22:14 +0200 Subject: [PATCH 69/80] drop feature 'switch' ...and replace given/when with classic if/elsif/else cascades. --- lib/Tapper/Benchmark/Query/SQLite.pm | 21 +++++++++------------ lib/Tapper/Benchmark/Query/mysql.pm | 21 +++++++++------------ 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index 2b0ffac..4b26459 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -3,7 +3,6 @@ package Tapper::Benchmark::Query::SQLite; use strict; use warnings; -use feature 'switch'; use base 'Tapper::Benchmark::Query'; use List::MoreUtils qw( any ); @@ -79,8 +78,7 @@ sub create_select_column { my $s_return_select = q##; AGGR: { - given( $s_aggr ) { - when ( q## ) { + if ( $s_aggr eq q## ) { # aggregate all columns if a single column is aggregated if ( $b_aggregate_all ) { $s_aggr = $or_self->{config}{default_aggregation}; @@ -88,36 +86,35 @@ sub create_select_column { } $s_return_select = '${COLUMN}'; } - when ( 'min' ) { + elsif ( $s_aggr eq 'min' ) { $s_return_select = 'MIN( ${COLUMN} )'; } - when ( 'max' ) { + elsif ( $s_aggr eq 'max' ) { $s_return_select = 'MAX( ${COLUMN} )'; } - when ( 'avg' ) { + elsif ( $s_aggr eq 'avg' ) { $s_return_select = 'AVG( ${COLUMN} )'; } # Geometric Mean, unsupported in SQLite due to lack of EXP(), # see http://stackoverflow.com/questions/13190064/how-to-find-power-of-a-number-in-sqlite # - # when ( 'gem' ) { + # elsif ( $s_aggr eq 'gem' ) { # $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; # } - when ( 'sum' ) { + elsif ( $s_aggr eq 'sum' ) { $s_return_select = 'SUM( ${COLUMN} )'; } - when ( 'cnt' ) { + elsif ( $s_aggr eq 'cnt' ) { $s_return_select = 'COUNT( ${COLUMN} )'; } - when ( 'cnd' ) { + elsif ( $s_aggr eq 'cnd' ) { $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; } - default { + else { require Carp; Carp::confess("unknown aggregate function '$s_aggr'"); return; } - } } # AGGR my ( $s_return_column ); diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index 99c8329..a2cfa46 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -3,7 +3,6 @@ package Tapper::Benchmark::Query::mysql; use strict; use warnings; -use feature 'switch'; use base 'Tapper::Benchmark::Query'; use List::MoreUtils qw( any ); @@ -79,8 +78,7 @@ sub create_select_column { my $s_return_select = q##; AGGR: { - given( $s_aggr ) { - when ( q## ) { + if ( $s_aggr eq q## ) { # aggregate all columns if a single column is aggregated if ( $b_aggregate_all ) { $s_aggr = $or_self->{config}{default_aggregation}; @@ -88,33 +86,32 @@ sub create_select_column { } $s_return_select = '${COLUMN}'; } - when ( 'min' ) { + elsif ( $s_aggr eq 'min' ) { $s_return_select = 'MIN( ${COLUMN} )'; } - when ( 'max' ) { + elsif ( $s_aggr eq 'max' ) { $s_return_select = 'MAX( ${COLUMN} )'; } - when ( 'avg' ) { + elsif ( $s_aggr eq 'avg' ) { $s_return_select = 'AVG( ${COLUMN} )'; } - when ( 'gem' ) { + elsif ( $s_aggr eq 'gem' ) { $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; } - when ( 'sum' ) { + elsif ( $s_aggr eq 'sum' ) { $s_return_select = 'SUM( ${COLUMN} )'; } - when ( 'cnt' ) { + elsif ( $s_aggr eq 'cnt' ) { $s_return_select = 'COUNT( ${COLUMN} )'; } - when ( 'cnd' ) { + elsif ( $s_aggr eq 'cnd' ) { $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; } - default { + else { require Carp; Carp::confess("unknown aggregate function '$s_aggr'"); return; } - } } # AGGR my ( $s_return_column ); From 6d68de47ffaa544a1d7610312e35e8091dc998ed Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 15 Sep 2015 14:23:18 +0200 Subject: [PATCH 70/80] changelog++ --- Changes | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes b/Changes index dc3a594..2eaaa21 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for {{$dist->name}} {{$NEXT}} + - drop feature 'switch' - replace given/when with classic if/else 0.009 2015-09-11 - DB tuning/tweaking: From d6c5cabf3024561458b076b03c2255917954e65a Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Tue, 15 Sep 2015 14:23:32 +0200 Subject: [PATCH 71/80] v0.010 - drop feature 'switch' - replace given/when with classic if/else --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 2eaaa21..91937a2 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.010 2015-09-15 - drop feature 'switch' - replace given/when with classic if/else 0.009 2015-09-11 From f9382660bb308d1016a5167020f0abff78a155bf Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 16 Sep 2015 16:36:50 +0200 Subject: [PATCH 72/80] fix getting db driver name --- lib/Tapper/Benchmark.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 1b9e499..e42253c 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -364,7 +364,7 @@ sub process_queued_multi_benchmark { my $ar_data_points; my $ar_results; my $or_result; - my $driver = $or_self->{query}{driver}; + my $driver = $or_self->{query}{dbh}{Driver}{Name}; # ===== exclusively pick single raw entry ===== # Lock single row via processing=1 so that only one worker handles it! From aabcb77d87131e67c142ddb2ab02c8044a7392b7 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 16 Sep 2015 16:37:53 +0200 Subject: [PATCH 73/80] do not compress in application side ...to save cpu cycles and let this handle the database. --- lib/Tapper/Benchmark.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index e42253c..79593f0 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -347,7 +347,7 @@ sub enqueue_multi_benchmark { require Sereal::Encoder; - my $s_serialized = Sereal::Encoder->new({compress => 1})->encode($ar_data_points); + my $s_serialized = Sereal::Encoder->new->encode($ar_data_points); $or_self->{query}->insert_raw_bench_bundle($s_serialized); return 1; From 6cad86f8cad1d2ccd28e5e418632c9bf2e5a3cea Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 16 Sep 2015 16:38:27 +0200 Subject: [PATCH 74/80] mysql: enable compression in raw queue --- share/tapper-benchmark-create-schema.mysql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/tapper-benchmark-create-schema.mysql b/share/tapper-benchmark-create-schema.mysql index 11e23d5..84a18fd 100644 --- a/share/tapper-benchmark-create-schema.mysql +++ b/share/tapper-benchmark-create-schema.mysql @@ -187,7 +187,7 @@ CREATE TABLE `raw_bench_bundles` ( `processing` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', `processed` tinyint(3) NOT NULL DEFAULT 0 COMMENT 'is entry processed (0=no,1=yes)', `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date' -) COMMENT='queue raw incoming data for later processing'; +) ROW_FORMAT=COMPRESSED COMMENT='queue raw incoming data for later processing'; -- Usage: -- * create elements: -- INSERT INTO raw_bench_bundles (raw_bench_bundle_serialized) VALUES ('hot content'); From f3598fe52a7621c0e0aca4e5fbaa1e9060dbdb69 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 16 Sep 2015 16:44:30 +0200 Subject: [PATCH 75/80] changelog++ --- Changes | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes b/Changes index 91937a2..8a76a0b 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for {{$dist->name}} {{$NEXT}} + - let the db take care of compression in raw queue 0.010 2015-09-15 - drop feature 'switch' - replace given/when with classic if/else From 793f143204f8d846f9b27f35735a808849895afc Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 16 Sep 2015 16:46:36 +0200 Subject: [PATCH 76/80] v0.011 - let the db take care of compression in raw queue --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index 8a76a0b..1d40b21 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.011 2015-09-16 - let the db take care of compression in raw queue 0.010 2015-09-15 From 412db2a47ceeb47be7a9339dfe88988764799510 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Wed, 16 Sep 2015 16:47:23 +0200 Subject: [PATCH 77/80] doc typo --- lib/Tapper/Benchmark/Query/SQLite.pm | 2 +- lib/Tapper/Benchmark/Query/mysql.pm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index 4b26459..c3652cd 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -1,5 +1,5 @@ package Tapper::Benchmark::Query::SQLite; -# ABSTRACT: Tapper::Benchmark - qerying - SQLite backend +# ABSTRACT: Tapper::Benchmark - querying - SQLite backend use strict; use warnings; diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index a2cfa46..be52fc3 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -1,5 +1,5 @@ package Tapper::Benchmark::Query::mysql; -# ABSTRACT: Tapper::Benchmark - qerying - MySQL backend +# ABSTRACT: Tapper::Benchmark - querying - MySQL backend use strict; use warnings; From d74f274d62962a8e7975dd80ffa2864ad489075a Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Thu, 17 Sep 2015 08:35:24 +0200 Subject: [PATCH 78/80] Refactor common query methods ...into a ::common.pm class. --- lib/Tapper/Benchmark/Query/SQLite.pm | 628 +++------------------------ lib/Tapper/Benchmark/Query/common.pm | 531 ++++++++++++++++++++++ lib/Tapper/Benchmark/Query/mysql.pm | 623 +++----------------------- 3 files changed, 645 insertions(+), 1137 deletions(-) create mode 100644 lib/Tapper/Benchmark/Query/common.pm diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm index c3652cd..a32baba 100644 --- a/lib/Tapper/Benchmark/Query/SQLite.pm +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -3,7 +3,7 @@ package Tapper::Benchmark::Query::SQLite; use strict; use warnings; -use base 'Tapper::Benchmark::Query'; +use base 'Tapper::Benchmark::Query::common'; use List::MoreUtils qw( any ); @@ -16,163 +16,6 @@ my %h_default_columns = ( 'CREATED' => 'bv.created_at', ); -sub default_columns { - return %h_default_columns; -} - -sub benchmark_operators { - return ( '=', '!=', 'like', 'not like', '<', '>', '<=', '>=' ); -} - -sub create_where_clause { - - my ( $s_column_name, $ar_value ) = @_; - - my $s_where_clause = q##; - if ( $ar_value->[0] eq 'not like' ) { - $s_where_clause = "$s_column_name NOT LIKE ?"; - } - elsif ( $ar_value->[0] eq 'like' ) { - $s_where_clause = "$s_column_name LIKE ?"; - } - elsif ( - $ar_value->[0] eq '<' - || $ar_value->[0] eq '>' - || $ar_value->[0] eq '<=' - || $ar_value->[0] eq '>=' - ) { - $s_where_clause = "$s_column_name $ar_value->[0] ?"; - } - elsif ( $ar_value->[0] eq '=' ) { - if ( $#{$ar_value} > 1 ) { - $s_where_clause = "$s_column_name IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; - } - else { - $s_where_clause = "$s_column_name = ?"; - } - } - elsif ( $ar_value->[0] eq '!=' ) { - if ( $#{$ar_value} > 1 ) { - $s_where_clause = "$s_column_name NOT IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; - } - else { - $s_where_clause = "$s_column_name != ?"; - } - } - else { - require Carp; - Carp::confess("unknown operator '$ar_value->[0]'"); - return; - } - - return $s_where_clause; - -} - -sub create_select_column { - - my ( $or_self, $ar_select, $i_counter, $b_aggregate_all ) = @_; - - my $s_aggr_func = q##; - my ( $s_aggr, $s_column ) = @{$ar_select}; - my $s_return_select = q##; - - AGGR: { - if ( $s_aggr eq q## ) { - # aggregate all columns if a single column is aggregated - if ( $b_aggregate_all ) { - $s_aggr = $or_self->{config}{default_aggregation}; - redo AGGR; - } - $s_return_select = '${COLUMN}'; - } - elsif ( $s_aggr eq 'min' ) { - $s_return_select = 'MIN( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'max' ) { - $s_return_select = 'MAX( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'avg' ) { - $s_return_select = 'AVG( ${COLUMN} )'; - } - # Geometric Mean, unsupported in SQLite due to lack of EXP(), - # see http://stackoverflow.com/questions/13190064/how-to-find-power-of-a-number-in-sqlite - # - # elsif ( $s_aggr eq 'gem' ) { - # $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; - # } - elsif ( $s_aggr eq 'sum' ) { - $s_return_select = 'SUM( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'cnt' ) { - $s_return_select = 'COUNT( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'cnd' ) { - $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; - } - else { - require Carp; - Carp::confess("unknown aggregate function '$s_aggr'"); - return; - } - } # AGGR - - my ( $s_return_column ); - my $s_replace_as = $s_aggr ? $s_aggr . "_$s_column" : $s_column; - - if ( $h_used_selects{$or_self}{$s_replace_as} ) { - return; - } - if ( any { $s_column eq $_ } keys %h_default_columns ) { - $h_used_selects{$or_self}{$s_replace_as} = $h_default_columns{$s_column}; - } - else { - $s_return_column = $s_column; - $h_used_selects{$or_self}{$s_replace_as} = "bav$i_counter.bench_additional_value"; - } - - $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; - - return ( $s_return_column, "$s_return_select AS '$s_replace_as'", ); - -} - -sub create_period_check { - - my ( $s_column, $dt_from, $dt_to ) = @_; - - my @a_vals; - my $s_where; - if ( $dt_from ) { - if ( my ( $s_date, $s_time ) = $dt_from =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { - $s_where .= "\nAND $s_column > ?"; - push @a_vals, $s_date . ( $s_time || ' 00:00:00' ); - } - else { - require Carp; - Carp::confess(q#unknown date format for 'date_from'#); - return; - } - } - if ( $dt_to ) { - if ( my ( $s_date, $s_time ) = $dt_to =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { - $s_where .= "\nAND $s_column < ?"; - push @a_vals, $s_date . ( $s_time || ' 23:59:59' ); - } - else { - require Carp; - Carp::confess(q#unknown date format for 'date_to'#); - return; - } - } - - return { - vals => \@a_vals, - where => $s_where, - }; - -} - sub select_benchmark_values { my ( $or_self, $hr_search ) = @_; @@ -232,7 +75,7 @@ sub select_benchmark_values { for my $ar_where ( @{$hr_search->{where}} ) { if ( any { $ar_where->[1] eq $_ } keys %h_default_columns ) { my $s_column = splice( @{$ar_where}, 1, 1 ); - push @a_where, create_where_clause( $h_default_columns{$s_column}, $ar_where ); + push @a_where, $or_self->create_where_clause( $h_default_columns{$s_column}, $ar_where ); push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; } else { @@ -258,7 +101,7 @@ sub select_benchmark_values { ) "; push @a_from_vals, $hr_additional_type->{bench_additional_type_id}; - push @a_where, create_where_clause( "bav$i_counter.bench_additional_value", $ar_where ); + push @a_where, $or_self->create_where_clause( "bav$i_counter.bench_additional_value", $ar_where ); push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; $i_counter++; } @@ -419,289 +262,71 @@ sub select_benchmark_values { } -sub select_benchmark_point_essentials { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT - b.bench, - bv.bench_value, - bu.bench_unit - FROM - $or_self->{config}{tables}{benchmark_table} b - JOIN - $or_self->{config}{tables}{benchmark_value_table} bv - ON - b.bench_id = bv.bench_id - LEFT JOIN - $or_self->{config}{tables}{unit_table} bu - ON - b.bench_unit_id = bu.bench_unit_id - WHERE - bv.bench_value_id = ? - ; - ", @a_vals ); - -} - -sub select_complete_benchmark_point { - - my ( $or_self, @a_vals ) = @_; - - my $query = " - SELECT - bat.bench_additional_type, - bav.bench_additional_value - FROM - benchs b - JOIN - bench_values bv - ON - b.bench_id = bv.bench_id - JOIN - bench_additional_type_relations batr - ON - bv.bench_id = batr.bench_id - JOIN - bench_additional_types bat - ON - batr.bench_additional_type_id = bat.bench_additional_type_id - JOIN - bench_additional_relations bar - ON - bv.bench_value_id = bar.bench_value_id - JOIN - bench_additional_values bav - ON - bar.bench_additional_value_id = bav.bench_additional_value_id AND - bat.bench_additional_type_id = bav.bench_additional_type_id - WHERE - bv.bench_value_id = ? - ORDER BY - bat.bench_additional_type"; - return $or_self->execute_query( $query, @a_vals ); -} - -sub select_addtype_by_name { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_additional_type_id - FROM $or_self->{config}{tables}{additional_type_table} - WHERE bench_additional_type = ? - ", @a_vals ); - -} - -sub select_min_subsume_type { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_subsume_type_id - FROM $or_self->{config}{tables}{subsume_type_table} - ORDER BY bench_subsume_type_rank ASC - LIMIT 1 - " ); - -} - -sub select_subsume_type { - - my ( $or_self, @a_vals ) = @_; +sub create_select_column { - return $or_self->execute_query( " - SELECT - bench_subsume_type_id, - bench_subsume_type_rank, - datetime_strftime_pattern - FROM - $or_self->{config}{tables}{subsume_type_table} - WHERE - bench_subsume_type = ? - ", @a_vals ); + my ( $or_self, $ar_select, $i_counter, $b_aggregate_all ) = @_; -} + my $s_aggr_func = q##; + my ( $s_aggr, $s_column ) = @{$ar_select}; + my $s_return_select = q##; -sub select_check_subsumed_values { + AGGR: { + if ( $s_aggr eq q## ) { + # aggregate all columns if a single column is aggregated + if ( $b_aggregate_all ) { + $s_aggr = $or_self->{config}{default_aggregation}; + redo AGGR; + } + $s_return_select = '${COLUMN}'; + } + elsif ( $s_aggr eq 'min' ) { + $s_return_select = 'MIN( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'max' ) { + $s_return_select = 'MAX( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'avg' ) { + $s_return_select = 'AVG( ${COLUMN} )'; + } + # Geometric Mean, unsupported in SQLite due to lack of EXP(), + # see http://stackoverflow.com/questions/13190064/how-to-find-power-of-a-number-in-sqlite + # + # elsif ( $s_aggr eq 'gem' ) { + # $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; + # } + elsif ( $s_aggr eq 'sum' ) { + $s_return_select = 'SUM( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'cnt' ) { + $s_return_select = 'COUNT( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'cnd' ) { + $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; + } + else { + require Carp; + Carp::confess("unknown aggregate function '$s_aggr'"); + return; + } + } # AGGR - my ( $or_self, $hr_vals ) = @_; + my ( $s_return_column ); + my $s_replace_as = $s_aggr ? $s_aggr . "_$s_column" : $s_column; - if (! $hr_vals->{subsume_type_id} ) { - require Carp; - Carp::confess(q#required parameter 'subsume_type_id' is missing#); + if ( $h_used_selects{$or_self}{$s_replace_as} ) { return; } - - my $hr_period_check = create_period_check( - 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} - ); - - return $or_self->execute_query( - " - SELECT - bv.bench_value_id - FROM - bench_values bv - JOIN bench_subsume_types bet - ON ( bv.bench_subsume_type_id = bet.bench_subsume_type_id ) - WHERE - bet.bench_subsume_type_rank > ( - SELECT beti.bench_subsume_type_rank - FROM bench_subsume_types beti - WHERE bench_subsume_type_id = ? - ) - $hr_period_check->{where} - LIMIT - 1 - ", - $hr_vals->{subsume_type_id}, - @{$hr_period_check->{vals}}, - ); - -} - -sub select_data_values_for_subsume { - - my ( $or_self, $hr_vals ) = @_; - - my $hr_period_check = create_period_check( - 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} - ); - - my @a_addexclude_vals; - my $s_addexclude_where = q##; - if ( $hr_vals->{exclude_additionals} && @{$hr_vals->{exclude_additionals}} ) { - $s_addexclude_where = 'AND bav.bench_additional_type_id NOT IN (' . (join ',', map {'?'} @{$hr_vals->{exclude_additionals}}) . ')'; - push @a_addexclude_vals, @{$hr_vals->{exclude_additionals}}; + if ( any { $s_column eq $_ } keys %h_default_columns ) { + $h_used_selects{$or_self}{$s_replace_as} = $h_default_columns{$s_column}; + } + else { + $s_return_column = $s_column; + $h_used_selects{$or_self}{$s_replace_as} = "bav$i_counter.bench_additional_value"; } - return $or_self->execute_query( - " - SELECT - b.bench_id, - bv.bench_value_id, - bv.created_at, - bv.bench_value, - bet.bench_subsume_type_rank, - GROUP_CONCAT( - bav.bench_additional_type_id, - '|', - bav.bench_additional_value_id - ORDER BY - bav.bench_additional_type_id, - bav.bench_additional_value_id - SEPARATOR - '-' - ) AS additionals - FROM - benchs b - JOIN bench_values bv - ON ( bv.bench_id = b.bench_id ) - JOIN bench_subsume_types bet - ON ( bet.bench_subsume_type_id = bv.bench_subsume_type_id ) - LEFT JOIN ( - bench_additional_relations bar - JOIN bench_additional_values bav - ON ( bav.bench_additional_value_id = bar.bench_additional_value_id ) - ) - ON ( - bar.active = 1 - AND bar.bench_value_id = bv.bench_value_id - $s_addexclude_where - ) - WHERE - b.active = 1 - AND bv.active = 1 - $hr_period_check->{where} - GROUP BY - bet.bench_subsume_type_rank, - b.bench_id, - bv.created_at, - bv.bench_value, - bv.bench_value_id - ORDER BY - b.bench_id, - additionals, - bv.created_at - ", - @a_addexclude_vals, - @{$hr_period_check->{vals}}, - ); -} - -sub select_benchmark { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_id - FROM $or_self->{config}{tables}{benchmark_table} - WHERE bench = ? - ", @a_vals ); - -} - -sub select_benchmark_names { - - my ( $or_self, @a_vals ) = @_; - - my $query = " - SELECT DISTINCT bench - FROM $or_self->{config}{tables}{benchmark_table}"; - $query .= " - WHERE bench LIKE ? " if @a_vals; - return $or_self->execute_query( $query, @a_vals ); - -} - -sub select_unit { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_unit_id - FROM $or_self->{config}{tables}{unit_table} - WHERE bench_unit = ? - ", @a_vals ); - -} - -sub select_addtype { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_additional_type_id - FROM $or_self->{config}{tables}{additional_type_table} - WHERE bench_additional_type = ? - ", @a_vals ); - -} - -sub select_addvalue { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_additional_value_id - FROM $or_self->{config}{tables}{additional_value_table} - WHERE bench_additional_type_id = ? AND bench_additional_value = ? - ", @a_vals ); - -} - -sub select_addtyperelation { - - my ( $or_self, @a_vals ) = @_; + $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; - return $or_self->execute_query( " - SELECT bench_id, bench_additional_type_id, created_at - FROM $or_self->{config}{tables}{additional_type_relation_table} - WHERE bench_id = ? AND bench_additional_type_id = ? - ", @a_vals ); + return ( $s_return_column, "$s_return_select AS '$s_replace_as'", ); } @@ -783,18 +408,6 @@ sub insert_benchmark_value { } -sub insert_raw_bench_bundle { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - INSERT INTO raw_bench_bundles - (raw_bench_bundle_serialized) - VALUES ( ? ) - ", @a_vals ); - -} - sub copy_additional_values { my ( $or_self, $hr_vals ) = @_; @@ -820,56 +433,6 @@ sub copy_additional_values { } -sub copy_benchmark_backup_value { - - my ( $or_self, $hr_vals ) = @_; - - for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { - if (! $hr_vals->{$s_param} ) { - require Carp; - Carp::confess("missing parameter '$s_param'"); - return; - } - } - - return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{benchmark_backup_value_table} - ( bench_value_id, bench_id, bench_subsume_type_id, bench_value, active, created_at ) - SELECT - ?, bench_id, bench_subsume_type_id, bench_value, active, created_at - FROM - $or_self->{config}{tables}{benchmark_value_table} - WHERE - bench_value_id = ? - ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); - -} - -sub copy_benchmark_backup_additional_relations { - - my ( $or_self, $hr_vals ) = @_; - - for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { - if (! $hr_vals->{$s_param} ) { - require Carp; - Carp::confess("missing parameter '$s_param'"); - return; - } - } - - return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{backup_additional_relation_table} - ( bench_backup_value_id, bench_additional_value_id, active, created_at ) - SELECT - ?, bench_additional_value_id, active, created_at - FROM - $or_self->{config}{tables}{additional_relation_table} - WHERE - bench_value_id = ? - ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); - -} - sub insert_addtype { my ( $or_self, @a_vals ) = @_; @@ -909,79 +472,4 @@ sub insert_addvaluerelation { } -sub update_benchmark_backup_value { - - my ( $or_self, $hr_vals ) = @_; - - return $or_self->execute_query( " - UPDATE $or_self->{config}{tables}{benchmark_backup_value_table} - SET bench_value_id = ? - WHERE bench_value_id = ? - ", @{$hr_vals}{qw/ - new_bench_value_id - old_bench_value_id - /} ); - -} - -sub start_processing_raw_bench_bundle { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - UPDATE raw_bench_bundles - SET processing = 1 - WHERE raw_bench_bundle_id = ? - ", @a_vals ); - -} - -sub update_raw_bench_bundle_set_processed { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - UPDATE raw_bench_bundles - SET processed=1, - processing=0 - WHERE raw_bench_bundle_id = ? - ", @a_vals ); - -} - -sub delete_benchmark_additional_relations { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - DELETE FROM $or_self->{config}{tables}{additional_relation_table} - WHERE bench_value_id = ? - ", @a_vals ); - -} - -sub delete_benchmark_value { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - DELETE FROM $or_self->{config}{tables}{benchmark_value_table} - WHERE bench_value_id = ? - ", @a_vals ); - -} - -# Garbage Collection -sub delete_processed_raw_bench_bundles { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - DELETE FROM raw_bench_bundles - WHERE processed=1 AND - processing=0 - ", @a_vals ); - -} - 1; diff --git a/lib/Tapper/Benchmark/Query/common.pm b/lib/Tapper/Benchmark/Query/common.pm new file mode 100644 index 0000000..42df21f --- /dev/null +++ b/lib/Tapper/Benchmark/Query/common.pm @@ -0,0 +1,531 @@ +package Tapper::Benchmark::Query::common; +# ABSTRACT: Tapper::Benchmark - querying - backend base class + +use strict; +use warnings; +use base 'Tapper::Benchmark::Query'; + +use List::MoreUtils qw( any ); + +my %h_used_selects; +my %h_default_columns = ( + 'NAME' => 'b.bench', + 'UNIT' => 'bu.bench_unit', + 'VALUE' => 'bv.bench_value', + 'VALUE_ID' => 'bv.bench_value_id', + 'CREATED' => 'bv.created_at', +); + +sub default_columns { + return %h_default_columns; +} + +sub benchmark_operators { + return ( '=', '!=', 'like', 'not like', '<', '>', '<=', '>=' ); +} + +sub create_where_clause { + + my ( $or_self, $s_column_name, $ar_value ) = @_; + + my $s_where_clause = q##; + if ( $ar_value->[0] eq 'not like' ) { + $s_where_clause = "$s_column_name NOT LIKE ?"; + } + elsif ( $ar_value->[0] eq 'like' ) { + $s_where_clause = "$s_column_name LIKE ?"; + } + elsif ( + $ar_value->[0] eq '<' + || $ar_value->[0] eq '>' + || $ar_value->[0] eq '<=' + || $ar_value->[0] eq '>=' + ) { + $s_where_clause = "$s_column_name $ar_value->[0] ?"; + } + elsif ( $ar_value->[0] eq '=' ) { + if ( $#{$ar_value} > 1 ) { + $s_where_clause = "$s_column_name IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; + } + else { + $s_where_clause = "$s_column_name = ?"; + } + } + elsif ( $ar_value->[0] eq '!=' ) { + if ( $#{$ar_value} > 1 ) { + $s_where_clause = "$s_column_name NOT IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; + } + else { + $s_where_clause = "$s_column_name != ?"; + } + } + else { + require Carp; + Carp::confess("unknown operator '$ar_value->[0]'"); + return; + } + + return $s_where_clause; + +} + +sub create_period_check { + + my ( $or_self, $s_column, $dt_from, $dt_to ) = @_; + + my @a_vals; + my $s_where; + if ( $dt_from ) { + if ( my ( $s_date, $s_time ) = $dt_from =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { + $s_where .= "\nAND $s_column > ?"; + push @a_vals, $s_date . ( $s_time || ' 00:00:00' ); + } + else { + require Carp; + Carp::confess(q#unknown date format for 'date_from'#); + return; + } + } + if ( $dt_to ) { + if ( my ( $s_date, $s_time ) = $dt_to =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { + $s_where .= "\nAND $s_column < ?"; + push @a_vals, $s_date . ( $s_time || ' 23:59:59' ); + } + else { + require Carp; + Carp::confess(q#unknown date format for 'date_to'#); + return; + } + } + + return { + vals => \@a_vals, + where => $s_where, + }; + +} + +sub select_benchmark_point_essentials { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT + b.bench, + bv.bench_value, + bu.bench_unit + FROM + $or_self->{config}{tables}{benchmark_table} b + JOIN + $or_self->{config}{tables}{benchmark_value_table} bv + ON + b.bench_id = bv.bench_id + LEFT JOIN + $or_self->{config}{tables}{unit_table} bu + ON + b.bench_unit_id = bu.bench_unit_id + WHERE + bv.bench_value_id = ? + ; + ", @a_vals ); + +} + +sub select_complete_benchmark_point { + + my ( $or_self, @a_vals ) = @_; + + my $query = " + SELECT + bat.bench_additional_type, + bav.bench_additional_value + FROM + benchs b + JOIN + bench_values bv + ON + b.bench_id = bv.bench_id + JOIN + bench_additional_type_relations batr + ON + bv.bench_id = batr.bench_id + JOIN + bench_additional_types bat + ON + batr.bench_additional_type_id = bat.bench_additional_type_id + JOIN + bench_additional_relations bar + ON + bv.bench_value_id = bar.bench_value_id + JOIN + bench_additional_values bav + ON + bar.bench_additional_value_id = bav.bench_additional_value_id AND + bat.bench_additional_type_id = bav.bench_additional_type_id + WHERE + bv.bench_value_id = ? + ORDER BY + bat.bench_additional_type"; + return $or_self->execute_query( $query, @a_vals ); +} + +sub select_addtype_by_name { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_type_id + FROM $or_self->{config}{tables}{additional_type_table} + WHERE bench_additional_type = ? + ", @a_vals ); + +} + +sub select_min_subsume_type { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_subsume_type_id + FROM $or_self->{config}{tables}{subsume_type_table} + ORDER BY bench_subsume_type_rank ASC + LIMIT 1 + " ); + +} + +sub select_subsume_type { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT + bench_subsume_type_id, + bench_subsume_type_rank, + datetime_strftime_pattern + FROM + $or_self->{config}{tables}{subsume_type_table} + WHERE + bench_subsume_type = ? + ", @a_vals ); + +} + +sub select_check_subsumed_values { + + my ( $or_self, $hr_vals ) = @_; + + if (! $hr_vals->{subsume_type_id} ) { + require Carp; + Carp::confess(q#required parameter 'subsume_type_id' is missing#); + return; + } + + my $hr_period_check = $or_self->create_period_check( + 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} + ); + + return $or_self->execute_query( + " + SELECT + bv.bench_value_id + FROM + bench_values bv + JOIN bench_subsume_types bet + ON ( bv.bench_subsume_type_id = bet.bench_subsume_type_id ) + WHERE + bet.bench_subsume_type_rank > ( + SELECT beti.bench_subsume_type_rank + FROM bench_subsume_types beti + WHERE bench_subsume_type_id = ? + ) + $hr_period_check->{where} + LIMIT + 1 + ", + $hr_vals->{subsume_type_id}, + @{$hr_period_check->{vals}}, + ); + +} + +sub select_data_values_for_subsume { + + my ( $or_self, $hr_vals ) = @_; + + my $hr_period_check = $or_self->create_period_check( + 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} + ); + + my @a_addexclude_vals; + my $s_addexclude_where = q##; + if ( $hr_vals->{exclude_additionals} && @{$hr_vals->{exclude_additionals}} ) { + $s_addexclude_where = 'AND bav.bench_additional_type_id NOT IN (' . (join ',', map {'?'} @{$hr_vals->{exclude_additionals}}) . ')'; + push @a_addexclude_vals, @{$hr_vals->{exclude_additionals}}; + } + + return $or_self->execute_query( + " + SELECT + b.bench_id, + bv.bench_value_id, + bv.created_at, + bv.bench_value, + bet.bench_subsume_type_rank, + GROUP_CONCAT( + bav.bench_additional_type_id, + '|', + bav.bench_additional_value_id + ORDER BY + bav.bench_additional_type_id, + bav.bench_additional_value_id + SEPARATOR + '-' + ) AS additionals + FROM + benchs b + JOIN bench_values bv + ON ( bv.bench_id = b.bench_id ) + JOIN bench_subsume_types bet + ON ( bet.bench_subsume_type_id = bv.bench_subsume_type_id ) + LEFT JOIN ( + bench_additional_relations bar + JOIN bench_additional_values bav + ON ( bav.bench_additional_value_id = bar.bench_additional_value_id ) + ) + ON ( + bar.active = 1 + AND bar.bench_value_id = bv.bench_value_id + $s_addexclude_where + ) + WHERE + b.active = 1 + AND bv.active = 1 + $hr_period_check->{where} + GROUP BY + bet.bench_subsume_type_rank, + b.bench_id, + bv.created_at, + bv.bench_value, + bv.bench_value_id + ORDER BY + b.bench_id, + additionals, + bv.created_at + ", + @a_addexclude_vals, + @{$hr_period_check->{vals}}, + ); +} + +sub select_benchmark { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_id + FROM $or_self->{config}{tables}{benchmark_table} + WHERE bench = ? + ", @a_vals ); + +} + +sub select_benchmark_names { + + my ( $or_self, @a_vals ) = @_; + + my $query = " + SELECT DISTINCT bench + FROM $or_self->{config}{tables}{benchmark_table}"; + $query .= " + WHERE bench LIKE ? " if @a_vals; + return $or_self->execute_query( $query, @a_vals ); + +} + +sub select_unit { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_unit_id + FROM $or_self->{config}{tables}{unit_table} + WHERE bench_unit = ? + ", @a_vals ); + +} + +sub select_addtype { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_type_id + FROM $or_self->{config}{tables}{additional_type_table} + WHERE bench_additional_type = ? + ", @a_vals ); + +} + +sub select_addvalue { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_value_id + FROM $or_self->{config}{tables}{additional_value_table} + WHERE bench_additional_type_id = ? AND bench_additional_value = ? + ", @a_vals ); + +} + +sub select_addtyperelation { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_id, bench_additional_type_id, created_at + FROM $or_self->{config}{tables}{additional_type_relation_table} + WHERE bench_id = ? AND bench_additional_type_id = ? + ", @a_vals ); + +} + +sub insert_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO raw_bench_bundles + (raw_bench_bundle_serialized) + VALUES ( ? ) + ", @a_vals ); + +} + +sub copy_benchmark_backup_value { + + my ( $or_self, $hr_vals ) = @_; + + for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { + if (! $hr_vals->{$s_param} ) { + require Carp; + Carp::confess("missing parameter '$s_param'"); + return; + } + } + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{benchmark_backup_value_table} + ( bench_value_id, bench_id, bench_subsume_type_id, bench_value, active, created_at ) + SELECT + ?, bench_id, bench_subsume_type_id, bench_value, active, created_at + FROM + $or_self->{config}{tables}{benchmark_value_table} + WHERE + bench_value_id = ? + ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); + +} + +sub copy_benchmark_backup_additional_relations { + + my ( $or_self, $hr_vals ) = @_; + + for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { + if (! $hr_vals->{$s_param} ) { + require Carp; + Carp::confess("missing parameter '$s_param'"); + return; + } + } + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{backup_additional_relation_table} + ( bench_backup_value_id, bench_additional_value_id, active, created_at ) + SELECT + ?, bench_additional_value_id, active, created_at + FROM + $or_self->{config}{tables}{additional_relation_table} + WHERE + bench_value_id = ? + ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); + +} + +sub update_benchmark_backup_value { + + my ( $or_self, $hr_vals ) = @_; + + return $or_self->execute_query( " + UPDATE $or_self->{config}{tables}{benchmark_backup_value_table} + SET bench_value_id = ? + WHERE bench_value_id = ? + ", @{$hr_vals}{qw/ + new_bench_value_id + old_bench_value_id + /} ); + +} + +sub start_processing_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + UPDATE raw_bench_bundles + SET processing = 1 + WHERE raw_bench_bundle_id = ? + ", @a_vals ); + +} + +sub update_raw_bench_bundle_set_processed { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + UPDATE raw_bench_bundles + SET processed=1, + processing=0 + WHERE raw_bench_bundle_id = ? + ", @a_vals ); + +} + +sub delete_benchmark_additional_relations { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM $or_self->{config}{tables}{additional_relation_table} + WHERE bench_value_id = ? + ", @a_vals ); + +} + +sub delete_benchmark_value { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM $or_self->{config}{tables}{benchmark_value_table} + WHERE bench_value_id = ? + ", @a_vals ); + +} + +# Garbage Collection +sub delete_processed_raw_bench_bundles { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM raw_bench_bundles + WHERE processed=1 AND + processing=0 + ", @a_vals ); + +} + +1; diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index be52fc3..00f3704 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -3,7 +3,7 @@ package Tapper::Benchmark::Query::mysql; use strict; use warnings; -use base 'Tapper::Benchmark::Query'; +use base 'Tapper::Benchmark::Query::common'; use List::MoreUtils qw( any ); @@ -16,160 +16,6 @@ my %h_default_columns = ( 'CREATED' => 'bv.created_at', ); -sub default_columns { - return %h_default_columns; -} - -sub benchmark_operators { - return ( '=', '!=', 'like', 'not like', '<', '>', '<=', '>=' ); -} - -sub create_where_clause { - - my ( $s_column_name, $ar_value ) = @_; - - my $s_where_clause = q##; - if ( $ar_value->[0] eq 'not like' ) { - $s_where_clause = "$s_column_name NOT LIKE ?"; - } - elsif ( $ar_value->[0] eq 'like' ) { - $s_where_clause = "$s_column_name LIKE ?"; - } - elsif ( - $ar_value->[0] eq '<' - || $ar_value->[0] eq '>' - || $ar_value->[0] eq '<=' - || $ar_value->[0] eq '>=' - ) { - $s_where_clause = "$s_column_name $ar_value->[0] ?"; - } - elsif ( $ar_value->[0] eq '=' ) { - if ( $#{$ar_value} > 1 ) { - $s_where_clause = "$s_column_name IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; - } - else { - $s_where_clause = "$s_column_name = ?"; - } - } - elsif ( $ar_value->[0] eq '!=' ) { - if ( $#{$ar_value} > 1 ) { - $s_where_clause = "$s_column_name NOT IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; - } - else { - $s_where_clause = "$s_column_name != ?"; - } - } - else { - require Carp; - Carp::confess("unknown operator '$ar_value->[0]'"); - return; - } - - return $s_where_clause; - -} - -sub create_select_column { - - my ( $or_self, $ar_select, $i_counter, $b_aggregate_all ) = @_; - - my $s_aggr_func = q##; - my ( $s_aggr, $s_column ) = @{$ar_select}; - my $s_return_select = q##; - - AGGR: { - if ( $s_aggr eq q## ) { - # aggregate all columns if a single column is aggregated - if ( $b_aggregate_all ) { - $s_aggr = $or_self->{config}{default_aggregation}; - redo AGGR; - } - $s_return_select = '${COLUMN}'; - } - elsif ( $s_aggr eq 'min' ) { - $s_return_select = 'MIN( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'max' ) { - $s_return_select = 'MAX( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'avg' ) { - $s_return_select = 'AVG( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'gem' ) { - $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; - } - elsif ( $s_aggr eq 'sum' ) { - $s_return_select = 'SUM( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'cnt' ) { - $s_return_select = 'COUNT( ${COLUMN} )'; - } - elsif ( $s_aggr eq 'cnd' ) { - $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; - } - else { - require Carp; - Carp::confess("unknown aggregate function '$s_aggr'"); - return; - } - } # AGGR - - my ( $s_return_column ); - my $s_replace_as = $s_aggr ? $s_aggr . "_$s_column" : $s_column; - - if ( $h_used_selects{$or_self}{$s_replace_as} ) { - return; - } - if ( any { $s_column eq $_ } keys %h_default_columns ) { - $h_used_selects{$or_self}{$s_replace_as} = $h_default_columns{$s_column}; - } - else { - $s_return_column = $s_column; - $h_used_selects{$or_self}{$s_replace_as} = "bav$i_counter.bench_additional_value"; - } - - $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; - - return ( $s_return_column, "$s_return_select AS '$s_replace_as'", ); - -} - -sub create_period_check { - - my ( $s_column, $dt_from, $dt_to ) = @_; - - my @a_vals; - my $s_where; - if ( $dt_from ) { - if ( my ( $s_date, $s_time ) = $dt_from =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { - $s_where .= "\nAND $s_column > ?"; - push @a_vals, $s_date . ( $s_time || ' 00:00:00' ); - } - else { - require Carp; - Carp::confess(q#unknown date format for 'date_from'#); - return; - } - } - if ( $dt_to ) { - if ( my ( $s_date, $s_time ) = $dt_to =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { - $s_where .= "\nAND $s_column < ?"; - push @a_vals, $s_date . ( $s_time || ' 23:59:59' ); - } - else { - require Carp; - Carp::confess(q#unknown date format for 'date_to'#); - return; - } - } - - return { - vals => \@a_vals, - where => $s_where, - }; - -} - sub select_benchmark_values { my ( $or_self, $hr_search ) = @_; @@ -229,7 +75,7 @@ sub select_benchmark_values { for my $ar_where ( @{$hr_search->{where}} ) { if ( any { $ar_where->[1] eq $_ } keys %h_default_columns ) { my $s_column = splice( @{$ar_where}, 1, 1 ); - push @a_where, create_where_clause( $h_default_columns{$s_column}, $ar_where ); + push @a_where, $or_self->create_where_clause( $h_default_columns{$s_column}, $ar_where ); push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; } else { @@ -255,7 +101,7 @@ sub select_benchmark_values { ) "; push @a_from_vals, $hr_additional_type->{bench_additional_type_id}; - push @a_where, create_where_clause( "bav$i_counter.bench_additional_value", $ar_where ); + push @a_where, $or_self->create_where_clause( "bav$i_counter.bench_additional_value", $ar_where ); push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; $i_counter++; } @@ -416,289 +262,69 @@ sub select_benchmark_values { } -sub select_benchmark_point_essentials { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT - b.bench, - bv.bench_value, - bu.bench_unit - FROM - $or_self->{config}{tables}{benchmark_table} b - JOIN - $or_self->{config}{tables}{benchmark_value_table} bv - ON - b.bench_id = bv.bench_id - LEFT JOIN - $or_self->{config}{tables}{unit_table} bu - ON - b.bench_unit_id = bu.bench_unit_id - WHERE - bv.bench_value_id = ? - ; - ", @a_vals ); - -} - -sub select_complete_benchmark_point { - - my ( $or_self, @a_vals ) = @_; - - my $query = " - SELECT - bat.bench_additional_type, - bav.bench_additional_value - FROM - benchs b - JOIN - bench_values bv - ON - b.bench_id = bv.bench_id - JOIN - bench_additional_type_relations batr - ON - bv.bench_id = batr.bench_id - JOIN - bench_additional_types bat - ON - batr.bench_additional_type_id = bat.bench_additional_type_id - JOIN - bench_additional_relations bar - ON - bv.bench_value_id = bar.bench_value_id - JOIN - bench_additional_values bav - ON - bar.bench_additional_value_id = bav.bench_additional_value_id AND - bat.bench_additional_type_id = bav.bench_additional_type_id - WHERE - bv.bench_value_id = ? - ORDER BY - bat.bench_additional_type"; - return $or_self->execute_query( $query, @a_vals ); -} - -sub select_addtype_by_name { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_additional_type_id - FROM $or_self->{config}{tables}{additional_type_table} - WHERE bench_additional_type = ? - ", @a_vals ); - -} - -sub select_min_subsume_type { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_subsume_type_id - FROM $or_self->{config}{tables}{subsume_type_table} - ORDER BY bench_subsume_type_rank ASC - LIMIT 1 - " ); - -} - -sub select_subsume_type { - - my ( $or_self, @a_vals ) = @_; +sub create_select_column { - return $or_self->execute_query( " - SELECT - bench_subsume_type_id, - bench_subsume_type_rank, - datetime_strftime_pattern - FROM - $or_self->{config}{tables}{subsume_type_table} - WHERE - bench_subsume_type = ? - ", @a_vals ); + my ( $or_self, $ar_select, $i_counter, $b_aggregate_all ) = @_; -} + warn ("***** mysql - create_select_column"); + my $s_aggr_func = q##; + my ( $s_aggr, $s_column ) = @{$ar_select}; + my $s_return_select = q##; -sub select_check_subsumed_values { + AGGR: { + if ( $s_aggr eq q## ) { + # aggregate all columns if a single column is aggregated + if ( $b_aggregate_all ) { + $s_aggr = $or_self->{config}{default_aggregation}; + redo AGGR; + } + $s_return_select = '${COLUMN}'; + } + elsif ( $s_aggr eq 'min' ) { + $s_return_select = 'MIN( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'max' ) { + $s_return_select = 'MAX( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'avg' ) { + $s_return_select = 'AVG( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'gem' ) { + $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; + } + elsif ( $s_aggr eq 'sum' ) { + $s_return_select = 'SUM( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'cnt' ) { + $s_return_select = 'COUNT( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'cnd' ) { + $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; + } + else { + require Carp; + Carp::confess("unknown aggregate function '$s_aggr'"); + return; + } + } # AGGR - my ( $or_self, $hr_vals ) = @_; + my ( $s_return_column ); + my $s_replace_as = $s_aggr ? $s_aggr . "_$s_column" : $s_column; - if (! $hr_vals->{subsume_type_id} ) { - require Carp; - Carp::confess(q#required parameter 'subsume_type_id' is missing#); + if ( $h_used_selects{$or_self}{$s_replace_as} ) { return; } - - my $hr_period_check = create_period_check( - 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} - ); - - return $or_self->execute_query( - " - SELECT - bv.bench_value_id - FROM - bench_values bv - JOIN bench_subsume_types bet - ON ( bv.bench_subsume_type_id = bet.bench_subsume_type_id ) - WHERE - bet.bench_subsume_type_rank > ( - SELECT beti.bench_subsume_type_rank - FROM bench_subsume_types beti - WHERE bench_subsume_type_id = ? - ) - $hr_period_check->{where} - LIMIT - 1 - ", - $hr_vals->{subsume_type_id}, - @{$hr_period_check->{vals}}, - ); - -} - -sub select_data_values_for_subsume { - - my ( $or_self, $hr_vals ) = @_; - - my $hr_period_check = create_period_check( - 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} - ); - - my @a_addexclude_vals; - my $s_addexclude_where = q##; - if ( $hr_vals->{exclude_additionals} && @{$hr_vals->{exclude_additionals}} ) { - $s_addexclude_where = 'AND bav.bench_additional_type_id NOT IN (' . (join ',', map {'?'} @{$hr_vals->{exclude_additionals}}) . ')'; - push @a_addexclude_vals, @{$hr_vals->{exclude_additionals}}; + if ( any { $s_column eq $_ } keys %h_default_columns ) { + $h_used_selects{$or_self}{$s_replace_as} = $h_default_columns{$s_column}; + } + else { + $s_return_column = $s_column; + $h_used_selects{$or_self}{$s_replace_as} = "bav$i_counter.bench_additional_value"; } - return $or_self->execute_query( - " - SELECT - b.bench_id, - bv.bench_value_id, - bv.created_at, - bv.bench_value, - bet.bench_subsume_type_rank, - GROUP_CONCAT( - bav.bench_additional_type_id, - '|', - bav.bench_additional_value_id - ORDER BY - bav.bench_additional_type_id, - bav.bench_additional_value_id - SEPARATOR - '-' - ) AS additionals - FROM - benchs b - JOIN bench_values bv - ON ( bv.bench_id = b.bench_id ) - JOIN bench_subsume_types bet - ON ( bet.bench_subsume_type_id = bv.bench_subsume_type_id ) - LEFT JOIN ( - bench_additional_relations bar - JOIN bench_additional_values bav - ON ( bav.bench_additional_value_id = bar.bench_additional_value_id ) - ) - ON ( - bar.active = 1 - AND bar.bench_value_id = bv.bench_value_id - $s_addexclude_where - ) - WHERE - b.active = 1 - AND bv.active = 1 - $hr_period_check->{where} - GROUP BY - bet.bench_subsume_type_rank, - b.bench_id, - bv.created_at, - bv.bench_value, - bv.bench_value_id - ORDER BY - b.bench_id, - additionals, - bv.created_at - ", - @a_addexclude_vals, - @{$hr_period_check->{vals}}, - ); -} - -sub select_benchmark { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_id - FROM $or_self->{config}{tables}{benchmark_table} - WHERE bench = ? - ", @a_vals ); - -} - -sub select_benchmark_names { - - my ( $or_self, @a_vals ) = @_; - - my $query = " - SELECT DISTINCT bench - FROM $or_self->{config}{tables}{benchmark_table}"; - $query .= " - WHERE bench LIKE ? " if @a_vals; - return $or_self->execute_query( $query, @a_vals ); - -} - -sub select_unit { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_unit_id - FROM $or_self->{config}{tables}{unit_table} - WHERE bench_unit = ? - ", @a_vals ); - -} - -sub select_addtype { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_additional_type_id - FROM $or_self->{config}{tables}{additional_type_table} - WHERE bench_additional_type = ? - ", @a_vals ); - -} - -sub select_addvalue { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_additional_value_id - FROM $or_self->{config}{tables}{additional_value_table} - WHERE bench_additional_type_id = ? AND bench_additional_value = ? - ", @a_vals ); - -} - -sub select_addtyperelation { - - my ( $or_self, @a_vals ) = @_; + $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; - return $or_self->execute_query( " - SELECT bench_id, bench_additional_type_id, created_at - FROM $or_self->{config}{tables}{additional_type_relation_table} - WHERE bench_id = ? AND bench_additional_type_id = ? - ", @a_vals ); + return ( $s_return_column, "$s_return_select AS '$s_replace_as'", ); } @@ -786,18 +412,6 @@ sub insert_benchmark_value { } -sub insert_raw_bench_bundle { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - INSERT INTO raw_bench_bundles - (raw_bench_bundle_serialized) - VALUES ( ? ) - ", @a_vals ); - -} - sub copy_additional_values { my ( $or_self, $hr_vals ) = @_; @@ -823,56 +437,6 @@ sub copy_additional_values { } -sub copy_benchmark_backup_value { - - my ( $or_self, $hr_vals ) = @_; - - for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { - if (! $hr_vals->{$s_param} ) { - require Carp; - Carp::confess("missing parameter '$s_param'"); - return; - } - } - - return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{benchmark_backup_value_table} - ( bench_value_id, bench_id, bench_subsume_type_id, bench_value, active, created_at ) - SELECT - ?, bench_id, bench_subsume_type_id, bench_value, active, created_at - FROM - $or_self->{config}{tables}{benchmark_value_table} - WHERE - bench_value_id = ? - ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); - -} - -sub copy_benchmark_backup_additional_relations { - - my ( $or_self, $hr_vals ) = @_; - - for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { - if (! $hr_vals->{$s_param} ) { - require Carp; - Carp::confess("missing parameter '$s_param'"); - return; - } - } - - return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{backup_additional_relation_table} - ( bench_backup_value_id, bench_additional_value_id, active, created_at ) - SELECT - ?, bench_additional_value_id, active, created_at - FROM - $or_self->{config}{tables}{additional_relation_table} - WHERE - bench_value_id = ? - ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); - -} - sub insert_addtype { my ( $or_self, @a_vals ) = @_; @@ -916,79 +480,4 @@ sub insert_addvaluerelation { } -sub update_benchmark_backup_value { - - my ( $or_self, $hr_vals ) = @_; - - return $or_self->execute_query( " - UPDATE $or_self->{config}{tables}{benchmark_backup_value_table} - SET bench_value_id = ? - WHERE bench_value_id = ? - ", @{$hr_vals}{qw/ - new_bench_value_id - old_bench_value_id - /} ); - -} - -sub start_processing_raw_bench_bundle { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - UPDATE raw_bench_bundles - SET processing = 1 - WHERE raw_bench_bundle_id = ? - ", @a_vals ); - -} - -sub update_raw_bench_bundle_set_processed { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - UPDATE raw_bench_bundles - SET processed=1, - processing=0 - WHERE raw_bench_bundle_id = ? - ", @a_vals ); - -} - -sub delete_benchmark_additional_relations { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - DELETE FROM $or_self->{config}{tables}{additional_relation_table} - WHERE bench_value_id = ? - ", @a_vals ); - -} - -sub delete_benchmark_value { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - DELETE FROM $or_self->{config}{tables}{benchmark_value_table} - WHERE bench_value_id = ? - ", @a_vals ); - -} - -# Garbage Collection -sub delete_processed_raw_bench_bundles { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - DELETE FROM raw_bench_bundles - WHERE processed=1 AND - processing=0 - ", @a_vals ); - -} - 1; From 52797e2b3462915c1a204b10c6566b04e5da97f8 Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Thu, 17 Sep 2015 08:45:48 +0200 Subject: [PATCH 79/80] changelog++ --- Changes | 1 + 1 file changed, 1 insertion(+) diff --git a/Changes b/Changes index 1d40b21..baf72dc 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for {{$dist->name}} {{$NEXT}} + - refactor common methods between db backends 0.011 2015-09-16 - let the db take care of compression in raw queue From dd639af360e20292033f17940d7bd90a24fd67bc Mon Sep 17 00:00:00 2001 From: Steffen Schwigon Date: Thu, 17 Sep 2015 08:46:01 +0200 Subject: [PATCH 80/80] v0.012 - refactor common methods between db backends --- Changes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Changes b/Changes index baf72dc..dac84dc 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,8 @@ Revision history for {{$dist->name}} {{$NEXT}} + +0.012 2015-09-17 - refactor common methods between db backends 0.011 2015-09-16