Merge branch 'dev'

Author: mstyushin

.github/workflows/mamonsu-tests-dev.yml

@@ -86,7 +86,7 @@ jobs:
           echo "zabbix_address=$(hostname -I | awk '{print $1}')" >> $GITHUB_OUTPUT
         id: zabbix_address
       - name: Edit Zabbix address in agent.conf
-        run: sed -i "s/\(address *= *\).*/\1 ${{ steps.zabbix_address.outputs.zabbix_address }}/" ${{ env.MAMONSU_PATH }}/github-actions-tests/sources/agent_3.5.10.conf
+        run: sed -i "s/\(address *= *\).*/\1 ${{ steps.zabbix_address.outputs.zabbix_address }}/" ${{ env.MAMONSU_PATH }}/github-actions-tests/sources/agent_3.5.11.conf
 
       - name: Copy test scripts to container
         run: docker exec $( echo "${{ matrix.docker_os }}" | sed 's/://' | sed 's/\.//' ) mkdir -p -m 755 /mamonsu/

.github/workflows/mamonsu-tests-master.yml

@@ -91,7 +91,7 @@ jobs:
           echo "zabbix_address=$(hostname -I | awk '{print $1}')" >> $GITHUB_OUTPUT
         id: zabbix_address
       - name: Edit Zabbix address in agent.conf
-        run: sed -i "s/\(address *= *\).*/\1 ${{ steps.zabbix_address.outputs.zabbix_address }}/" ${{ env.MAMONSU_PATH }}/github-actions-tests/sources/agent_3.5.10.conf
+        run: sed -i "s/\(address *= *\).*/\1 ${{ steps.zabbix_address.outputs.zabbix_address }}/" ${{ env.MAMONSU_PATH }}/github-actions-tests/sources/agent_3.5.11.conf
 
       - name: Copy test scripts to container
         run: docker exec $( echo "${{ matrix.docker_os }}" | sed 's/://' | sed 's/\.//' ) mkdir -p -m 755 /mamonsu/

github-actions-tests/mamonsu_build.sh

@@ -41,7 +41,7 @@ if [ "${OS%:*}" = "centos" ]; then
     python3 setup.py build && python3 setup.py install
     make rpm
     sudo rpm -i ./mamonsu*.rpm
-    cat /mamonsu/github-actions-tests/sources/agent_3.5.10.conf > /etc/mamonsu/agent.conf
+    cat /mamonsu/github-actions-tests/sources/agent_3.5.11.conf > /etc/mamonsu/agent.conf
     # ensuring mamonsu can actually start
     sudo su -s /bin/bash -c "mamonsu bootstrap -x --user postgres -d mamonsu_test_db" mamonsu
     /etc/init.d/mamonsu restart
@@ -65,7 +65,7 @@ elif [ "${OS%:*}" = "ubuntu" ]; then
     python3 setup.py build && python3 setup.py install
     make deb
     sudo dpkg -i ./mamonsu*.deb
-    cat /mamonsu/github-actions-tests/sources/agent_3.5.10.conf > /etc/mamonsu/agent.conf
+    cat /mamonsu/github-actions-tests/sources/agent_3.5.11.conf > /etc/mamonsu/agent.conf
     # ensuring mamonsu can actually start
     sudo su -s /bin/bash -c "mamonsu bootstrap -x --user postgres -d mamonsu_test_db" mamonsu
     service mamonsu restart

github-actions-tests/sources/agent_3.5.10.conf renamed to github-actions-tests/sources/agent_3.5.11.conf

@@ -1,7 +1,7 @@
 __author__ = 'Dmitry Vasilyev'
 __author_email__ = '[email protected]'
 __description__ = 'Monitoring agent for PostgreSQL'
-__version__ = '3.5.10'
+__version__ = '3.5.11'
 __licence__ = 'BSD'
 
 __url__ = 'https://github.com/postgrespro/mamonsu'

mamonsu/__init__.py

@@ -110,6 +110,7 @@ def __init__(self, params=None):
             "bootstrap": {"storage": {}, "counter": 0, "cache": 10, "version": False},
             "recovery": {"storage": {}, "counter": 0, "cache": 10},
             "extension_schema": {"pg_buffercache": {}, "pg_stat_statements": {}, "pg_wait_sampling": {}, "pgpro_stats": {}},
+            "extension_versions" : {},
             "pgpro": {"storage": {}},
             "pgproee": {"storage": {}}
         }
@@ -135,6 +136,19 @@ def server_version(self, db=None):
             result.decode("ascii"))
         return self._cache["server_version"]["storage"][db]
 
+    def extension_version(self, extension, db=None):
+        db = self._normalize_db(db)
+        if extension in self._cache["extension_versions"] and db in self._cache["extension_versions"][extension][db]:
+            return self._cache["extension_versions"][extension][db]
+
+        version_string = self.query("select extversion from pg_catalog.pg_extension where lower(extname) = lower('{0}');".format(extension), db)[0][0]
+        result = bytes(
+            version_string.split(" ")[0], "utf-8")
+        self._cache["extension_versions"][extension] = {}
+        self._cache["extension_versions"][extension][db] = "{0}".format(
+            result.decode("ascii"))
+        return self._cache["extension_versions"][extension][db]
+
     def server_version_greater(self, version, db=None):
         db = self._normalize_db(db)
         return packaging.version.parse(self.server_version(db)) >= packaging.version.parse(version)
@@ -229,6 +243,14 @@ def is_pgpro_ee(self, db=None):
             self._cache["pgproee"][db] = False
         return self._cache["pgproee"][db]
 
+    def extension_version_greater(self, extension, version, db=None):
+        db = self._normalize_db(db)
+        return packaging.version.parse(self.extension_version(extension, db)) >= packaging.version.parse(version)
+
+    def extension_version_less(self, extension, version, db=None):
+        db = self._normalize_db(db)
+        return packaging.version.parse(self.extension_version(extension, db)) <= packaging.version.parse(version)
+
     def extension_installed(self, ext, db=None):
         db = self._normalize_db(db)
         result = self.query("""

mamonsu/plugins/pgsql/driver/pool.py

@@ -27,7 +27,7 @@ class Statements(Plugin):
 
     query_info = """
     SELECT {metrics}
-    FROM {extension_schema}.pg_stat_statements_info;
+    FROM {extension_schema}.{info_view_name};
     """
     key = "pgsql."
     # zbx_key, sql, desc, unit, delta, (Graph, color, side)
@@ -88,6 +88,32 @@ class Statements(Plugin):
          ("PostgreSQL Statements Info: Last Statistics Reset Time", "9C8A4E", 0))
     ]
 
+    Items_pgpro_stats_1_8 = [
+        ("stat[read_bytes]",
+         "(sum(shared_blks_read+local_blks_read+temp_blks_read)*8*1024)::bigint",
+         "Read bytes/s", Plugin.UNITS.bytes_per_second, Plugin.DELTA.speed_per_second,
+         ("PostgreSQL Statements: Bytes", "87C2B9", 0)),
+        ("stat[write_bytes]",
+         "(sum(shared_blks_written+local_blks_written+temp_blks_written)*8*1024)::bigint",
+         "Write bytes/s", Plugin.UNITS.bytes_per_second, Plugin.DELTA.speed_per_second,
+         ("PostgreSQL Statements: Bytes", "793F5D", 0)),
+        ("stat[dirty_bytes]",
+         "(sum(shared_blks_dirtied+local_blks_dirtied)*8*1024)::bigint",
+         "Dirty bytes/s", Plugin.UNITS.bytes_per_second, Plugin.DELTA.speed_per_second,
+         ("PostgreSQL Statements: Bytes", "9C8A4E", 0)),
+        ("stat[read_time]",
+         "(sum(shared_blk_read_time+local_blk_read_time+temp_blk_read_time)/float4(100))::bigint",
+         "Read IO Time", Plugin.UNITS.s, Plugin.DELTA.speed_per_second,
+         ("PostgreSQL Statements: Spent Time", "87C2B9", 0)),
+        ("stat[write_time]",
+         "(sum(shared_blk_write_time+local_blk_write_time+temp_blk_write_time)/float4(100))::bigint",
+         "Write IO Time", Plugin.UNITS.s, Plugin.DELTA.speed_per_second,
+         ("PostgreSQL Statements: Spent Time", "793F5D", 0)),
+        ["stat[other_time]",
+         "(sum(total_exec_time+total_plan_time-shared_blk_read_time-local_blk_read_time-temp_blk_read_time-shared_blk_write_time-local_blk_write_time-temp_blk_write_time)/float4(100))::bigint",
+         "Other (mostly CPU) Time", Plugin.UNITS.s, Plugin.DELTA.speed_per_second,
+         ("PostgreSQL Statements: Spent Time", "9C8A4E", 0)]]
+
     all_graphs = [
         ("PostgreSQL Statements: Bytes", None),
         ("PostgreSQL Statements: Spent Time", 1),
@@ -115,21 +141,45 @@ def run(self, zbx):
 
         # TODO: add 13 and 14 items when pgpro_stats added new WAL metrics
         all_items = self.Items.copy()
-        if Pooler.server_version_greater("14"):
+
+        if Pooler.extension_installed("pgpro_stats") and Pooler.extension_version_greater("pgpro_stats", "1.8"):
+            info_view = 'pg_stat_statements_info'
+            if self.extension == "pgpro_stats":
+                info_view = 'pgpro_stats_info'
+
+            info_items = self.Items_pg_14
+            info_params = [x[1] for x in info_items]
+            info_result = Pooler.query(
+                self.query_info.format(metrics=(", ".join(info_params)), extension_schema=extension_schema, info_view_name=info_view))
+            for key, value in enumerate(info_result[0]):
+                zbx_key, value = "pgsql.{0}".format(
+                    info_items[key][0]), int(value)
+                zbx.send(zbx_key, value, info_items[key][4])
+
+            all_items = self.Items_pgpro_stats_1_8.copy()
+            all_items += self.Items_pg_13
+
+        elif Pooler.server_version_greater("14"):
             self.Items[5][1] = self.Items[5][1].format("total_exec_time+total_plan_time")
             all_items += self.Items_pg_13
+            info_view = 'pgpro_stats_info'
             if self.extension == "pg_stat_statements":
-                info_items = self.Items_pg_14
-                info_params = [x[1] for x in info_items]
-                info_result = Pooler.query(
-                    self.query_info.format(metrics=(", ".join(info_params)), extension_schema=extension_schema))
-                for key, value in enumerate(info_result[0]):
-                    zbx_key, value = "pgsql.{0}".format(
-                        info_items[key][0]), int(value)
-                    zbx.send(zbx_key, value, info_items[key][4])
+                info_view = 'pg_stat_statements_info'
+            info_items = self.Items_pg_14
+            info_params = [x[1] for x in info_items]
+            info_result = Pooler.query(
+                self.query_info.format(metrics=(", ".join(info_params)),
+                                       extension_schema=extension_schema,
+                                       info_view_name=info_view))
+            for key, value in enumerate(info_result[0]):
+                zbx_key, value = "pgsql.{0}".format(
+                    info_items[key][0]), int(value)
+                zbx.send(zbx_key, value, info_items[key][4])
+
         elif Pooler.server_version_greater("13"):
             self.Items[5][1] = self.Items[5][1].format("total_exec_time+total_plan_time")
             all_items += self.Items_pg_13
+
         else:
             self.Items[5][1] = self.Items[5][1].format("total_time")
         columns = [x[1] for x in all_items]

mamonsu/plugins/pgsql/statements.py

@@ -7,7 +7,7 @@
 $do$
 BEGIN
    IF NOT EXISTS (
-      SELECT FROM pg_catalog.pg_roles 
+      SELECT FROM pg_catalog.pg_roles
       WHERE rolname = '{0}') THEN
       CREATE ROLE {0} LOGIN PASSWORD '{0}';
       IF EXISTS (
@@ -166,15 +166,15 @@
 DROP FUNCTION IF EXISTS mamonsu.get_oldest_transaction();
 CREATE or REPLACE FUNCTION mamonsu.get_oldest_transaction()
 RETURNS DOUBLE PRECISION AS $$
-    SELECT 
-        CASE WHEN extract(epoch from max(now() - xact_start)) IS NOT null 
+    SELECT
+        CASE WHEN extract(epoch from max(now() - xact_start)) IS NOT null
               AND extract(epoch from max(now() - xact_start))>0
-            THEN extract(epoch from max(now() - xact_start)) 
-            ELSE 0 
-        END 
-    FROM pg_catalog.pg_stat_activity 
-    WHERE 
-        pid NOT IN(select pid from pg_stat_replication) AND 
+            THEN extract(epoch from max(now() - xact_start))
+            ELSE 0
+        END
+    FROM pg_catalog.pg_stat_activity
+    WHERE
+        pid NOT IN(select pid from pg_stat_replication) AND
         pid <> pg_backend_pid()
 $$ LANGUAGE SQL SECURITY DEFINER;
 
@@ -225,15 +225,15 @@
 CREATE OR REPLACE FUNCTION mamonsu.prepared_transaction()
 RETURNS TABLE(count_prepared BIGINT, oldest_prepared BIGINT) AS $$
 SELECT COUNT(*) AS count_prepared,
-coalesce (ROUND(MAX(EXTRACT (EPOCH FROM (now() - prepared)))),0)::bigint AS oldest_prepared  
+coalesce (ROUND(MAX(EXTRACT (EPOCH FROM (now() - prepared)))),0)::bigint AS oldest_prepared
 FROM pg_catalog.pg_prepared_xacts$$ LANGUAGE SQL SECURITY DEFINER;
 
 DROP FUNCTION IF EXISTS mamonsu.count_{3}_lag_lsn();
 CREATE OR REPLACE FUNCTION mamonsu.count_{3}_lag_lsn()
-RETURNS TABLE(application_name TEXT, {8} total_lag INTEGER) AS $$
+RETURNS TABLE(application_name TEXT, {8} total_lag BIGINT) AS $$
 SELECT application_name,
-       {6} 
-       coalesce((pg_{7}_diff(pg_current_{7}(), replay_{9}))::int, 0) AS total_lag
+       {6}
+       coalesce((pg_{7}_diff(pg_current_{7}(), replay_{9}))::bigint, 0) AS total_lag
 FROM pg_stat_replication
 $$ LANGUAGE SQL SECURITY DEFINER;
 """
@@ -287,7 +287,7 @@
          FROM pg_extension e
          JOIN pg_namespace n
          ON e.extnamespace = n.oid
-         WHERE e.extname = 'pgpro_stats';   
+         WHERE e.extname = 'pgpro_stats';
          EXECUTE 'DROP FUNCTION IF EXISTS mamonsu.wait_sampling_all_locks();
          CREATE OR REPLACE FUNCTION mamonsu.wait_sampling_all_locks()
          RETURNS TABLE(lock_type text, count bigint) AS $$
@@ -298,7 +298,7 @@
             FROM (SELECT key, value AS locktuple
                   FROM jsonb_each((SELECT wait_stats
                                    FROM ' || extension_schema || '.pgpro_stats_totals()
-                                   WHERE object_type = ''cluster''))) setoflocks, 
+                                   WHERE object_type = ''cluster''))) setoflocks,
             jsonb_each(setoflocks.locktuple) AS json_data)
             SELECT
                 CASE
@@ -327,7 +327,7 @@
             FROM (SELECT key, value AS locktuple
                   FROM jsonb_each((SELECT wait_stats
                                    FROM ' || extension_schema || '.pgpro_stats_totals()
-                                   WHERE object_type = ''cluster''))) setoflocks, 
+                                   WHERE object_type = ''cluster''))) setoflocks,
             jsonb_each(setoflocks.locktuple) AS json_data)
             SELECT
                 lock_type,
@@ -347,7 +347,7 @@
             FROM (SELECT key, value AS locktuple
                   FROM jsonb_each((SELECT wait_stats
                                    FROM ' || extension_schema || '.pgpro_stats_totals()
-                                   WHERE object_type = ''cluster''))) setoflocks, 
+                                   WHERE object_type = ''cluster''))) setoflocks,
             jsonb_each(setoflocks.locktuple) AS json_data
             WHERE setoflocks.key IN (''Lock'', ''LWLock'', ''LWLockTranche'', ''LWLockNamed''))
             SELECT
@@ -415,13 +415,13 @@
          FROM pg_extension e
          JOIN pg_namespace n
          ON e.extnamespace = n.oid
-         WHERE e.extname = 'pgpro_stats';   
+         WHERE e.extname = 'pgpro_stats';
          EXECUTE 'DROP FUNCTION IF EXISTS mamonsu.statements_pro();
                   CREATE OR REPLACE FUNCTION mamonsu.statements_pro()
                   RETURNS TABLE({columns}) AS $$
                       SELECT {metrics}
                       FROM ' || extension_schema || '.pgpro_stats_totals()
-                      WHERE object_type = ''cluster'';        
+                      WHERE object_type = ''cluster'';
                   $$ LANGUAGE SQL SECURITY DEFINER;';
       ELSE
          EXIT functions_creation;

mamonsu/tools/bootstrap/sql.py

@@ -237,7 +237,13 @@ def run_deploy():
             if Pooler.is_pgpro() or Pooler.is_pgpro_ee():
                 bootstrap_extension_queries = fill_query_params(CreateWaitSamplingFunctionsSQL)
                 Pooler.query(bootstrap_extension_queries)
-                if Pooler.server_version_greater("12"):
+                if Pooler.extension_installed("pgpro_stats") and Pooler.extension_version_greater("pgpro_stats", "1.8"):
+                    statements_items = [x[1] for x in Statements.Items_pgpro_stats_1_8] + [x[1] for x in Statements.Items_pg_13]
+                    statements_columns = [x[0][x[0].find("[")+1:x[0].find("]")] for x in Statements.Items_pgpro_stats_1_8] + [x[0][x[0].find("[")+1:x[0].find("]")] for x in Statements.Items_pg_13]
+                    bootstrap_extension_queries = CreateStatementsFunctionsSQL.format(
+                        columns=" bigint, ".join(statements_columns) + " bigint", metrics=(", ".join(statements_items)))
+                    Pooler.query(bootstrap_extension_queries)
+                elif Pooler.server_version_greater("12"):
                     statements_items = [x[1] for x in Statements.Items] + ([x[1] for x in Statements.Items_pg_13] if Pooler.server_version_greater("13") else [])
                     statements_items[5] = statements_items[5].format("total_exec_time+total_plan_time")
                     statements_columns = [x[0][x[0].find("[")+1:x[0].find("]")] for x in Statements.Items] + ([x[0][x[0].find("[")+1:x[0].find("]")] for x in Statements.Items_pg_13] if Pooler.server_version_greater("13") else [])

mamonsu/tools/bootstrap/start.py

@@ -1,3 +1,7 @@
+mamonsu (3.5.11-1) stable; urgency=low
+  * Updated statements plugin: added support for pgpro_stats 1.8;
+  * Fixed types for count_wal_lag_lsn() function (int to bigint);
+
 mamonsu (3.5.10-1) stable; urgency=low
   * Updated checkpoint plugin: added support for new view pg_stat_checkpointer;
   * Updated bgwriter plugin: consider updated view pg_stat_bgwriter in postgres 17;