From 3f7791d3baad7966570bd357ebbafbc2c09ffdaf Mon Sep 17 00:00:00 2001 From: Armand Parajon Date: Wed, 2 Sep 2026 11:15:51 -0400 Subject: [PATCH 1/2] refactor(tern): read table catalog and size estimates in one scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DETAILED pull scanned `information_schema.tables` twice per namespace: once for each table's kind and comment, once for its row-count and on-disk-size estimates. Both scans read the same rows keyed by table_name, and selecting `table_rows` / `data_length` / `index_length` makes that the most expensive query a pull issues — it forces InnoDB to open every table's tablespace metadata, and with `innodb_stats_on_metadata=ON` it recalculates statistics per table. Fold the two into a single query and populate both halves in one pass, halving the dominant target-side cost of a pull. Co-Authored-By: Claude Opus 5 --- pkg/tern/local_client.go | 59 ++++++++++++---------------------------- 1 file changed, 18 insertions(+), 41 deletions(-) diff --git a/pkg/tern/local_client.go b/pkg/tern/local_client.go index 7dd577b7e..e45d0b511 100644 --- a/pkg/tern/local_client.go +++ b/pkg/tern/local_client.go @@ -1071,15 +1071,22 @@ func (c *LocalClient) pullNamespaceCatalog(ctx context.Context, db *sql.DB, name if err := c.loadForeignKeyCatalog(ctx, db, physical, pulledTables, catalog.tables); err != nil { return nil, err } - if err := c.loadTableEstimates(ctx, db, physical, pulledTables, catalog.tables); err != nil { - return nil, err - } return catalog, nil } +// loadTableCatalog reads each pulled table's kind and comment together with its +// engine-maintained row-count and on-disk-size estimates. The estimates are +// approximations (NULL for views, stale until statistics are refreshed). +// +// Selecting table_rows / data_length / index_length makes this the most +// expensive query a pull issues: it forces InnoDB to open every table's +// tablespace metadata, and on a server with innodb_stats_on_metadata=ON it +// recalculates statistics per table. Both halves are keyed by table_name over +// the same rows, so they are read in a single scan of information_schema.tables +// rather than two. func (c *LocalClient) loadTableCatalog(ctx context.Context, db *sql.DB, physicalSchema string, pulledTables map[string]string, catalog map[string]*ternv1.TableCatalog) error { rows, err := db.QueryContext(ctx, ` - SELECT table_name, table_type, table_comment + SELECT table_name, table_type, table_comment, table_rows, data_length, index_length FROM information_schema.tables WHERE table_schema = ? ORDER BY table_name`, physicalSchema) @@ -1090,14 +1097,17 @@ func (c *LocalClient) loadTableCatalog(ctx context.Context, db *sql.DB, physical for rows.Next() { var tableName, tableType, tableComment string - if err := rows.Scan(&tableName, &tableType, &tableComment); err != nil { + var tableRows, dataLength, indexLength sql.NullInt64 + if err := rows.Scan(&tableName, &tableType, &tableComment, &tableRows, &dataLength, &indexLength); err != nil { return fmt.Errorf("scan table catalog for database %s physical schema %s: %w", c.config.Database, physicalSchema, err) } if _, ok := pulledTables[tableName]; ok { catalog[tableName] = &ternv1.TableCatalog{ - Name: tableName, - Kind: normalizedTableKind(tableType), - Comment: tableComment, + Name: tableName, + Kind: normalizedTableKind(tableType), + Comment: tableComment, + EstimatedRowCount: tableRows.Int64, + DataSizeBytes: dataLength.Int64 + indexLength.Int64, } } } @@ -1107,39 +1117,6 @@ func (c *LocalClient) loadTableCatalog(ctx context.Context, db *sql.DB, physical return nil } -// loadTableEstimates populates engine-maintained row-count and on-disk-size -// estimates from information_schema.tables. These are approximations (NULL for -// views, stale until statistics are refreshed) and are only loaded at DETAILED -// catalog detail. -func (c *LocalClient) loadTableEstimates(ctx context.Context, db *sql.DB, physicalSchema string, pulledTables map[string]string, catalog map[string]*ternv1.TableCatalog) error { - rows, err := db.QueryContext(ctx, ` - SELECT table_name, table_rows, data_length, index_length - FROM information_schema.tables - WHERE table_schema = ? - ORDER BY table_name`, physicalSchema) - if err != nil { - return fmt.Errorf("load table estimates for database %s physical schema %s: %w", c.config.Database, physicalSchema, err) - } - defer utils.CloseAndLog(rows) - - for rows.Next() { - var tableName string - var tableRows, dataLength, indexLength sql.NullInt64 - if err := rows.Scan(&tableName, &tableRows, &dataLength, &indexLength); err != nil { - return fmt.Errorf("scan table estimates for database %s physical schema %s: %w", c.config.Database, physicalSchema, err) - } - if _, ok := pulledTables[tableName]; ok { - tableCatalog := ensurePulledTableCatalog(catalog, tableName) - tableCatalog.EstimatedRowCount = tableRows.Int64 - tableCatalog.DataSizeBytes = dataLength.Int64 + indexLength.Int64 - } - } - if err := rows.Err(); err != nil { - return fmt.Errorf("iterate table estimates for database %s physical schema %s: %w", c.config.Database, physicalSchema, err) - } - return nil -} - func (c *LocalClient) loadColumnCatalog(ctx context.Context, db *sql.DB, physicalSchema string, pulledTables map[string]string, catalog map[string]*ternv1.TableCatalog) error { rows, err := db.QueryContext(ctx, ` SELECT table_name, column_name, column_type, is_nullable, column_default, column_comment, extra From 011021a1e4c6c64efdad0ea9d18f8d319614d711 Mon Sep 17 00:00:00 2001 From: Armand Parajon Date: Wed, 2 Sep 2026 11:55:13 -0400 Subject: [PATCH 2/2] docs(tern): name the variable that governs table-estimate cost The cost note on loadTableCatalog attributed the expense of table_rows / data_length / index_length to innodb_stats_on_metadata. That is the 5.6-era lever: on 8.0 it defaults to OFF, and it only applies when innodb_stats_persistent is OFF, which also defaults the other way. The variable that actually decides whether these columns are served from the data dictionary's cached statistics or force a per-table refresh is information_schema_stats_expiry. Name that one instead, and describe the cost profile it produces -- cheap while the cached statistics are fresh, expensive on the first read after they expire -- so a slow DETAILED pull sends the reader to the right knob. Co-Authored-By: Claude Opus 5 --- pkg/tern/local_client.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/tern/local_client.go b/pkg/tern/local_client.go index e45d0b511..a683f3e70 100644 --- a/pkg/tern/local_client.go +++ b/pkg/tern/local_client.go @@ -1076,14 +1076,16 @@ func (c *LocalClient) pullNamespaceCatalog(ctx context.Context, db *sql.DB, name // loadTableCatalog reads each pulled table's kind and comment together with its // engine-maintained row-count and on-disk-size estimates. The estimates are -// approximations (NULL for views, stale until statistics are refreshed). +// approximations: NULL for views, and served from the data dictionary's cached +// table statistics, so they lag the live table until those statistics are +// refreshed. // -// Selecting table_rows / data_length / index_length makes this the most -// expensive query a pull issues: it forces InnoDB to open every table's -// tablespace metadata, and on a server with innodb_stats_on_metadata=ON it -// recalculates statistics per table. Both halves are keyed by table_name over -// the same rows, so they are read in a single scan of information_schema.tables -// rather than two. +// The estimate columns are what make this read more than the kind and comment +// alone would. information_schema_stats_expiry governs how long the cached +// statistics are reused; the first read after they expire makes the server +// refresh them per table. Kind and comment come from the data dictionary and +// are cheap either way, so they ride along on this query rather than paying for +// a second traversal of the view. func (c *LocalClient) loadTableCatalog(ctx context.Context, db *sql.DB, physicalSchema string, pulledTables map[string]string, catalog map[string]*ternv1.TableCatalog) error { rows, err := db.QueryContext(ctx, ` SELECT table_name, table_type, table_comment, table_rows, data_length, index_length