feat: add CSVQ support for rule parsing

cmd/inspect.go

@@ -1,37 +1,25 @@
 package cmd
 
 import (
-	"fmt"
 	"os"
 	"os/signal"
-	"path/filepath"
 
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
+	"github.com/supabase/cli/internal/inspect"
 	"github.com/supabase/cli/internal/inspect/bloat"
 	"github.com/supabase/cli/internal/inspect/blocking"
-	"github.com/supabase/cli/internal/inspect/cache"
-	"github.com/supabase/cli/internal/utils"
-	"github.com/supabase/cli/internal/utils/flags"
-
-	"github.com/supabase/cli/internal/inspect"
 	"github.com/supabase/cli/internal/inspect/calls"
-	"github.com/supabase/cli/internal/inspect/index_sizes"
-	"github.com/supabase/cli/internal/inspect/index_usage"
+	"github.com/supabase/cli/internal/inspect/db_stats"
+	"github.com/supabase/cli/internal/inspect/index_stats"
 	"github.com/supabase/cli/internal/inspect/locks"
 	"github.com/supabase/cli/internal/inspect/long_running_queries"
 	"github.com/supabase/cli/internal/inspect/outliers"
 	"github.com/supabase/cli/internal/inspect/replication_slots"
-	"github.com/supabase/cli/internal/inspect/role_configs"
-	"github.com/supabase/cli/internal/inspect/role_connections"
-	"github.com/supabase/cli/internal/inspect/seq_scans"
-	"github.com/supabase/cli/internal/inspect/table_index_sizes"
-	"github.com/supabase/cli/internal/inspect/table_record_counts"
-	"github.com/supabase/cli/internal/inspect/table_sizes"
-	"github.com/supabase/cli/internal/inspect/total_index_size"
-	"github.com/supabase/cli/internal/inspect/total_table_sizes"
-	"github.com/supabase/cli/internal/inspect/unused_indexes"
+	"github.com/supabase/cli/internal/inspect/role_stats"
+	"github.com/supabase/cli/internal/inspect/table_stats"
 	"github.com/supabase/cli/internal/inspect/vacuum_stats"
+	"github.com/supabase/cli/internal/utils/flags"
 )
 
 var (
@@ -51,11 +39,11 @@ var (
 		},
 	}
 
-	inspectCacheHitCmd = &cobra.Command{
-		Use:   "cache-hit",
-		Short: "Show cache hit rates for tables and indices",
+	inspectDBStatsCmd = &cobra.Command{
+		Use:   "db-stats",
+		Short: "Show stats such as cache hit rates, total sizes, and WAL size",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return cache.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
@@ -67,14 +55,6 @@ var (
 		},
 	}
 
-	inspectIndexUsageCmd = &cobra.Command{
-		Use:   "index-usage",
-		Short: "Show information about the efficiency of indexes",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			return index_usage.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
-		},
-	}
-
 	inspectLocksCmd = &cobra.Command{
 		Use:   "locks",
 		Short: "Show queries which have taken out an exclusive lock on a relation",
@@ -107,107 +87,159 @@ var (
 		},
 	}
 
-	inspectTotalIndexSizeCmd = &cobra.Command{
-		Use:   "total-index-size",
-		Short: "Show total size of all indexes",
+	inspectIndexStatsCmd = &cobra.Command{
+		Use:   "index-stats",
+		Short: "Show combined index size, usage percent, scan counts, and unused status",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return total_index_size.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectIndexSizesCmd = &cobra.Command{
-		Use:   "index-sizes",
-		Short: "Show index sizes of individual indexes",
+	inspectLongRunningQueriesCmd = &cobra.Command{
+		Use:   "long-running-queries",
+		Short: "Show currently running queries running for longer than 5 minutes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return long_running_queries.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectTableSizesCmd = &cobra.Command{
-		Use:   "table-sizes",
-		Short: "Show table sizes of individual tables without their index sizes",
+	inspectBloatCmd = &cobra.Command{
+		Use:   "bloat",
+		Short: "Estimates space allocated to a relation that is full of dead tuples",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return bloat.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectTableIndexSizesCmd = &cobra.Command{
-		Use:   "table-index-sizes",
-		Short: "Show index sizes of individual tables",
+	inspectRoleStatsCmd = &cobra.Command{
+		Use:   "role-stats",
+		Short: "Show information about roles on the database",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return table_index_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return role_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectTotalTableSizesCmd = &cobra.Command{
-		Use:   "total-table-sizes",
-		Short: "Show total table sizes, including table index sizes",
+	inspectVacuumStatsCmd = &cobra.Command{
+		Use:   "vacuum-stats",
+		Short: "Show statistics related to vacuum operations per table",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return total_table_sizes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return vacuum_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectUnusedIndexesCmd = &cobra.Command{
-		Use:   "unused-indexes",
-		Short: "Show indexes with low usage",
+	inspectTableStatsCmd = &cobra.Command{
+		Use:   "table-stats",
+		Short: "Show combined table size, index size, and estimated row count",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return unused_indexes.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectSeqScansCmd = &cobra.Command{
-		Use:   "seq-scans",
-		Short: "Show number of sequential scans recorded against all tables",
+	inspectCacheHitCmd = &cobra.Command{
+		Deprecated: `use "db-stats" instead.`,
+		Use:        "cache-hit",
+		Short:      "Show cache hit rates for tables and indices",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return seq_scans.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return db_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectLongRunningQueriesCmd = &cobra.Command{
-		Use:   "long-running-queries",
-		Short: "Show currently running queries running for longer than 5 minutes",
+	inspectIndexUsageCmd = &cobra.Command{
+		Deprecated: `use "index-stats" instead.`,
+		Use:        "index-usage",
+		Short:      "Show information about the efficiency of indexes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return long_running_queries.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectTableRecordCountsCmd = &cobra.Command{
-		Use:   "table-record-counts",
-		Short: "Show estimated number of rows per table",
+	inspectTotalIndexSizeCmd = &cobra.Command{
+		Deprecated: `use "index-stats" instead.`,
+		Use:        "total-index-size",
+		Short:      "Show total size of all indexes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return table_record_counts.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectBloatCmd = &cobra.Command{
-		Use:   "bloat",
-		Short: "Estimates space allocated to a relation that is full of dead tuples",
+	inspectIndexSizesCmd = &cobra.Command{
+		Deprecated: `use "index-stats" instead.`,
+		Use:        "index-sizes",
+		Short:      "Show index sizes of individual indexes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return bloat.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
-	inspectVacuumStatsCmd = &cobra.Command{
-		Use:   "vacuum-stats",
-		Short: "Show statistics related to vacuum operations per table",
+	inspectTableSizesCmd = &cobra.Command{
+		Deprecated: `use "table-stats" instead.`,
+		Use:        "table-sizes",
+		Short:      "Show table sizes of individual tables without their index sizes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return vacuum_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+		},
+	}
+
+	inspectTableIndexSizesCmd = &cobra.Command{
+		Deprecated: `use "table-stats" instead.`,
+		Use:        "table-index-sizes",
+		Short:      "Show index sizes of individual tables",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+		},
+	}
+
+	inspectTotalTableSizesCmd = &cobra.Command{
+		Deprecated: `use "table-stats" instead.`,
+		Use:        "total-table-sizes",
+		Short:      "Show total table sizes, including table index sizes",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return table_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+		},
+	}
+
+	inspectUnusedIndexesCmd = &cobra.Command{
+		Deprecated: `use "index-stats" instead.`,
+		Use:        "unused-indexes",
+		Short:      "Show indexes with low usage",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+		},
+	}
+
+	inspectTableRecordCountsCmd = &cobra.Command{
+		Deprecated: `use "table-stats" instead.`,
+		Use:        "table-record-counts",
+		Short:      "Show estimated number of rows per table",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+		},
+	}
+
+	inspectSeqScansCmd = &cobra.Command{
+		Deprecated: `use "index-stats" instead.`,
+		Use:        "seq-scans",
+		Short:      "Show number of sequential scans recorded against all tables",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return index_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
 	inspectRoleConfigsCmd = &cobra.Command{
-		Use:   "role-configs",
-		Short: "Show configuration settings for database roles when they have been modified",
+		Deprecated: `use "role-stats" instead.`,
+		Use:        "role-configs",
+		Short:      "Show configuration settings for database roles when they have been modified",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return role_configs.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return role_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
 	inspectRoleConnectionsCmd = &cobra.Command{
-		Use:   "role-connections",
-		Short: "Show number of active connections for all database roles",
+		Deprecated: `use "role-stats" instead.`,
+		Use:        "role-connections",
+		Short:      "Show number of active connections for all database roles",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return role_connections.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
+			return role_stats.Run(cmd.Context(), flags.DbConfig, afero.NewOsFs())
 		},
 	}
 
@@ -217,17 +249,7 @@ var (
 		Use:   "report",
 		Short: "Generate a CSV output for all inspect commands",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := cmd.Context()
-			if len(outputDir) == 0 {
-				defaultPath := filepath.Join(utils.CurrentDirAbs, "report")
-				title := fmt.Sprintf("Enter a directory to save output files (or leave blank to use %s): ", utils.Bold(defaultPath))
-				if dir, err := utils.NewConsole().PromptText(ctx, title); err != nil {
-					return err
-				} else if len(dir) == 0 {
-					outputDir = defaultPath
-				}
-			}
-			return inspect.Report(ctx, outputDir, flags.DbConfig, afero.NewOsFs())
+			return inspect.Report(cmd.Context(), outputDir, flags.DbConfig, afero.NewOsFs())
 		},
 	}
 )
@@ -238,28 +260,33 @@ func init() {
 	inspectFlags.Bool("linked", true, "Inspect the linked project.")
 	inspectFlags.Bool("local", false, "Inspect the local database.")
 	inspectCmd.MarkFlagsMutuallyExclusive("db-url", "linked", "local")
-	inspectDBCmd.AddCommand(inspectCacheHitCmd)
 	inspectDBCmd.AddCommand(inspectReplicationSlotsCmd)
-	inspectDBCmd.AddCommand(inspectIndexUsageCmd)
+	inspectDBCmd.AddCommand(inspectIndexStatsCmd)
 	inspectDBCmd.AddCommand(inspectLocksCmd)
 	inspectDBCmd.AddCommand(inspectBlockingCmd)
 	inspectDBCmd.AddCommand(inspectOutliersCmd)
 	inspectDBCmd.AddCommand(inspectCallsCmd)
+	inspectDBCmd.AddCommand(inspectLongRunningQueriesCmd)
+	inspectDBCmd.AddCommand(inspectBloatCmd)
+	inspectDBCmd.AddCommand(inspectVacuumStatsCmd)
+	inspectDBCmd.AddCommand(inspectTableStatsCmd)
+	inspectDBCmd.AddCommand(inspectRoleStatsCmd)
+	inspectDBCmd.AddCommand(inspectDBStatsCmd)
+	// DEPRECATED
+	inspectDBCmd.AddCommand(inspectCacheHitCmd)
+	inspectDBCmd.AddCommand(inspectIndexUsageCmd)
+	inspectDBCmd.AddCommand(inspectSeqScansCmd)
+	inspectDBCmd.AddCommand(inspectUnusedIndexesCmd)
+	inspectDBCmd.AddCommand(inspectTotalTableSizesCmd)
+	inspectDBCmd.AddCommand(inspectTableIndexSizesCmd)
 	inspectDBCmd.AddCommand(inspectTotalIndexSizeCmd)
 	inspectDBCmd.AddCommand(inspectIndexSizesCmd)
 	inspectDBCmd.AddCommand(inspectTableSizesCmd)
-	inspectDBCmd.AddCommand(inspectTableIndexSizesCmd)
-	inspectDBCmd.AddCommand(inspectTotalTableSizesCmd)
-	inspectDBCmd.AddCommand(inspectUnusedIndexesCmd)
-	inspectDBCmd.AddCommand(inspectSeqScansCmd)
-	inspectDBCmd.AddCommand(inspectLongRunningQueriesCmd)
 	inspectDBCmd.AddCommand(inspectTableRecordCountsCmd)
-	inspectDBCmd.AddCommand(inspectBloatCmd)
-	inspectDBCmd.AddCommand(inspectVacuumStatsCmd)
 	inspectDBCmd.AddCommand(inspectRoleConfigsCmd)
 	inspectDBCmd.AddCommand(inspectRoleConnectionsCmd)
 	inspectCmd.AddCommand(inspectDBCmd)
-	reportCmd.Flags().StringVar(&outputDir, "output-dir", "", "Path to save CSV files in")
+	reportCmd.Flags().StringVar(&outputDir, "output-dir", ".", "Path to save CSV files in")
 	inspectCmd.AddCommand(reportCmd)
 	rootCmd.AddCommand(inspectCmd)
 }

go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/jackc/pgproto3/v2 v2.3.3
 	github.com/jackc/pgx/v4 v4.18.3
 	github.com/joho/godotenv v1.5.1
+	github.com/mithrandie/csvq-driver v1.7.0
 	github.com/muesli/reflow v0.3.0
 	github.com/oapi-codegen/oapi-codegen/v2 v2.4.1
 	github.com/slack-go/slack v0.17.1
@@ -223,6 +224,10 @@ require (
 	github.com/microcosm-cc/bluemonday v1.0.27 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
+	github.com/mithrandie/csvq v1.18.1 // indirect
+	github.com/mithrandie/go-file/v2 v2.1.0 // indirect
+	github.com/mithrandie/go-text v1.6.0 // indirect
+	github.com/mithrandie/ternary v1.1.1 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/sys/atomicwriter v0.1.0 // indirect
 	github.com/moby/sys/sequential v0.6.0 // indirect

go.sum

@@ -696,6 +696,16 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
 github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
 github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mithrandie/csvq v1.18.1 h1:f7NB2scbb7xx2ffPduJ2VtZ85RpWXfvanYskAkGlCBU=
+github.com/mithrandie/csvq v1.18.1/go.mod h1:MRJj7AtcXfk7jhNGxLuJGP3LORmh4lpiPWxQ7VyCRn8=
+github.com/mithrandie/csvq-driver v1.7.0 h1:ejiavXNWwTPMyr3fJFnhcqd1L1cYudA0foQy9cZrqhw=
+github.com/mithrandie/csvq-driver v1.7.0/go.mod h1:HcN3xL9UCJnBYA/AIQOOB/KlyfXAiYr5yxDmiwrGk5o=
+github.com/mithrandie/go-file/v2 v2.1.0 h1:XA5Tl+73GXMDvgwSE3Sg0uC5FkLr3hnXs8SpUas0hyg=
+github.com/mithrandie/go-file/v2 v2.1.0/go.mod h1:9YtTF3Xo59GqC1Pxw6KyGVcM/qubAMlxVsqI/u9r++c=
+github.com/mithrandie/go-text v1.6.0 h1:8gOXTMPbMY8DJbKMTv8kHhADcJlDWXqS/YQH4SyWO6s=
+github.com/mithrandie/go-text v1.6.0/go.mod h1:xCgj1xiNbI/d4xA9sLVvXkjh5B2tNx2ZT2/3rpmh8to=
+github.com/mithrandie/ternary v1.1.1 h1:k/joD6UGVYxHixYmSR8EGgDFNONBMqyD373xT4QRdC4=
+github.com/mithrandie/ternary v1.1.1/go.mod h1:0D9Ba3+09K2TdSZO7/bFCC0GjSXetCvYuYq0u8FY/1g=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw=