diff --git a/atomic_write_error_test.go b/atomic_write_error_test.go new file mode 100644 index 0000000..4791ee1 --- /dev/null +++ b/atomic_write_error_test.go @@ -0,0 +1,137 @@ +package filesql + +import ( + "io" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestWriteFileAtomically_ReportsARefusedCommit drives the failure that the +// staging exists for: everything is written, and only the last step — putting +// the staged file where the caller asked — is refused. The destination here is a +// directory, which no rename can replace. +func TestWriteFileAtomically_ReportsARefusedCommit(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + dest := filepath.Join(dir, "a-directory") + require.NoError(t, os.Mkdir(dest, 0o750)) + + err := writeFileAtomically(dest, func(w io.Writer) error { + _, writeErr := w.Write([]byte("payload")) + return writeErr + }) + require.Error(t, err, "a destination that cannot be replaced must be reported") + assert.ErrorIs(t, err, ErrIOOperation) + + entries, err := os.ReadDir(dir) + require.NoError(t, err) + assert.Len(t, entries, 1, "the staged file must not be left behind: %v", entries) +} + +// TestCommitStagedFile_FallsBackWhenTheDestinationIsInTheWay covers the branch +// that tells the two rename failures apart. A destination that is still there +// after a failed rename is the Windows case the copy fallback exists for, so the +// fallback runs and reports its own failure; a destination that is not there +// cannot be helped by copying, and the rename error is returned as is. +func TestCommitStagedFile_FallsBackWhenTheDestinationIsInTheWay(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + dest := filepath.Join(dir, "dest") + require.NoError(t, os.WriteFile(dest, []byte("precious"), 0o600)) + + err := commitStagedFile(filepath.Join(dir, "missing"), dest) + require.Error(t, err, "a staged file that is gone cannot be committed by either route") + + got, err := os.ReadFile(dest) //nolint:gosec // Test path from t.TempDir() + require.NoError(t, err) + assert.Equal(t, "precious", string(got), "the destination must survive a refused commit") +} + +// TestCommitByCopy_ReportsAnUnbackupableDestination covers the first step of the +// fallback. Without a backup there is nothing to restore from, so the copy must +// not start at all. +func TestCommitByCopy_ReportsAnUnbackupableDestination(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + staged := filepath.Join(dir, "staged") + require.NoError(t, os.WriteFile(staged, []byte("new"), 0o600)) + + err := commitByCopy(staged, filepath.Join(dir, "no-such-directory", "dest")) + assert.Error(t, err, "a destination whose directory does not exist cannot be backed up") +} + +// TestCopyToBackup covers the two answers of the backup step. +func TestCopyToBackup(t *testing.T) { + t.Parallel() + + t.Run("copies the file beside itself", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "dest") + require.NoError(t, os.WriteFile(path, []byte("content"), 0o600)) + + backup, err := copyToBackup(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.Remove(backup)) }) + + assert.Equal(t, dir, filepath.Dir(backup), "the backup belongs in the same directory as the file") + got, err := os.ReadFile(backup) //nolint:gosec // Test path from t.TempDir() + require.NoError(t, err) + assert.Equal(t, "content", string(got)) + }) + + t.Run("reports a file that cannot be read", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + _, err := copyToBackup(filepath.Join(dir, "missing")) + require.Error(t, err) + + assertNoBackupLeft(t, dir) + }) + + t.Run("reports a directory it cannot write the backup into", func(t *testing.T) { + t.Parallel() + + _, err := copyToBackup(filepath.Join(t.TempDir(), "no-such-directory", "dest")) + assert.Error(t, err) + }) +} + +// TestCopyOnto covers the failures of the copy itself, which is what the +// fallback restores from. +func TestCopyOnto(t *testing.T) { + t.Parallel() + + t.Run("reports a destination that cannot be opened for writing", func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + src := filepath.Join(dir, "src") + require.NoError(t, os.WriteFile(src, []byte("content"), 0o600)) + + assert.Error(t, copyOnto(src, dir), "a directory cannot be opened as an output file") + }) + + t.Run("reports a source it cannot read", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows refuses to open a directory as a file, so the read never starts") + } + t.Parallel() + + dir := t.TempDir() + dest := filepath.Join(dir, "dest") + require.NoError(t, os.WriteFile(dest, []byte("old"), 0o600)) + + assert.Error(t, copyOnto(dir, dest), "reading a directory as a file must be reported") + }) +} diff --git a/autosave_driver_test.go b/autosave_driver_test.go new file mode 100644 index 0000000..3716822 --- /dev/null +++ b/autosave_driver_test.go @@ -0,0 +1,239 @@ +package filesql + +import ( + "context" + "database/sql/driver" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// errStub is the failure a stub connection reports when a test asks it to fail. +var errStub = errors.New("stub failure") + +// plainConn implements only what driver.Conn requires. A driver this small is +// what the fallbacks in the auto-save wrapper exist for: the wrapper cannot +// assume the connection it wraps implements the context-aware interfaces. +type plainConn struct { + closeErr error + beginErr error + begun bool +} + +func (c *plainConn) Prepare(string) (driver.Stmt, error) { return nil, errStub } +func (c *plainConn) Close() error { return c.closeErr } + +func (c *plainConn) Begin() (driver.Tx, error) { + if c.beginErr != nil { + return nil, c.beginErr + } + c.begun = true + return stubTx{}, nil +} + +// legacyConn adds the pre-context Execer and Queryer interfaces, which is the +// other shape the wrapper has to handle. +type legacyConn struct { + plainConn + execCalled bool + queryCalled bool + lastArgs []driver.Value +} + +func (c *legacyConn) Exec(_ string, args []driver.Value) (driver.Result, error) { + c.execCalled = true + c.lastArgs = args + return driver.RowsAffected(1), nil +} + +func (c *legacyConn) Query(_ string, args []driver.Value) (driver.Rows, error) { + c.queryCalled = true + c.lastArgs = args + return stubRows{}, nil +} + +// stubTx is a transaction that accepts both outcomes. +type stubTx struct{} + +func (stubTx) Commit() error { return nil } +func (stubTx) Rollback() error { return nil } + +// stubRows is an empty result set. +type stubRows struct{} + +func (stubRows) Columns() []string { return nil } +func (stubRows) Close() error { return nil } +func (stubRows) Next([]driver.Value) error { return errStub } +func (stubRows) ColumnTypeScanType(int) any { return nil } +func (stubRows) ColumnTypeDatabaseTypeName(int) string { + return "" +} + +// TestAutoSaveConnection_BeginTxFallsBackToBegin covers a wrapped driver that +// predates ConnBeginTx. Without the fallback such a driver could not start a +// transaction at all once auto-save wrapped it. +func TestAutoSaveConnection_BeginTxFallsBackToBegin(t *testing.T) { + t.Parallel() + + t.Run("the legacy Begin is used", func(t *testing.T) { + t.Parallel() + + inner := &plainConn{} + conn := &autoSaveConnection{conn: inner} + + tx, err := conn.BeginTx(context.Background(), driver.TxOptions{}) + require.NoError(t, err) + assert.IsType(t, &autoSaveTransaction{}, tx, "the transaction stays wrapped so a commit can still auto-save") + assert.True(t, inner.begun, "the legacy Begin is what starts the transaction") + }) + + t.Run("a refused Begin is reported", func(t *testing.T) { + t.Parallel() + + conn := &autoSaveConnection{conn: &plainConn{beginErr: errStub}} + + _, err := conn.BeginTx(context.Background(), driver.TxOptions{}) + assert.ErrorIs(t, err, errStub) + }) + + t.Run("the deprecated Begin goes through BeginTx", func(t *testing.T) { + t.Parallel() + + inner := &plainConn{} + conn := &autoSaveConnection{conn: inner} + + tx, err := conn.Begin() + require.NoError(t, err) + assert.IsType(t, &autoSaveTransaction{}, tx) + assert.True(t, inner.begun) + }) +} + +// TestAutoSaveConnection_LegacyExecAndQuery covers the pre-context statement +// interfaces. A driver that implements only those still has to be usable, and +// the named arguments it cannot take have to be converted rather than dropped. +func TestAutoSaveConnection_LegacyExecAndQuery(t *testing.T) { + t.Parallel() + + t.Run("exec", func(t *testing.T) { + t.Parallel() + + inner := &legacyConn{} + conn := &autoSaveConnection{conn: inner} + + _, err := conn.ExecContext(context.Background(), "UPDATE t SET a = ?", []driver.NamedValue{{Ordinal: 1, Value: int64(7)}}) + require.NoError(t, err) + assert.True(t, inner.execCalled) + assert.Equal(t, []driver.Value{int64(7)}, inner.lastArgs, "the named values must reach the legacy driver as plain ones") + }) + + t.Run("query", func(t *testing.T) { + t.Parallel() + + inner := &legacyConn{} + conn := &autoSaveConnection{conn: inner} + + _, err := conn.QueryContext(context.Background(), "SELECT ?", []driver.NamedValue{{Ordinal: 1, Value: "x"}}) + require.NoError(t, err) + assert.True(t, inner.queryCalled) + assert.Equal(t, []driver.Value{"x"}, inner.lastArgs) + }) + + t.Run("a connection with neither interface asks database/sql to take over", func(t *testing.T) { + t.Parallel() + + conn := &autoSaveConnection{conn: &plainConn{}} + + _, err := conn.ExecContext(context.Background(), "UPDATE t SET a = 1", nil) + assert.ErrorIs(t, err, driver.ErrSkip, "database/sql falls back to Prepare when the driver skips") + + _, err = conn.QueryContext(context.Background(), "SELECT 1", nil) + assert.ErrorIs(t, err, driver.ErrSkip) + }) +} + +// TestAutoSaveConnection_Prepare checks that preparing is handed straight to the +// wrapped connection. +func TestAutoSaveConnection_Prepare(t *testing.T) { + t.Parallel() + + conn := &autoSaveConnection{conn: &plainConn{}} + + _, err := conn.Prepare("SELECT 1") + assert.ErrorIs(t, err, errStub) +} + +// TestAutoSaveConnection_CloseReportsBothFailures covers a close where the save +// and the close itself both fail. The save error is the one a caller acts on, so +// it leads, but a connection that also failed to close is worth saying. +func TestAutoSaveConnection_CloseReportsBothFailures(t *testing.T) { + t.Parallel() + + inner := &plainConn{closeErr: errStub} + conn := &autoSaveConnection{ + conn: inner, + // Overwrite mode with no original paths: the save has nowhere to write, so + // it fails without touching the filesystem. + autoSaveConfig: &autoSaveConfig{enabled: true, timing: autoSaveOnClose}, + } + + err := conn.Close() + require.Error(t, err) + assert.Contains(t, err.Error(), "auto-save failed") + assert.Contains(t, err.Error(), "also failed to close connection", "a connection left open is worth reporting too") +} + +// TestAutoSaveTransaction_CommitReportsAFailedSave covers a commit that +// succeeded followed by a save that did not. The rows are already committed, so +// the caller has to be told that only the file is out of date. +func TestAutoSaveTransaction_CommitReportsAFailedSave(t *testing.T) { + t.Parallel() + + tx := &autoSaveTransaction{ + tx: stubTx{}, + conn: &autoSaveConnection{ + conn: &plainConn{}, + autoSaveConfig: &autoSaveConfig{enabled: true, timing: autoSaveOnCommit}, + }, + } + + err := tx.Commit() + require.Error(t, err) + assert.Contains(t, err.Error(), "transaction committed successfully") +} + +// TestAutoSaveTransaction_RollbackNeverSaves checks that a rollback reaches the +// wrapped transaction and does not run the save a commit would. +func TestAutoSaveTransaction_RollbackNeverSaves(t *testing.T) { + t.Parallel() + + tx := &autoSaveTransaction{ + tx: stubTx{}, + conn: &autoSaveConnection{ + conn: &plainConn{}, + autoSaveConfig: &autoSaveConfig{enabled: true, timing: autoSaveOnCommit}, + }, + } + + assert.NoError(t, tx.Rollback()) +} + +// TestPerformAutoSave_DisabledDoesNothing covers the two states in which a close +// has nothing to save. +func TestPerformAutoSave_DisabledDoesNothing(t *testing.T) { + t.Parallel() + + t.Run("no configuration", func(t *testing.T) { + t.Parallel() + conn := &autoSaveConnection{conn: &plainConn{}} + assert.NoError(t, conn.performAutoSave()) + }) + + t.Run("configuration turned off", func(t *testing.T) { + t.Parallel() + conn := &autoSaveConnection{conn: &plainConn{}, autoSaveConfig: &autoSaveConfig{enabled: false}} + assert.NoError(t, conn.performAutoSave()) + }) +} diff --git a/builder_error_test.go b/builder_error_test.go new file mode 100644 index 0000000..371d31a --- /dev/null +++ b/builder_error_test.go @@ -0,0 +1,149 @@ +package filesql + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// csvFixture writes a small CSV for a test and returns its path. +func csvFixture(t *testing.T) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "users.csv") + require.NoError(t, os.WriteFile(path, []byte("id,name\n1,Alice\n"), 0o600)) + return path +} + +// canceledContext returns a context that is already done. +func canceledContext(t *testing.T) context.Context { + t.Helper() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} + +// builtBuilder returns a builder that has collected path, which is the state a +// load starts from. +func builtBuilder(t *testing.T, path string) *DBBuilder { + t.Helper() + + builder, err := NewBuilder().AddPath(path).Build(context.Background()) + require.NoError(t, err) + return builder +} + +// TestBuilderEntryPoints_CanceledContext checks that each way of loading stops +// on a context that is already done, before it opens files or writes tables. A +// load that ignored cancellation would leave half the tables of an abandoned +// request behind. +func TestBuilderEntryPoints_CanceledContext(t *testing.T) { + t.Parallel() + + path := csvFixture(t) + + t.Run("Open", func(t *testing.T) { + t.Parallel() + + _, err := builtBuilder(t, path).Open(canceledContext(t)) + assert.ErrorIs(t, err, context.Canceled) + }) + + t.Run("LoadInto", func(t *testing.T) { + t.Parallel() + + err := builtBuilder(t, path).LoadInto(canceledContext(t), openTestDB(t)) + assert.ErrorIs(t, err, context.Canceled) + }) + + t.Run("LoadIntoTx", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + tx, err := db.BeginTx(context.Background(), nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + + assert.ErrorIs(t, builtBuilder(t, path).LoadIntoTx(canceledContext(t), tx), context.Canceled) + }) +} + +// TestLoadIntoTx_Refusals covers what LoadIntoTx cannot do. The caller owns the +// transaction, so there is nothing for auto-save to attach its close to, and a +// nil transaction has to be named rather than panicking. +func TestLoadIntoTx_Refusals(t *testing.T) { + t.Parallel() + + ctx := context.Background() + path := csvFixture(t) + + t.Run("a nil transaction", func(t *testing.T) { + t.Parallel() + + err := builtBuilder(t, path).LoadIntoTx(ctx, nil) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run("auto-save", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + + err = builtBuilder(t, path).EnableAutoSave(t.TempDir()).LoadIntoTx(ctx, tx) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run("no input at all", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + tx, err := db.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + + assert.Error(t, NewBuilder().LoadIntoTx(ctx, tx), "a builder with no input has nothing to load") + }) +} + +// TestOpenReadOnly_PassesTheOpenFailureThrough checks that the read-only wrapper +// does not swallow the failure of the load it wraps. +func TestOpenReadOnly_PassesTheOpenFailureThrough(t *testing.T) { + t.Parallel() + + rodb, err := NewBuilder().OpenReadOnly(context.Background()) + require.Error(t, err, "a builder with no input has nothing to open") + assert.Nil(t, rodb) +} + +// TestValidateDatabaseConnection covers the health check a load runs before it +// hands the database back. +func TestValidateDatabaseConnection(t *testing.T) { + t.Parallel() + + ctx := context.Background() + builder := NewBuilder() + + t.Run("a working database passes", func(t *testing.T) { + t.Parallel() + assert.NoError(t, builder.validateDatabaseConnection(ctx, openTestDB(t))) + }) + + t.Run("a closed database is reported", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + assert.Error(t, builder.validateDatabaseConnection(ctx, db)) + }) +} diff --git a/compression_unsupported_test.go b/compression_unsupported_test.go new file mode 100644 index 0000000..b2854c7 --- /dev/null +++ b/compression_unsupported_test.go @@ -0,0 +1,59 @@ +package filesql + +import ( + "bytes" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// unknownCompression is a value outside the set this package defines. A caller +// can produce one by converting an int, so both directions have to answer rather +// than fall through with a nil reader or writer. +const unknownCompression CompressionType = 99 + +// TestCompressionHandler_UnknownType covers the refusal of a compression type +// this package does not know. +func TestCompressionHandler_UnknownType(t *testing.T) { + t.Parallel() + + handler := NewCompressionHandler(unknownCompression) + + t.Run("reading", func(t *testing.T) { + t.Parallel() + + reader, cleanup, err := handler.CreateReader(bytes.NewReader(nil)) + require.Error(t, err) + assert.ErrorIs(t, err, ErrCompression) + assert.Nil(t, reader) + assert.Nil(t, cleanup) + }) + + t.Run("writing", func(t *testing.T) { + t.Parallel() + + writer, cleanup, err := handler.CreateWriter(&bytes.Buffer{}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrCompression) + assert.Nil(t, writer) + assert.Nil(t, cleanup) + }) +} + +// TestCreateWriterForFile_UnsupportedCompression covers the path where the file +// has already been created and the compression is what is refused. bzip2 is the +// real case: this package reads it but has no writer for it. +func TestCreateWriterForFile_UnsupportedCompression(t *testing.T) { + t.Parallel() + + factory := NewCompressionFactory() + path := filepath.Join(t.TempDir(), "out.csv.bz2") + + writer, cleanup, err := factory.CreateWriterForFile(path, CompressionBZ2) + require.Error(t, err) + assert.ErrorIs(t, err, ErrUnsupportedFormat) + assert.Nil(t, writer) + assert.Nil(t, cleanup) +} diff --git a/dump_error_test.go b/dump_error_test.go new file mode 100644 index 0000000..239b916 --- /dev/null +++ b/dump_error_test.go @@ -0,0 +1,108 @@ +package filesql + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDumpDatabase_UnusableConnection covers the first thing a dump asks for. +// Without a connection there is nothing to read, and the caller's output +// directory must be left as it was. +func TestDumpDatabase_UnusableConnection(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + outputDir := filepath.Join(t.TempDir(), "out") + err := DumpDatabase(db, outputDir) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + assert.NoDirExists(t, outputDir, "a dump that never started must not leave a directory behind") +} + +// TestDumpSQLiteDatabase_Failures covers the steps between the connection and +// the first table. +func TestDumpSQLiteDatabase_Failures(t *testing.T) { + t.Parallel() + + t.Run("the output directory cannot be created", func(t *testing.T) { + t.Parallel() + + blocked := filepath.Join(t.TempDir(), "in-the-way") + require.NoError(t, os.WriteFile(blocked, nil, 0o600)) + + err := dumpSQLiteDatabase(openTestDB(t), filepath.Join(blocked, "out"), NewDumpOptions()) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIOOperation) + }) + + t.Run("the tables cannot be listed", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + err := dumpSQLiteDatabase(db, filepath.Join(t.TempDir(), "out"), NewDumpOptions()) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run("a database with no table", func(t *testing.T) { + t.Parallel() + + err := dumpSQLiteDatabase(openTestDB(t), filepath.Join(t.TempDir(), "out"), NewDumpOptions()) + assert.ErrorIs(t, err, ErrNoTables) + }) +} + +// TestDumpSQLiteDatabase_WriteBackSourceIsGone covers a dump of a database whose +// ACH or Fedwire source file has disappeared since the load. Those files are +// rebuilt from the original, so the dump fails naming the format rather than +// writing a file with the fields only the original carries left empty. +func TestDumpSQLiteDatabase_WriteBackSourceIsGone(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + tests := []struct { + name string + source string + format sourceFormat + want error + }{ + {"ACH source", "payment.ach", sourceFormatACH, ErrACH}, + {"Fedwire", "payment.fed", sourceFormatFedWire, ErrWire}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + _, err := db.ExecContext(ctx, `CREATE TABLE payment_entries (id TEXT)`) + require.NoError(t, err) + require.NoError(t, recordFileSource(ctx, db, "payment", filepath.Join(t.TempDir(), tt.source), tt.format)) + + err = dumpSQLiteDatabase(db, filepath.Join(t.TempDir(), "out"), NewDumpOptions()) + require.Error(t, err) + assert.ErrorIs(t, err, tt.want) + }) + } +} + +// TestDumpSQLiteTable_UnreadableTable covers the per-table step of a dump. +func TestDumpSQLiteTable_UnreadableTable(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + err := dumpSQLiteTable(db, "users", t.TempDir(), NewDumpOptions()) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) +} diff --git a/file_processor_error_test.go b/file_processor_error_test.go new file mode 100644 index 0000000..a026d8e --- /dev/null +++ b/file_processor_error_test.go @@ -0,0 +1,94 @@ +package filesql + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "runtime" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCollectFilesFromPaths_UnsupportedFile covers a named file whose extension +// is not a format this package reads. Naming a file explicitly is a request to +// load it, so it is refused rather than skipped the way an unrelated file inside +// a directory is. +func TestCollectFilesFromPaths_UnsupportedFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "notes.docx") + require.NoError(t, os.WriteFile(path, []byte("content"), 0o600)) + + _, err := newFileProcessor(100).collectFilesFromPaths([]string{path}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrUnsupportedFormat) +} + +// TestCollectFilesFromDirectory_UnreadableDirectory covers a directory the +// process cannot walk. The load stops rather than returning the files it managed +// to reach, because a partial set of tables is worse than no load at all. +func TestCollectFilesFromDirectory_UnreadableDirectory(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("directory permissions do not stop a walk on Windows") + } + if os.Geteuid() == 0 { + t.Skip("root reads a directory whatever its mode says") + } + t.Parallel() + + dir := filepath.Join(t.TempDir(), "closed") + require.NoError(t, os.Mkdir(dir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "users.csv"), []byte("id\n1\n"), 0o600)) + require.NoError(t, os.Chmod(dir, 0o000)) + // The directory has to be traversable again, or the temporary directory it + // lives in cannot be removed. + t.Cleanup(func() { require.NoError(t, os.Chmod(dir, 0o700)) }) //nolint:gosec // A directory needs its execute bit to be walked and removed + + _, err := newFileProcessor(100).collectFilesFromPaths([]string{dir}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIOOperation) +} + +// TestProcessFSToReaders_FindsFilesInSubdirectories covers the walk that runs +// after the glob. A glob pattern matches one directory level, so a workbook or +// CSV one directory down is only found by the walk. +func TestProcessFSToReaders_FindsFilesInSubdirectories(t *testing.T) { + t.Parallel() + + filesystem := fstest.MapFS{ + "top.csv": &fstest.MapFile{Data: []byte("id\n1\n")}, + "nested/deep/in.csv": &fstest.MapFile{Data: []byte("id\n2\n")}, + "nested/notes.txt": &fstest.MapFile{Data: []byte("ignored")}, + } + + readers, err := newFileProcessor(100).processFSToReaders(context.Background(), filesystem) + require.NoError(t, err) + t.Cleanup(func() { + for _, r := range readers { + if r.closer != nil { + _ = r.closer.Close() + } + } + }) + + names := make([]string, 0, len(readers)) + for _, r := range readers { + names = append(names, r.tableName) + } + assert.ElementsMatch(t, []string{"top", "in"}, names, "the file one directory down belongs in the load too") +} + +// TestProcessFilesystemsToReaders_NilFilesystem covers the argument check. A nil +// filesystem is a caller mistake that would otherwise surface as a panic deep in +// the walk. +func TestProcessFilesystemsToReaders_NilFilesystem(t *testing.T) { + t.Parallel() + + _, err := newFileProcessor(100).processFilesystemsToReaders(context.Background(), []fs.FS{nil}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrNilInput) +} diff --git a/memory_fallback_test.go b/memory_fallback_test.go new file mode 100644 index 0000000..6d107a7 --- /dev/null +++ b/memory_fallback_test.go @@ -0,0 +1,83 @@ +package filesql + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestMemoryPool_ForeignValueInThePool checks the type assertions in the pool +// getters. A sync.Pool holds any, so nothing stops a value of another type from +// reaching a getter; the getters answer with a fresh slice rather than panicking, +// and these cases pin that down. +func TestMemoryPool_ForeignValueInThePool(t *testing.T) { + t.Parallel() + + t.Run("byte buffer", func(t *testing.T) { + t.Parallel() + pool := newMemoryPool(1024) + pool.bytePool.Put(new(pooledStringSlice)) + + buf := pool.getByteBuffer() + assert.NotNil(t, buf, "a foreign value must not stop the pool from answering") + assert.Empty(t, buf, "the buffer is handed back with no leftover length") + }) + + t.Run("record slice", func(t *testing.T) { + t.Parallel() + pool := newMemoryPool(1024) + pool.recordPool.Put(new(pooledByteSlice)) + + slice := pool.getRecordSlice() + assert.NotNil(t, slice, "a foreign value must not stop the pool from answering") + assert.Empty(t, slice, "the slice is handed back with no leftover length") + }) + + t.Run("string slice", func(t *testing.T) { + t.Parallel() + pool := newMemoryPool(1024) + pool.stringPool.Put(new(pooledRecordSlice)) + + slice := pool.getStringSlice() + assert.NotNil(t, slice, "a foreign value must not stop the pool from answering") + assert.Empty(t, slice, "the slice is handed back with no leftover length") + }) +} + +// TestMemoryLimit_ShouldReduceChunkSizeUnderPressure covers the two reducing +// answers. The status comes from the live heap, so the limit is set from the +// heap this process already holds instead of trying to allocate up to a fixed +// one: a limit at the current usage reads as exceeded, and one just above it +// reads as a warning. +func TestMemoryLimit_ShouldReduceChunkSizeUnderPressure(t *testing.T) { + t.Parallel() + + t.Run("exceeded cuts the chunk to a quarter", func(t *testing.T) { + t.Parallel() + limit := newMemoryLimit(defaultMemoryLimit) + // A limit at the heap this process already holds is exceeded however the + // heap moves afterwards, because it can only grow past it. + limit.maxMemoryMB = limit.getMemoryInfo().currentMB + require.Equal(t, memoryStatusExceeded, limit.checkMemoryUsage()) + + shouldReduce, size := limit.shouldReduceChunkSize(1000) + assert.True(t, shouldReduce) + assert.Equal(t, 250, size, "an exceeded limit cuts the chunk to a quarter") + }) + + t.Run("warning halves the chunk", func(t *testing.T) { + t.Parallel() + limit := newMemoryLimit(defaultMemoryLimit) + // A gigabyte of headroom keeps the limit out of reach, and a threshold of + // zero makes any usage at all a warning, so neither answer depends on what + // the heap does while the test runs. + limit.maxMemoryMB = limit.getMemoryInfo().currentMB + 1024 + limit.warningThreshold = 0 + require.Equal(t, memoryStatusWarning, limit.checkMemoryUsage()) + + shouldReduce, size := limit.shouldReduceChunkSize(1000) + assert.True(t, shouldReduce) + assert.Equal(t, 500, size, "a warning cuts the chunk in half") + }) +} diff --git a/parser/wire/allfields_test.go b/parser/wire/allfields_test.go new file mode 100644 index 0000000..e06776e --- /dev/null +++ b/parser/wire/allfields_test.go @@ -0,0 +1,98 @@ +package wire + +import ( + "reflect" + "slices" + "testing" + + "github.com/moov-io/wire" + "github.com/nao1215/filesql/parser" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// validateOptionsField is the one pointer field of FEDWireMessage that is not a +// message section: it carries the reader's validation switches and has no +// columns, so the section rules below do not apply to it. +const validateOptionsField = "ValidateOptions" + +// fullRecord returns a record that gives every column a distinct non-empty +// value. Using the column's own name as its value means a value that lands in +// the wrong field is visible in the failure message instead of matching by +// accident. +func fullRecord(headers []string) []string { + return slices.Clone(headers) +} + +// sectionFields returns the names of the message sections of fwm that are nil +// and the ones that are set. +func sectionFields(fwm *wire.FEDWireMessage) (nilSections, setSections []string) { + value := reflect.ValueOf(*fwm) + for _, f := range reflect.VisibleFields(value.Type()) { + if f.Type.Kind() != reflect.Pointer || f.Name == validateOptionsField { + continue + } + if value.FieldByIndex(f.Index).IsNil() { + nilSections = append(nilSections, f.Name) + continue + } + setSections = append(setSections, f.Name) + } + return nilSections, setSections +} + +// TestApplyModifications_EveryColumnRoundTrips writes a value into every column +// of the message table and reads the message back out. A column that +// applyModifications forgets, or that messageRecord writes to a different +// position, shows up here as a mismatch on that column; the per-section tests +// reach only the sections they name. +func TestApplyModifications_EveryColumnRoundTrips(t *testing.T) { + t.Parallel() + + headers := messageHeaders() + record := fullRecord(headers) + + fwm := &wire.FEDWireMessage{} + applyModifications(fwm, &parser.TableData{ + Headers: headers, + Records: [][]string{record}, + }) + + got := messageRecord(fwm) + require.Len(t, got, len(headers), "messageRecord must return one value per header") + for i, h := range headers { + assert.Equalf(t, record[i], got[i], "column %q did not survive the round trip", h) + } +} + +// TestEnsureNonNilSubStructs_AllocatesEverySection checks that a record with a +// value in every column leaves no section nil. A section that stays nil silently +// drops the values a caller wrote into its columns. +func TestEnsureNonNilSubStructs_AllocatesEverySection(t *testing.T) { + t.Parallel() + + headers := messageHeaders() + record := fullRecord(headers) + + fwm := &wire.FEDWireMessage{} + ensureNonNilSubStructs(fwm, buildHeaderIndex(headers), record) + + nilSections, _ := sectionFields(fwm) + assert.Emptyf(t, nilSections, "these sections stay nil even though their columns hold values: %v", nilSections) +} + +// TestEnsureNonNilSubStructs_AllocatesNothingForEmptyRecord is the other half of +// the rule: an untouched row must not invent sections, because an allocated +// empty section is written back out as an empty tag. +func TestEnsureNonNilSubStructs_AllocatesNothingForEmptyRecord(t *testing.T) { + t.Parallel() + + headers := messageHeaders() + record := make([]string, len(headers)) + + fwm := &wire.FEDWireMessage{} + ensureNonNilSubStructs(fwm, buildHeaderIndex(headers), record) + + _, setSections := sectionFields(fwm) + assert.Emptyf(t, setSections, "these sections were allocated for an all-empty record: %v", setSections) +} diff --git a/save_encoding_unit_test.go b/save_encoding_unit_test.go new file mode 100644 index 0000000..5d53399 --- /dev/null +++ b/save_encoding_unit_test.go @@ -0,0 +1,165 @@ +package filesql + +import ( + "bytes" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The names the encodings answer with. Each is written once here so the two +// tables below agree on what a given encoding is called. +const ( + nameUTF8 = "utf-8" + nameShiftJIS = "shift-jis" + nameEUCJP = "euc-jp" + nameISO2022JP = "iso-2022-jp" + nameUTF16LE = "utf-16le" + nameUTF16BE = "utf-16be" +) + +// TestEncoding_String pins the name of every encoding. The name is what a save +// error quotes back to the caller, so an encoding that answers with someone +// else's name misdirects whoever reads the failure. +func TestEncoding_String(t *testing.T) { + t.Parallel() + + tests := []struct { + encoding Encoding + want string + }{ + {EncodingUTF8, nameUTF8}, + {EncodingShiftJIS, nameShiftJIS}, + {EncodingEUCJP, nameEUCJP}, + {EncodingISO2022JP, nameISO2022JP}, + {EncodingUTF16LE, nameUTF16LE}, + {EncodingUTF16BE, nameUTF16BE}, + {Encoding(99), "unknown"}, + } + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, tt.encoding.String()) + }) + } +} + +// TestEncoding_Encoder checks which encodings need a transformer. UTF-8 needs +// none because the values are already UTF-8, and an unknown value is treated the +// same way rather than being guessed at. +func TestEncoding_Encoder(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + encoding Encoding + wantEncoder bool + }{ + {nameUTF8 + " needs no transformer", EncodingUTF8, false}, + {nameShiftJIS, EncodingShiftJIS, true}, + {nameEUCJP, EncodingEUCJP, true}, + {nameISO2022JP, EncodingISO2022JP, true}, + {nameUTF16LE, EncodingUTF16LE, true}, + {nameUTF16BE, EncodingUTF16BE, true}, + {"an unknown encoding needs no transformer", Encoding(99), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + transformer, ok := tt.encoding.encoder() + assert.Equal(t, tt.wantEncoder, ok) + if tt.wantEncoder { + assert.NotNil(t, transformer) + return + } + assert.Nil(t, transformer) + }) + } +} + +// TestEncoding_EncodingWriter covers both shapes of the writer wrapper: the +// encodings that need one get a writer whose failures are attributed to the +// encoder, and the ones that do not get their own writer back untouched. +func TestEncoding_EncodingWriter(t *testing.T) { + t.Parallel() + + t.Run(nameUTF8+" hands back the same writer", func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + w, encoded := EncodingUTF8.encodingWriter(&buf) + assert.Nil(t, encoded, "there is nothing to attribute a failure to without an encoder") + assert.False(t, encoded.encoderFailed(), "a nil encoded writer reports no failure") + + _, err := w.Write([]byte("hello")) + require.NoError(t, err) + assert.Equal(t, "hello", buf.String(), "UTF-8 values pass through unchanged") + }) + + t.Run(nameShiftJIS+" encodes what it writes", func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + w, encoded := EncodingShiftJIS.encodingWriter(&buf) + require.NotNil(t, encoded) + + _, err := w.Write([]byte("あ")) + require.NoError(t, err) + require.NoError(t, encoded.Close()) + assert.Equal(t, []byte{0x82, 0xa0}, buf.Bytes(), "あ is 0x82a0 in Shift-JIS") + assert.False(t, encoded.encoderFailed()) + }) +} + +// TestEncodedWriter_RecordsItsOwnFailures checks the bookkeeping that separates +// "this encoding cannot write this table" from a failure to write the bytes at +// all. x/text reports an unwritable rune with an unexported error type, so the +// only exact record of it is the one taken here. +func TestEncodedWriter_RecordsItsOwnFailures(t *testing.T) { + t.Parallel() + + t.Run("a failed write is recorded", func(t *testing.T) { + t.Parallel() + + wantErr := errors.New("refused") + w := &encodedWriter{ + w: writerFunc(func([]byte) (int, error) { return 0, wantErr }), + closer: func() error { return nil }, + } + + _, err := w.Write([]byte("あ")) + require.ErrorIs(t, err, wantErr) + assert.True(t, w.encoderFailed(), "the refusal must be attributed to the encoder") + }) + + t.Run("a failed close is recorded", func(t *testing.T) { + t.Parallel() + + wantErr := errors.New("held back a partial sequence") + w := &encodedWriter{ + w: &bytes.Buffer{}, + closer: func() error { return wantErr }, + } + + require.ErrorIs(t, w.Close(), wantErr) + assert.True(t, w.encoderFailed(), "a rune refused at flush time is still the encoder's refusal") + }) + + t.Run("a clean writer reports no failure", func(t *testing.T) { + t.Parallel() + + w := &encodedWriter{w: &bytes.Buffer{}, closer: func() error { return nil }} + _, err := w.Write([]byte("ok")) + require.NoError(t, err) + require.NoError(t, w.Close()) + assert.False(t, w.encoderFailed()) + }) +} + +// writerFunc turns a function into an io.Writer. +type writerFunc func(p []byte) (int, error) + +func (f writerFunc) Write(p []byte) (int, error) { return f(p) } diff --git a/save_overwrite_error_test.go b/save_overwrite_error_test.go new file mode 100644 index 0000000..a032138 --- /dev/null +++ b/save_overwrite_error_test.go @@ -0,0 +1,227 @@ +package filesql + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestPerformFedWireAutoSave covers the Fedwire branch of a directory save. +// Fedwire is rebuilt from the file it was loaded from, so a database that +// records no such file has nothing to write and says so. +func TestPerformFedWireAutoSave(t *testing.T) { + t.Parallel() + + t.Run("writes one file per loaded Fedwire source", func(t *testing.T) { + t.Parallel() + + db, err := Open(filepath.Join("testdata", "customer-transfer.fed")) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + outputDir := filepath.Join(t.TempDir(), "out") + require.NoError(t, (&autoSaveConnection{}).performFedWireAutoSave(db, outputDir)) + + // The file is named after the base table, which is the sanitized file name. + assert.FileExists(t, filepath.Join(outputDir, "customer_transfer.fed")) + }) + + t.Run("reports a database with no Fedwire source", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + err := (&autoSaveConnection{}).performFedWireAutoSave(db, t.TempDir()) + assert.ErrorContains(t, err, "no Fedwire tables found to save") + }) + + t.Run("reports an output directory it cannot create", func(t *testing.T) { + t.Parallel() + + db, err := Open(filepath.Join("testdata", "customer-transfer.fed")) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + // A regular file cannot also be a directory. + blocked := filepath.Join(t.TempDir(), "in-the-way") + require.NoError(t, os.WriteFile(blocked, nil, 0o600)) + + err = (&autoSaveConnection{}).performFedWireAutoSave(db, filepath.Join(blocked, "out")) + assert.ErrorContains(t, err, "failed to create output directory") + }) +} + +// TestPerformACHAutoSave_UncreatableOutputDirectory is the same refusal on the +// ACH branch. +func TestPerformACHAutoSave_UncreatableOutputDirectory(t *testing.T) { + t.Parallel() + + db, err := Open(filepath.Join("testdata", "ppd-debit.ach")) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + blocked := filepath.Join(t.TempDir(), "in-the-way") + require.NoError(t, os.WriteFile(blocked, nil, 0o600)) + + err = (&autoSaveConnection{}).performACHAutoSave(db, filepath.Join(blocked, "out")) + assert.ErrorContains(t, err, "failed to create output directory") +} + +// TestOverwriteOriginalFiles_NothingToOverwrite covers a save in overwrite mode +// on a database that has no file behind it, which is what a load from an +// io.Reader leaves. +func TestOverwriteOriginalFiles_NothingToOverwrite(t *testing.T) { + t.Parallel() + + err := (&autoSaveConnection{}).overwriteOriginalFiles(openTestDB(t)) + assert.ErrorContains(t, err, "no original paths available for overwrite") +} + +// TestOverwriteOriginalFile_WriteBackFormatFailures covers the two formats that +// are rebuilt from their source file. Each is reported with the path it failed +// on, because a save of several files has to say which one did not land. +func TestOverwriteOriginalFile_WriteBackFormatFailures(t *testing.T) { + t.Parallel() + + ctx := context.Background() + db := openTestDB(t) + + t.Run("ACH overwrite", func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "payment.ach") + err := (&autoSaveConnection{}).overwriteOriginalFile(ctx, db, path) + assert.ErrorContains(t, err, "failed to overwrite ACH file") + }) + + t.Run("Fedwire", func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "payment.fed") + err := (&autoSaveConnection{}).overwriteOriginalFile(ctx, db, path) + assert.ErrorContains(t, err, "failed to overwrite Fedwire file") + }) +} + +// TestOverwriteFormatFor pins which source formats can be written back. A format +// this package reads but cannot write is refused by name: quietly writing CSV +// instead left the caller's file untouched and the change in a file they never +// named. +func TestOverwriteFormatFor(t *testing.T) { + t.Parallel() + + tests := []struct { + path string + want OutputFormat + }{ + {"data.csv", OutputFormatCSV}, + {"data.tsv", OutputFormatTSV}, + {"data.ltsv", OutputFormatLTSV}, + {"data.parquet", OutputFormatParquet}, + {"data.xlsx", OutputFormatXLSX}, + {"data.csv.gz", OutputFormatCSV}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + t.Parallel() + + got, err := overwriteFormatFor(tt.path) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + + t.Run("a format with no writer is refused", func(t *testing.T) { + t.Parallel() + + _, err := overwriteFormatFor("data.json") + require.Error(t, err) + assert.ErrorIs(t, err, ErrUnsupportedFormat) + assert.Contains(t, err.Error(), "data.json", "the refusal names the file it is about") + }) +} + +// TestOverwriteWorkbookAtPath_Failures covers the workbook branch, which writes +// every sheet of a file in one staged write. +func TestOverwriteWorkbookAtPath_Failures(t *testing.T) { + t.Parallel() + + t.Run("the tables cannot be listed", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + err := overwriteWorkbookAtPath(db, filepath.Join(t.TempDir(), "book.xlsx"), "book", NewDumpOptions()) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run("no table of the workbook is left", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + err := overwriteWorkbookAtPath(db, filepath.Join(t.TempDir(), "book.xlsx"), "book", NewDumpOptions()) + require.Error(t, err) + assert.ErrorIs(t, err, ErrEmptyData) + }) +} + +// TestWriteXLSXWorkbookCompressed_UnknownCodec covers the refusal of a codec the +// workbook writer cannot open, before any sheet is written. +func TestWriteXLSXWorkbookCompressed_UnknownCodec(t *testing.T) { + t.Parallel() + + err := writeXLSXWorkbookCompressed(&bytes.Buffer{}, "book.xlsx", nil, unknownCompression) + require.Error(t, err) + assert.ErrorIs(t, err, ErrCompression) +} + +// TestTablesFromWorkbook_UnreadableCatalog covers the listing behind a workbook +// save. +func TestTablesFromWorkbook_UnreadableCatalog(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + _, err := tablesFromWorkbook(db, "book") + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) +} + +// TestOverwriteTableAtPath_Failures covers the single-table write-back. A table +// that is gone by save time cannot be written, and truncating the caller's file +// to nothing would be worse than refusing. +func TestOverwriteTableAtPath_Failures(t *testing.T) { + t.Parallel() + + t.Run("the columns cannot be read", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + err := overwriteTableAtPath(db, filepath.Join(t.TempDir(), "data.csv"), "data", NewDumpOptions()) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run("the table no longer exists", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + path := filepath.Join(t.TempDir(), "data.csv") + err := overwriteTableAtPath(db, path, "data", NewDumpOptions()) + require.Error(t, err) + assert.ErrorIs(t, err, ErrEmptyData) + assert.NoFileExists(t, path, "a refused save must not create the file it could not write") + }) +} diff --git a/small_units_test.go b/small_units_test.go new file mode 100644 index 0000000..c21b640 --- /dev/null +++ b/small_units_test.go @@ -0,0 +1,111 @@ +package filesql + +import ( + "context" + "database/sql/driver" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/nao1215/filesql/dialect" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestExcelSheetsInFile_UnreadableWorkbook covers the two ways the sheet listing +// can fail before it has a workbook to read: no file at that path, and a file +// that is not a workbook. +func TestExcelSheetsInFile_UnreadableWorkbook(t *testing.T) { + t.Parallel() + + t.Run("a file that is not there", func(t *testing.T) { + t.Parallel() + + _, err := ExcelSheetsInFile(filepath.Join(t.TempDir(), "missing.xlsx")) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIOOperation) + }) + + t.Run("a file that is not a workbook", func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "fake.xlsx") + require.NoError(t, os.WriteFile(path, []byte("id,name\n1,Alice\n"), 0o600)) + + _, err := ExcelSheetsInFile(path) + require.Error(t, err) + assert.ErrorIs(t, err, ErrParsing) + }) +} + +// TestExcelSheetsInReader_NotAWorkbook is the same refusal for a workbook that +// has no path. +func TestExcelSheetsInReader_NotAWorkbook(t *testing.T) { + t.Parallel() + + _, err := ExcelSheetsInReader(strings.NewReader("id,name\n1,Alice\n")) + require.Error(t, err) + assert.ErrorIs(t, err, ErrParsing) +} + +// TestReadOnlyTx_QueryRowRejectsWrite covers the last read entry point of a +// read-only transaction. QueryRow has no error to return, so the write is turned +// into a query whose Scan reports the refusal. +func TestReadOnlyTx_QueryRowRejectsWrite(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + _, err := db.ExecContext(context.Background(), `CREATE TABLE users (id TEXT)`) + require.NoError(t, err) + + tx, err := NewReadOnlyDB(db).Begin() + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + + var id string + err = tx.QueryRow(`DELETE FROM users`).Scan(&id) + require.Error(t, err, "a write must not be executed by the read-only wrapper") + assert.Contains(t, err.Error(), "read-only") +} + +// TestDialectConnector_UnusableDSN covers the connector that opens the +// translating connections. A DSN the driver refuses has to be reported when the +// connection is made rather than at the first query. +func TestDialectConnector_UnusableDSN(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + connector := &dialectConnector{ + drv: db.Driver(), + dsn: "file:/nonexistent-directory/db.sqlite?mode=rw", + sqlDialect: dialect.PostgreSQL, + } + + _, err := connector.Connect(context.Background()) + assert.Error(t, err) +} + +// TestDialectConnection_LegacyDriverFallbacks covers a wrapped connection that +// implements neither of the context-aware interfaces. Preparing and beginning +// still have to work, through the pre-context methods. +func TestDialectConnection_LegacyDriverFallbacks(t *testing.T) { + t.Parallel() + + conn := &dialectConnection{conn: &plainConn{}, sqlDialect: dialect.PostgreSQL} + + t.Run("prepare", func(t *testing.T) { + t.Parallel() + + _, err := conn.PrepareContext(context.Background(), "SELECT 1") + assert.ErrorIs(t, err, errStub, "the legacy Prepare is what answers") + }) + + t.Run("begin", func(t *testing.T) { + t.Parallel() + + tx, err := conn.BeginTx(context.Background(), driver.TxOptions{}) + require.NoError(t, err) + assert.NotNil(t, tx) + }) +} diff --git a/source_registry_error_test.go b/source_registry_error_test.go new file mode 100644 index 0000000..584ad65 --- /dev/null +++ b/source_registry_error_test.go @@ -0,0 +1,134 @@ +package filesql + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// openTestDB opens an empty on-disk database for a test and closes it afterwards. +func openTestDB(t *testing.T) *sql.DB { + t.Helper() + + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "test.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + return db +} + +// TestRecordFileSource_ReportsAnUnusableDatabase covers the two writes the +// bookkeeping makes. They run on the caller's own DBTX, so a database that +// cannot take them has to be reported rather than leaving tables whose source is +// silently unrecorded — a later dump would then refuse with a puzzling +// "no source recorded". +func TestRecordFileSource_ReportsAnUnusableDatabase(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("the source table cannot be created", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + err := recordFileSource(ctx, db, "payment", "payment.ach", sourceFormatACH) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run("the row cannot be inserted", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + // A table of that name with other columns is left alone by CREATE TABLE IF + // NOT EXISTS, so the insert is what fails. + _, err := db.ExecContext(ctx, `CREATE TABLE "`+sourceTableName+`" (unrelated TEXT)`) + require.NoError(t, err) + + err = recordFileSource(ctx, db, "payment", "payment.ach", sourceFormatACH) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run("a reader load records nothing", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, recordFileSource(ctx, db, "payment", "", sourceFormatACH)) + + _, ok := fileSourcePath(ctx, db, "payment", sourceFormatACH) + assert.False(t, ok, "a load with no file behind it has no source to go back to") + }) +} + +// TestFileSourceBaseNames_UnreadableRows checks the listing used by a dump of +// every loaded file. A row it cannot read means the set of files to write is +// unknown, so it answers with nothing rather than a partial set that would dump +// some files and silently skip others. +func TestFileSourceBaseNames_UnreadableRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + db := openTestDB(t) + + _, err := db.ExecContext(ctx, `CREATE TABLE "`+sourceTableName+`" (base_table_name TEXT, source_path TEXT, format TEXT)`) + require.NoError(t, err) + _, err = db.ExecContext(ctx, `INSERT INTO "`+sourceTableName+`" VALUES (NULL, '/tmp/payment.ach', 'ach')`) + require.NoError(t, err) + + assert.Nil(t, fileSourceBaseNames(ctx, db, sourceFormatACH), "a row that cannot be read yields no names") +} + +// TestTableSetForDump_UnparsableSource covers the reread a write-back format +// depends on. The tables alone cannot rebuild the file, so a source that no +// longer parses has to be reported instead of writing a file built from +// whatever was salvageable. +func TestTableSetForDump_UnparsableSource(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("ACH reread", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + path := filepath.Join(t.TempDir(), "payment.ach") + require.NoError(t, os.WriteFile(path, []byte("this is not an ACH file"), 0o600)) + require.NoError(t, recordFileSource(ctx, db, "payment", path, sourceFormatACH)) + + _, err := achTableSetForDump(ctx, db, "payment") + require.Error(t, err) + assert.ErrorIs(t, err, ErrACH) + }) + + t.Run("Fedwire", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + path := filepath.Join(t.TempDir(), "payment.fed") + require.NoError(t, os.WriteFile(path, []byte("this is not a Fedwire file"), 0o600)) + require.NoError(t, recordFileSource(ctx, db, "payment", path, sourceFormatFedWire)) + + _, err := wireTableSetForDump(ctx, db, "payment") + require.Error(t, err) + assert.ErrorIs(t, err, ErrWire) + }) + + t.Run("the recorded file is gone", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + path := filepath.Join(t.TempDir(), "payment.ach") + require.NoError(t, recordFileSource(ctx, db, "payment", path, sourceFormatACH)) + + _, err := achTableSetForDump(ctx, db, "payment") + require.Error(t, err) + assert.ErrorIs(t, err, ErrSourceUnavailable) + }) +} diff --git a/stream_parse_error_test.go b/stream_parse_error_test.go new file mode 100644 index 0000000..14a3eba --- /dev/null +++ b/stream_parse_error_test.go @@ -0,0 +1,123 @@ +package filesql + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestParseFromReader_EmptyInput covers what each format's parser answers for an +// input with nothing in it. JSON and JSONL are excluded on purpose: an empty one +// is a valid zero-row table, which the loader turns into an empty table rather +// than a failure. XLSX is excluded because no bytes at all is not an empty +// workbook but an unreadable one, which the case below covers. +func TestParseFromReader_EmptyInput(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fileType FileType + }{ + {"CSV", FileTypeCSV}, + {"TSV", FileTypeTSV}, + {"Parquet", FileTypeParquet}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + parser := newStreamingParser(tt.fileType, CompressionNone, "empty", 100) + _, err := parser.parseFromReader(strings.NewReader("")) + require.Error(t, err) + assert.ErrorIs(t, err, ErrEmptyData) + }) + } +} + +// TestParseFromReader_UnparsableInput covers the binary formats given bytes that +// are not the format at all, which is what a mislabelled file looks like. +func TestParseFromReader_UnparsableInput(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fileType FileType + }{ + {"Parquet", FileTypeParquet}, + {"XLSX", FileTypeXLSX}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + parser := newStreamingParser(tt.fileType, CompressionNone, "wrong", 100) + _, err := parser.parseFromReader(strings.NewReader("id,name\n1,Alice\n")) + assert.Error(t, err, "bytes that are not the format must not load as a table") + }) + } +} + +// TestParseDelimitedStream_MalformedRowPolicies covers what a ragged row does +// under each policy. The counts matter as much as the outcome: a load that +// dropped rows reports how many, so a caller can tell a clean load from a lossy +// one. +func TestParseDelimitedStream_MalformedRowPolicies(t *testing.T) { + t.Parallel() + + // The second row has one field too few and the third one too many. + const content = "id,name,email\n1,Alice\n3,Carol,c@example.com,extra\n4,Dave,d@example.com\n" + + t.Run("stop refuses the file", func(t *testing.T) { + t.Parallel() + + parser := newStreamingParser(FileTypeCSV, CompressionNone, "users", 100) + parser.malformedRowPolicy = MalformedRowStop + + _, err := parser.parseFromReader(strings.NewReader(content)) + require.Error(t, err) + assert.ErrorIs(t, err, ErrColumnMismatch) + }) + + t.Run("skip drops the ragged rows and counts them", func(t *testing.T) { + t.Parallel() + + parser := newStreamingParser(FileTypeCSV, CompressionNone, "users", 100) + parser.malformedRowPolicy = MalformedRowSkip + + table, err := parser.parseFromReader(strings.NewReader(content)) + require.NoError(t, err) + assert.Len(t, table.getRecords(), 1, "only the well-formed row is kept") + assert.Equal(t, 2, parser.skippedRows) + assert.Equal(t, 3, parser.totalRows) + }) + + t.Run("fill pads a short row and still refuses a long one", func(t *testing.T) { + t.Parallel() + + parser := newStreamingParser(FileTypeCSV, CompressionNone, "users", 100) + parser.malformedRowPolicy = MalformedRowFill + + short := newStreamingParser(FileTypeCSV, CompressionNone, "users", 100) + short.malformedRowPolicy = MalformedRowFill + table, err := short.parseFromReader(strings.NewReader("id,name,email\n1,Alice\n")) + require.NoError(t, err) + require.Len(t, table.getRecords(), 1) + assert.Equal(t, []string{"1", "Alice", ""}, []string(table.getRecords()[0]), "a missing field becomes an empty one") + + _, err = parser.parseFromReader(strings.NewReader(content)) + require.Error(t, err, "a row with more fields than the header would lose data if it were reshaped") + assert.ErrorIs(t, err, ErrColumnMismatch) + }) +} + +// TestMalformedRowPolicy_String pins the names the policies are configured by. +func TestMalformedRowPolicy_String(t *testing.T) { + t.Parallel() + + assert.Equal(t, "stop", MalformedRowStop.String()) + assert.Equal(t, "skip", MalformedRowSkip.String()) + assert.Equal(t, "fill", MalformedRowFill.String()) + assert.Equal(t, "MalformedRowPolicy(9)", MalformedRowPolicy(9).String()) +} diff --git a/stream_processor_error_test.go b/stream_processor_error_test.go new file mode 100644 index 0000000..8b0fe80 --- /dev/null +++ b/stream_processor_error_test.go @@ -0,0 +1,283 @@ +package filesql + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// plainExecutor is a DBTX that is neither *sql.DB nor *sql.Tx. The chunk loader +// needs one of those two to open its own transaction, so a caller's own +// implementation has to be refused by name rather than crashing on a type +// assertion. +type plainExecutor struct { + db *sql.DB +} + +func (e plainExecutor) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + return e.db.ExecContext(ctx, query, args...) +} + +func (e plainExecutor) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + return e.db.QueryContext(ctx, query, args...) +} + +func (e plainExecutor) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + return e.db.QueryRowContext(ctx, query, args...) +} + +func (e plainExecutor) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) { + return e.db.PrepareContext(ctx, query) +} + +// failingCloser reports a failure when the loader closes a reader it opened. +type failingCloser struct{ closed bool } + +func (c *failingCloser) Close() error { + c.closed = true + return errStub +} + +// TestStreamFileToDatabase_UnsupportedFormat covers the refusal of a file whose +// extension names no format this package reads. +func TestStreamFileToDatabase_UnsupportedFormat(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "notes.docx") + require.NoError(t, os.WriteFile(path, []byte("content"), 0o600)) + + err := newStreamProcessor(100).streamFileToDatabase(context.Background(), openTestDB(t), path) + require.Error(t, err) + assert.ErrorIs(t, err, ErrUnsupportedFormat) +} + +// TestStreamWriteBackFormatFiles_Failures covers the two formats that are read +// from a path rather than through the chunk loader. Both are opened and measured +// before parsing, so a missing or empty file is reported as such instead of as a +// parse failure with nothing in it. +func TestStreamWriteBackFormatFiles_Failures(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + tests := []struct { + name string + ext string + }{ + {"ACH input", extACH}, + {"Fedwire", extFED}, + } + for _, tt := range tests { + t.Run(tt.name+" file that is not there", func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "missing"+tt.ext) + err := newStreamProcessor(100).streamFileToDatabase(ctx, openTestDB(t), path) + require.Error(t, err) + assert.ErrorIs(t, err, ErrIOOperation) + }) + + t.Run(tt.name+" file with no bytes in it", func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "empty"+tt.ext) + require.NoError(t, os.WriteFile(path, nil, 0o600)) + + err := newStreamProcessor(100).streamFileToDatabase(ctx, openTestDB(t), path) + require.Error(t, err) + assert.ErrorIs(t, err, ErrEmptyData) + }) + } +} + +// TestStreamReaderToDatabase_UnsupportedExecutor covers a DBTX the loader cannot +// start a transaction on. It is refused with the type in the message, because a +// caller who passed their own wrapper has no other way to tell what was wrong. +func TestStreamReaderToDatabase_UnsupportedExecutor(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + err := newStreamProcessor(100).streamReaderToDatabase(context.Background(), plainExecutor{db: db}, readerInput{ + reader: strings.NewReader("id,name\n1,Alice\n"), + tableName: "users", + fileType: FileTypeCSV, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + assert.Contains(t, err.Error(), "unsupported database executor") +} + +// TestStreamReaderToDatabase_UnusableDatabase covers the check for a table of +// the same name, which is the first thing a load asks the database. +func TestStreamReaderToDatabase_UnusableDatabase(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + err := newStreamProcessor(100).streamReaderToDatabase(context.Background(), db, readerInput{ + reader: strings.NewReader("id,name\n1,Alice\n"), + tableName: "users", + fileType: FileTypeCSV, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) +} + +// TestStreamReaderToDatabase_ReservedTableName pins that the reserved namespace +// is refused for readers too, not only for paths. +func TestStreamReaderToDatabase_ReservedTableName(t *testing.T) { + t.Parallel() + + err := newStreamProcessor(100).streamReaderToDatabase(context.Background(), openTestDB(t), readerInput{ + reader: strings.NewReader("id\n1\n"), + tableName: sourceTablePrefix + "report", + fileType: FileTypeCSV, + }) + require.Error(t, err) + assert.ErrorIs(t, err, ErrReservedTableName) +} + +// TestCloseReaderInput_ReportsNothingToTheCaller checks that a reader this +// package opened itself is closed, and that a failure to close it does not fail +// the load: the rows are already in the database by then. +func TestCloseReaderInput_ReportsNothingToTheCaller(t *testing.T) { + t.Parallel() + + closer := &failingCloser{} + newStreamProcessor(100).closeReaderInput(readerInput{tableName: "users", closer: closer}) + assert.True(t, closer.closed, "a reader opened by this package must be closed") +} + +// TestDropIfReplacing covers the drop that lets a reload install its own schema. +func TestDropIfReplacing(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("does nothing in open mode", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + // A closed database would fail any statement, so a successful call proves + // none was sent. + assert.NoError(t, newStreamProcessor(100).dropIfReplacing(ctx, db, "users")) + }) + + t.Run("reports a drop the database refused", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + sp := newStreamProcessor(100) + sp.replaceExisting = true + + err := sp.dropIfReplacing(ctx, db, "users") + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) +} + +// TestCreateEmptyTable covers the header-only file, which is a valid input that +// produces a table with no rows. +func TestCreateEmptyTable(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("creates the columns the header names", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + input := readerInput{ + reader: strings.NewReader("id,name\n"), + tableName: "users", + fileType: FileTypeCSV, + } + require.NoError(t, newStreamProcessor(100).createEmptyTable(ctx, db, input)) + + var count int + require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM users`).Scan(&count)) + assert.Equal(t, 0, count, "a header-only file loads as a table with no rows") + + rows, err := db.QueryContext(ctx, `SELECT * FROM users`) + require.NoError(t, err) + defer rows.Close() + columns, err := rows.Columns() + require.NoError(t, err) + assert.Equal(t, []string{"id", "name"}, columns) + require.NoError(t, rows.Err()) + }) + + t.Run("keeps a duplicate column refusal", func(t *testing.T) { + t.Parallel() + + input := readerInput{ + reader: strings.NewReader("id,id\n"), + tableName: "users", + fileType: FileTypeCSV, + } + err := newStreamProcessor(100).createEmptyTable(ctx, openTestDB(t), input) + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate column name", "the parser's own refusal must not be replaced by a fallback table") + }) + + t.Run("reports a database that cannot take the table", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + input := readerInput{ + reader: strings.NewReader("id,name\n"), + tableName: "users", + fileType: FileTypeCSV, + } + err := newStreamProcessor(100).createEmptyTable(ctx, db, input) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) +} + +// TestCreateTableFromHeaders covers the fallback used when the header cannot be +// parsed at all: the file still becomes a table, so a later query names a table +// that exists instead of failing on a missing one. +func TestCreateTableFromHeaders(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("creates a single-column table", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + input := readerInput{tableName: "users", fileType: FileTypeCSV} + require.NoError(t, newStreamProcessor(100).createTableFromHeaders(ctx, db, input)) + + var name string + require.NoError(t, db.QueryRowContext(ctx, + `SELECT name FROM sqlite_master WHERE type='table' AND name='users'`).Scan(&name)) + assert.Equal(t, "users", name) + }) + + t.Run("reports a database that cannot take the table", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + input := readerInput{tableName: "users", fileType: FileTypeCSV} + err := newStreamProcessor(100).createTableFromHeaders(ctx, db, input) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) +} diff --git a/types_edge_test.go b/types_edge_test.go new file mode 100644 index 0000000..0d8cf66 --- /dev/null +++ b/types_edge_test.go @@ -0,0 +1,109 @@ +package filesql + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestNewChunkSize_BelowTheMinimum checks the floor on a chunk size. A chunk of +// zero or fewer rows would read a file forever, so anything under the minimum +// falls back to the default. +func TestNewChunkSize_BelowTheMinimum(t *testing.T) { + t.Parallel() + + assert.Equal(t, chunkSizeValue(DefaultRowsPerChunk), newChunkSize(0)) + assert.Equal(t, chunkSizeValue(DefaultRowsPerChunk), newChunkSize(-1)) + assert.Equal(t, chunkSizeValue(MinChunkSize), newChunkSize(MinChunkSize)) +} + +// TestNewColumnInfoList_NoColumns covers a header with nothing in it, which is +// what an input with no columns produces. +func TestNewColumnInfoList_NoColumns(t *testing.T) { + t.Parallel() + + assert.Nil(t, newColumnInfoList(newHeader(nil), nil)) + assert.Nil(t, inferColumnsInfo(newHeader(nil), nil)) +} + +// TestColumnInfoList_EqualTypes covers the comparison that decides whether a +// later chunk widens the table already created. +func TestColumnInfoList_EqualTypes(t *testing.T) { + t.Parallel() + + integers := columnInfoList{{Name: "a", Type: columnTypeInteger}} + texts := columnInfoList{{Name: "a", Type: columnTypeText}} + + assert.True(t, integers.equalTypes(columnInfoList{{Name: "a", Type: columnTypeInteger}})) + assert.False(t, integers.equalTypes(texts), "a widened column is not the same schema") + assert.False(t, integers.equalTypes(columnInfoList{}), "a different column count is not the same schema") +} + +// TestInferColumnType_NoValues covers a column with no values to judge by. Text +// is the only type that holds anything a later row can bring. +func TestInferColumnType_NoValues(t *testing.T) { + t.Parallel() + + assert.Equal(t, columnTypeText, inferColumnType(nil)) +} + +// TestSelectColumnType_WithoutAConfidentMajority covers the fallbacks used when +// no type reaches the confidence threshold. The column still has to be declared +// as something, and the numeric types are preferred over text in the order that +// keeps values readable. +func TestSelectColumnType_WithoutAConfidentMajority(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + typeCounts map[columnType]int + totalCount int + want columnType + }{ + { + name: "a few reals among empty values", + typeCounts: map[columnType]int{columnTypeReal: 1}, + totalCount: 100, + want: columnTypeReal, + }, + { + name: "a few integers among empty values", + typeCounts: map[columnType]int{columnTypeInteger: 1}, + totalCount: 100, + want: columnTypeInteger, + }, + { + name: "a few datetimes among empty values", + typeCounts: map[columnType]int{columnTypeDatetime: 1}, + totalCount: 100, + want: columnTypeDatetime, + }, + { + name: "nothing classified at all", + typeCounts: map[columnType]int{}, + totalCount: 100, + want: columnTypeText, + }, + { + name: "a datetime beside a number has no type covering both", + typeCounts: map[columnType]int{columnTypeDatetime: 5, columnTypeInteger: 5}, + totalCount: 10, + want: columnTypeText, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, selectColumnType(tt.typeCounts, tt.totalCount)) + }) + } +} + +// TestIsIntegerLiteralOverflowingInt64_SignOnly covers a value that is a sign +// and nothing else, which is not a number at all. +func TestIsIntegerLiteralOverflowingInt64_SignOnly(t *testing.T) { + t.Parallel() + + assert.False(t, isIntegerLiteralOverflowingInt64("+")) + assert.False(t, isIntegerLiteralOverflowingInt64("-")) +} diff --git a/writeback_stream_test.go b/writeback_stream_test.go new file mode 100644 index 0000000..0df9c21 --- /dev/null +++ b/writeback_stream_test.go @@ -0,0 +1,243 @@ +package filesql + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// achFixture returns the bytes of a small ACH file that parses. +func achFixture(t *testing.T) []byte { + t.Helper() + + data, err := os.ReadFile(filepath.Join("testdata", "ppd-debit.ach")) + require.NoError(t, err) + return data +} + +// wireFixture returns the bytes of a small Fedwire file that parses. +func wireFixture(t *testing.T) []byte { + t.Helper() + + data, err := os.ReadFile(filepath.Join("testdata", "customer-transfer.fed")) + require.NoError(t, err) + return data +} + +// TestStreamWriteBackFormatsToDatabase covers the two loaders that build tables +// from a whole file at once. They share a shape — validate the name, parse, +// then create and fill one table per section — so the refusals are checked for +// both. +func TestStreamWriteBackFormatsToDatabase(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + tests := []struct { + name string + ext string + load func(ctx context.Context, db DBTX, content []byte, filePath string, replaceExisting bool) error + data func(t *testing.T) []byte + }{ + { + name: "ACH file", + ext: extACH, + data: achFixture, + load: func(ctx context.Context, db DBTX, content []byte, filePath string, replaceExisting bool) error { + return streamACHFileToDatabase(ctx, db, strings.NewReader(string(content)), filePath, "", replaceExisting) + }, + }, + { + name: "Fedwire", + ext: extFED, + data: wireFixture, + load: func(ctx context.Context, db DBTX, content []byte, filePath string, replaceExisting bool) error { + return streamWireFileToDatabase(ctx, db, strings.NewReader(string(content)), filePath, "", replaceExisting) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name+" refuses a reserved table name", func(t *testing.T) { + t.Parallel() + + err := tt.load(ctx, openTestDB(t), tt.data(t), sourceTablePrefix+"payment"+tt.ext, false) + require.Error(t, err) + assert.ErrorIs(t, err, ErrReservedTableName) + }) + + t.Run(tt.name+" reports content it cannot parse", func(t *testing.T) { + t.Parallel() + + err := tt.load(ctx, openTestDB(t), []byte("this is not a payment file"), "payment"+tt.ext, false) + assert.Error(t, err) + }) + + t.Run(tt.name+" reports a database it cannot query", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + err := tt.load(ctx, db, tt.data(t), "payment"+tt.ext, false) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run(tt.name+" refuses to load twice over its own tables", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + content := tt.data(t) + require.NoError(t, tt.load(ctx, db, content, "payment"+tt.ext, false)) + + err := tt.load(ctx, db, content, "payment"+tt.ext, false) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDuplicateTable) + }) + + t.Run(tt.name+" replaces its own tables when asked", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + content := tt.data(t) + require.NoError(t, tt.load(ctx, db, content, "payment"+tt.ext, false)) + + assert.NoError(t, tt.load(ctx, db, content, "payment"+tt.ext, true), + "a reload in replace mode drops the tables it is about to rebuild") + }) + } +} + +// TestParseACHFile covers the parse step on its own, which is what turns file +// bytes into tables and into the structure a later dump rebuilds the file from. +func TestParseACHFile(t *testing.T) { + t.Parallel() + + t.Run("returns the tables and the structure behind them", func(t *testing.T) { + t.Parallel() + + tables, tableSet, err := parseACHFile(strings.NewReader(string(achFixture(t))), "payment") + require.NoError(t, err) + require.NotNil(t, tableSet, "a dump needs the structure the tables came from") + assert.NotEmpty(t, tables) + assert.NotNil(t, tableSet.GetFileHeaderTable(), "the file header is what an ACH file starts with") + }) + + t.Run("reports content that is not an ACH file", func(t *testing.T) { + t.Parallel() + + tables, tableSet, err := parseACHFile(strings.NewReader("this is not an ACH file"), "payment") + require.Error(t, err) + assert.ErrorIs(t, err, ErrACH) + assert.Nil(t, tables) + assert.Nil(t, tableSet) + }) +} + +// TestParseFedWireFile is the same for Fedwire, which is one message table. +func TestParseFedWireFile(t *testing.T) { + t.Parallel() + + t.Run("returns the message table and the structure behind it", func(t *testing.T) { + t.Parallel() + + tables, tableSet, err := parseFedWireFile(strings.NewReader(string(wireFixture(t))), "payment") + require.NoError(t, err) + require.NotNil(t, tableSet) + require.Len(t, tables, 1, "a Fedwire file holds one message") + assert.Equal(t, "payment_message", tables[0].getName()) + assert.NotNil(t, tableSet.GetMessageTable()) + }) + + t.Run("reports content that is not a Fedwire file", func(t *testing.T) { + t.Parallel() + + tables, tableSet, err := parseFedWireFile(strings.NewReader("this is not a Fedwire file"), "payment") + require.Error(t, err) + assert.ErrorIs(t, err, ErrWire) + assert.Nil(t, tables) + assert.Nil(t, tableSet) + }) +} + +// TestDumpWithTableSet_NilTableSet covers the argument neither dump can work +// without: the file is rebuilt from the structure, so there is nothing to write +// without one. +func TestDumpWithTableSet_NilTableSet(t *testing.T) { + t.Parallel() + + ctx := context.Background() + db := openTestDB(t) + out := filepath.Join(t.TempDir(), "payment") + + assert.ErrorIs(t, DumpACHWithTableSet(ctx, db, "payment", out+extACH, nil), ErrNilInput) + assert.ErrorIs(t, DumpFedWireWithTableSet(ctx, db, "payment", out+extFED, nil), ErrNilInput) +} + +// TestInsertRecordsIntoTable_Failures covers the insert step used by the +// write-back loaders. +func TestInsertRecordsIntoTable_Failures(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("the statement cannot be prepared", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + require.NoError(t, db.Close()) + + err := insertRecordsIntoTable(ctx, db, "users", newHeader([]string{"id"}), []record{newRecord([]string{"1"})}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) + + t.Run("a row the table refuses", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + _, err := db.ExecContext(ctx, `CREATE TABLE users (id TEXT CHECK (id <> 'refused'))`) + require.NoError(t, err) + + err = insertRecordsIntoTable(ctx, db, "users", newHeader([]string{"id"}), []record{newRecord([]string{"refused"})}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrDatabaseOperation) + }) +} + +// TestReadTableToTableData covers the read-back a dump starts from. +func TestReadTableToTableData(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("reads rows and turns a NULL into an empty value", func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + _, err := db.ExecContext(ctx, `CREATE TABLE users (id TEXT, name TEXT)`) + require.NoError(t, err) + _, err = db.ExecContext(ctx, `INSERT INTO users VALUES ('1', NULL)`) + require.NoError(t, err) + + data, err := readTableToTableData(ctx, db, "users") + require.NoError(t, err) + assert.Equal(t, []string{"id", "name"}, data.Headers) + require.Len(t, data.Records, 1) + assert.Equal(t, []string{"1", ""}, data.Records[0], "a NULL has no text of its own to write back") + }) + + t.Run("reports a table that is not there", func(t *testing.T) { + t.Parallel() + + _, err := readTableToTableData(ctx, openTestDB(t), "missing") + require.Error(t, err) + assert.ErrorIs(t, err, ErrTableNotFound) + }) +}