diff --git a/docs/schema-definition.md b/docs/schema-definition.md index 89f250b..e438b32 100644 --- a/docs/schema-definition.md +++ b/docs/schema-definition.md @@ -75,7 +75,7 @@ return [ * NOTE: This approach is not compatible with Laravel's config caching. */ 'default' => DumpSchema::define() - ->allTables() + ->allTables() ->table('users', function (TableDefinition $table) { $table->replace('name', function (Faker $faker) { return $faker->name; @@ -96,19 +96,38 @@ This ensures that all of your database tables will be represented in the dump. Y ```php return [ 'default' => DumpSchema::define() - ->allTables(), + ->allTables(), ]; ``` -## Exclude specific tables from dumps +If you don't want to dump all of your tables, you may add specific tables that you wish to include in the dump by using the `include()` method. You don't need to add a table here if you will be customizing it. -The `exclude()` method allows you to exclude specific tables from the dump. This can be useful if you want to exclude certain tables from the dump: +```php +return [ + 'default' => DumpSchema::define() + ->include('audit_logs') + ->include(['user_logins', 'failed_jobs']), +]; +``` + +If you've started out by adding all tables with `allTables()`, you can remove some of them with the `exclude()` method. This can be useful if you have certain tables which aren't required for this dump. ```php return [ 'default' => DumpSchema::define() - ->allTables() - ->exclude('password_resets'), + ->allTables() + ->exclude('password_resets'), +]; +``` + +Consider the case where you have a set of tables that you never want to dump. Perhaps they contain complex nested JSON that is too complex to anonymise, or huge amounts of analytic data that just won't be necessary. Well, you can also pass an array to `exclude()`: + +```php +return [ + 'default' => DumpSchema::define() + ->allTables() + ->exclude(['password_resets', 'secrets', 'lies', 'cat_birthdays']), + ->exclude(config('database.forbidden_tables')), ]; ``` @@ -170,7 +189,7 @@ When dumping your data, the dump will now contain a safe, randomly generated ema ## Optimizing large datasets -The method TableDefinition::outputInChunksOf(int $chunkSize) allows for chunked inserts for large datasets, +The method `TableDefinition::outputInChunksOf(int $chunkSize)` allows for chunked inserts for large datasets, improving performance and reducing memory consumption during the dump process. ```php @@ -191,7 +210,7 @@ You can pass the connection to the `DumpSchema::define` method, in order to spec ```php return [ 'default' => DumpSchema::define('sqlite') - ->allTables() + ->allTables() ]; ``` @@ -203,10 +222,10 @@ The key in the configuration array is the identifier that will be used when you ```php return [ 'default' => DumpSchema::define() - ->allTables(), + ->allTables(), 'sqlite' => DumpSchema::define('sqlite') - ->schemaOnly('custom_table'), + ->schemaOnly('custom_table'), ]; ``` diff --git a/src/DumpSchema.php b/src/DumpSchema.php index 5a7c623..7714b75 100644 --- a/src/DumpSchema.php +++ b/src/DumpSchema.php @@ -8,6 +8,8 @@ use Doctrine\DBAL\Types\Types; use Illuminate\Support\Facades\Schema; +use function collect; + class DumpSchema { protected $connectionName; @@ -42,9 +44,42 @@ public function allTables() return $this; } - public function exclude(string $tableName) + /** + * @param string|string[] $tableName Table name(s) to exclude from the dump + * @return $this + */ + public function exclude(string|array $tableName) + { + collect($tableName) + ->flatten() + ->unique() + ->filter(fn ($table) => is_string($table)) + ->each(fn ($table) => $this->excludedTables[] = $table); + + return $this; + } + + public function include(string|array $tableName) { - $this->excludedTables[] = $tableName; + // We're kinda fooling the `load()` and `loadAvailableTables()` method here; + // by setting `loadAllTables` to true, and adding tables directly to `availableTables`, + // the `load()` method still calls `loadAvailableTables()` but that returns early + // because there's already our tables in `availableTables`. Then the `load()` + // method sees that `loadAllTables` is true, and loads the tables from + // `availableTables` (which we just put there!) into `dumpTables`. + $this->loadAllTables = true; + $tables = collect($tableName) + ->flatten() + ->unique() + ->filter(fn ($table) => is_string($table)) + ->filter(fn ($table) => $this->getBuilder()->hasTable($table)) + ->map(fn ($table) => ['name' => $table]) + ->toArray(); + + $doctrineTables = $this->createDoctrineTables($tables); + foreach ($doctrineTables as $doctrineTable) { + $this->availableTables[] = $doctrineTable; + } return $this; } diff --git a/tests/DumperTest.php b/tests/DumperTest.php index ca20e52..558360a 100644 --- a/tests/DumperTest.php +++ b/tests/DumperTest.php @@ -6,7 +6,6 @@ use BeyondCode\LaravelMaskedDumper\LaravelMaskedDumpServiceProvider; use BeyondCode\LaravelMaskedDumper\TableDefinitions\TableDefinition; use Faker\Generator; -use Illuminate\Auth\Authenticatable; use Illuminate\Support\Facades\DB; use Orchestra\Testbench\TestCase; use Spatie\Snapshots\MatchesSnapshots; @@ -197,6 +196,32 @@ public function it_does_remove_excluded_tables_from_allTables() $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); } + /** @test */ + public function it_does_remove_excluded_array_of_tables_from_allTables() + { + $this->loadLaravelMigrations(); + + DB::table('users') + ->insert([ + 'name' => 'Marcel', + 'email' => 'marcel@beyondco.de', + 'password' => 'test', + 'created_at' => '2021-01-01 00:00:00', + 'updated_at' => '2021-01-01 00:00:00', + ]); + + $outputFile = base_path('test.sql'); + + $this->app['config']['masked-dump.default'] = DumpSchema::define() + ->allTables() + ->exclude(['users']); + + $this->artisan('db:masked-dump', [ + 'output' => $outputFile + ]); + + $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); + } /** @test */ public function it_creates_chunked_insert_statements_for_a_table() { @@ -224,12 +249,91 @@ public function it_creates_chunked_insert_statements_for_a_table() ]); $outputFile = base_path('test.sql'); - + + $this->app['config']['masked-dump.default'] = DumpSchema::define() + ->allTables() + ->table('users', function($table) { + return $table->outputInChunksOf(3); + }); + + $this->artisan('db:masked-dump', [ + 'output' => $outputFile + ]); + + $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); + } + + /** @test */ + public function it_can_include_individual_unmodified_tables() + { + $this->loadLaravelMigrations(); + + DB::table('users') + ->insert([ + 'name' => 'Marcel', + 'email' => 'marcel@beyondco.de', + 'password' => 'test', + 'created_at' => '2021-01-01 00:00:00', + 'updated_at' => '2021-01-01 00:00:00', + ]); + + $outputFile = base_path('test.sql'); + + $this->app['config']['masked-dump.default'] = DumpSchema::define() + ->include('users'); + + $this->artisan('db:masked-dump', [ + 'output' => $outputFile + ]); + + $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); + } + + /** @test */ + public function it_can_include_an_array_of_unmodified_tables() + { + $this->loadLaravelMigrations(); + + DB::table('users') + ->insert([ + 'name' => 'Marcel', + 'email' => 'marcel@beyondco.de', + 'password' => 'test', + 'created_at' => '2021-01-01 00:00:00', + 'updated_at' => '2021-01-01 00:00:00', + ]); + + $outputFile = base_path('test.sql'); + + $this->app['config']['masked-dump.default'] = DumpSchema::define() + ->include(['users', 'migrations']); + + $this->artisan('db:masked-dump', [ + 'output' => $outputFile + ]); + + $this->assertMatchesTextSnapshot(file_get_contents($outputFile)); + } + + /** @test */ + public function it_still_includes_tables_with_repeated_use_of_include() + { + $this->loadLaravelMigrations(); + + DB::table('users') + ->insert([ + 'name' => 'Marcel', + 'email' => 'marcel@beyondco.de', + 'password' => 'test', + 'created_at' => '2021-01-01 00:00:00', + 'updated_at' => '2021-01-01 00:00:00', + ]); + + $outputFile = base_path('test.sql'); + $this->app['config']['masked-dump.default'] = DumpSchema::define() - ->allTables() - ->table('users', function($table) { - return $table->outputInChunksOf(3); - }); + ->include('users') + ->include('migrations'); $this->artisan('db:masked-dump', [ 'output' => $outputFile diff --git a/tests/__snapshots__/DumperTest__it_can_include_an_array_of_unmodified_tables__1.txt b/tests/__snapshots__/DumperTest__it_can_include_an_array_of_unmodified_tables__1.txt new file mode 100644 index 0000000..6eabb3f --- /dev/null +++ b/tests/__snapshots__/DumperTest__it_can_include_an_array_of_unmodified_tables__1.txt @@ -0,0 +1,4 @@ +INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', 'marcel@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('1', '0001_01_01_000000_testbench_create_users_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('2', '0001_01_01_000001_testbench_create_cache_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('3', '0001_01_01_000002_testbench_create_jobs_table', '1'); diff --git a/tests/__snapshots__/DumperTest__it_can_include_individual_unmodified_tables__1.txt b/tests/__snapshots__/DumperTest__it_can_include_individual_unmodified_tables__1.txt new file mode 100644 index 0000000..1fcebe4 --- /dev/null +++ b/tests/__snapshots__/DumperTest__it_can_include_individual_unmodified_tables__1.txt @@ -0,0 +1 @@ +INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', 'marcel@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'); diff --git a/tests/__snapshots__/DumperTest__it_does_remove_excluded_array_of_tables_from_allTables__1.txt b/tests/__snapshots__/DumperTest__it_does_remove_excluded_array_of_tables_from_allTables__1.txt new file mode 100644 index 0000000..e1088c5 --- /dev/null +++ b/tests/__snapshots__/DumperTest__it_does_remove_excluded_array_of_tables_from_allTables__1.txt @@ -0,0 +1,3 @@ +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('1', '0001_01_01_000000_testbench_create_users_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('2', '0001_01_01_000001_testbench_create_cache_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('3', '0001_01_01_000002_testbench_create_jobs_table', '1'); diff --git a/tests/__snapshots__/DumperTest__it_still_includes_tables_with_repeated_use_of_include__1.txt b/tests/__snapshots__/DumperTest__it_still_includes_tables_with_repeated_use_of_include__1.txt new file mode 100644 index 0000000..6eabb3f --- /dev/null +++ b/tests/__snapshots__/DumperTest__it_still_includes_tables_with_repeated_use_of_include__1.txt @@ -0,0 +1,4 @@ +INSERT INTO `users` (`id`, `name`, `email`, `email_verified_at`, `password`, `remember_token`, `created_at`, `updated_at`) VALUES ('1', 'Marcel', 'marcel@beyondco.de', NULL, 'test', NULL, '2021-01-01 00:00:00', '2021-01-01 00:00:00'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('1', '0001_01_01_000000_testbench_create_users_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('2', '0001_01_01_000001_testbench_create_cache_table', '1'); +INSERT INTO `migrations` (`id`, `migration`, `batch`) VALUES ('3', '0001_01_01_000002_testbench_create_jobs_table', '1');