diff --git a/CHANGELOG.md b/CHANGELOG.md index f1036476b..89699f1e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- `tt pack `: support `.packignore` file to specify files that should not be included + in package (works the same as `.gitignore`). + ### Changed ### Fixed diff --git a/cli/pack/common.go b/cli/pack/common.go index 25c778c8a..c34094993 100644 --- a/cli/pack/common.go +++ b/cli/pack/common.go @@ -1,6 +1,7 @@ package pack import ( + "errors" "fmt" "io/fs" "os" @@ -33,6 +34,8 @@ const ( versionLuaFileName = "VERSION.lua" rocksManifestPath = ".rocks/share/tarantool/rocks/manifest" + + ignoreFile = ".packignore" ) var ( @@ -51,6 +54,8 @@ var ( } ) +type skipFilter func(srcInfo os.FileInfo, src string) bool + type RocksVersions map[string][]string // packFileInfo contains information to set for files/dirs in rpm/deb packages. @@ -76,9 +81,8 @@ func skipDefaults(srcInfo os.FileInfo, src string) bool { } // appArtifactsFilters returns a slice of skip functions to avoid copying application artifacts. -func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func( - srcInfo os.FileInfo, src string) bool { - filters := make([]func(srcInfo os.FileInfo, src string) bool, 0) +func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []skipFilter { + filters := make([]skipFilter, 0) if cliOpts.App == nil { return filters } @@ -102,9 +106,8 @@ func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func( } // ttEnvironmentFilters prepares a slice of filters for tt environment directories/files. -func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func( - srcInfo os.FileInfo, src string) bool { - filters := make([]func(srcInfo os.FileInfo, src string) bool, 0) +func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []skipFilter { + filters := make([]skipFilter, 0) if cliOpts == nil { return filters } @@ -139,10 +142,9 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func( } // previousPackageFilters returns filters for the previously built packages. -func previousPackageFilters(packCtx *PackCtx) []func( - srcInfo os.FileInfo, src string) bool { +func previousPackageFilters(packCtx *PackCtx) []skipFilter { pkgName := packCtx.Name - return []func(srcInfo os.FileInfo, src string) bool{ + return []skipFilter{ func(srcInfo os.FileInfo, src string) bool { name := srcInfo.Name() if strings.HasPrefix(name, pkgName) { @@ -159,13 +161,18 @@ func previousPackageFilters(packCtx *PackCtx) []func( // appSrcCopySkip returns a filter func to filter out artifacts paths. func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts, - srcAppPath string) func(srcinfo os.FileInfo, src, dest string) (bool, error) { + srcAppPath string) (func(srcinfo os.FileInfo, src, dest string) (bool, error), error) { appCopyFilters := appArtifactsFilters(cliOpts, srcAppPath) appCopyFilters = append(appCopyFilters, ttEnvironmentFilters(packCtx, cliOpts)...) appCopyFilters = append(appCopyFilters, previousPackageFilters(packCtx)...) appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool { return skipDefaults(srcInfo, src) }) + if f, err := ignoreFilter(util.GetOsFS(), filepath.Join(srcAppPath, ignoreFile)); err == nil { + appCopyFilters = append(appCopyFilters, f) + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("failed to load %q: %w", ignoreFile, err) + } return func(srcinfo os.FileInfo, src, dest string) (bool, error) { for _, shouldSkip := range appCopyFilters { @@ -174,7 +181,7 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts, } } return false, nil - } + }, nil } // getAppNamesToPack generates application names list to pack. @@ -430,7 +437,10 @@ func copyAppSrc(packCtx *PackCtx, cliOpts *config.CliOpts, srcAppPath, dstAppPat return err } - skipFunc := appSrcCopySkip(packCtx, cliOpts, resolvedAppPath) + skipFunc, err := appSrcCopySkip(packCtx, cliOpts, resolvedAppPath) + if err != nil { + return err + } // Copying application. log.Debugf("Copying application source %q -> %q", resolvedAppPath, dstAppPath) diff --git a/cli/pack/ignore.go b/cli/pack/ignore.go new file mode 100644 index 000000000..41acbd8bf --- /dev/null +++ b/cli/pack/ignore.go @@ -0,0 +1,139 @@ +package pack + +import ( + "bufio" + "bytes" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" +) + +// ignorePattern corresponds to a single ignore pattern from .packignore file. +type ignorePattern struct { + // re holds the "matching" part of ignore pattern (i.e. w/o trailing spaces, directory and + // negate markers) in the form of regular expression. + re *regexp.Regexp + // dirOnly defines whether this pattern should be applied to all entries (false) or + // to directory entries only (true). + dirOnly bool + // isNegate defines how to interpret the match (false means that it's an ordinary pattern + // that excludes entry, true - it's a negate pattern that includes entry). + isNegate bool +} + +func turnEscapedToHexCode(s string, c rune) string { + return strings.ReplaceAll(s, `\`+string(c), fmt.Sprintf(`\x%x`, c)) +} + +func splitIgnorePattern(pattern string) (cleanPattern string, dirOnly bool, isNegate bool) { + // Remove trailing spaces (unless escaped one). + cleanPattern = turnEscapedToHexCode(pattern, ' ') + cleanPattern = strings.TrimRight(cleanPattern, " ") + // Parse negate and directory markers. + cleanPattern, dirOnly = strings.CutSuffix(cleanPattern, "/") + cleanPattern, isNegate = strings.CutPrefix(cleanPattern, "!") + return +} + +func createIgnorePattern(pattern string, basepath string) (ignorePattern, error) { + // First, get rid of `\\` to simplify further handling of escaped sequences. + // From now on any `\c` always means escaped 'c' (previously it might also + // occur as a part of `\\c` sequence which denotes '\' followed by ). + pattern = turnEscapedToHexCode(pattern, '\\') + + cleanPattern, dirOnly, isNegate := splitIgnorePattern(pattern) + + // Translate pattern to regex expression. + expr := cleanPattern + // Turn escaped '*' and '?' to their hex representation to simplify the translation. + expr = turnEscapedToHexCode(expr, '*') + expr = turnEscapedToHexCode(expr, '?') + // Escape symbols that designate themselves in pattern, but have special meaning in regex. + for _, s := range []string{"(", ")", "{", "}", "+"} { + // Do unescape first to avoid double escaping of the ones that are already escaped. + expr = strings.ReplaceAll(expr, "\\"+s, s) + expr = strings.ReplaceAll(expr, s, "\\"+s) + } + // Replace wildcards with the corresponding regex representation. + // Note that '{0,}' (not '*') is used while replacing '**' to avoid confusing + // in the subsequent replacement of a single '*'. + expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/){0,}") + expr, found := strings.CutPrefix(expr, "**/") + if found || !strings.Contains(cleanPattern, "/") { + expr = "([^/]+/){0,}" + expr + } + expr, found = strings.CutSuffix(expr, "/**") + if found { + expr = expr + "/([^/]+/){0,}[^/]+" + } + expr = strings.ReplaceAll(expr, "*", "[^/]*") + expr = strings.ReplaceAll(expr, "?", "[^/]") + + re, err := regexp.Compile("^" + basepath + expr + "$") + if err != nil { + return ignorePattern{}, fmt.Errorf("failed to compile expression: %w", err) + } + + return ignorePattern{ + re: re, + dirOnly: dirOnly, + isNegate: isNegate, + }, nil +} + +// loadIgnorePatterns reads ignore patterns from the patternsFile. +func loadIgnorePatterns(fsys fs.FS, patternsFile string) ([]ignorePattern, error) { + contents, err := fs.ReadFile(fsys, patternsFile) + if err != nil { + return nil, err + } + + basepath, _ := filepath.Split(patternsFile) + + var patterns []ignorePattern + s := bufio.NewScanner(bytes.NewReader(contents)) + for s.Scan() { + pattern := s.Text() + if pattern == "" || strings.HasPrefix(pattern, "#") { + continue + } + + p, err := createIgnorePattern(pattern, basepath) + if err != nil { + return nil, err + } + + patterns = append(patterns, p) + } + return patterns, nil +} + +// ignoreFilter returns filter function that implements .gitignore approach of filtering files. +func ignoreFilter(fsys fs.FS, patternsFile string) (skipFilter, error) { + patterns, err := loadIgnorePatterns(fsys, patternsFile) + if err != nil { + return nil, err + } + + // According to .gitignore documentation "the last matching pattern decides the outcome" + // so we need to iterate in reverse order until the first match. + slices.Reverse(patterns) + + return func(srcInfo os.FileInfo, src string) bool { + // Skip ignore file itself. + if src == patternsFile { + return true + } + for _, p := range patterns { + isApplicable := srcInfo.IsDir() || !p.dirOnly + if isApplicable && p.re.MatchString(src) { + return !p.isNegate + } + } + return false + }, nil +} diff --git a/cli/pack/ignore_test.go b/cli/pack/ignore_test.go new file mode 100644 index 000000000..03b8a5d19 --- /dev/null +++ b/cli/pack/ignore_test.go @@ -0,0 +1,900 @@ +package pack + +import ( + "errors" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strings" + "testing" + "testing/fstest" + + "github.com/otiai10/copy" + "github.com/stretchr/testify/assert" +) + +// transformMapValues is a generic mapper function that returns map with the same keys +// and values mapped from the originals with fn function. +func transformMapValues[K comparable, V0, V any](src map[K]V0, fn func(V0) V) map[K]V { + result := make(map[K]V, len(src)) + for k, v := range src { + result[k] = fn(v) + } + return result +} + +// There are 3 ways to prepare the bunch of test cases for the concrete function: +// 1. Generate from the common test data (transformMapValues against common test data). +// 2. Transform from another bunch (transformMapValues against another bunch). +// 3. Manual (just initialize bunch with the desired cases). + +// ignoreTestData is used to define the very base data set that can be used as a source +// to generate the actual testcases suitable for the concrete test functions. +type ignoreTestData struct { + pattern string + matches []string + mismatches []string +} + +// The 'pattern' field of any item from this data set refers to name (no path separator). +// The 'matches'/'mismatches' fields must contain only names as well. This constraint allows +// to expand corresponding test cases for the certain function in a more convenient way. +var ignoreTestData_namesOnly = map[string]ignoreTestData{ + "simple_name": { + pattern: "foo", + matches: []string{ + "foo", + }, + mismatches: []string{ + "foo2", + ".foo", + "blabla_foo", + "foo_blabla", + "bla_foo_bla", + }, + }, + "name_with_space": { + pattern: "foo with space", + matches: []string{ + "foo with space", + }, + mismatches: []string{ + "foo with space2", + ".foo with space", + "blabla_foo with space", + "foo with space_blabla", + "bla_foo with space_bla", + }, + }, + "name_ends_with_space": { + pattern: "foo_ends_with_space\\ ", + matches: []string{ + "foo_ends_with_space ", + }, + mismatches: []string{ + "foo_ends_with_space", + "foo_ends_with_space ", + ".foo_ends_with_space ", + "blabla_foo_ends_with_space ", + "foo_ends_with_space blabla", + "bla_foo_ends_with_space bla", + }, + }, + "name_with_brackets": { + pattern: "foo(with_brackets)", + matches: []string{ + "foo(with_brackets)", + }, + mismatches: []string{ + "foo(with_brackets)2", + ".foo(with_brackets)", + "blabla_foo(with_brackets)", + "foo(with_brackets)_blabla", + "bla_foo(with_brackets)_bla", + }, + }, + "name_with_curly_brackets": { + pattern: "foo{with_curly_brackets}", + matches: []string{ + "foo{with_curly_brackets}", + }, + mismatches: []string{ + "foo{with_curly_brackets}2", + ".foo{with_curly_brackets}", + "blabla_foo{with_curly_brackets}", + "foo{with_curly_brackets}_blabla", + "bla_foo{with_curly_brackets}_bla", + }, + }, + "name_with_plus": { + pattern: "f+oo", + matches: []string{ + "f+oo", + }, + mismatches: []string{ + "f+oo2", + "ffoo", + ".f+oo", + "blabla_f+oo2", + "f+oo2_blabla", + "bla_f+oo2_bla", + }, + }, + "name_with_escaped_square_brackets": { + pattern: "foo\\[with_escaped_square_brackets\\]", + matches: []string{ + "foo[with_escaped_square_brackets]", + }, + mismatches: []string{ + "foo[with_escaped_square_brackets]2", + ".foo[with_escaped_square_brackets]", + "blabla_foo[with_escaped_square_brackets]2", + "foo[with_escaped_square_brackets]2_blabla", + "bla_foo[with_escaped_square_brackets]2_bla", + }, + }, + "name_with_escaped_question": { + pattern: "foo\\?with_escaped_question", + matches: []string{ + "foo?with_escaped_question", + }, + mismatches: []string{ + "foo?with_escaped_question2", + ".foo?with_escaped_question", + "foo2with_escaped_question", + "blabla_foo?with_escaped_question2", + "foo?with_escaped_question2_blabla", + "bla_foo?with_escaped_question2_bla", + }, + }, + "name_with_escaped_asterisk": { + pattern: "foo\\*with_escaped_asterisk", + matches: []string{ + "foo*with_escaped_asterisk", + }, + mismatches: []string{ + "foo*with_escaped_asterisk2", + ".foo*with_escaped_asterisk", + "blabla_foo*with_escaped_asterisk2", + "foo*with_escaped_asterisk2_blabla", + "bla_foo*with_escaped_asterisk2_bla", + }, + }, + "name_with_question_prefix": { + pattern: "?foo", + matches: []string{ + "2foo", + "?foo", + ".foo", + "*foo", + }, + mismatches: []string{ + "foo", + "foo2", + "blabla_2foo", + "2foo_blabla", + "bla_2foo_bla", + }, + }, + "name_with_question_suffix": { + pattern: "foo?", + matches: []string{ + "foo2", + "foo?", + "foo*", + "foo ", + }, + mismatches: []string{ + "foo", + "blabla_foo2", + "foo2_blabla", + "bla_foo2_bla", + }, + }, + "name_with_question_between": { + pattern: "f?oo", + matches: []string{ + "f2oo", + "fooo", + "f?oo", + "f*oo", + }, + mismatches: []string{ + "foo", + "blabla_f2oo", + "f2oo_blabla", + "bla_f2oo_bla", + }, + }, + "name_with_asterisk_prefix": { + pattern: "*foo", + matches: []string{ + "blabla_foo", + "foo", + ".foo", + "*foo", + "?foo", + }, + mismatches: []string{ + "foo2", + "2foo_blabla", + "bla_2foo_bla", + }, + }, + "name_with_asterisk_suffix": { + pattern: "foo*", + matches: []string{ + "foo_blabla", + "foo", + "foo*", + "foo?", + }, + mismatches: []string{ + "2foo", + "blabla_2foo", + "2foo_blabla", + "bla_2foo_bla", + }, + }, + "name_with_asterisk_between": { + pattern: "f*oo", + matches: []string{ + "f2oo", + "foo", + "f*oo", + "f?oo", + }, + mismatches: []string{ + "foo2", + "blabla_foo2", + "foo2_blabla", + "bla_foo2_bla", + }, + }, + "name_with_range_basic": { + pattern: "f[n-p]o", + matches: []string{ + "fno", + "foo", + "fpo", + }, + mismatches: []string{ + "f2o", + "fmo", + "fqo", + "f?o", + "blabla_foo", + "foo_blabla", + "bla_foo_bla", + }, + }, + "name_with_range_inverted": { + pattern: "f[^n-p]o", + matches: []string{ + "f2o", + "fmo", + "fqo", + "f?o", + }, + mismatches: []string{ + "foo", + "fno", + "fpo", + }, + }, + "name_with_set_basic": { + pattern: "[fgm]oo", + matches: []string{ + "foo", + "goo", + "moo", + }, + mismatches: []string{ + "zoo", + "ooo", + "?oo", + "blabla_foo", + "foo_blabla", + "bla_foo_bla", + }, + }, + "name_with_set_inverted": { + pattern: "[^fgm]oo", + matches: []string{ + "zoo", + "ooo", + "?oo", + }, + mismatches: []string{ + "foo", + "goo", + "moo", + "blabla_zoo", + "zoo_blabla", + "bla_zoo_bla", + }, + }, +} + +type ignorePatternCase struct { + pattern string + matches []string + mismatches []string + dirOnly bool + isNegate bool +} + +type ignorePatternCases map[string]ignorePatternCase + +func checkIgnorePattern(t *testing.T, testCases ignorePatternCases) { + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + p, err := createIgnorePattern(tc.pattern, "") + assert.Nil(t, err) + assert.NotNil(t, p.re) + assert.Equal(t, tc.dirOnly, p.dirOnly) + assert.Equal(t, tc.isNegate, p.isNegate) + for _, s := range tc.matches { + assert.Truef(t, p.re.MatchString(s), "%q doesn't match %s", s, p.re.String()) + } + for _, s := range tc.mismatches { + assert.False(t, p.re.MatchString(s), "%q matches %s", s, p.re.String()) + } + }) + } +} + +// Prepare a kind of base test set for createIgnorePattern function. +// It is generated from the common test data and thus consists of ordinary patterns only. +// Negate and directory patterns are tested in separate functions. Cases for them are generated +// by adjusting cases of this base set (note that these adjustments don't affect +// matches/mismatches because matching part of every pattern stays the same). +var ignorePatternBaseCases = transformMapValues(ignoreTestData_namesOnly, + func(td ignoreTestData) ignorePatternCase { + return ignorePatternCase{ + pattern: td.pattern, + // Expand with some meaningful paths (td.matches itself has no path-item). + matches: append(td.matches, + "in_subdir/"+td.matches[0], + "in/deep/nested/subdir/"+td.matches[0], + ), + mismatches: td.mismatches, + dirOnly: false, + isNegate: false, + } + }) + +func Test_createIgnorePattern_basic(t *testing.T) { + checkIgnorePattern(t, ignorePatternBaseCases) +} + +func Test_createIgnorePattern_negate(t *testing.T) { + turnToNegate := func(tc ignorePatternCase) ignorePatternCase { + tc.pattern = "!" + tc.pattern + tc.isNegate = true + return tc + } + testCases := transformMapValues(ignorePatternBaseCases, turnToNegate) + checkIgnorePattern(t, testCases) +} + +func Test_createIgnorePattern_dirOnly(t *testing.T) { + turnToDirOnly := func(tc ignorePatternCase) ignorePatternCase { + tc.pattern = tc.pattern + "/" + tc.dirOnly = true + return tc + } + testCases := transformMapValues(ignorePatternBaseCases, turnToDirOnly) + checkIgnorePattern(t, testCases) +} + +func Test_createIgnorePattern_trailingSpace(t *testing.T) { + addTrailingSpace := func(tc ignorePatternCase) ignorePatternCase { + tc.pattern = tc.pattern + strings.Repeat(" ", 1+len(tc.pattern)%3) + return tc + } + testCases := transformMapValues(ignorePatternBaseCases, addTrailingSpace) + checkIgnorePattern(t, testCases) +} + +// NOTE: For a new test that is not based on the base set the below snippet can be used. +// func Test_createIgnorePattern_someNewTest(t *testing.T) { +// testCases := ignorePatternCases{ +// "case_name1": {...}, +// "case_name2": {...}, +// } +// checkIgnorePattern(t, testCases) +// } + +type ignoreFilterCase struct { + // Ignore patterns. + patterns []string + // Files that are expected to be ignored/copied during copy. + // Every item here denotes file (not directory). + ignored []string + copied []string +} + +type ignoreFilterCases map[string]ignoreFilterCase + +// Check that no entry ends with '/' and all files are able to coexist within a single FS +// (the same path should not refer to a file and a directory simultaneously). +func validateIgnoreFilterCase(t *testing.T, tc ignoreFilterCase) { + files := slices.Concat(tc.ignored, tc.copied) + slices.Sort(files) + for i, f := range files { + assert.Falsef(t, strings.HasSuffix(f, "/"), "Invalid test case: %q ends with '/'", f) + if i > 0 { + assert.Falsef(t, strings.HasPrefix(f, files[i-1]+"/"), + "Invalid test case: %q and %q are not able to coexist within single FS", + f, + files[i-1], + ) + } + } +} + +func ignoreFilterCreateMockFS(t *testing.T, tc ignoreFilterCase) fs.FS { + validateIgnoreFilterCase(t, tc) + + fsys := fstest.MapFS{} + if tc.patterns != nil { + fsys[ignoreFile] = &fstest.MapFile{ + Data: []byte(strings.Join(tc.patterns, "\n")), + Mode: fs.FileMode(0644), + } + } + for _, name := range slices.Concat(tc.copied, tc.ignored) { + fsys[name] = &fstest.MapFile{ + Mode: fs.FileMode(0644), + } + } + return fsys +} + +func checkIgnoreFilter(t *testing.T, testCases ignoreFilterCases) { + basedst := t.TempDir() + + // Do test + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + fsys := ignoreFilterCreateMockFS(t, tc) + + filter, err := ignoreFilter(fsys, ignoreFile) + assert.Nil(t, err) + assert.NotNil(t, filter) + + dst := filepath.Join(basedst, name) + err = os.MkdirAll(dst, 0755) + assert.Nil(t, err) + + err = copy.Copy(".", dst, copy.Options{ + FS: fsys, + Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + return filter(srcinfo, src), nil + }, + PermissionControl: copy.AddPermission(0755), + }) + assert.Nil(t, err) + for _, name := range tc.ignored { + assert.NoFileExists(t, path.Join(dst, name)) + } + for _, name := range tc.copied { + assert.FileExists(t, path.Join(dst, name)) + } + }) + } +} + +// Prepare a kind of base test set for ignoreFilter function. +// It is generated from the common test data and contains only cases with single ordinary pattern. +var ignoreFilterBaseCases = transformMapValues(ignoreTestData_namesOnly, + func(td ignoreTestData) ignoreFilterCase { + // Make sure pattern contains no path separator. + if strings.Contains(td.pattern, "/") { + panic("unexpected path separator in pattern") + } + return ignoreFilterCase{ + patterns: []string{ + td.pattern, + }, + // Expand with some meaningful paths (td.matches itself has no path-item). + ignored: append(td.matches, + "in_subdir/"+td.matches[0], + "in/deep/nested/subdir/"+td.matches[0], + "as_subdir/"+td.matches[0]+"/bar", + "as_subdir/"+td.matches[0]+"/with_nested_subdir/bar", + "as/deep/nested/subdir/"+td.matches[0]+"/bar", + "as/deep/nested/subdir/"+td.matches[0]+"/with_nested_subdir/bar", + ), + // Expand with some meaningful paths (td.mismatches itself has no path-item). + copied: append(td.mismatches, + "in_subdir/"+td.mismatches[0], + "in/deep/nested/subdir/"+td.mismatches[0], + ), + } + }) + +func Test_ignoreFilter_noIgnoreFile(t *testing.T) { + f, err := ignoreFilter(fstest.MapFS{}, ignoreFile) + assert.NotNil(t, err) + assert.True(t, errors.Is(err, fs.ErrNotExist)) + assert.Nil(t, f) +} + +func Test_ignoreFilter_singleBasic(t *testing.T) { + checkIgnoreFilter(t, ignoreFilterBaseCases) +} + +func Test_ignoreFilter_singleNegate(t *testing.T) { + // Single negate pattern has no effect (i.e. all files are copied). + toSingleNegate := func(tc ignoreFilterCase) ignoreFilterCase { + return ignoreFilterCase{ + patterns: []string{ + "!" + tc.patterns[0], + }, + ignored: nil, + copied: slices.Concat(tc.copied, tc.ignored), + } + } + testCases := transformMapValues(ignoreFilterBaseCases, toSingleNegate) + checkIgnoreFilter(t, testCases) +} + +func Test_ignoreFilter_selfNegate(t *testing.T) { + // An ignore pattern followed by the same but negated (thus it just reinclude all). + toSelfNegate := func(tc ignoreFilterCase) ignoreFilterCase { + return ignoreFilterCase{ + patterns: []string{ + tc.patterns[0], + "!" + tc.patterns[0], + }, + ignored: nil, + copied: slices.Concat(tc.copied, tc.ignored), + } + } + testCases := transformMapValues(ignoreFilterBaseCases, toSelfNegate) + checkIgnoreFilter(t, testCases) +} + +func Test_ignoreFilter_negateWrongOrder(t *testing.T) { + // A negate pattern in a wrong position doesn't affect the original result. + toWrongOrderNegate := func(tc ignoreFilterCase) ignoreFilterCase { + return ignoreFilterCase{ + patterns: []string{ + "!" + tc.patterns[0], + tc.patterns[0], + }, + ignored: tc.ignored, + copied: tc.copied, + } + } + testCases := transformMapValues(ignoreFilterBaseCases, toWrongOrderNegate) + checkIgnoreFilter(t, testCases) +} + +func Test_ignoreFilter_singleDir(t *testing.T) { + // Generate test set from the common test data rather than from the base set + // because result for this set differ significantly from the original. + testCases := transformMapValues(ignoreTestData_namesOnly, + func(td ignoreTestData) ignoreFilterCase { + return ignoreFilterCase{ + patterns: []string{ + td.pattern + "/", + }, + // td.matches (as well as td.mismatches) are represented as files so they don't + // match directory pattern as-is and should be appended as below to become + // directories that do match. + ignored: []string{ + td.matches[0] + "/as_dir", + td.matches[0] + "/as_dir_with_nested_subdir/bar", + "as_subdir/" + td.matches[0] + "/bar", + "as_subdir/" + td.matches[0] + "/with_nested_subdir/bar", + "as/deep/nested/subdir/" + td.matches[0] + "/bar", + "as/deep/nested/subdir/" + td.matches[0] + "/with_nested_subdir/bar", + }, + // Note that matches[0] is excluded because it can't coexist with ignored[0] and + // ignored[1] within a single FS. + copied: slices.Concat( + td.mismatches, + td.matches[1:], + []string{ + "in_subdir/" + td.matches[0], + "in/deep/nested/subdir/" + td.matches[0], + }, + ), + } + }) + checkIgnoreFilter(t, testCases) +} + +func Test_ignoreFilter_multiNames(t *testing.T) { + testCases := ignoreFilterCases{ + "any": { + patterns: []string{ + "name1", + "name2", + }, + ignored: []string{ + "name1", + "in_subdir/name1", + "in/deep/nested/subdir/name1", + "as_subdir/name1/foo", + "as/deep/nested/subdir/name1/bar", + "name2", + "in_subdir/name2", + "in/deep/nested/subdir/name2", + "as_subdir/name2/foo", + "as/deep/nested/subdir/name2/bar", + }, + copied: []string{ + "name3", + "name4", + }, + }, + "dironly": { + patterns: []string{ + "name1/", + "name2/", + }, + ignored: []string{ + "as_subdir/name1/foo", + "as/deep/nested/subdir/name1/bar", + "as_subdir/name2/foo", + "as/deep/nested/subdir/name2/bar", + }, + copied: []string{ + "name1", + "in_subdir/name1", + "in/deep/nested/subdir/name1", + "name2", + "in_subdir/name2", + "in/deep/nested/subdir/name2", + "name3", + "name4", + }, + }, + "mixed": { + patterns: []string{ + "name1", + "name2/", + }, + ignored: []string{ + "name1", + "in_subdir/name1", + "in/deep/nested/subdir/name1", + "as_subdir/name1/foo", + "as/deep/nested/subdir/name1/bar", + "as_subdir/name2/bar", + "as/deep/nested/subdir/name2/bar", + }, + copied: []string{ + "name2", + "in_subdir/name2", + "in/deep/nested/subdir/name2", + "name3", + "name4", + }, + }, + } + checkIgnoreFilter(t, testCases) +} + +func Test_ignoreFilter_fixedDepth(t *testing.T) { + testCases := ignoreFilterCases{ + "name_at_depth1": { + patterns: []string{ + "*/foo", + }, + ignored: []string{ + "in_subdir/foo", + "in_another_subdir/foo", + "as_subdir/foo/bar", + "as_another_subdir/foo/bar", + }, + copied: []string{ + "foo", + "in/subdir/of/another/depth/foo", + "as/subdir/of/another/depth/foo/bar", + "foo2", + "similar_in_subdir/foo2", + "similar_as_subdir/foo2/bar", + }, + }, + "name_at_depth2": { + patterns: []string{ + "*/*/foo", + }, + ignored: []string{ + "in_subdir/of_depth2/foo", + "in_another_subdir/of_depth2/foo", + "as_subdir/of_depth2/foo/bar", + "as_another_subdir/of_depth2/foo/bar", + }, + copied: []string{ + "foo", + "in/subdir/of/another/depth/foo", + "as/subdir/of/another/depth/foo/bar", + "foo2", + "similar_in_subdir/of_depth2/foo2", + "similar_as_subdir/of_depth2/foo2/bar", + }, + }, + "under_name_depth1": { + patterns: []string{ + "foo/*", + }, + ignored: []string{ + "foo/bar", + "foo/blabla", + "foo/with_subdir/bar", + "foo/with_subdir/blabla", + }, + copied: []string{ + "as_subdir/foo/bar", + "as/subdir/of/another/depth/foo/bar", + "foo2/bar", + "foo2/blabla", + }, + }, + "under_name_depth2": { + patterns: []string{ + "foo/*/*", + }, + ignored: []string{ + "foo/subdir/bar", + "foo/subdir/blabla", + "foo/another_subdir/bar", + }, + copied: []string{ + "as_subdir/foo/subdir/bar", + "as/subdir/of/another/depth/foo/subdir/bar", + "foo/bar", + "foo/blabla", + "foo2/subdir/bar", + }, + }, + } + checkIgnoreFilter(t, testCases) +} + +func Test_ignoreFilter_reinclude(t *testing.T) { + testCases := ignoreFilterCases{ + "by_name": { + patterns: []string{ + "*name?", + "!renamed", + }, + ignored: []string{ + "name1", + "in_subdir/filename2", + "in/deep/nested/subdir/rename3", + "as_subdir/dirname4/bar", + "as_subdir/dirname4/renamed", + "as/deep/nested/subdir/newname5/bar", + "as/deep/nested/subdir/newname5/renamed", + }, + copied: []string{ + "renamed", + "in_subdir/renamed", + "as_subdir/renamed/bar", + "name13", + "rename14", + }, + }, + "by_names": { + patterns: []string{ + "*name?", + "!renamed", + "!unnamed", + }, + ignored: []string{ + "name1", + "newname2", + "oldname3", + "in_subdir/filename2", + "in/deep/nested/subdir/rename3", + "as_subdir/dirname4/bar", + "as_subdir/dirname4/renamed", + "as/deep/nested/subdir/newname5/bar", + "as/deep/nested/subdir/newname5/renamed", + }, + copied: []string{ + "renamed", + "in_subdir/renamed", + "as_subdir/renamed/bar", + "unnamed", + "in_subdir/unnamed", + "as_subdir/unnamed/bar", + "name13", + }, + }, + "by_pattern": { + patterns: []string{ + "*name?", + "!*named", + }, + ignored: []string{ + "name1", + "newname2", + "oldname3", + "in_subdir/filename2", + "in/deep/nested/subdir/rename3", + "as_subdir/dirname4/bar", + "as_subdir/dirname4/renamed", + "as/deep/nested/subdir/newname5/bar", + "as/deep/nested/subdir/newname5/renamed", + "as/deep/nested/subdir/newname5/unnamed", + }, + copied: []string{ + "renamed", + "in_subdir/renamed", + "as_subdir/renamed/bar", + "unnamed", + "in_subdir/unnamed", + "as_subdir/unnamed/bar", + "name13", + }, + }, + } + checkIgnoreFilter(t, testCases) +} + +func Test_ignoreFilter_doubleAsterisk(t *testing.T) { + testCases := ignoreFilterCases{ + "leading": { + patterns: []string{ + "**/foo", + }, + ignored: []string{ + "foo", + "in_subdir/foo", + "in/deep/nested/subdir/foo", + }, + copied: []string{ + "foo2", + "similar_in_subdir/foo2", + "similar/in/deep/nested/subdir/foo2", + "subdir/foo2/bar", + }, + }, + "trailing": { + patterns: []string{ + "foo/**", + }, + ignored: []string{ + "foo/bar", + "foo/with_subdir/bar", + "foo/with/deep/nested/subdir/bar", + }, + copied: []string{ + "foo_blabla", + "file_in_subdir/foo", + "file/in/deep/nested/subdir/foo", + "similar_subdir/foo2/bar", + }, + }, + "inner": { + patterns: []string{ + "foo/**/bar", + }, + ignored: []string{ + "foo/bar", + "foo/subdir/bar", + "foo/deep/nested/subdir/bar", + }, + copied: []string{ + "foo/bar2", + "foo/with_subdir/bar2", + "foo/with/deep/nested/subdir/bar2", + "foo2", + "similar_in_subdir/foo2", + "similar/in/deep/nested/subdir/foo2", + "subdir/foo2/bar", + }, + }, + } + checkIgnoreFilter(t, testCases) +} diff --git a/cli/util/osfs.go b/cli/util/osfs.go new file mode 100644 index 000000000..cc0ea4a31 --- /dev/null +++ b/cli/util/osfs.go @@ -0,0 +1,25 @@ +package util + +import ( + "io/fs" + "os" +) + +type osFS struct{} + +// GetOsFS returns a default implementation of fs.FS interface. In general interface fs.FS +// should be added as an argument to any function where you need to be able to substitute +// non-default FS. The most obvious scenario is using mock FS for testing. In such a case +// while general code uses this default implementation, test code is able to substitute +// some mock FS (like fstest.MapFS). +func GetOsFS() fs.FS { + return osFS{} +} + +func (fs osFS) Open(name string) (fs.File, error) { + return os.Open(name) +} + +func (fs osFS) ReadFile(name string) ([]byte, error) { + return os.ReadFile(name) +} diff --git a/test/integration/pack/test_pack.py b/test/integration/pack/test_pack.py index bba754919..e9d4894eb 100644 --- a/test/integration/pack/test_pack.py +++ b/test/integration/pack/test_pack.py @@ -1,11 +1,13 @@ import filecmp import glob +import itertools import os import re import shutil import stat import subprocess import tarfile +from pathlib import Path import pytest import yaml @@ -1875,3 +1877,114 @@ def test_pack_app_local_tarantool(tt_cmd, tmpdir_with_tarantool, tmp_path): build_output = tt_process.stdout.read() assert "Bundle is packed successfully" in build_output + + +@pytest.mark.slow +def test_pack_ignore(tt_cmd, tmp_path): + shutil.copytree(os.path.join(os.path.dirname(__file__), "test_bundles"), + tmp_path, symlinks=True, ignore=None, + copy_function=shutil.copy2, ignore_dangling_symlinks=True, + dirs_exist_ok=True) + + bundle_src = "single_app" + base_dir = tmp_path / bundle_src + + files_to_ignore = [ + " ", + "#", + "#hash_name1", + "name1", + "subdir/name1", + "deep/nested/subdir/name1", + "subdir2/name1/file", + "name2", + "subdir/name3_blabla", + "dir1/file", + "subdir/dir1/file", + "dir2/file", + "subdir/dir3_blabla/file", + "name11", + "subdir/name11", + "deep/nested/subdir/name11", + "dir/name11/file_bla", + "dir/name11/file_blabla", + "dir12/name12", + "subdir/dir12/name12", + "deep/nested/subdir/dir12/name12", + ] + + files_to_pack = [ + "# ", + "#comment", + "#hash_name_reincluded", + "mismatched_name1", + ".mismatched_name2", + "subdir/mismatched_name3", + "deep/nested/subdir/mismatched_name4", + "name2_reincluded", + "subdir/name3_reincluded1", + "dir4/mismatched_dir_name", + "subdir/as_file/dir1", + "subdir/mismatched_dir2/file", + "dir2_reincluded/file", + "subdir/dir3_reincluded1/file" + "name12", + "mismatched_parent_dir/name12", + "deep/nested/mismatched_parent_dir/name12", + ] + + ignore_patterns = [ + " ", + "\\ ", + "#comment", + "\\#", + "\\#hash_name*", + "!#hash_name_reincluded", + "name1", + "name[2-3]*", + "!name2_reincluded", + "!name3_reincluded[1-9]", + "dir1/", + "dir[2-3]*/", + "!dir2_reincluded/", + "!dir3_reincluded[1-9]/", + "**/name11", + "**/dir12/name12", + ] + + # Prepare .packignore layout. + (base_dir / ".packignore").write_text("\n".join(ignore_patterns) + "\n") + for f in itertools.chain(files_to_ignore, files_to_pack): + fpath = Path(base_dir, f) + fpath.parent.mkdir(parents=True, exist_ok=True) + fpath.write_text("") + + packages_wildcard = os.path.join(base_dir, "*.tar.gz") + packages = set(glob.glob(packages_wildcard)) + + rc, _ = run_command_and_get_output( + [tt_cmd, "pack", "tgz"], + cwd=base_dir, + env=dict(os.environ, PWD=base_dir), + ) + assert rc == 0 + + # Find the newly generated package. + new_packages = set(glob.glob(packages_wildcard)) - packages + assert len(new_packages) == 1 + package_file = Path(next(iter(new_packages))) + + extract_path = os.path.join(base_dir, "tmp") + os.mkdir(extract_path) + + tar = tarfile.open(package_file) + tar.extractall(extract_path) + tar.close() + + extract_base_dir = os.path.join(extract_path, bundle_src) + for file_path in [".packignore"] + files_to_ignore: + assert not os.path.exists(os.path.join(extract_base_dir, file_path)), \ + f"'{os.path.join(extract_base_dir, file_path)}' unexpectedly exists" + for file_path in files_to_pack: + assert os.path.exists(os.path.join(extract_base_dir, file_path)), \ + f"'{os.path.join(extract_base_dir, file_path)}' doesn't exist"