pack: support .packignore

In some cases, there are files that may be useful, but should not
be part of artifact. The ability to add files and directories to
the .packignore file has been added, which allows you to ignore
these files and directories when packing.

Closes #812

@TarantoolBot document
Title: `tt pack` support `.packignore`

Use `.packignore` in the same way as `.gitignore` allows to exclude
unnecessary files while preparing application package with `tt pack
command.

Author: elhimov

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - `tt pack `: added TCM file packaging.
+- `tt pack `: support `.packignore` file to specify files that should not be included
+  in package (works the same as `.gitignore`).
 
 ### Changed
 

cli/pack/common.go

@@ -1,6 +1,7 @@
 package pack
 
 import (
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -33,6 +34,8 @@ const (
 	versionLuaFileName = "VERSION.lua"
 
 	rocksManifestPath = ".rocks/share/tarantool/rocks/manifest"
+
+	ignoreFile = ".packignore"
 )
 
 var (
@@ -51,6 +54,8 @@ var (
 	}
 )
 
+type skipFilter func(srcInfo os.FileInfo, src string) bool
+
 type RocksVersions map[string][]string
 
 // packFileInfo contains information to set for files/dirs in rpm/deb packages.
@@ -76,9 +81,8 @@ func skipDefaults(srcInfo os.FileInfo, src string) bool {
 }
 
 // appArtifactsFilters returns a slice of skip functions to avoid copying application artifacts.
-func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func(
-	srcInfo os.FileInfo, src string) bool {
-	filters := make([]func(srcInfo os.FileInfo, src string) bool, 0)
+func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []skipFilter {
+	filters := make([]skipFilter, 0)
 	if cliOpts.App == nil {
 		return filters
 	}
@@ -102,9 +106,8 @@ func appArtifactsFilters(cliOpts *config.CliOpts, srcAppPath string) []func(
 }
 
 // ttEnvironmentFilters prepares a slice of filters for tt environment directories/files.
-func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func(
-	srcInfo os.FileInfo, src string) bool {
-	filters := make([]func(srcInfo os.FileInfo, src string) bool, 0)
+func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []skipFilter {
+	filters := make([]skipFilter, 0)
 	if cliOpts == nil {
 		return filters
 	}
@@ -139,10 +142,9 @@ func ttEnvironmentFilters(packCtx *PackCtx, cliOpts *config.CliOpts) []func(
 }
 
 // previousPackageFilters returns filters for the previously built packages.
-func previousPackageFilters(packCtx *PackCtx) []func(
-	srcInfo os.FileInfo, src string) bool {
+func previousPackageFilters(packCtx *PackCtx) []skipFilter {
 	pkgName := packCtx.Name
-	return []func(srcInfo os.FileInfo, src string) bool{
+	return []skipFilter{
 		func(srcInfo os.FileInfo, src string) bool {
 			name := srcInfo.Name()
 			if strings.HasPrefix(name, pkgName) {
@@ -159,13 +161,18 @@ func previousPackageFilters(packCtx *PackCtx) []func(
 
 // appSrcCopySkip returns a filter func to filter out artifacts paths.
 func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
-	srcAppPath string) func(srcinfo os.FileInfo, src, dest string) (bool, error) {
+	srcAppPath string) (func(srcinfo os.FileInfo, src, dest string) (bool, error), error) {
 	appCopyFilters := appArtifactsFilters(cliOpts, srcAppPath)
 	appCopyFilters = append(appCopyFilters, ttEnvironmentFilters(packCtx, cliOpts)...)
 	appCopyFilters = append(appCopyFilters, previousPackageFilters(packCtx)...)
 	appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool {
 		return skipDefaults(srcInfo, src)
 	})
+	if f, err := ignoreFilter(util.GetOsFS(), filepath.Join(srcAppPath, ignoreFile)); err == nil {
+		appCopyFilters = append(appCopyFilters, f)
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return nil, fmt.Errorf("failed to load %q: %w", ignoreFile, err)
+	}
 
 	return func(srcinfo os.FileInfo, src, dest string) (bool, error) {
 		for _, shouldSkip := range appCopyFilters {
@@ -174,7 +181,7 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
 			}
 		}
 		return false, nil
-	}
+	}, nil
 }
 
 // getAppNamesToPack generates application names list to pack.
@@ -430,7 +437,10 @@ func copyAppSrc(packCtx *PackCtx, cliOpts *config.CliOpts, srcAppPath, dstAppPat
 		return err
 	}
 
-	skipFunc := appSrcCopySkip(packCtx, cliOpts, resolvedAppPath)
+	skipFunc, err := appSrcCopySkip(packCtx, cliOpts, resolvedAppPath)
+	if err != nil {
+		return err
+	}
 
 	// Copying application.
 	log.Debugf("Copying application source %q -> %q", resolvedAppPath, dstAppPath)

cli/pack/ignore.go

@@ -0,0 +1,123 @@
+package pack
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+)
+
+type ignorePattern struct {
+	re       *regexp.Regexp
+	dirOnly  bool
+	isNegate bool
+}
+
+func createIgnorePattern(pattern string, basepath string) (ignorePattern, error) {
+	var p ignorePattern
+	var err error
+
+	pattern, p.isNegate = strings.CutPrefix(pattern, "!")
+	if !p.isNegate && (strings.HasPrefix(pattern, "\\!") || strings.HasPrefix(pattern, "\\#")) {
+		pattern = pattern[1:]
+	}
+	pattern, p.dirOnly = strings.CutSuffix(pattern, "/")
+	if !p.dirOnly && !strings.HasSuffix(pattern, "\\ ") {
+		pattern = strings.TrimRight(pattern, " ")
+	}
+
+	// Turn pattern to regex expression.
+	expr := pattern
+	// First, escape symbols that designate themselves in pattern, but have
+	// special meaning in regex.
+	for _, s := range []string{"(", ")", "{", "}", "+"} {
+		expr = strings.ReplaceAll(expr, s, "\\"+s)
+	}
+	// Then replace wildcards with the corresponding regex representation.
+	// Note that '{0,}' (not '*') is used while replacing '**' to avoid confusing
+	// in the subsequent replacement of a single '*'.
+	expr, found := strings.CutPrefix(expr, "**/")
+	if found {
+		expr = "([^/]+/){0,}" + expr
+	}
+	expr, found = strings.CutSuffix(expr, "/**")
+	if found {
+		expr = expr + "(/[^/]+){0,}"
+	}
+	expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/){0,}")
+	expr = strings.ReplaceAll(expr, "\\*", fmt.Sprintf("\\x%x", "*"[0]))
+	expr = strings.ReplaceAll(expr, "*", "[^/]*")
+	expr = strings.ReplaceAll(expr, "\\?", fmt.Sprintf("\\x%x", "?"[0]))
+	expr = strings.ReplaceAll(expr, "?", "[^/]")
+
+	if strings.Contains(pattern, "/") {
+		expr = basepath + expr
+	} else {
+		expr = basepath + "([^/]+/)*" + expr
+	}
+
+	p.re, err = regexp.Compile("^" + expr + "$")
+	if err != nil {
+		return ignorePattern{}, fmt.Errorf("failed to compile expression: %w", err)
+	}
+
+	return p, nil
+}
+
+// loadIgnorePatterns reads ignore patterns from the patternsFile.
+func loadIgnorePatterns(fsys fs.FS, patternsFile string) ([]ignorePattern, error) {
+	contents, err := fs.ReadFile(fsys, patternsFile)
+	if err != nil {
+		return nil, err
+	}
+
+	basepath, _ := filepath.Split(ignoreFile)
+
+	var patterns []ignorePattern
+	s := bufio.NewScanner(bytes.NewReader(contents))
+	for s.Scan() {
+		pattern := strings.TrimSpace(s.Text())
+		if pattern == "" || strings.HasPrefix(pattern, "#") {
+			continue
+		}
+
+		p, err := createIgnorePattern(pattern, basepath)
+		if err != nil {
+			return nil, err
+		}
+
+		patterns = append(patterns, p)
+	}
+	return patterns, nil
+}
+
+// ignoreFilter returns filter function that implements .gitignore approach of filtering files.
+func ignoreFilter(fsys fs.FS, patternsFile string) (skipFilter, error) {
+	patterns, err := loadIgnorePatterns(fsys, patternsFile)
+	if err != nil {
+		return nil, err
+	}
+
+	// According to .gitignore documentation "the last matching pattern decides the outcome"
+	// so we need to iterate in reverse order until the first match.
+	slices.Reverse(patterns)
+
+	return func(srcInfo os.FileInfo, src string) bool {
+		// Skip ignore file itself.
+		if src == patternsFile {
+			return true
+		}
+		for _, p := range patterns {
+			isApplicable := srcInfo.IsDir() || !p.dirOnly
+			if isApplicable && p.re.MatchString(src) {
+				return !p.isNegate
+			}
+		}
+		return false
+	}, nil
+}