Skip to content

Commit f7a3cdc

Browse files
elhimovelhimov
committed
pack: support .packignore
In some cases, there are files that may be useful, but should not be part of artifact. The ability to add files and directories to the .packignore file has been added, which allows you to ignore these files and directories when packing. Closes #812
1 parent b7ebf79 commit f7a3cdc

File tree

2 files changed

+303
-0
lines changed

2 files changed

+303
-0
lines changed

‎cli/pack/common.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package pack
22

33
import (
4+
"bufio"
5+
"errors"
46
"fmt"
57
"io/fs"
68
"os"
@@ -33,6 +35,8 @@ const (
3335
versionLuaFileName = "VERSION.lua"
3436

3537
rocksManifestPath = ".rocks/share/tarantool/rocks/manifest"
38+
39+
ignoreFile = ".packignore"
3640
)
3741

3842
var (
@@ -157,6 +161,139 @@ func previousPackageFilters(packCtx *PackCtx) []func(
157161
}
158162
}
159163

164+
func ignorePatternToRegex(pattern string, ignoreLocation string) (string, bool, bool) {
165+
onlyDirs := strings.HasSuffix(pattern, "/")
166+
if onlyDirs {
167+
pattern = pattern[:len(pattern)-1]
168+
}
169+
170+
negate := false
171+
if strings.HasPrefix(pattern, "!") {
172+
negate = true
173+
pattern = pattern[1:]
174+
} else {
175+
if strings.HasPrefix(pattern, "\\!") || strings.HasPrefix(pattern, "\\#") {
176+
pattern = pattern[1:]
177+
}
178+
}
179+
180+
expr := pattern
181+
expr = strings.ReplaceAll(expr, "/**/", "/([^/]+/)*")
182+
// expr = strings.ReplaceAll(expr, "**/", "([^/]+/)*")
183+
// expr = strings.ReplaceAll(expr, "/**", "[^/]*")
184+
expr = strings.ReplaceAll(expr, "*", "[^/]*")
185+
expr = strings.ReplaceAll(expr, "?", "[^/]")
186+
187+
if strings.HasPrefix(pattern, "/") {
188+
expr = ignoreLocation + expr
189+
} else {
190+
expr = "/?([^/]+/)*" + expr
191+
}
192+
193+
return expr, negate, onlyDirs
194+
}
195+
196+
// ignoreFilter returns filter that excludes files based on the patterns.
197+
func ignoreFilter(fsys fs.FS, ignoreFile string) func(
198+
srcInfo os.FileInfo, src string) bool {
199+
log.Infof("ignoreFilter: %q", ignoreFile)
200+
ignoreFileDir := filepath.Dir(ignoreFile)
201+
202+
f, err := fsys.Open(ignoreFile)
203+
if err != nil {
204+
if !errors.Is(err, fs.ErrNotExist) {
205+
log.Errorf("Failed to open %q: %s", ignoreFile, err.Error())
206+
}
207+
return nil
208+
}
209+
210+
defer f.Close()
211+
212+
var exprExclude, exprInclude, exprExcludeDirs, exprIncludeDirs []string
213+
214+
s := bufio.NewScanner(f)
215+
for s.Scan() {
216+
pattern := strings.TrimSpace(s.Text())
217+
if pattern == "" || strings.HasPrefix(pattern, "#") {
218+
continue
219+
}
220+
221+
exprPart, negate, onlyDirs := ignorePatternToRegex(pattern, ignoreFileDir)
222+
log.Infof("exprPart: %q %v %v", exprPart, negate, onlyDirs)
223+
224+
var expr *[]string
225+
if onlyDirs {
226+
if negate {
227+
expr = &exprIncludeDirs
228+
} else {
229+
expr = &exprExcludeDirs
230+
}
231+
} else {
232+
if negate {
233+
expr = &exprInclude
234+
} else {
235+
expr = &exprExclude
236+
}
237+
}
238+
*expr = append(*expr, "("+exprPart+")")
239+
}
240+
241+
compileRegexp := func(expr []string) *regexp.Regexp {
242+
log.Infof("compileRegexp: %v", expr)
243+
if len(expr) == 0 {
244+
return nil
245+
}
246+
re, err := regexp.Compile("^" + strings.Join(expr, "|") + "$")
247+
if err != nil {
248+
log.Infof(" failed to compile expression: %s", err.Error())
249+
return nil
250+
}
251+
log.Infof(" expr=%s", re.String())
252+
return re
253+
}
254+
255+
reInclude := compileRegexp(exprInclude)
256+
reExclude := compileRegexp(exprExclude)
257+
reIncludeDirs := compileRegexp(exprIncludeDirs)
258+
reExcludeDirs := compileRegexp(exprExcludeDirs)
259+
// log.Infof("reInclude: %s", reInclude.String())
260+
// log.Infof("reExclude: %s", reExclude.String())
261+
// log.Infof("reIncludeDirs: %s", reIncludeDirs.String())
262+
// log.Infof("reExcludeDirs: %s", reExcludeDirs.String())
263+
264+
return func(srcInfo os.FileInfo, src string) bool {
265+
log.Infof("ignoreFilter(): %q", src)
266+
267+
// Skip ignore file itself.
268+
if filepath.Base(src) == ignoreFile {
269+
log.Infof(" } true (ignore file itself)")
270+
return true
271+
}
272+
// If it's directory first check patterns that only match directories.
273+
if srcInfo.IsDir() {
274+
log.Infof(" is dir")
275+
if reIncludeDirs != nil && reIncludeDirs.MatchString(src) {
276+
log.Infof(" } false (include dirs) %q", reIncludeDirs.String())
277+
return false
278+
}
279+
if reExcludeDirs != nil && reExcludeDirs.MatchString(src) {
280+
log.Infof(" } true (exclude dirs) %q", reExcludeDirs.String())
281+
return true
282+
}
283+
}
284+
if reInclude != nil && reInclude.MatchString(src) {
285+
log.Infof(" } false (include) %q", reInclude.String())
286+
return false
287+
}
288+
if reExclude != nil && reExclude.MatchString(src) {
289+
log.Infof(" } true (exclude) %q", reExclude.String())
290+
return true
291+
}
292+
log.Infof(" } false")
293+
return false
294+
}
295+
}
296+
160297
// appSrcCopySkip returns a filter func to filter out artifacts paths.
161298
func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
162299
srcAppPath string) func(srcinfo os.FileInfo, src, dest string) (bool, error) {
@@ -166,13 +303,20 @@ func appSrcCopySkip(packCtx *PackCtx, cliOpts *config.CliOpts,
166303
appCopyFilters = append(appCopyFilters, func(srcInfo os.FileInfo, src string) bool {
167304
return skipDefaults(srcInfo, src)
168305
})
306+
log.Infof("appSrcCopySkip: srcAppPath: %q", srcAppPath)
307+
log.Infof("appSrcCopySkip: ignoreFile: %q", ignoreFile)
308+
if f := ignoreFilter(os.DirFS(srcAppPath), ignoreFile); f != nil {
309+
appCopyFilters = append(appCopyFilters, f)
310+
}
169311

170312
return func(srcinfo os.FileInfo, src, dest string) (bool, error) {
171313
for _, shouldSkip := range appCopyFilters {
172314
if shouldSkip(srcinfo, src) {
315+
log.Infof("skip: %q > %q isDir=%v... SKIPPED", src, dest, srcinfo.IsDir())
173316
return true, nil
174317
}
175318
}
319+
log.Infof("skip: %q > %q isDir=%v... COPIED", src, dest, srcinfo.IsDir())
176320
return false, nil
177321
}
178322
}

‎cli/pack/common_test.go‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"path/filepath"
88
"strings"
99
"testing"
10+
"testing/fstest"
1011

1112
"github.com/stretchr/testify/assert"
1213
"github.com/stretchr/testify/require"
@@ -1106,3 +1107,161 @@ func Test_prepareBundle(t *testing.T) {
11061107
})
11071108
}
11081109
}
1110+
1111+
type testFile struct {
1112+
path string
1113+
expected bool
1114+
}
1115+
1116+
type testCase struct {
1117+
name string
1118+
patterns []string
1119+
files []testFile
1120+
}
1121+
1122+
func createTestFS(tc testCase) fstest.MapFS {
1123+
testFS := fstest.MapFS{}
1124+
if tc.patterns != nil {
1125+
testFS[".packignore"] = &fstest.MapFile{
1126+
Data: []byte(strings.Join(tc.patterns, "\n")),
1127+
}
1128+
}
1129+
for _, file := range tc.files {
1130+
testFS[file.path] = &fstest.MapFile{}
1131+
}
1132+
return testFS
1133+
}
1134+
1135+
func Test_ignoreFilterSinglePattern(t *testing.T) {
1136+
testCases := []testCase{
1137+
{
1138+
name: "simple",
1139+
patterns: []string{
1140+
"test",
1141+
},
1142+
files: []testFile{
1143+
{"test", true},
1144+
{"test_blabla", false},
1145+
{"blabla_test", false},
1146+
{"bla_test_bla", false},
1147+
{"foo", false},
1148+
{"d/test", true},
1149+
{"d/test_blabla", false},
1150+
{"d/blabla_test", false},
1151+
{"d/bla_test_bla", false},
1152+
{"d/foo", false},
1153+
},
1154+
},
1155+
{
1156+
name: "question_prefix",
1157+
patterns: []string{
1158+
"?test",
1159+
},
1160+
files: []testFile{
1161+
{"test", false},
1162+
{"2test", true},
1163+
{".test", true},
1164+
{"test_blabla", false},
1165+
{"blabla_test", false},
1166+
{"bla_test_bla", false},
1167+
{"foo", false},
1168+
{"d/test", false},
1169+
{"d/2test", true},
1170+
{"d/.test", true},
1171+
{"d/test_blabla", false},
1172+
{"d/blabla_test", false},
1173+
{"d/bla_test_bla", false},
1174+
{"d/foo", false},
1175+
},
1176+
},
1177+
{
1178+
name: "question_suffix",
1179+
patterns: []string{
1180+
"test?",
1181+
},
1182+
files: []testFile{
1183+
{"test", false},
1184+
{"test2", true},
1185+
{"test.", true},
1186+
{"test_blabla", false},
1187+
{"blabla_test", false},
1188+
{"bla_test_bla", false},
1189+
{"foo", false},
1190+
{"d/test", false},
1191+
{"d/test2", true},
1192+
{"d/test.", true},
1193+
{"d/test_blabla", false},
1194+
{"d/blabla_test", false},
1195+
{"d/bla_test_bla", false},
1196+
{"d/foo", false},
1197+
},
1198+
},
1199+
{
1200+
name: "asterisk_prefix",
1201+
patterns: []string{
1202+
"*test",
1203+
},
1204+
files: []testFile{
1205+
{"test", true},
1206+
{"test_blabla", false},
1207+
{"blabla_test", true},
1208+
{"bla_test_bla", false},
1209+
{"foo", false},
1210+
{"d/test", true},
1211+
{"d/test_blabla", false},
1212+
{"d/blabla_test", true},
1213+
{"d/bla_test_bla", false},
1214+
{"d/foo", false},
1215+
},
1216+
},
1217+
{
1218+
name: "asterisk_suffix",
1219+
patterns: []string{
1220+
"test*",
1221+
},
1222+
files: []testFile{
1223+
{"test", true},
1224+
{"test_blabla", true},
1225+
{"blabla_test", false},
1226+
{"bla_test_bla", false},
1227+
{"foo", false},
1228+
{"d/test", true},
1229+
{"d/test_blabla", true},
1230+
{"d/blabla_test", false},
1231+
{"d/bla_test_bla", false},
1232+
{"d/foo", false},
1233+
},
1234+
},
1235+
}
1236+
1237+
t.Run("no ignore file", func(t *testing.T) {
1238+
f := ignoreFilter(fstest.MapFS{}, ".packignore")
1239+
assert.Nil(t, f)
1240+
})
1241+
1242+
for _, tc := range testCases {
1243+
t.Run(tc.name, func(t *testing.T) {
1244+
testFS := createTestFS(tc)
1245+
f := ignoreFilter(testFS, ".packignore")
1246+
files := append(tc.files, testFile{".packignore", true})
1247+
for _, file := range files {
1248+
fi, err := testFS.Stat(file.path)
1249+
assert.Nil(t, err)
1250+
actual := f(fi, file.path)
1251+
assert.Equal(t, file.expected, actual)
1252+
}
1253+
})
1254+
// t.Run(tc.name+"_negate", func(t *testing.T) {
1255+
// tc.patterns[0] = "!" + tc.patterns[0]
1256+
// testFS := createTestFS(tc)
1257+
// f := ignoreFilter(testFS, ".packignore")
1258+
// files := append(tc.files, testFile{".packignore", true})
1259+
// for _, file := range files {
1260+
// fi, err := testFS.Stat(file.path)
1261+
// assert.Nil(t, err)
1262+
// actual := f(fi, file.path)
1263+
// assert.Equal(t, !file.expected, actual)
1264+
// }
1265+
// })
1266+
}
1267+
}

0 commit comments

Comments
 (0)