From 84b2062ce405198e1f9c9547dd967c6b0dec01b9 Mon Sep 17 00:00:00 2001 From: Kim Eik Date: Tue, 9 Sep 2025 14:34:07 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(files=5Fglob):=20Add=20files?= =?UTF-8?q?=5Fglob=20and=20watch=5Fglob=20functions=20for=20pattern=20expa?= =?UTF-8?q?nsion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated the `README.md` to include an entry for the `files_glob` extension. The `files_glob` extension now supports expanding glob-like patterns into real file paths at Tiltfile load time, enhancing dependency management in builds. It includes two functions: `files_glob` for path expansion and `watch_glob` for file change detection, ensuring Tilt reloads as files are added. New tests have been added in `files_glob/test/Tiltfile` to validate functionality, including deduplication of file paths from overlapping patterns. The helper now utilizes bash/find for practical pattern resolution. Signed-off-by: Kim Eik --- README.md | 1 + files_glob/README.md | 44 ++++++++++++ files_glob/Tiltfile | 106 +++++++++++++++++++++++++++++ files_glob/test/Tiltfile | 34 +++++++++ files_glob/test/dir1/a.txt | 2 + files_glob/test/dir1/b.txt | 2 + files_glob/test/dir2/nested/t1.txt | 2 + files_glob/test/dir2/nested/t2.txt | 2 + files_glob/test/root.txt | 1 + 9 files changed, 194 insertions(+) create mode 100644 files_glob/README.md create mode 100644 files_glob/Tiltfile create mode 100644 files_glob/test/Tiltfile create mode 100644 files_glob/test/dir1/a.txt create mode 100644 files_glob/test/dir1/b.txt create mode 100644 files_glob/test/dir2/nested/t1.txt create mode 100644 files_glob/test/dir2/nested/t2.txt create mode 100644 files_glob/test/root.txt diff --git a/README.md b/README.md index d948286f9..3427f243a 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ All extensions have been vetted and approved by the Tilt team. - [`dotenv`](/dotenv): Load environment variables from `.env` or another file. - [`earthly`](/earthly): Build container images using [earthly](https://earthly.dev) - [`execute_in_pod`](/execute_in_pod): Execute a command on a pod container. +- [`files_glob`](/files_glob): Expand glob-like patterns into real file paths for use in deps and other APIs. - [`file_sync_only`](/file_sync_only): No-build, no-push, file sync-only development. Useful when you want to live-reload a single config file into an existing public image, like nginx. - [`get_obj`](/get_obj): Get object yaml and the container's registry and image from an existing k8s resource such as deployment or statefulset - [`git_resource`](/git_resource): Deploy a dockerfile from a remote repository -- or specify the path to a local checkout for local development. diff --git a/files_glob/README.md b/files_glob/README.md new file mode 100644 index 000000000..ef8570e0e --- /dev/null +++ b/files_glob/README.md @@ -0,0 +1,44 @@ +# files_glob + +Author: Kim Eik + +Expand glob-like patterns (e.g., `**/*.go`, `dir/*.templ`) into real file paths at Tiltfile load time for use in APIs that require concrete paths (like `deps=` in `docker_build`/`custom_build`). + +It also ensures Tilt reloads when matching files are added by watching the base directories implied by the patterns. + +## Functions + +- files_glob(*patterns) -> [str] + - Expands common glob-style patterns into a de-duplicated list of file paths. + - Examples: + - `dir/**/*.ext` → recursively find files under `dir` matching `*.ext` + - `dir/*.ext` → non-recursive match within `dir` + - `*.ext` → match in current directory only + - literal paths → included as-is +- watch_glob(*patterns) + - Adds `watch_file()` entries for base directories implied by patterns so Tilt reloads when new files appear. `files_glob()` calls this for you automatically. + +Note: Implementation uses `bash`/`find` under the hood; it’s meant to be practical for common dev workflows, not a full glob engine. + +## Usage + +```python path=null start=null +# Optionally set your default extension repo first if not using the shared repo: +# v1alpha1.extension_repo(name='default', url='file:///path/to/tilt-extensions') + +load('ext://files_glob', 'files_glob') + +# Use with docker_build/custom_build deps +srcs = files_glob('**/*.go', 'web/*.templ', 'scripts/*.sh') + +docker_build('myimg', '.', deps=srcs) +``` + +You can also import `watch_glob` explicitly if you want to add watches without expanding files: + +```python path=null start=null +load('ext://files_glob', 'watch_glob') +watch_glob('**/*.sql', 'migrations/**/up/*.sql') +``` + + diff --git a/files_glob/Tiltfile b/files_glob/Tiltfile new file mode 100644 index 000000000..5b05b416a --- /dev/null +++ b/files_glob/Tiltfile @@ -0,0 +1,106 @@ +# -*- mode: Python -*- + +# Local Tilt extension: expand glob-like patterns into real file paths for use in deps +# Note: deps only accepts real paths; this helper uses bash/find to resolve common +# patterns at Tiltfile load time (e.g., '**/*.ext', 'dir/*.ext', '*.ext'). + + +def _base_dir_from_pattern(pat): + # Determine a base directory from a pattern for watch_file + # Treat patterns that start with '**/' (or are '**' variants) as repo root + if pat.startswith('**/') or pat == '**' or pat == '**/*' or pat.startswith('./**/'): + return '.' + # Handle patterns ending with '/**' or '/**/*' -> base dir before the ** + if pat.endswith('/**') or pat.endswith('/**/*'): + base = pat.rsplit('/**', 1)[0] + return base if base else '.' + # Handle recursive patterns with '/**/' inside + if '/**/' in pat: + base = pat.split('/**/', 1)[0] + return base if base else '.' + # For simple 'dir/*.ext' or 'dir/name' + if '/' in pat: + base = pat.rsplit('/', 1)[0] + return base if base else '.' + # For '*.ext' or literal file name, base is current dir + return '.' + + +def watch_glob(*patterns): + # Add Tiltfile reload watches for base dirs implied by patterns + seen = {} + for pat in patterns: + base = _base_dir_from_pattern(pat) + if base not in seen: + seen[base] = True + watch_file(base) + + +def files_glob(*glob_patterns, **kwargs): + """ + Expand one or more patterns into a list of real file paths using bash/find. + Supported patterns (good enough for most Tilt workflows): + - 'dir/**/*.ext' => find dir -type f -name "*.ext" + - 'dir/*.ext' => find dir -maxdepth 1 -type f -name "*.ext" + - '*.ext' => find . -maxdepth 1 -type f -name "*.ext" + - literal file/dir => included as-is + Returns a de-duplicated list of paths. + """ + # Ensure Tiltfile reloads when new files matching these patterns are added + watch_glob(*glob_patterns) + results = [] + for pat in glob_patterns: + # Handle recursive patterns with '**/' anywhere in the string (e.g., 'dir/**/x', '**/*.go') + if '/**/' in pat: + parts = pat.split('/**/', 1) + base = parts[0] if parts[0] else '.' + tail = parts[1] if len(parts) > 1 else '' + if not tail or tail == '*': + find_cmd = "bash --noprofile --norc -lc 'find " + base + " -type f -print'" + else: + if '/' in tail: + # Fallback to -path for complex tails + find_cmd = "bash --noprofile --norc -lc 'find " + base + " -type f -path \"*" + tail + "\" -print'" + else: + find_cmd = "bash --noprofile --norc -lc 'find " + base + " -type f -name \"" + tail + "\" -print'" + # Handle patterns that START with '**/' (e.g., '**/*.go') + elif pat.startswith('**/'): + tail = pat[len('**/'):] + if not tail or tail == '*': + find_cmd = "bash --noprofile --norc -lc 'find . -type f -print'" + else: + if '/' in tail: + find_cmd = "bash --noprofile --norc -lc 'find . -type f -path \"*" + tail + "\" -print'" + else: + find_cmd = "bash --noprofile --norc -lc 'find . -type f -name \"" + tail + "\" -print'" + # Root-level wildcard like '*.go' + elif pat.startswith('*.') or (pat and '*' in pat and '/' not in pat): + find_cmd = "bash --noprofile --norc -lc 'find . -maxdepth 1 -type f -name \"" + pat + "\" -print'" + # Directory-limited wildcard like 'dir/*.templ' + elif '*' in pat and '/' in pat: + dir_part, name_pat = pat.rsplit('/', 1) + find_cmd = "bash --noprofile --norc -lc 'find " + dir_part + " -maxdepth 1 -type f -name \"" + name_pat + "\" -print'" + else: + # Literal path (file or dir). deps can watch directories too; include as-is. + results.append(pat) + continue + + out = local(command=find_cmd, quiet=True) + s = str(out).strip() + if s: + for line in s.split('\n'): + p = line.strip() + if p.startswith('./'): + p = p[2:] + if p: + results.append(p) + + # Deduplicate while preserving order + seen = {} + deduped = [] + for p in results: + if p not in seen: + seen[p] = True + deduped.append(p) + return deduped + diff --git a/files_glob/test/Tiltfile b/files_glob/test/Tiltfile new file mode 100644 index 000000000..81d201f20 --- /dev/null +++ b/files_glob/test/Tiltfile @@ -0,0 +1,34 @@ +load('../Tiltfile', 'files_glob') + +# Doesn't use any kubernetes context, therefore safe +allow_k8s_contexts(k8s_context()) + +# Create some sample files for matching +# (These are committed to the repo; tests assume files exist on disk.) + +# Validate root-level pattern +paths_root_txt = files_glob('*.txt') +expected_root_txt = ['root.txt'] +if set(paths_root_txt) != set(expected_root_txt): + fail('root-level pattern failed: got %s, expected %s' % (paths_root_txt, expected_root_txt)) + +# Validate non-recursive dir pattern +paths_dir1_txt = files_glob('dir1/*.txt') +expected_dir1_txt = ['dir1/a.txt', 'dir1/b.txt'] +if set(paths_dir1_txt) != set(expected_dir1_txt): + fail('dir1/*.txt pattern failed: got %s, expected %s' % (paths_dir1_txt, expected_dir1_txt)) + +# Validate recursive pattern +paths_dir2_txt = files_glob('dir2/**/*.txt') +expected_dir2_txt = ['dir2/nested/t1.txt', 'dir2/nested/t2.txt'] +if set(paths_dir2_txt) != set(expected_dir2_txt): + fail('dir2/**/*.txt pattern failed: got %s, expected %s' % (paths_dir2_txt, expected_dir2_txt)) + +# Validate deduplication (same files via two overlapping patterns) +paths_dedupe = files_glob('dir1/*.txt', 'dir1/**/*.txt') +if len(paths_dedupe) != len(set(paths_dedupe)): + fail('deduplication failed: got %d items with duplicates: %s' % (len(paths_dedupe), paths_dedupe)) + +# CI expects at least one resource +local_resource('dummy', 'echo ok') + diff --git a/files_glob/test/dir1/a.txt b/files_glob/test/dir1/a.txt new file mode 100644 index 000000000..dede4985a --- /dev/null +++ b/files_glob/test/dir1/a.txt @@ -0,0 +1,2 @@ +alpha + diff --git a/files_glob/test/dir1/b.txt b/files_glob/test/dir1/b.txt new file mode 100644 index 000000000..ead5105e4 --- /dev/null +++ b/files_glob/test/dir1/b.txt @@ -0,0 +1,2 @@ +bravo + diff --git a/files_glob/test/dir2/nested/t1.txt b/files_glob/test/dir2/nested/t1.txt new file mode 100644 index 000000000..0f0c7232c --- /dev/null +++ b/files_glob/test/dir2/nested/t1.txt @@ -0,0 +1,2 @@ +t1 text + diff --git a/files_glob/test/dir2/nested/t2.txt b/files_glob/test/dir2/nested/t2.txt new file mode 100644 index 000000000..cbe853647 --- /dev/null +++ b/files_glob/test/dir2/nested/t2.txt @@ -0,0 +1,2 @@ +t2 text + diff --git a/files_glob/test/root.txt b/files_glob/test/root.txt new file mode 100644 index 000000000..a1e2013ea --- /dev/null +++ b/files_glob/test/root.txt @@ -0,0 +1 @@ +root sample