Skip to content

Commit 456ada3

Browse files
ianicianic
committed
package manager: don't strip components in tar
Unpack tar without removing leading root folder. Then find package root in unpacked tmp folder. Fixes #17779
1 parent 43d7fdb commit 456ada3

File tree

2 files changed

+147
-55
lines changed

2 files changed

+147
-55
lines changed

‎src/Package/Fetch.zig‎

Lines changed: 43 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -440,33 +440,36 @@ fn runResource(
440440
const s = fs.path.sep_str;
441441
const cache_root = f.job_queue.global_cache;
442442
const rand_int = std.crypto.random.int(u64);
443+
// temporary directory for unpacking; sub path of cache_root
443444
const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
445+
// sub path of cache_root, inside temporary directory, if null package root
446+
// is tmp_dir_sub_path
447+
var package_sub_path: ?[]const u8 = null;
444448

445449
{
446-
const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
447-
var tmp_directory: Cache.Directory = .{
448-
.path = tmp_directory_path,
449-
.handle = handle: {
450-
const dir = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{
451-
.iterate = true,
452-
}) catch |err| {
453-
try eb.addRootErrorMessage(.{
454-
.msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{
455-
tmp_directory_path, @errorName(err),
456-
}),
457-
});
458-
return error.FetchFailed;
459-
};
460-
break :handle dir;
461-
},
462-
};
450+
var tmp_directory = try f.makeOpenCacheDirectory(tmp_dir_sub_path);
463451
defer tmp_directory.handle.close();
464452

465453
// Fetch and unpack a URL into a temporary directory.
466454
var unpack = Unpack.init(gpa, tmp_directory.handle);
467455
try unpackResource(f, resource, &unpack, uri_path);
468456
defer unpack.deinit();
469457

458+
// Position tmp_directory to sub_path if package root is deeper inside
459+
// temporary directory.
460+
if (unpack.package_sub_path) |sub_path| {
461+
package_sub_path = try fs.path.join(arena, &.{ tmp_dir_sub_path, sub_path });
462+
tmp_directory.handle.close();
463+
tmp_directory = try f.makeOpenCacheDirectory(package_sub_path.?);
464+
} else {
465+
// btrfs workaround; reopen tmp_directory
466+
if (native_os == .linux and f.job_queue.work_around_btrfs_bug) {
467+
// https://github.com/ziglang/zig/issues/17095
468+
tmp_directory.handle.close();
469+
tmp_directory = try f.makeOpenCacheDirectory(tmp_dir_sub_path);
470+
}
471+
}
472+
470473
// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
471474
// for the file to be missing, in which case this fetched package is
472475
// considered to be a "naked" package.
@@ -488,15 +491,6 @@ fn runResource(
488491
// It will also apply the manifest's inclusion rules to the temporary
489492
// directory by deleting excluded files.
490493
// Empty directories have already been omitted by `unpackResource`.
491-
492-
if (native_os == .linux and f.job_queue.work_around_btrfs_bug) {
493-
// https://github.com/ziglang/zig/issues/17095
494-
tmp_directory.handle.close();
495-
tmp_directory.handle = cache_root.handle.makeOpenPath(tmp_dir_sub_path, .{
496-
.iterate = true,
497-
}) catch @panic("btrfs workaround failed");
498-
}
499-
500494
f.actual_hash = try computeHash(f, tmp_directory, filter);
501495
}
502496

@@ -510,7 +504,11 @@ fn runResource(
510504
.root_dir = cache_root,
511505
.sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
512506
};
513-
renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, f.package_root.sub_path) catch |err| {
507+
renameTmpIntoCache(
508+
cache_root.handle,
509+
if (package_sub_path) |p| p else tmp_dir_sub_path,
510+
f.package_root.sub_path,
511+
) catch |err| {
514512
const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
515513
const dest = try cache_root.join(arena, &.{f.package_root.sub_path});
516514
try eb.addRootErrorMessage(.{ .msg = try eb.printString(
@@ -519,6 +517,9 @@ fn runResource(
519517
) });
520518
return error.FetchFailed;
521519
};
520+
if (package_sub_path) |_| {
521+
cache_root.handle.deleteTree(tmp_dir_sub_path) catch {};
522+
}
522523

523524
// Validate the computed hash against the expected hash. If invalid, this
524525
// job is done.
@@ -551,6 +552,21 @@ fn runResource(
551552
return queueJobsForDeps(f);
552553
}
553554

555+
fn makeOpenCacheDirectory(f: *Fetch, sub_path: []const u8) RunError!Cache.Directory {
556+
const arena = f.arena.allocator();
557+
const cache_root = f.job_queue.global_cache;
558+
559+
const path = try cache_root.join(arena, &.{sub_path});
560+
return .{
561+
.path = path,
562+
.handle = cache_root.handle.makeOpenPath(sub_path, .{ .iterate = true }) catch |err| {
563+
return f.failMsg(err, "unable to create temporary directory '{s}': {s}", .{
564+
path, @errorName(err),
565+
});
566+
},
567+
};
568+
}
569+
554570
/// `computeHash` gets a free check for the existence of `build.zig`, but when
555571
/// not computing a hash, we need to do a syscall to check for it.
556572
fn checkBuildFileExistence(f: *Fetch) RunError!void {

‎src/Package/Unpack.zig‎

Lines changed: 104 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const Filter = @import("Fetch.zig").Filter;
55

66
allocator: std.mem.Allocator,
77
root: fs.Dir,
8+
package_sub_path: ?[]const u8 = null,
89
errors: Errors,
910

1011
pub const Error = union(enum) {
@@ -69,7 +70,7 @@ pub const Errors = struct {
6970
} });
7071
}
7172

72-
pub fn filterWith(self: *Errors, filter: Filter) !void {
73+
fn filterWith(self: *Errors, filter: Filter) !void {
7374
var i = self.list.items.len;
7475
while (i > 0) {
7576
i -= 1;
@@ -80,6 +81,25 @@ pub const Errors = struct {
8081
}
8182
}
8283
}
84+
85+
fn stripRoot(self: *Errors) !void {
86+
if (self.count() == 0) return;
87+
88+
var old_list = self.list;
89+
self.list = .{};
90+
for (old_list.items) |item| {
91+
switch (item) {
92+
.unable_to_create_sym_link => |info| {
93+
try self.symLink("", stripComponents(info.target_path, 1), info.sym_link_path, info.code);
94+
},
95+
.unable_to_create_file => |info| {
96+
try self.createFile("", stripComponents(info.file_name, 1), info.code);
97+
},
98+
}
99+
self.free(item);
100+
}
101+
old_list.deinit(self.allocator);
102+
}
83103
};
84104

85105
pub fn init(allocator: std.mem.Allocator, root: fs.Dir) Self {
@@ -92,13 +112,14 @@ pub fn init(allocator: std.mem.Allocator, root: fs.Dir) Self {
92112

93113
pub fn deinit(self: *Self) void {
94114
self.errors.deinit();
115+
if (self.package_sub_path) |package_sub_path| {
116+
self.allocator.free(package_sub_path);
117+
}
95118
}
96119

97120
const Self = @This();
98121

99122
pub fn tarball(self: *Self, reader: anytype) !void {
100-
const strip_components = 1;
101-
102123
var file_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
103124
var link_name_buffer: [std.fs.MAX_PATH_BYTES]u8 = undefined;
104125
var iter = std.tar.iterator(reader, .{
@@ -114,22 +135,59 @@ pub fn tarball(self: *Self, reader: anytype) !void {
114135
.directory => {}, // skip empty
115136
.file => {
116137
if (entry.size == 0 and entry.name.len == 0) continue;
117-
const file_name = stripComponents(entry.name, strip_components);
118-
if (file_name.len == 0) return error.BadFileName;
119-
if (try self.createFile("", file_name)) |file| {
138+
if (try self.createFile("", entry.name)) |file| {
120139
defer file.close();
121140
try entry.writeAll(file);
122141
}
123142
},
124143
.sym_link => {
125-
const file_name = stripComponents(entry.name, strip_components);
126-
if (file_name.len == 0) return error.BadFileName;
127-
const link_name = entry.link_name;
128-
try self.symLink("", link_name, file_name);
144+
try self.symLink("", entry.link_name, entry.name);
129145
},
130146
}
131147
} else break;
132148
}
149+
try self.findPackageSubPath();
150+
}
151+
152+
fn findPackageSubPath(self: *Self) !void {
153+
var iter = self.root.iterate();
154+
if (try iter.next()) |entry| {
155+
if (try iter.next() != null) return;
156+
if (entry.kind == .directory) { // single directory below root
157+
self.package_sub_path = try self.allocator.dupe(u8, entry.name);
158+
try self.errors.stripRoot();
159+
}
160+
}
161+
}
162+
163+
test findPackageSubPath {
164+
var tmp = testing.tmpDir(.{ .iterate = true });
165+
defer tmp.cleanup();
166+
167+
// folder1
168+
// ├── folder2
169+
// ├── file1
170+
//
171+
try tmp.dir.makePath("folder1/folder2");
172+
(try tmp.dir.createFile("folder1/file1", .{})).close();
173+
174+
var unpack = init(testing.allocator, tmp.dir);
175+
try unpack.findPackageSubPath();
176+
// start at root returns folder1 as package root
177+
try testing.expectEqualStrings("folder1", unpack.package_sub_path.?);
178+
unpack.deinit();
179+
180+
// start at folder1 returns null
181+
unpack = init(testing.allocator, try tmp.dir.openDir("folder1", .{ .iterate = true }));
182+
try unpack.findPackageSubPath();
183+
try testing.expect(unpack.package_sub_path == null);
184+
unpack.deinit();
185+
186+
// start at folder1/folder2 returns null
187+
unpack = init(testing.allocator, try tmp.dir.openDir("folder1/folder2", .{ .iterate = true }));
188+
try unpack.findPackageSubPath();
189+
try testing.expect(unpack.package_sub_path == null);
190+
unpack.deinit();
133191
}
134192

135193
pub fn gitPack(self: *Self, commit_oid: git.Oid, reader: anytype) !void {
@@ -322,22 +380,36 @@ test tarball {
322380
};
323381
var buf: [paths.len * @sizeOf(TarHeader)]u8 = undefined;
324382

325-
// create tarball
326-
try createTarball(paths, &buf);
383+
// tarball with leading root folder
384+
{
385+
try createTarball("package_root", paths, &buf);
386+
var tmp = testing.tmpDir(.{ .iterate = true });
387+
defer tmp.cleanup();
327388

328-
var tmp = testing.tmpDir(.{ .iterate = true });
329-
defer tmp.cleanup();
389+
var fbs = std.io.fixedBufferStream(&buf);
390+
391+
var unpack = Unpack.init(testing.allocator, tmp.dir);
392+
defer unpack.deinit();
393+
try unpack.tarball(fbs.reader());
394+
try testing.expectEqualStrings("package_root", unpack.package_sub_path.?);
330395

331-
// unpack tarball to tmp dir, will strip root dir
396+
try expectDirFiles(try tmp.dir.openDir("package_root", .{ .iterate = true }), paths);
397+
}
398+
// tarball without root
332399
{
400+
try createTarball("", paths, &buf);
401+
var tmp = testing.tmpDir(.{ .iterate = true });
402+
defer tmp.cleanup();
403+
333404
var fbs = std.io.fixedBufferStream(&buf);
334405

335406
var unpack = Unpack.init(testing.allocator, tmp.dir);
336407
defer unpack.deinit();
337408
try unpack.tarball(fbs.reader());
338-
}
409+
try testing.expect(unpack.package_sub_path == null);
339410

340-
try expectDirFiles(tmp.dir, paths);
411+
try expectDirFiles(tmp.dir, paths);
412+
}
341413
}
342414

343415
test directory {
@@ -357,14 +429,14 @@ test directory {
357429
f.close();
358430
}
359431

360-
var dest = testing.tmpDir(.{ .iterate = true });
361-
defer dest.cleanup();
432+
var tmp = testing.tmpDir(.{ .iterate = true });
433+
defer tmp.cleanup();
362434

363-
var unpack = Unpack.init(testing.allocator, dest.dir);
435+
var unpack = Unpack.init(testing.allocator, tmp.dir);
364436
defer unpack.deinit();
365437
try unpack.directory(source.dir);
366438

367-
try expectDirFiles(dest.dir, paths);
439+
try expectDirFiles(tmp.dir, paths);
368440
}
369441

370442
test "collect/filter errors" {
@@ -378,7 +450,7 @@ test "collect/filter errors" {
378450
"dir1/file1",
379451
};
380452
var buf: [paths.len * @sizeOf(TarHeader)]u8 = undefined;
381-
try createTarball(paths, &buf);
453+
try createTarball("package_root", paths, &buf);
382454

383455
var tmp = testing.tmpDir(.{ .iterate = true });
384456
defer tmp.cleanup();
@@ -388,12 +460,12 @@ test "collect/filter errors" {
388460
defer unpack.deinit();
389461
try unpack.tarball(fbs.reader());
390462
try testing.expect(unpack.hasErrors());
391-
392-
try expectDirFiles(tmp.dir, paths[0..2]);
463+
try testing.expectEqualStrings("package_root", unpack.package_sub_path.?);
464+
try expectDirFiles(try tmp.dir.openDir("package_root", .{ .iterate = true }), paths[0..2]);
393465

394466
try testing.expectEqual(2, unpack.errors.count());
395-
try testing.expectEqualStrings(paths[2], unpack.errors.list.items[0].unable_to_create_file.file_name);
396-
try testing.expectEqualStrings(paths[3], unpack.errors.list.items[1].unable_to_create_file.file_name);
467+
try testing.expectEqualStrings("dir/file", unpack.errors.list.items[0].unable_to_create_file.file_name);
468+
try testing.expectEqualStrings("dir1/file1", unpack.errors.list.items[1].unable_to_create_file.file_name);
397469

398470
{
399471
var filter: Filter = .{};
@@ -420,13 +492,17 @@ test "collect/filter errors" {
420492
}
421493
}
422494

423-
fn createTarball(paths: []const []const u8, buf: []u8) !void {
495+
fn createTarball(prefix: []const u8, paths: []const []const u8, buf: []u8) !void {
424496
var fbs = std.io.fixedBufferStream(buf);
425497
const writer = fbs.writer();
426498
for (paths) |path| {
427499
var hdr = TarHeader.init();
428500
hdr.typeflag = .regular;
429-
try hdr.setPath("stripped_root", path);
501+
if (prefix.len > 0) {
502+
try hdr.setPath(prefix, path);
503+
} else {
504+
hdr.setName(path);
505+
}
430506
try hdr.updateChecksum();
431507
try writer.writeAll(std.mem.asBytes(&hdr));
432508
}

0 commit comments

Comments
 (0)