From 629e28d8d9a4452d59c4888eef926313b34a4bc7 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Tue, 13 Oct 2020 10:25:06 -0700 Subject: [PATCH 01/15] add globs.glob; fixed https://github.com/nim-lang/RFCs/issues/261 --- lib/std/globs.nim | 38 +++++++++++++++++++++++++ lib/std/private/globs.nim | 46 ++++--------------------------- testament/lib/stdtest/osutils.nim | 13 +++++++++ tests/stdlib/tglobs.nim | 17 ++++++++++++ 4 files changed, 73 insertions(+), 41 deletions(-) create mode 100644 lib/std/globs.nim create mode 100644 testament/lib/stdtest/osutils.nim create mode 100644 tests/stdlib/tglobs.nim diff --git a/lib/std/globs.nim b/lib/std/globs.nim new file mode 100644 index 0000000000000..be459f38d8baa --- /dev/null +++ b/lib/std/globs.nim @@ -0,0 +1,38 @@ +import std/os + +type + PathEntry* = object + kind*: PathComponent + path*: string + +iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, + relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect].} = + ## Improved `os.walkDirRec`. + #[ + note: a yieldFilter isn't needed because caller can filter at call site, without + loss of generality, unlike `follow`. + + Future work: + * need to document + * add a `sort` option, which can be implemented efficiently only here, not at call site. + * provide a way to do error reporting, which is tricky because iteration cannot be resumed + * `walkDirRec` can be implemented in terms of this to avoid duplication, + modulo some refactoring. + ]# + var stack = @["."] + var checkDir = checkDir + var entry: PathEntry + while stack.len > 0: + let d = stack.pop() + for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): + let rel = d / p + entry.kind = k + if relative: entry.path = rel + else: entry.path = dir / rel + if k in {pcDir, pcLinkToDir}: + if follow == nil or follow(entry): stack.add rel + yield entry + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to + # continue iteration. diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index e697ca91cf1ae..6561ca42ade63 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -1,45 +1,14 @@ ##[ unstable API, internal use only for now. -this can eventually be moved to std/os and `walkDirRec` can be implemented in terms of this -to avoid duplication ]## -import std/[os,strutils] +#[ +this module should be renamed to reflect its scope, eg to osutils. +]# -type - PathEntry* = object - kind*: PathComponent - path*: string +import std/[os,strutils,globs] -iterator walkDirRecFilter*(dir: string, follow: proc(entry: PathEntry): bool = nil, - relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect].} = - ## Improved `os.walkDirRec`. - #[ - note: a yieldFilter isn't needed because caller can filter at call site, without - loss of generality, unlike `follow`. - - Future work: - * need to document - * add a `sort` option, which can be implemented efficiently only here, not at call site. - * provide a way to do error reporting, which is tricky because iteration cannot be resumed - ]# - var stack = @["."] - var checkDir = checkDir - var entry: PathEntry - while stack.len > 0: - let d = stack.pop() - for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): - let rel = d / p - entry.kind = k - if relative: entry.path = rel - else: entry.path = dir / rel - if k in {pcDir, pcLinkToDir}: - if follow == nil or follow(entry): stack.add rel - yield entry - checkDir = false - # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong - # permissions), it'll abort iteration and there would be no way to - # continue iteration. +{.deprecated: [walkDirRecFilter: glob].} proc nativeToUnixPath*(path: string): string = # pending https://github.com/nim-lang/Nim/pull/13265 @@ -47,8 +16,3 @@ proc nativeToUnixPath*(path: string): string = when DirSep == '\\': result = replace(path, '\\', '/') else: result = path - -when isMainModule: - import sugar - for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", ".csources", "bin"]): - echo a diff --git a/testament/lib/stdtest/osutils.nim b/testament/lib/stdtest/osutils.nim new file mode 100644 index 0000000000000..dbe90d4d0e73d --- /dev/null +++ b/testament/lib/stdtest/osutils.nim @@ -0,0 +1,13 @@ +import std/[os] + +proc genTestPaths*(dir: string, paths: seq[string]) = + ## generates a filesystem rooted under `dir` from given relative `paths`. + ## `paths` ending in `/` are treated as directories. + # xxx use this in tos.nim + for a in paths: + doAssert a.isRelativePath + if a.endsWith("/"): + createDir(a) + else: + createDir(a.parentDir) + diff --git a/tests/stdlib/tglobs.nim b/tests/stdlib/tglobs.nim new file mode 100644 index 0000000000000..3483632e1006e --- /dev/null +++ b/tests/stdlib/tglobs.nim @@ -0,0 +1,17 @@ +import std/[sugar,globs,os,strutils] +import stdtest/specialpaths + +block: # glob + let dir = buildDir/"D20201013T100140" + defer: removeDir(dir) + let files = """ +d1/f1.txt +d1/d2/f2.txt +d1/d2/f3.txt +d1/d3/ +d4/ +""".splitLines + for + + for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", ".csources", "bin"]): + echo a From 4044ad72bd79166a0eb28393d99c62d2e87cb507 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 14 Oct 2020 01:19:34 -0700 Subject: [PATCH 02/15] _ --- lib/std/globs.nim | 32 ++++++++++++++++++++++++------- testament/lib/stdtest/osutils.nim | 8 +++++--- tests/stdlib/tglobs.nim | 28 +++++++++++++++------------ 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index be459f38d8baa..5d7b1e57242a2 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -4,33 +4,51 @@ type PathEntry* = object kind*: PathComponent path*: string + ## absolute or relative path wrt globbed dir + depth*: int + ## depth wrt globbed dir iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect].} = - ## Improved `os.walkDirRec`. + ## Recursively walks `dir` which must exist when checkDir=true (else raises `OSError`). + ## Paths in `result.path` are relative to `dir` unless `relative=false`, + ## `result.depth >= 1` is the tree depth relative to the root `dir` (at depth 0). + ## if `follow != nil`, `glob` visits `entry` if `filter(entry) == true`. + ## This is more flexible than `os.walkDirRec`. + runnableExamples: + import os,sugar + if defined(nimGlobsEnableExample): # `if` intentional + # list hidden files of depth <= 2 + 1 in your home. + for e in glob(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2): + if e.kind in {pcFile, pcLinkToFile}: echo e.path #[ - note: a yieldFilter isn't needed because caller can filter at call site, without - loss of generality, unlike `follow`. + note: a yieldFilter, regex match etc isn't needed because caller can filter at + call site, without loss of generality, unlike `follow`; this simplifies the API. Future work: * need to document + * add `includeRoot = false` (ie, depth = 0) to optionally add the root dir; + this must be done while preserving a single `yield` to avoid code bloat. * add a `sort` option, which can be implemented efficiently only here, not at call site. * provide a way to do error reporting, which is tricky because iteration cannot be resumed * `walkDirRec` can be implemented in terms of this to avoid duplication, - modulo some refactoring. + modulo some refactoring. ]# - var stack = @["."] + var stack = @[(0, ".")] var checkDir = checkDir var entry: PathEntry + # if includeRoot: while stack.len > 0: - let d = stack.pop() + let (depth, d) = stack.pop() for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): let rel = d / p + entry.depth = depth + 1 entry.kind = k if relative: entry.path = rel else: entry.path = dir / rel + normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 if k in {pcDir, pcLinkToDir}: - if follow == nil or follow(entry): stack.add rel + if follow == nil or follow(entry): stack.add (depth + 1, rel) yield entry checkDir = false # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong diff --git a/testament/lib/stdtest/osutils.nim b/testament/lib/stdtest/osutils.nim index dbe90d4d0e73d..c998ebd017749 100644 --- a/testament/lib/stdtest/osutils.nim +++ b/testament/lib/stdtest/osutils.nim @@ -1,13 +1,15 @@ -import std/[os] +import std/[os,strutils] proc genTestPaths*(dir: string, paths: seq[string]) = ## generates a filesystem rooted under `dir` from given relative `paths`. ## `paths` ending in `/` are treated as directories. # xxx use this in tos.nim for a in paths: - doAssert a.isRelativePath + doAssert not a.isAbsolute + doAssert a.len > 0 + let a = dir / a if a.endsWith("/"): createDir(a) else: createDir(a.parentDir) - + writeFile(a, "") diff --git a/tests/stdlib/tglobs.nim b/tests/stdlib/tglobs.nim index 3483632e1006e..ff03b18aa029d 100644 --- a/tests/stdlib/tglobs.nim +++ b/tests/stdlib/tglobs.nim @@ -1,17 +1,21 @@ -import std/[sugar,globs,os,strutils] -import stdtest/specialpaths +import std/[sugar,globs,os,strutils,sequtils,algorithm] +import stdtest/[specialpaths,osutils] block: # glob let dir = buildDir/"D20201013T100140" defer: removeDir(dir) - let files = """ + let paths = """ d1/f1.txt -d1/d2/f2.txt -d1/d2/f3.txt -d1/d3/ -d4/ -""".splitLines - for - - for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", ".csources", "bin"]): - echo a +d1/d1a/f2.txt +d1/d1a/f3 +d1/d1a/d1a1/ +d1/d1b/d1b1/f4 +d2/ +""".splitLines.filter(a=>a.len>0) + genTestPaths(dir, paths) + doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) + .filterIt(it.kind == pcFile).mapIt(it.path).sorted == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt"] + doAssert toSeq(glob(dir, relative = true)) + .filterIt(it.kind == pcDir).mapIt(it.path).sorted == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + doAssertRaises(OSError): discard toSeq(glob("nonexistant")) + doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] From ae04071df0329ca3398d7fabac966ffd790db4c9 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Wed, 14 Oct 2020 15:11:57 -0700 Subject: [PATCH 03/15] fixup --- lib/std/globs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index 5d7b1e57242a2..614b6d301bc53 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -17,7 +17,7 @@ iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, ## This is more flexible than `os.walkDirRec`. runnableExamples: import os,sugar - if defined(nimGlobsEnableExample): # `if` intentional + if false: # list hidden files of depth <= 2 + 1 in your home. for e in glob(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2): if e.kind in {pcFile, pcLinkToFile}: echo e.path From 78683970208bd0b58a2c684682ae22f1ec3f2dee Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 15 Oct 2020 14:10:33 -0700 Subject: [PATCH 04/15] _ --- lib/std/globs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index 614b6d301bc53..fe9a05b1bbcbf 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -9,7 +9,7 @@ type ## depth wrt globbed dir iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, - relative = false, checkDir = true): PathEntry {.tags: [ReadDirEffect].} = + relative = false, checkDir = true): PathEntry {.closure, tags: [ReadDirEffect].} = ## Recursively walks `dir` which must exist when checkDir=true (else raises `OSError`). ## Paths in `result.path` are relative to `dir` unless `relative=false`, ## `result.depth >= 1` is the tree depth relative to the root `dir` (at depth 0). From 5b25236dd8aacb3fb8778f3dfa5702f2d8a90bdf Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 15 Oct 2020 16:44:36 -0700 Subject: [PATCH 05/15] _ --- lib/std/globs.nim | 26 +++++++++++++++++++++++--- lib/std/private/globs.nim | 3 ++- tests/stdlib/tglobs.nim | 21 +++++++++++++++++++-- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index fe9a05b1bbcbf..edf7837e75bc3 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -9,7 +9,7 @@ type ## depth wrt globbed dir iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, - relative = false, checkDir = true): PathEntry {.closure, tags: [ReadDirEffect].} = + relative = false, checkDir = true, includeRoot = false): PathEntry {.closure, tags: [ReadDirEffect].} = ## Recursively walks `dir` which must exist when checkDir=true (else raises `OSError`). ## Paths in `result.path` are relative to `dir` unless `relative=false`, ## `result.depth >= 1` is the tree depth relative to the root `dir` (at depth 0). @@ -22,8 +22,11 @@ iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, for e in glob(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2): if e.kind in {pcFile, pcLinkToFile}: echo e.path #[ - note: a yieldFilter, regex match etc isn't needed because caller can filter at + note: + * a yieldFilter, regex match etc isn't needed because caller can filter at call site, without loss of generality, unlike `follow`; this simplifies the API. + * a closure iterator is used since we need multiple yield statements and it simplifies code. + In practice optimized performance drops by less than 2%, likely 0 when filesystem is not "hot". Future work: * need to document @@ -37,9 +40,26 @@ iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, var stack = @[(0, ".")] var checkDir = checkDir var entry: PathEntry - # if includeRoot: + if not dirExists(dir): + if checkDir: + raise newException(OSError, "invalid root dir: " & dir) + else: + return + + if includeRoot: + if symlinkExists(dir): + entry.kind = pcLinkToDir + else: + entry.kind = pcDir + entry.path = if relative: "." else: dir + normalizePath(entry.path) + entry.depth = 0 + yield entry + while stack.len > 0: let (depth, d) = stack.pop() + # checkDir is still needed here in first iteration because things could + # fail for reasons other than `not dirExists`. for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): let rel = d / p entry.depth = depth + 1 diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim index 6561ca42ade63..f2c95daa30fa2 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/globs.nim @@ -8,7 +8,8 @@ this module should be renamed to reflect its scope, eg to osutils. import std/[os,strutils,globs] -{.deprecated: [walkDirRecFilter: glob].} +# {.deprecated: [walkDirRecFilter: glob].} +# export glob proc nativeToUnixPath*(path: string): string = # pending https://github.com/nim-lang/Nim/pull/13265 diff --git a/tests/stdlib/tglobs.nim b/tests/stdlib/tglobs.nim index ff03b18aa029d..fcae1ae15dd6a 100644 --- a/tests/stdlib/tglobs.nim +++ b/tests/stdlib/tglobs.nim @@ -1,6 +1,19 @@ import std/[sugar,globs,os,strutils,sequtils,algorithm] +from std/private/globs as globsOld import nativeToUnixPath import stdtest/[specialpaths,osutils] +# proc nativeToUnixPath*(path: string): string = +# # pending https://github.com/nim-lang/Nim/pull/13265 +# doAssert not path.isAbsolute # not implemented here; absolute files need more care for the drive +# when DirSep == '\\': +# result = replace(path, '\\', '/') +# else: result = path + +# import timn/exp/taps + +proc process[T](a: T): seq[string] = + a.mapIt(it.path.nativeToUnixPath).sorted + block: # glob let dir = buildDir/"D20201013T100140" defer: removeDir(dir) @@ -11,11 +24,15 @@ d1/d1a/f3 d1/d1a/d1a1/ d1/d1b/d1b1/f4 d2/ +f5 """.splitLines.filter(a=>a.len>0) genTestPaths(dir, paths) doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) - .filterIt(it.kind == pcFile).mapIt(it.path).sorted == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt"] + .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"] doAssert toSeq(glob(dir, relative = true)) - .filterIt(it.kind == pcDir).mapIt(it.path).sorted == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + doAssert toSeq(glob(dir, relative = true, includeRoot = true)) + .filterIt(it.kind == pcDir).process == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] doAssertRaises(OSError): discard toSeq(glob("nonexistant")) + doAssertRaises(OSError): discard toSeq(glob("f5")) doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] From 62eff55d7370f2f5f54e49875c3dbef457d94f17 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 15 Oct 2020 18:52:40 -0700 Subject: [PATCH 06/15] _ --- lib/std/globs.nim | 50 ++++++++++++++++------ lib/std/private/{globs.nim => osutils.nim} | 8 +--- tests/stdlib/tglobs.nim | 21 ++++----- 3 files changed, 50 insertions(+), 29 deletions(-) rename lib/std/private/{globs.nim => osutils.nim} (73%) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index edf7837e75bc3..437c468979bfe 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -1,6 +1,10 @@ import std/os +import std/algorithm type + PathEntrySub* = object + kind*: PathComponent + path*: string PathEntry* = object kind*: PathComponent path*: string @@ -8,8 +12,11 @@ type depth*: int ## depth wrt globbed dir -iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, - relative = false, checkDir = true, includeRoot = false): PathEntry {.closure, tags: [ReadDirEffect].} = +iterator glob*(dir: string, relative = false, checkDir = true, includeRoot = false, + follow: proc(entry: PathEntry): bool = nil, + sortCmp: proc (x, y: PathEntrySub): int = nil, + topFirst = true): + PathEntry {.closure, tags: [ReadDirEffect].} = ## Recursively walks `dir` which must exist when checkDir=true (else raises `OSError`). ## Paths in `result.path` are relative to `dir` unless `relative=false`, ## `result.depth >= 1` is the tree depth relative to the root `dir` (at depth 0). @@ -30,16 +37,16 @@ iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, Future work: * need to document - * add `includeRoot = false` (ie, depth = 0) to optionally add the root dir; - this must be done while preserving a single `yield` to avoid code bloat. * add a `sort` option, which can be implemented efficiently only here, not at call site. * provide a way to do error reporting, which is tricky because iteration cannot be resumed * `walkDirRec` can be implemented in terms of this to avoid duplication, modulo some refactoring. ]# + echo() var stack = @[(0, ".")] var checkDir = checkDir var entry: PathEntry + var dirsLevel: seq[PathEntrySub] if not dirExists(dir): if checkDir: raise newException(OSError, "invalid root dir: " & dir) @@ -56,20 +63,37 @@ iterator glob*(dir: string, follow: proc(entry: PathEntry): bool = nil, entry.depth = 0 yield entry + template processEntry(): untyped = + let rel = d / p + entry.depth = depth + 1 + entry.kind = k + if relative: entry.path = rel + else: entry.path = dir / rel + normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 + if k in {pcDir, pcLinkToDir}: + if follow == nil or follow(entry): stack.add (depth + 1, rel) + entry + while stack.len > 0: let (depth, d) = stack.pop() # checkDir is still needed here in first iteration because things could # fail for reasons other than `not dirExists`. + + if sortCmp != nil: + dirsLevel.setLen 0 + # echo "...: ", d for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): - let rel = d / p - entry.depth = depth + 1 - entry.kind = k - if relative: entry.path = rel - else: entry.path = dir / rel - normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 - if k in {pcDir, pcLinkToDir}: - if follow == nil or follow(entry): stack.add (depth + 1, rel) - yield entry + if sortCmp != nil: + dirsLevel.add PathEntrySub(kind: k, path: p) + else: + yield processEntry() + if sortCmp != nil: + sort(dirsLevel, sortCmp) + for ai in dirsLevel: + let k = ai.kind + let p = ai.path + yield processEntry() + checkDir = false # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong # permissions), it'll abort iteration and there would be no way to diff --git a/lib/std/private/globs.nim b/lib/std/private/osutils.nim similarity index 73% rename from lib/std/private/globs.nim rename to lib/std/private/osutils.nim index f2c95daa30fa2..21e88f38b2ddd 100644 --- a/lib/std/private/globs.nim +++ b/lib/std/private/osutils.nim @@ -2,14 +2,10 @@ unstable API, internal use only for now. ]## -#[ -this module should be renamed to reflect its scope, eg to osutils. -]# - import std/[os,strutils,globs] -# {.deprecated: [walkDirRecFilter: glob].} -# export glob +{.deprecated: [walkDirRecFilter: glob].} +export glob proc nativeToUnixPath*(path: string): string = # pending https://github.com/nim-lang/Nim/pull/13265 diff --git a/tests/stdlib/tglobs.nim b/tests/stdlib/tglobs.nim index fcae1ae15dd6a..51446c5bae076 100644 --- a/tests/stdlib/tglobs.nim +++ b/tests/stdlib/tglobs.nim @@ -1,18 +1,13 @@ import std/[sugar,globs,os,strutils,sequtils,algorithm] -from std/private/globs as globsOld import nativeToUnixPath +from std/private/osutils as osutils2 import nativeToUnixPath import stdtest/[specialpaths,osutils] -# proc nativeToUnixPath*(path: string): string = -# # pending https://github.com/nim-lang/Nim/pull/13265 -# doAssert not path.isAbsolute # not implemented here; absolute files need more care for the drive -# when DirSep == '\\': -# result = replace(path, '\\', '/') -# else: result = path - -# import timn/exp/taps +import timn/exp/taps +proc processAux[T](a: T): seq[string] = + a.mapIt(it.path.nativeToUnixPath) proc process[T](a: T): seq[string] = - a.mapIt(it.path.nativeToUnixPath).sorted + a.processAux.sorted block: # glob let dir = buildDir/"D20201013T100140" @@ -36,3 +31,9 @@ f5 doAssertRaises(OSError): discard toSeq(glob("nonexistant")) doAssertRaises(OSError): discard toSeq(glob("f5")) doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] + + proc mySort(a, b: PathEntrySub): int = cmp(a.path, b.path) + proc mySort2(a, b: PathEntrySub): int = -cmp(a.path, b.path) + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort2)).processAux == @["f5", "d2", "d1", "d1/f1.txt", "d1/d1b", "d1/d1a", "d1/d1a/f3", "d1/d1a/f2.txt", "d1/d1a/d1a1", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"] + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort)) + .processAux.tap == @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3"] From d72cb38861902345f16211058f955c5f59f21fd1 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 15 Oct 2020 20:08:49 -0700 Subject: [PATCH 07/15] works --- lib/std/globs.nim | 99 ++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 57 deletions(-) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index 437c468979bfe..3377a627d2adf 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -11,12 +11,13 @@ type ## absolute or relative path wrt globbed dir depth*: int ## depth wrt globbed dir + epilogue*: bool -iterator glob*(dir: string, relative = false, checkDir = true, includeRoot = false, +iterator glob*(dir: string, relative = false, checkDir = true, includeRoot = false, includeEpilogue = false, follow: proc(entry: PathEntry): bool = nil, sortCmp: proc (x, y: PathEntrySub): int = nil, topFirst = true): - PathEntry {.closure, tags: [ReadDirEffect].} = + PathEntry {.tags: [ReadDirEffect].} = ## Recursively walks `dir` which must exist when checkDir=true (else raises `OSError`). ## Paths in `result.path` are relative to `dir` unless `relative=false`, ## `result.depth >= 1` is the tree depth relative to the root `dir` (at depth 0). @@ -32,69 +33,53 @@ iterator glob*(dir: string, relative = false, checkDir = true, includeRoot = fal note: * a yieldFilter, regex match etc isn't needed because caller can filter at call site, without loss of generality, unlike `follow`; this simplifies the API. - * a closure iterator is used since we need multiple yield statements and it simplifies code. - In practice optimized performance drops by less than 2%, likely 0 when filesystem is not "hot". Future work: - * need to document - * add a `sort` option, which can be implemented efficiently only here, not at call site. * provide a way to do error reporting, which is tricky because iteration cannot be resumed * `walkDirRec` can be implemented in terms of this to avoid duplication, modulo some refactoring. ]# - echo() - var stack = @[(0, ".")] - var checkDir = checkDir - var entry: PathEntry - var dirsLevel: seq[PathEntrySub] - if not dirExists(dir): - if checkDir: - raise newException(OSError, "invalid root dir: " & dir) - else: - return + var entry = PathEntry(depth: 0, path: ".") + entry.kind = if symlinkExists(dir): pcLinkToDir else: pcDir + var stack: seq[PathEntry] - if includeRoot: - if symlinkExists(dir): - entry.kind = pcLinkToDir - else: - entry.kind = pcDir - entry.path = if relative: "." else: dir - normalizePath(entry.path) - entry.depth = 0 - yield entry - - template processEntry(): untyped = - let rel = d / p - entry.depth = depth + 1 - entry.kind = k - if relative: entry.path = rel - else: entry.path = dir / rel - normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 - if k in {pcDir, pcLinkToDir}: - if follow == nil or follow(entry): stack.add (depth + 1, rel) - entry + var checkDir = checkDir + if dirExists(dir): + stack.add entry + elif checkDir: + raise newException(OSError, "invalid root dir: " & dir) + var dirsLevel: seq[PathEntrySub] while stack.len > 0: - let (depth, d) = stack.pop() - # checkDir is still needed here in first iteration because things could - # fail for reasons other than `not dirExists`. + let current = stack.pop() + # let current = stack.popFront() + entry.epilogue = current.epilogue + entry.depth = current.depth + entry.kind = current.kind + entry.path = if relative: current.path else: dir / current.path + normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 - if sortCmp != nil: - dirsLevel.setLen 0 - # echo "...: ", d - for k, p in walkDir(dir / d, relative = true, checkDir = checkDir): - if sortCmp != nil: - dirsLevel.add PathEntrySub(kind: k, path: p) - else: - yield processEntry() - if sortCmp != nil: - sort(dirsLevel, sortCmp) - for ai in dirsLevel: - let k = ai.kind - let p = ai.path - yield processEntry() + if includeRoot or current.depth > 0: + yield entry - checkDir = false - # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong - # permissions), it'll abort iteration and there would be no way to - # continue iteration. + if current.kind in {pcDir, pcLinkToDir} and not current.epilogue: + if follow == nil or follow(current): + if sortCmp != nil: + dirsLevel.setLen 0 + if includeEpilogue: + stack.add PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) + # checkDir is still needed here in first iteration because things could + # fail for reasons other than `not dirExists`. + for k, p in walkDir(dir / current.path, relative = true, checkDir = checkDir): + if sortCmp != nil: + dirsLevel.add PathEntrySub(kind: k, path: p) + else: + stack.add PathEntry(depth: current.depth + 1, path: current.path / p, kind: k) + checkDir = false + # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong + # permissions), it'll abort iteration and there would be no way to resume iteration. + if sortCmp != nil: + sort(dirsLevel, sortCmp) + for i in countdown(dirsLevel.len-1, 0): + let ai = dirsLevel[i] + stack.add PathEntry(depth: current.depth + 1, path: current.path / ai.path, kind: ai.kind) From 3c49166a6153f1c68ea3a69b9bbbeb440c6e9527 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 15 Oct 2020 21:27:45 -0700 Subject: [PATCH 08/15] tests --- lib/std/globs.nim | 32 +++++++++++++++++++------------- tests/stdlib/tglobs.nim | 37 +++++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index 3377a627d2adf..7bf2d69b78533 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -1,5 +1,6 @@ import std/os import std/algorithm +import std/deques type PathEntrySub* = object @@ -12,10 +13,14 @@ type depth*: int ## depth wrt globbed dir epilogue*: bool + GlobMode* = enum + gDfs, gBfs + FollowCallback* = proc(entry: PathEntry): bool + SortCmpCallback* = proc (x, y: PathEntrySub): int -iterator glob*(dir: string, relative = false, checkDir = true, includeRoot = false, includeEpilogue = false, - follow: proc(entry: PathEntry): bool = nil, - sortCmp: proc (x, y: PathEntrySub): int = nil, +iterator glob*(dir: string, relative = false, checkDir = true, globMode = gDfs, includeRoot = false, includeEpilogue = false, followSymlinks = false, + follow: FollowCallback = nil, + sortCmp: SortCmpCallback = nil, topFirst = true): PathEntry {.tags: [ReadDirEffect].} = ## Recursively walks `dir` which must exist when checkDir=true (else raises `OSError`). @@ -41,18 +46,18 @@ iterator glob*(dir: string, relative = false, checkDir = true, includeRoot = fal ]# var entry = PathEntry(depth: 0, path: ".") entry.kind = if symlinkExists(dir): pcLinkToDir else: pcDir - var stack: seq[PathEntry] + # var stack: seq[PathEntry] + var stack = initDeque[PathEntry]() var checkDir = checkDir if dirExists(dir): - stack.add entry + stack.addLast entry elif checkDir: raise newException(OSError, "invalid root dir: " & dir) var dirsLevel: seq[PathEntrySub] while stack.len > 0: - let current = stack.pop() - # let current = stack.popFront() + let current = if globMode == gDfs: stack.popLast() else: stack.popFirst() entry.epilogue = current.epilogue entry.depth = current.depth entry.kind = current.kind @@ -62,24 +67,25 @@ iterator glob*(dir: string, relative = false, checkDir = true, includeRoot = fal if includeRoot or current.depth > 0: yield entry - if current.kind in {pcDir, pcLinkToDir} and not current.epilogue: + if (current.kind == pcDir or current.kind == pcLinkToDir and followSymlinks) and not current.epilogue: if follow == nil or follow(current): if sortCmp != nil: dirsLevel.setLen 0 if includeEpilogue: - stack.add PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) + stack.addLast PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) # checkDir is still needed here in first iteration because things could # fail for reasons other than `not dirExists`. for k, p in walkDir(dir / current.path, relative = true, checkDir = checkDir): if sortCmp != nil: dirsLevel.add PathEntrySub(kind: k, path: p) else: - stack.add PathEntry(depth: current.depth + 1, path: current.path / p, kind: k) + stack.addLast PathEntry(depth: current.depth + 1, path: current.path / p, kind: k) checkDir = false # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong # permissions), it'll abort iteration and there would be no way to resume iteration. if sortCmp != nil: sort(dirsLevel, sortCmp) - for i in countdown(dirsLevel.len-1, 0): - let ai = dirsLevel[i] - stack.add PathEntry(depth: current.depth + 1, path: current.path / ai.path, kind: ai.kind) + for i in 0..a.len>0) genTestPaths(dir, paths) - doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) - .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"] - doAssert toSeq(glob(dir, relative = true)) - .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] - doAssert toSeq(glob(dir, relative = true, includeRoot = true)) + + block: # follow + # filter by pcFile + doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true)) + .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"] + # filter by pcDir + doAssert toSeq(glob(dir, relative = true)) + .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + + block: # includeRoot + doAssert toSeq(glob(dir, relative = true, includeRoot = true)) .filterIt(it.kind == pcDir).process == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] - doAssertRaises(OSError): discard toSeq(glob("nonexistant")) - doAssertRaises(OSError): discard toSeq(glob("f5")) - doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] + block: # checkDir + doAssertRaises(OSError): discard toSeq(glob("nonexistant")) + doAssertRaises(OSError): discard toSeq(glob("f5")) + doAssert toSeq(glob("nonexistant", checkDir = false)) == @[] + + # sortCmp proc mySort(a, b: PathEntrySub): int = cmp(a.path, b.path) - proc mySort2(a, b: PathEntrySub): int = -cmp(a.path, b.path) - doAssert toSeq(glob(dir, relative = true, sortCmp = mySort2)).processAux == @["f5", "d2", "d1", "d1/f1.txt", "d1/d1b", "d1/d1a", "d1/d1a/f3", "d1/d1a/f2.txt", "d1/d1a/d1a1", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"] - doAssert toSeq(glob(dir, relative = true, sortCmp = mySort)) - .processAux.tap == @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3"] + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort)).processAux == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/f1.txt", "d2", "f5"] + + # gBfs + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, globMode = gBfs)).processAux == @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"] + + # includeEpilogue + doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux.tap == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] From 8ce0c6aabd62c18f19d647deebee6c7011f06d9b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Thu, 15 Oct 2020 22:04:02 -0700 Subject: [PATCH 09/15] new interface --- lib/std/globs.nim | 77 ++++++++++++++++++++++++++--------------- tests/stdlib/tglobs.nim | 4 +-- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index 7bf2d69b78533..21483088a9c2d 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -11,23 +11,41 @@ type path*: string ## absolute or relative path wrt globbed dir depth*: int - ## depth wrt globbed dir + ## depth wrt GlobOpt.dir (which is at depth 0) epilogue*: bool GlobMode* = enum - gDfs, gBfs + gDfs ## depth first search + gBfs ## breadth first search FollowCallback* = proc(entry: PathEntry): bool SortCmpCallback* = proc (x, y: PathEntrySub): int + GlobOpt* = object + dir*: string ## root of glob + relative: bool ## when true, paths are are returned relative to `dir`, else they start with `dir` + checkDir: bool ## if true, raises `OSError` when `dir` can't be listed. Deeper + ## directories do not cause `OSError`, and currently no error reporting is done for those. + globMode: GlobMode ## controls how paths are returned + includeRoot: bool ## whether to include root `dir` + includeEpilogue: bool + ## when false, yields: someDir, + ## when true, yields: someDir, , someDir: each dir is + ## yielded a 2nd time. This is useful in applications that aggregate data over dirs. + followSymlinks: bool ## whether to follow symlinks + follow: FollowCallback + ## if not `nil`, `glob` visits `entry` if `follow(entry) == true`. + sortCmp: SortCmpCallback + ## if not `nil`, immediate children of a dir are sorted using `sortCmp` -iterator glob*(dir: string, relative = false, checkDir = true, globMode = gDfs, includeRoot = false, includeEpilogue = false, followSymlinks = false, - follow: FollowCallback = nil, - sortCmp: SortCmpCallback = nil, - topFirst = true): - PathEntry {.tags: [ReadDirEffect].} = - ## Recursively walks `dir` which must exist when checkDir=true (else raises `OSError`). - ## Paths in `result.path` are relative to `dir` unless `relative=false`, - ## `result.depth >= 1` is the tree depth relative to the root `dir` (at depth 0). - ## if `follow != nil`, `glob` visits `entry` if `filter(entry) == true`. - ## This is more flexible than `os.walkDirRec`. +proc initGlobOpt*( + dir: string, relative = false, checkDir = true, globMode = gDfs, + includeRoot = false, includeEpilogue = false, followSymlinks = false, + follow: FollowCallback = nil, sortCmp: SortCmpCallback = nil): GlobOpt = + result = GlobOpt(dir: dir, relative: relative, checkDir: checkDir, globMode: globMode, includeRoot: includeRoot, includeEpilogue: includeEpilogue, followSymlinks: followSymlinks, follow: follow, sortCmp: sortCmp) + +iterator globOpt*(opt: GlobOpt): PathEntry = + ##[ + Recursively walks `dir`. + This is more flexible than `os.walkDirRec`. + ]## runnableExamples: import os,sugar if false: @@ -45,47 +63,50 @@ iterator glob*(dir: string, relative = false, checkDir = true, globMode = gDfs, modulo some refactoring. ]# var entry = PathEntry(depth: 0, path: ".") - entry.kind = if symlinkExists(dir): pcLinkToDir else: pcDir + entry.kind = if symlinkExists(opt.dir): pcLinkToDir else: pcDir # var stack: seq[PathEntry] var stack = initDeque[PathEntry]() - var checkDir = checkDir - if dirExists(dir): + var checkDir = opt.checkDir + if dirExists(opt.dir): stack.addLast entry elif checkDir: - raise newException(OSError, "invalid root dir: " & dir) + raise newException(OSError, "invalid root dir: " & opt.dir) var dirsLevel: seq[PathEntrySub] while stack.len > 0: - let current = if globMode == gDfs: stack.popLast() else: stack.popFirst() + let current = if opt.globMode == gDfs: stack.popLast() else: stack.popFirst() entry.epilogue = current.epilogue entry.depth = current.depth entry.kind = current.kind - entry.path = if relative: current.path else: dir / current.path + entry.path = if opt.relative: current.path else: opt.dir / current.path normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343 - if includeRoot or current.depth > 0: + if opt.includeRoot or current.depth > 0: yield entry - if (current.kind == pcDir or current.kind == pcLinkToDir and followSymlinks) and not current.epilogue: - if follow == nil or follow(current): - if sortCmp != nil: + if (current.kind == pcDir or current.kind == pcLinkToDir and opt.followSymlinks) and not current.epilogue: + if opt.follow == nil or opt.follow(current): + if opt.sortCmp != nil: dirsLevel.setLen 0 - if includeEpilogue: + if opt.includeEpilogue: stack.addLast PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true) # checkDir is still needed here in first iteration because things could # fail for reasons other than `not dirExists`. - for k, p in walkDir(dir / current.path, relative = true, checkDir = checkDir): - if sortCmp != nil: + for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir): + if opt.sortCmp != nil: dirsLevel.add PathEntrySub(kind: k, path: p) else: stack.addLast PathEntry(depth: current.depth + 1, path: current.path / p, kind: k) checkDir = false # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong # permissions), it'll abort iteration and there would be no way to resume iteration. - if sortCmp != nil: - sort(dirsLevel, sortCmp) + if opt.sortCmp != nil: + sort(dirsLevel, opt.sortCmp) for i in 0.. Date: Thu, 15 Oct 2020 22:16:11 -0700 Subject: [PATCH 10/15] _ --- lib/std/globs.nim | 27 +++++++++++++++------------ tests/stdlib/tglobs.nim | 6 +++++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/lib/std/globs.nim b/lib/std/globs.nim index 21483088a9c2d..8d861be0c6405 100644 --- a/lib/std/globs.nim +++ b/lib/std/globs.nim @@ -1,3 +1,13 @@ +#[ +## Design rationale +* an intermediate `GlobOpt` is used to allow easier proc forwarding +* a yieldFilter, regex match etc isn't needed because caller can filter at + call site, without loss of generality, unlike `follow`; this simplifies the API. + +## Future work: +* provide a way to do error reporting, which is tricky because iteration cannot be resumed +]# + import std/os import std/algorithm import std/deques @@ -52,23 +62,15 @@ iterator globOpt*(opt: GlobOpt): PathEntry = # list hidden files of depth <= 2 + 1 in your home. for e in glob(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2): if e.kind in {pcFile, pcLinkToFile}: echo e.path - #[ - note: - * a yieldFilter, regex match etc isn't needed because caller can filter at - call site, without loss of generality, unlike `follow`; this simplifies the API. - Future work: - * provide a way to do error reporting, which is tricky because iteration cannot be resumed - * `walkDirRec` can be implemented in terms of this to avoid duplication, - modulo some refactoring. - ]# var entry = PathEntry(depth: 0, path: ".") - entry.kind = if symlinkExists(opt.dir): pcLinkToDir else: pcDir - # var stack: seq[PathEntry] + # entry.kind = if symlinkExists(opt.dir): pcLinkToDir else: pcDir + entry.kind = if false: pcLinkToDir else: pcDir var stack = initDeque[PathEntry]() var checkDir = opt.checkDir - if dirExists(opt.dir): + # if dirExists(opt.dir): + if true: stack.addLast entry elif checkDir: raise newException(OSError, "invalid root dir: " & opt.dir) @@ -109,4 +111,5 @@ iterator globOpt*(opt: GlobOpt): PathEntry = stack.addLast PathEntry(depth: current.depth + 1, path: current.path / ai.path, kind: ai.kind) template glob*(args: varargs[untyped]): untyped = + ## convenience wrapper globOpt(initGlobOpt(args)) diff --git a/tests/stdlib/tglobs.nim b/tests/stdlib/tglobs.nim index 5fa225a5e42e0..ee44fd9312bce 100644 --- a/tests/stdlib/tglobs.nim +++ b/tests/stdlib/tglobs.nim @@ -9,7 +9,7 @@ proc process[T](a: T): seq[string] = a.processAux.sorted block: # glob - let dir = buildDir/"D20201013T100140" + const dir = buildDir/"D20201013T100140" defer: removeDir(dir) let paths = """ d1/f1.txt @@ -29,6 +29,9 @@ f5 # filter by pcDir doAssert toSeq(glob(dir, relative = true)) .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] + + const s = toSeq(glob(dir, relative = true)).filterIt(it.kind == pcDir).process + echo s block: # includeRoot doAssert toSeq(glob(dir, relative = true, includeRoot = true)) @@ -48,3 +51,4 @@ f5 # includeEpilogue doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] + From fb15b34c6ec120eb68a33d5193561e4963f0cc53 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Oct 2020 01:11:57 -0700 Subject: [PATCH 11/15] fixup --- tests/stdlib/tglobs.nim | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/stdlib/tglobs.nim b/tests/stdlib/tglobs.nim index ee44fd9312bce..10aa0e8570520 100644 --- a/tests/stdlib/tglobs.nim +++ b/tests/stdlib/tglobs.nim @@ -8,8 +8,9 @@ proc processAux[T](a: T): seq[string] = proc process[T](a: T): seq[string] = a.processAux.sorted +const dir = buildDir/"D20201013T100140" + block: # glob - const dir = buildDir/"D20201013T100140" defer: removeDir(dir) let paths = """ d1/f1.txt @@ -29,9 +30,6 @@ f5 # filter by pcDir doAssert toSeq(glob(dir, relative = true)) .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"] - - const s = toSeq(glob(dir, relative = true)).filterIt(it.kind == pcDir).process - echo s block: # includeRoot doAssert toSeq(glob(dir, relative = true, includeRoot = true)) @@ -52,3 +50,12 @@ f5 # includeEpilogue doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."] + +proc main()= + let s = toSeq(glob(dir, relative = true)) + +when false: + #[ + pending https://github.com/nim-lang/Nim/issues/15597 and https://github.com/nim-lang/Nim/issues/15595 + ]# + static: main() From 06af93909418bd3c9cb5cc00060911e1fec89141 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Oct 2020 01:13:17 -0700 Subject: [PATCH 12/15] changelog --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index f585f4e15b10b..181e3049f655d 100644 --- a/changelog.md +++ b/changelog.md @@ -3,7 +3,7 @@ ## Standard library additions and changes - +- Added `globs.glob`, more flexible than `walkDirRec`. ## Language changes From 2eddda767fbae077af58805df8e879919f161778 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Oct 2020 01:19:52 -0700 Subject: [PATCH 13/15] fixup --- compiler/docgen.nim | 2 +- lib/std/private/osutils.nim | 5 +---- tests/misc/trunner.nim | 2 +- tools/kochdocs.nim | 7 ++++--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index fb15f382677f1..c8521491807fb 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -19,7 +19,7 @@ import typesrenderer, astalgo, lineinfos, intsets, pathutils, trees, tables, nimpaths, renderverbatim, osproc -from std/private/globs import nativeToUnixPath +from std/private/osutils import nativeToUnixPath const exportSection = skField diff --git a/lib/std/private/osutils.nim b/lib/std/private/osutils.nim index 21e88f38b2ddd..eaa4ac88c23a6 100644 --- a/lib/std/private/osutils.nim +++ b/lib/std/private/osutils.nim @@ -2,10 +2,7 @@ unstable API, internal use only for now. ]## -import std/[os,strutils,globs] - -{.deprecated: [walkDirRecFilter: glob].} -export glob +import std/[os,strutils] proc nativeToUnixPath*(path: string): string = # pending https://github.com/nim-lang/Nim/pull/13265 diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index d67547d62cd79..41774ea92f6a2 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -11,7 +11,7 @@ import std/[strformat,os,osproc,unittest] from std/sequtils import toSeq,mapIt from std/algorithm import sorted import stdtest/[specialpaths, unittest_light] -from std/private/globs import nativeToUnixPath +from std/private/osutils import nativeToUnixPath import "$lib/../compiler/nimpaths" diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index ba2963cfd31bc..1e16f25abbe4d 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,7 +1,8 @@ ## Part of 'koch' responsible for the documentation generation. import os, strutils, osproc, sets, pathnorm, pegs -from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry +from std/private/osutils import nativeToUnixPath +from std/globs import globs, PathEntry import "../compiler/nimpaths" const @@ -99,7 +100,7 @@ proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options execFold(desc, cmd) proc getRst2html(): seq[string] = - for a in walkDirRecFilter("doc"): + for a in glob("doc"): let path = a.path if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin ["docs.rst", "nimfix.rst"]: @@ -188,7 +189,7 @@ lib/system/widestrs.nim proc follow(a: PathEntry): bool = a.path.lastPathPart notin ["nimcache", "htmldocs", "includes", "deprecated", "genode"] - for entry in walkDirRecFilter("lib", follow = follow): + for entry in glob("lib", follow = follow): let a = entry.path if entry.kind != pcFile or a.splitFile.ext != ".nim" or (a.isRelativeTo("lib/system") and a.nativeToUnixPath notin goodSystem) or From 4fe5ba382458e6caa533030c3e18e3c1d5c092f2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Oct 2020 01:20:43 -0700 Subject: [PATCH 14/15] _ --- tools/kochdocs.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 1e16f25abbe4d..125eb07f05d90 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -2,7 +2,7 @@ import os, strutils, osproc, sets, pathnorm, pegs from std/private/osutils import nativeToUnixPath -from std/globs import globs, PathEntry +from std/globs import glob, PathEntry import "../compiler/nimpaths" const From 60a7628ffc42da7a8daeaa64dd0c0d369b3f90a2 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Fri, 16 Oct 2020 12:45:20 -0700 Subject: [PATCH 15/15] fix for windows --- testament/lib/stdtest/osutils.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/testament/lib/stdtest/osutils.nim b/testament/lib/stdtest/osutils.nim index c998ebd017749..66eed75a3a39a 100644 --- a/testament/lib/stdtest/osutils.nim +++ b/testament/lib/stdtest/osutils.nim @@ -7,9 +7,9 @@ proc genTestPaths*(dir: string, paths: seq[string]) = for a in paths: doAssert not a.isAbsolute doAssert a.len > 0 - let a = dir / a + let a2 = dir / a if a.endsWith("/"): - createDir(a) + createDir(a2) else: - createDir(a.parentDir) - writeFile(a, "") + createDir(a2.parentDir) + writeFile(a2, "")