glob: more flexible than walkDirRec

changelog.md

@@ -3,7 +3,7 @@
 
 
 ## Standard library additions and changes
-
+- Added `globs.glob`, more flexible than `walkDirRec`.
 
 
 ## Language changes

compiler/docgen.nim

@@ -19,7 +19,7 @@ import
   typesrenderer, astalgo, lineinfos, intsets,
   pathutils, trees, tables, nimpaths, renderverbatim, osproc
 
-from std/private/globs import nativeToUnixPath
+from std/private/osutils import nativeToUnixPath
 
 const
   exportSection = skField

lib/std/globs.nim

@@ -0,0 +1,115 @@
+#[
+## Design rationale
+* an intermediate `GlobOpt` is used to allow easier proc forwarding
+* a yieldFilter, regex match etc isn't needed because caller can filter at
+  call site, without loss of generality, unlike `follow`; this simplifies the API.
+
+## Future work:
+* provide a way to do error reporting, which is tricky because iteration cannot be resumed
+]#
+
+import std/os
+import std/algorithm
+import std/deques
+
+type
+  PathEntrySub* = object
+    kind*: PathComponent
+    path*: string
+  PathEntry* = object
+    kind*: PathComponent
+    path*: string
+      ## absolute or relative path wrt globbed dir
+    depth*: int
+      ## depth wrt GlobOpt.dir (which is at depth 0)
+    epilogue*: bool
+  GlobMode* = enum
+    gDfs ## depth first search
+    gBfs ## breadth first search
+  FollowCallback* = proc(entry: PathEntry): bool
+  SortCmpCallback* = proc (x, y: PathEntrySub): int
+  GlobOpt* = object
+    dir*: string ## root of glob
+    relative: bool ## when true, paths are are returned relative to `dir`, else they start with `dir`
+    checkDir: bool ## if true, raises `OSError` when `dir` can't be listed. Deeper
+      ## directories do not cause `OSError`, and currently no error reporting is done for those.
+    globMode: GlobMode ## controls how paths are returned
+    includeRoot: bool ## whether to include root `dir`
+    includeEpilogue: bool
+      ## when false, yields: someDir, <children of someDir>
+      ## when true, yields: someDir, <children of someDir>, someDir: each dir is
+      ## yielded a 2nd time. This is useful in applications that aggregate data over dirs.
+    followSymlinks: bool ## whether to follow symlinks
+    follow: FollowCallback
+      ## if not `nil`, `glob` visits `entry` if `follow(entry) == true`.
+    sortCmp: SortCmpCallback
+      ## if not `nil`, immediate children of a dir are sorted using `sortCmp`
+
+proc initGlobOpt*(
+  dir: string, relative = false, checkDir = true, globMode = gDfs,
+  includeRoot = false, includeEpilogue = false, followSymlinks = false,
+  follow: FollowCallback = nil, sortCmp: SortCmpCallback = nil): GlobOpt =
+  result = GlobOpt(dir: dir, relative: relative, checkDir: checkDir, globMode: globMode, includeRoot: includeRoot, includeEpilogue: includeEpilogue, followSymlinks: followSymlinks, follow: follow, sortCmp: sortCmp)
+
+iterator globOpt*(opt: GlobOpt): PathEntry =
+  ##[
+  Recursively walks `dir`.
+  This is more flexible than `os.walkDirRec`.
+  ]##
+  runnableExamples:
+    import os,sugar
+    if false:
+      # list hidden files of depth <= 2 + 1 in your home.
+      for e in glob(getHomeDir(), follow = a=>a.path.isHidden and a.depth <= 2):
+        if e.kind in {pcFile, pcLinkToFile}: echo e.path
+
+  var entry = PathEntry(depth: 0, path: ".")
+  # entry.kind = if symlinkExists(opt.dir): pcLinkToDir else: pcDir
+  entry.kind = if false: pcLinkToDir else: pcDir
+  var stack = initDeque[PathEntry]()
+
+  var checkDir = opt.checkDir
+  # if dirExists(opt.dir):
+  if true:
+    stack.addLast entry
+  elif checkDir:
+    raise newException(OSError, "invalid root dir: " & opt.dir)
+
+  var dirsLevel: seq[PathEntrySub]
+  while stack.len > 0:
+    let current = if opt.globMode == gDfs: stack.popLast() else: stack.popFirst()
+    entry.epilogue = current.epilogue
+    entry.depth = current.depth
+    entry.kind = current.kind
+    entry.path = if opt.relative: current.path else: opt.dir / current.path
+    normalizePath(entry.path) # pending https://github.com/timotheecour/Nim/issues/343
+
+    if opt.includeRoot or current.depth > 0:
+      yield entry
+
+    if (current.kind == pcDir or current.kind == pcLinkToDir and opt.followSymlinks) and not current.epilogue:
+      if opt.follow == nil or opt.follow(current):
+        if opt.sortCmp != nil:
+          dirsLevel.setLen 0
+        if opt.includeEpilogue:
+          stack.addLast PathEntry(depth: current.depth, path: current.path, kind: current.kind, epilogue: true)
+        # checkDir is still needed here in first iteration because things could
+        # fail for reasons other than `not dirExists`.
+        for k, p in walkDir(opt.dir / current.path, relative = true, checkDir = checkDir):
+          if opt.sortCmp != nil:
+            dirsLevel.add PathEntrySub(kind: k, path: p)
+          else:
+            stack.addLast PathEntry(depth: current.depth + 1, path: current.path / p, kind: k)
+        checkDir = false
+          # We only check top-level dir, otherwise if a subdir is invalid (eg. wrong
+          # permissions), it'll abort iteration and there would be no way to resume iteration.
+        if opt.sortCmp != nil:
+          sort(dirsLevel, opt.sortCmp)
+          for i in 0..<dirsLevel.len:
+            let j = if opt.globMode == gDfs: dirsLevel.len-1-i else: i
+            let ai = dirsLevel[j]
+            stack.addLast PathEntry(depth: current.depth + 1, path: current.path / ai.path, kind: ai.kind)
+
+template glob*(args: varargs[untyped]): untyped =
+  ## convenience wrapper
+  globOpt(initGlobOpt(args))

lib/std/private/osutils.nim

@@ -0,0 +1,12 @@
+##[
+unstable API, internal use only for now.
+]##
+
+import std/[os,strutils]
+
+proc nativeToUnixPath*(path: string): string =
+  # pending https://github.com/nim-lang/Nim/pull/13265
+  doAssert not path.isAbsolute # not implemented here; absolute files need more care for the drive
+  when DirSep == '\\':
+    result = replace(path, '\\', '/')
+  else: result = path

testament/lib/stdtest/osutils.nim

@@ -0,0 +1,15 @@
+import std/[os,strutils]
+
+proc genTestPaths*(dir: string, paths: seq[string]) =
+  ## generates a filesystem rooted under `dir` from given relative `paths`.
+  ## `paths` ending in `/` are treated as directories.
+  # xxx use this in tos.nim
+  for a in paths:
+    doAssert not a.isAbsolute
+    doAssert a.len > 0
+    let a2 = dir / a
+    if a.endsWith("/"):
+      createDir(a2)
+    else:
+      createDir(a2.parentDir)
+      writeFile(a2, "")

tests/misc/trunner.nim

@@ -11,7 +11,7 @@ import std/[strformat,os,osproc,unittest]
 from std/sequtils import toSeq,mapIt
 from std/algorithm import sorted
 import stdtest/[specialpaths, unittest_light]
-from std/private/globs import nativeToUnixPath
+from std/private/osutils import nativeToUnixPath
 
 import "$lib/../compiler/nimpaths"
 

tests/stdlib/tglobs.nim

@@ -0,0 +1,61 @@
+import std/[sugar,globs,os,strutils,sequtils,algorithm]
+from std/private/osutils as osutils2 import nativeToUnixPath
+import stdtest/[specialpaths,osutils]
+
+proc processAux[T](a: T): seq[string] =
+  a.mapIt(it.path.nativeToUnixPath)
+
+proc process[T](a: T): seq[string] =
+  a.processAux.sorted
+
+const dir = buildDir/"D20201013T100140"
+
+block: # glob
+  defer: removeDir(dir)
+  let paths = """
+d1/f1.txt
+d1/d1a/f2.txt
+d1/d1a/f3
+d1/d1a/d1a1/
+d1/d1b/d1b1/f4
+d2/
+f5
+""".splitLines.filter(a=>a.len>0)
+  genTestPaths(dir, paths)
+
+  block: # follow
+    # filter by pcFile
+    doAssert toSeq(glob(dir, follow = a=>a.path.lastPathPart != "d1b", relative = true))
+      .filterIt(it.kind == pcFile).process == @["d1/d1a/f2.txt", "d1/d1a/f3", "d1/f1.txt", "f5"]
+    # filter by pcDir
+    doAssert toSeq(glob(dir, relative = true))
+      .filterIt(it.kind == pcDir).process == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"]
+
+  block: # includeRoot
+    doAssert toSeq(glob(dir, relative = true, includeRoot = true))
+    .filterIt(it.kind == pcDir).process == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1b", "d1/d1b/d1b1", "d2"]
+
+  block: # checkDir
+    doAssertRaises(OSError): discard toSeq(glob("nonexistant"))
+    doAssertRaises(OSError): discard toSeq(glob("f5"))
+    doAssert toSeq(glob("nonexistant", checkDir = false)) == @[]
+
+  # sortCmp
+  proc mySort(a, b: PathEntrySub): int = cmp(a.path, b.path)
+  doAssert toSeq(glob(dir, relative = true, sortCmp = mySort)).processAux == @["d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/f1.txt", "d2", "f5"]
+
+  # gBfs
+  doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, globMode = gBfs)).processAux == @["d1", "d2", "f5", "d1/d1a", "d1/d1b", "d1/f1.txt", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1b/d1b1", "d1/d1b/d1b1/f4"]
+
+  # includeEpilogue
+  doAssert toSeq(glob(dir, relative = true, sortCmp = mySort, includeEpilogue = true, includeRoot = true)).processAux == @[".", "d1", "d1/d1a", "d1/d1a/d1a1", "d1/d1a/d1a1", "d1/d1a/f2.txt", "d1/d1a/f3", "d1/d1a", "d1/d1b", "d1/d1b/d1b1", "d1/d1b/d1b1/f4", "d1/d1b/d1b1", "d1/d1b", "d1/f1.txt", "d1", "d2", "d2", "f5", "."]
+
+
+proc main()=
+  let s = toSeq(glob(dir, relative = true))
+
+when false:
+  #[
+  pending https://github.com/nim-lang/Nim/issues/15597 and https://github.com/nim-lang/Nim/issues/15595
+  ]#
+  static: main()

tools/kochdocs.nim

@@ -1,7 +1,8 @@
 ## Part of 'koch' responsible for the documentation generation.
 
 import os, strutils, osproc, sets, pathnorm, pegs
-from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry
+from std/private/osutils import nativeToUnixPath
+from std/globs import glob, PathEntry
 import "../compiler/nimpaths"
 
 const
@@ -99,7 +100,7 @@ proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options
   execFold(desc, cmd)
 
 proc getRst2html(): seq[string] =
-  for a in walkDirRecFilter("doc"):
+  for a in glob("doc"):
     let path = a.path
     if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin
         ["docs.rst", "nimfix.rst"]:
@@ -188,7 +189,7 @@ lib/system/widestrs.nim
 
   proc follow(a: PathEntry): bool =
     a.path.lastPathPart notin ["nimcache", "htmldocs", "includes", "deprecated", "genode"]
-  for entry in walkDirRecFilter("lib", follow = follow):
+  for entry in glob("lib", follow = follow):
     let a = entry.path
     if entry.kind != pcFile or a.splitFile.ext != ".nim" or
        (a.isRelativeTo("lib/system") and a.nativeToUnixPath notin goodSystem) or