nim-lang · a-mr · Jan 15, 2020 · Jan 19, 2020 · Jan 19, 2020 · Jan 20, 2020
diff --git a/changelog.md b/changelog.md
@@ -80,6 +80,7 @@
 - `htmlgen` adds [MathML](https://wikipedia.org/wiki/MathML) support
   (ISO 40314).
 - `macros.eqIdent` is now invariant to export markers and backtick quotes.
+- Added `os.tryWalkDir` iterator to traverse directories with error checking. 
 - `htmlgen.html` allows `lang` on the `<html>` tag and common valid attributes.
 - `macros.basename` and `basename=` got support for `PragmaExpr`,
   so that an expression like `MyEnum {.pure.}` is handled correctly.

diff --git a/lib/pure/os.nim b/lib/pure/os.nim
@@ -2020,7 +2020,14 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
   ##   dirA/fileA1.txt
   ##   dirA/fileA2.txt
   ##
+  ## Error checking is aimed at silent dropping: if path `dir` is missing
+  ## (or its access is denied) then the iterator will yield nothing;
+  ## all broken symlinks inside `dir` get default type ``pcLinkToFile``.
+  ## However, OSException may be raised if the directory handle is
+  ## invalidated during walking.
+  ##
   ## See also:
+  ## * `tryWalkDir iterator <#tryWalkDir.i,string>`_
   ## * `walkPattern iterator <#walkPattern.i,string>`_
   ## * `walkFiles iterator <#walkFiles.i,string>`_
   ## * `walkDirs iterator <#walkDirs.i,string>`_
@@ -2087,6 +2094,239 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path:
               k = getSymlinkFileKind(path)
             yield (k, y)
 
+type
+  WalkStatus* = enum  ## status of a step got from walking through directory.
+                      ## Due to differences between operating systems,
+                      ## this status may not have exactly the same meaning.
+                      ## It's yielded by
+                      ## `tryWalkDir iterator <#tryWalkDir.i,string>`.
+    wsOpenUnknown,  ## open directory error: unrecognized OS-specific one
+    wsOpenNotFound, ## open directory error: no such path
+    wsOpenNoAccess, ## open directory error: access denied a.k.a.
+                    ## permission denied error for the specified path
+                    ## (it may have happened at its parent directory on posix)
+    wsOpenNotDir,   ## open directory error: not a directory
+                    ## (a normal file with the same path exists or path is a
+                    ## loop of symlinks)
+    wsOpenBadPath,  ## open directory error: path is invalid (too long or,
+                    ## in Windows, it contains illegal characters)
+    wsOpenOk,       ## open directory OK: the directory can be read
+    wsEntryOk,      ## get entry OK: path component is correct
+    wsEntrySpecial  ## get entry OK: OS-specific entry
+                    ## ("special" or "device" file, not a normal data file)
+                    ## like Posix domain sockets, FIFOs, etc
+    wsEntryBad,     ## get entry error: its path is a broken symlink (on Posix),
+                    ## where it's unclear if it points to a file or directory
+    wsInterrupted   ## walking was interrupted while getting the next entry
+
+iterator tryWalkDir*(dir: string, relative=false):
+  tuple[status: WalkStatus, kind: PathComponent, path: string,
+        code: OSErrorCode] {.
+  tags: [ReadDirEffect], raises: [], noNimScript, since: (1, 1).} =
+  ## Walks over the directory `dir` and yields *steps* for each
+  ## directory or file in `dir`, non-recursively. It's a version of
+  ## `walkDir iterator <#walkDir.i,string>`_  with more thorough error
+  ## checking and reporting of "special" files or "defice" files.
+  ## Never raises an exception.
+  ##
+  ## Each step is a tuple containing ``status: WalkStatus``, path ``path``,
+  ## entry type ``kind: PathComponent``, OS error code ``code``.
+  ## 
+  ## - it yields open directory status **once** after opening
+  ##   (when `relative=true` the path is '.'),
+  ##   ``status`` is one of ``wsOpenOk``, ``wsOpenNoAccess``,
+  ##   ``wsOpenNotFound``, ``wsOpenNotDir``, ``wsOpenUnknown``
+  ## - then it yields zero or more entries,
+  ##   each can be one of the following type:
+  ##    - ``status=wsEntryOk``: signifies normal entries with
+  ##      path component ``kind``
+  ##    - ``status=wsEntrySpecial``: signifies special (non-data) files with
+  ##      path component ``kind``
+  ##    - ``status=wsEntryBad``: broken symlink (without path component)
+  ##    - ``status=wsInterrupted``: signifies that a rare OS-specific I/O error
+  ##      happenned and the walking was terminated.
+  ##
+  ## Path component ``kind`` value is reliable only when ``status=wsEntryOk``
+  ## or ``wsEntrySpecial``.
+  ##
+  ## **Examples:**
+  ##
+  ## .. code-block:: Nim
+  ##   # An example of usage with just a minimal error logging:
+  ##   for status, kind, path, code in tryWalkDir("dirA"):
+  ##     case status
+  ##     of wsOpenOk: discard
+  ##     of wsEntryOk: echo path & " is entry of kind " & $kind
+  ##     of wsEntrySpecial: echo path & " is a special file " & $kind
+  ##     else: echo "got error " & osErrorMsg(code) & " on " & path
+  ##
+  ##   # To just check whether the directory can be opened or not:
+  ##   proc tryOpenDir(dir: string): WalkStatus =
+  ##     for status, _, _, _ in tryWalkDir(dir):
+  ##       case status
+  ##       of wsOpenOk, wsOpenUnknown, wsOpenNotFound,
+  ##          wsOpenNoAccess, wsOpenNotDir, wsOpenBadPath: return status
+  ##       else: continue  # can not happen
+  ##   echo "can be opened: ", tryOpenDir("dirA") == wsOpenOk
+  ##
+  ##   # Iterator walkDir itself may be implemented using tryWalkDir:
+  ##   iterator myWalkDir(path: string, relative: bool):
+  ##                     tuple[kind: PathComponent, path: string] =
+  ##     for status, kind, path, code in tryWalkDir(path, relative):
+  ##       case status
+  ##       of wsOpenOk, wsOpenUnknown, wsOpenNotFound,
+  ##          wsOpenNoAccess, wsOpenNotDir, wsOpenBadPath: discard
+  ##       of wsEntryOk, wsEntrySpecial: yield (kind, path)
+  ##       of wsEntryBad: yield (pcLinkToFile, path)
+  ##       of wsInterrupted: raiseOSError(code)
+
+  var step: tuple[status: WalkStatus, kind: PathComponent,
+                  path: string, code: OSErrorCode]
+  var skip = false
+  let outDir = if relative: "." else: dir
+  template openStatus(s, p): auto =
+    let code = if s == wsOpenOk: OSErrorCode(0) else: osLastError()
+    (status: s, kind: pcDir, path: p, code: code)
+  template entryOk(pc, p): auto =
+    (status: wsEntryOk, kind: pc, path: p, code: OSErrorCode(0))
+  template entrySpecial(pc, p): auto =
+    (status: wsEntrySpecial, kind: pc, path: p, code: OSErrorCode(0))
+  template entryError(s, p): auto =
+    (status: s, kind: pcLinkToFile, path: p, code: osLastError())
+
+  when defined(windows):
+    template resolvePath(fdat: WIN32_FIND_DATA, relative: bool): type(step) =
+      var k: PathComponent
+      if (fdat.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32:
+        k = pcDir
+      let xx = if relative: extractFilename(getFilename(fdat))
+               else: dir / extractFilename(getFilename(fdat))
+      if (fdat.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32:
+        if (fdat.dwReserved0 and REPARSE_TAG_NAME_SURROGATE) != 0'u32:
+          k = succ(k)  # it's a "surrogate", that is symlink (or junction)
+          entryOk(k, xx)
+        else:  # some strange reparse point, considering it a normal file
+          entryOk(k, xx)
+      else:
+        entryOk(k, xx)
+
+    var f: WIN32_FIND_DATA
+    var h: Handle = findFirstFile(dir / "*", f)
+    let status =
+      if h != -1:
+        wsOpenOk
+      else:
+        case getLastError()
+        of ERROR_PATH_NOT_FOUND: wsOpenNotFound
+        of ERROR_INVALID_NAME: wsOpenBadPath
+        of ERROR_DIRECTORY: wsOpenNotDir
+        of ERROR_ACCESS_DENIED: wsOpenNoAccess
+        else: wsOpenUnknown
+    step = openStatus(status, outDir)
+    var firstFile = true
+
+    try:
+      while true:
+        if not skip:
+          yield step
+        if h == -1 or step.status == wsInterrupted:
+          break
+        if firstFile:  # use file obtained by findFirstFile
+          skip = skipFindData(f)
+          if not skip:
+            step = resolvePath(f, relative)
+          firstFile = false
+        else:  # load next file
+          if findNextFile(h, f) != 0'i32:
+            skip = skipFindData(f)
+            if not skip:
+              step = resolvePath(f, relative)
+          else:
+            let errCode = getLastError()
+            if errCode == ERROR_NO_MORE_FILES:
+              break  # normal end, yielding nothing
+            else:
+              skip = false
+              step = entryError(wsInterrupted, outDir)
+    finally:
+      if h != -1:
+        findClose(h)
+
+  else:
+    template resolveSymlink(p, y: string): type(step) =
+      var s: Stat
+      if stat(p, s) >= 0'i32:
+        if S_ISDIR(s.st_mode): entryOk(pcLinkToDir, y)
+        elif S_ISREG(s.st_mode): entryOk(pcLinkToFile, y)
+        else: entrySpecial(pcLinkToFile, y)
+      else: entryError(wsEntryBad, y)
+    template resolvePathLstat(path, y: string): type(step) =
+      var s: Stat
+      if lstat(path, s) < 0'i32:
+        entryError(wsEntryBad, y)
+      else:
+        if S_ISREG(s.st_mode): entryOk(pcFile, y)
+        elif S_ISDIR(s.st_mode): entryOk(pcDir, y)
+        elif S_ISLNK(s.st_mode): resolveSymlink(path, y)
+        else: entrySpecial(pcFile, y)
+    template resolvePath(ent: ptr Dirent; dir, entName: string;
+                         rel: bool): type(step) =
+      let path = dir / entName
+      let y = if rel: entName else: path
+      # fall back to pure-posix stat/lstat when d_type is not available or in
+      # case of d_type==DT_UNKNOWN (some filesystems can't report entry type)
+      when defined(linux) or defined(macosx) or
+           defined(bsd) or defined(genode) or defined(nintendoswitch):
+        if ent.d_type != DT_UNKNOWN:
+            if ent.d_type == DT_REG: entryOk(pcFile, y)
+            elif ent.d_type == DT_DIR: entryOk(pcDir, y)
+            elif ent.d_type == DT_LNK: resolveSymlink(path, y)
+            else: entrySpecial(pcFile, y)
+        else:
+          resolvePathLstat(path, y)
+      else:
+          resolvePathLstat(path, y)
+
+    var d: ptr DIR = opendir(dir)
+    let status =
+      if d != nil:
+        wsOpenOk
+      else:
+        if errno == ENOENT: wsOpenNotFound
+        elif errno == ENOTDIR or errno == ELOOP: wsOpenNotDir
+        elif errno == EACCES: wsOpenNoAccess
+        elif errno == ENAMETOOLONG: wsOpenBadPath
+        else: wsOpenUnknown
+    step = openStatus(status, outDir)
+    var name: string
+    var ent: ptr Dirent
+
+    try:
+      while true:
+        if not skip:
+          yield step
+        if d == nil or step.status == wsInterrupted:
+          break
+        let errnoSave = errno
+        errno = 0
+        ent = readdir(d)
+        if ent == nil:
+          if errno == 0:
+            errno = errnoSave
+            break  # normal end, yielding nothing
+          else:
+            skip = false
+            step = entryError(wsInterrupted, outDir)
+            errno = errnoSave
+        else:
+          name = $cstring(addr ent.d_name)
+          skip = (name == "." or name == "..")
+          if not skip:
+            step = resolvePath(ent, dir, name, relative)
+    finally:
+      if d != nil:
+        discard closedir(d)
+
 iterator walkDirRec*(dir: string,
                      yieldFilter = {pcFile}, followFilter = {pcDir},
                      relative = false): string {.tags: [ReadDirEffect].} =

diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim
@@ -312,6 +312,7 @@ const
   MOVEFILE_FAIL_IF_NOT_TRACKABLE* = 0x20'i32
   MOVEFILE_REPLACE_EXISTING* = 0x1'i32
   MOVEFILE_WRITE_THROUGH* = 0x8'i32
+  REPARSE_TAG_NAME_SURROGATE* = 0x20000000'u32
 
 type
   WIN32_FIND_DATA* {.pure.} = object
@@ -321,7 +322,7 @@ type
     ftLastWriteTime*: FILETIME
     nFileSizeHigh*: int32
     nFileSizeLow*: int32
-    dwReserved0: int32
+    dwReserved0*: uint32
     dwReserved1: int32
     cFileName*: array[0..(MAX_PATH) - 1, WinChar]
     cAlternateFileName*: array[0..13, WinChar]
@@ -703,7 +704,9 @@ const
   ERROR_NO_MORE_FILES* = 18
   ERROR_LOCK_VIOLATION* = 33
   ERROR_HANDLE_EOF* = 38
+  ERROR_INVALID_NAME* = 123
   ERROR_BAD_ARGUMENTS* = 165
+  ERROR_DIRECTORY* = 267
 
 proc duplicateHandle*(hSourceProcessHandle: Handle, hSourceHandle: Handle,
                       hTargetProcessHandle: Handle,

diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
@@ -25,7 +25,7 @@ Raises
 """
 # test os path creation, iteration, and deletion
 
-import os, strutils, pathnorm
+import os, strutils, pathnorm, algorithm
 
 block fileOperations:
   let files = @["these.txt", "are.x", "testing.r", "files.q"]
@@ -166,10 +166,41 @@ block walkDirRec:
 when not defined(windows):
   block walkDirRelative:
     createDir("walkdir_test")
-    createSymlink(".", "walkdir_test/c")
+    createSymlink(".", "walkdir_test/c_goodlink")
     for k, p in walkDir("walkdir_test", true):
       doAssert k == pcLinkToDir
+
+    var dir: seq[(WalkStatus, PathComponent, string, OSErrorCode)]
+    createDir("walkdir_test/d_dir")
+    createSymlink("walkdir_test/non_existent", "walkdir_test/e_broken")
+    open("walkdir_test/f_file.txt", fmWrite).close()
+    for step in tryWalkDir("walkdir_test", true):
+      dir.add(step)
+    proc myCmp(x, y: (WalkStatus, PathComponent, string, OSErrorCode)): int =
+      system.cmp(x[2], y[2])  # sort by entry name, self "." will be the first
+    dir.sort(myCmp)
+    doAssert dir.len == 5
+    # open step
+    doAssert dir[0][0] == wsOpenOk
+    doAssert dir[0][2] == "."
+    doAssert dir[0][3] == OSErrorCode(0)
+    # c_goodlink
+    doAssert dir[1] == (wsEntryOk, pcLinkToDir, "c_goodlink", OSErrorCode(0))
+    # d_dir
+    doAssert dir[2] == (wsEntryOk, pcDir, "d_dir", OSErrorCode(0))
+    # e_broken
+    doAssert dir[3][0] == wsEntryBad
+    doAssert dir[3][2] == "e_broken"
+    # f_file.txt
+    doAssert dir[4] == (wsEntryOk, pcFile, "f_file.txt", OSErrorCode(0))
     removeDir("walkdir_test")
+    # after remove tryWalkDir returns error:
+    dir = @[]
+    for step in tryWalkDir("walkdir_test", true):
+      dir.add(step)
+    doAssert dir.len == 1
+    doAssert dir[0][0] == wsOpenNotFound
+    doAssert dir[0][2] == "."
 
 block normalizedPath:
   doAssert normalizedPath("") == ""