From 3956d605d0d1d30480f823f8bdb074a11c4a8a11 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 16 Jan 2020 00:08:24 +0300 Subject: [PATCH 01/26] add error-checked version of walkDir --- lib/pure/os.nim | 98 +++++++++++++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 35 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 452c856a60463..ea1a8cc6c85ce 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1995,48 +1995,33 @@ proc staticWalkDir(dir: string; relative: bool): seq[ tuple[kind: PathComponent, path: string]] = discard -iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. - tags: [ReadDirEffect].} = - ## Walks over the directory `dir` and yields for each directory or file in - ## `dir`. The component type and full path for each item are returned. - ## - ## Walking is not recursive. If ``relative`` is true (default: false) - ## the resulting path is shortened to be relative to ``dir``. - ## Example: This directory structure:: - ## dirA / dirB / fileB1.txt - ## / dirC - ## / fileA1.txt - ## / fileA2.txt - ## - ## and this code: +iterator walkDirErr*(dir: string; relative=false): + tuple[kind: PathComponent, path: string, error: OSErrorCode] + {.tags: [ReadDirEffect].} = + ## The same as `walkDir iterator <#walkDir.i,string>`_ but with full error + ## checking: when an error at getting information about some path happened + ## then yields the path and the error code (`kind` may be wrong in this case). ## ## .. code-block:: Nim - ## for kind, path in walkDir("dirA"): - ## echo(path) - ## - ## produce this output (but not necessarily in this order!):: - ## dirA/dirB - ## dirA/dirC - ## dirA/fileA1.txt - ## dirA/fileA2.txt - ## - ## See also: - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + ## for kind, path, error in walkDirErr("dirA"): + ## if error == OSErrorCode(0): + ## echo path & " of kind " & $kind + ## else: + ## echo path & " GET INFO FAILED: " & osErrorMsg(error) when nimvm: for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) + yield (k, v, osLastError()) else: when weirdTarget: for k, v in items(staticWalkDir(dir, relative)): - yield (k, v) + yield (k, v, osLastError()) elif defined(windows): var f: WIN32_FIND_DATA var h = findFirstFile(dir / "*", f) - if h != -1: + if h == -1: + yield (pcFile, dir, osLastError()) + else: defer: findClose(h) while true: var k = pcFile @@ -2047,14 +2032,16 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: k = succ(k) let xx = if relative: extractFilename(getFilename(f)) else: dir / extractFilename(getFilename(f)) - yield (k, xx) + yield (k, xx, OSErrorCode(0)) if findNextFile(h, f) == 0'i32: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break else: raiseOSError(errCode.OSErrorCode) else: var d = opendir(dir) - if d != nil: + if d == nil: + yield(pcFile, dir, osLastError()) + else: defer: discard closedir(d) while true: var x = readdir(d) @@ -2074,18 +2061,59 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: defined(bsd) or defined(genode) or defined(nintendoswitch): if x.d_type != DT_UNKNOWN: if x.d_type == DT_DIR: k = pcDir + errno = 0 if x.d_type == DT_LNK: if dirExists(path): k = pcLinkToDir else: k = pcLinkToFile - yield (k, y) + yield (k, y, osLastError()) # check error in dirExists continue + errno = 0 if lstat(path, s) < 0'i32: break if S_ISDIR(s.st_mode): k = pcDir elif S_ISLNK(s.st_mode): k = getSymlinkFileKind(path) - yield (k, y) + yield (k, y, osLastError()) # check error in getSymlinkFileKind + +iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. + tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. If ``relative`` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``. + ## Example: This directory structure:: + ## dirA / dirB / fileB1.txt + ## / dirC + ## / fileA1.txt + ## / fileA2.txt + ## + ## and this code: + ## + ## .. code-block:: Nim + ## for kind, path in walkDir("dirA"): + ## echo(path) + ## + ## produce this output (but not necessarily in this order!):: + ## dirA/dirB + ## dirA/dirC + ## dirA/fileA1.txt + ## dirA/fileA2.txt + ## + ## Error checking is minimal and aimed at dropping: if `dir` is not an open + ## directory the iterator yields nothing; all closed symlinks inside `dir` + ## are silently omitted. + ## + ## See also: + ## * `walkDirErr iterator <#walkDirErr.i,string>`_ + ## * `walkPattern iterator <#walkPattern.i,string>`_ + ## * `walkFiles iterator <#walkFiles.i,string>`_ + ## * `walkDirs iterator <#walkDirs.i,string>`_ + ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + for kind, path, error in walkDirErr(dir, relative): + if error == OSErrorCode(0): + yield (kind, path) iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}, From f2c700e3a0e93584e59f876fcbff59786fc5fb2b Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 19 Jan 2020 19:20:21 +0300 Subject: [PATCH 02/26] next iteration: return variant type --- lib/pure/os.nim | 125 +++++++++++++++++++++++++++++++--------- lib/windows/winlean.nim | 1 + 2 files changed, 100 insertions(+), 26 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index ea1a8cc6c85ce..a6065c8621cf7 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1995,34 +1995,73 @@ proc staticWalkDir(dir: string; relative: bool): seq[ tuple[kind: PathComponent, path: string]] = discard -iterator walkDirErr*(dir: string; relative=false): - tuple[kind: PathComponent, path: string, error: OSErrorCode] - {.tags: [ReadDirEffect].} = +type openDirStatus* = enum + odOpenOk, + odNotDir, + odAccessDenied, + odNotFound, + odUnknownError + +type WalkStepKind* = enum + wiOpenDir, ## attempt to open directory + wiEntryOk, ## the entry is OK + wiEntryBad ## (Posix-specific) dangling symlink, so unclear if it + ## points to a file or directory + wiInterrupted ## the directory handle got invalidated during reading + +type WalkStep = ref object + code*: OSErrorCode + path*: string + case kind*: WalkStepKind + of wiOpenDir: + openStatus*: openDirStatus + of wiEntryOk: + entryType*: PathComponent + of wiEntryBad, wiInterrupted: + discard + +iterator tryWalkDir*(dir: string; relative=false): WalkStep {. + tags: [ReadDirEffect], raises: [].} = ## The same as `walkDir iterator <#walkDir.i,string>`_ but with full error ## checking: when an error at getting information about some path happened ## then yields the path and the error code (`kind` may be wrong in this case). ## ## .. code-block:: Nim - ## for kind, path, error in walkDirErr("dirA"): - ## if error == OSErrorCode(0): - ## echo path & " of kind " & $kind - ## else: - ## echo path & " GET INFO FAILED: " & osErrorMsg(error) + ## for info, path in tryWalkDir("dirA"): + ## case info.kind + ## of wiEntryOk: echo "found " & path & " of kind " & $info.entryType + ## of wiOpenError: echo "directory " & dir & " could not be opened " & + ## $info.openError & " / " & osErrorMsg(info.code) + ## of wiEntryBad: echo "bad symlink " & path & " with error " & + ## osErrorMsg(info.code) + + proc noErrors(pc: PathComponent, path: string): WalkStep = + WalkStep(kind: wiEntryOk, entryType: pc, code: OSErrorCode(0), path: path) + proc openDir(s: openDirStatus, path: string): WalkStep = + WalkStep(kind: wiOpenDir, openStatus: s, code: osLastError(), path: path) + proc getError(k: WalkStepKind, path: string): WalkStep = + WalkStep(kind: k, code: osLastError(), path: path) when nimvm: for k, v in items(staticWalkDir(dir, relative)): - yield (k, v, osLastError()) + yield noErrors(k, v) else: when weirdTarget: for k, v in items(staticWalkDir(dir, relative)): - yield (k, v, osLastError()) + yield noErrors(k, v) elif defined(windows): var f: WIN32_FIND_DATA var h = findFirstFile(dir / "*", f) if h == -1: - yield (pcFile, dir, osLastError()) + let errCode = getLastError() + case errCode + of ERROR_PATH_NOT_FOUND: yield openDir(odNotFound, dir) + of ERROR_DIRECTORY: yield openDir(odNotDir, dir) + of ERROR_ACCESS_DENIED: yield openDir(odAccessDenied, dir) + else: yield openDir(odUnknownError, dir) else: defer: findClose(h) + yield openDir(odOpenOk, dir) while true: var k = pcFile if not skipFindData(f): @@ -2032,20 +2071,28 @@ iterator walkDirErr*(dir: string; relative=false): k = succ(k) let xx = if relative: extractFilename(getFilename(f)) else: dir / extractFilename(getFilename(f)) - yield (k, xx, OSErrorCode(0)) + yield noErrors(k, xx) if findNextFile(h, f) == 0'i32: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break - else: raiseOSError(errCode.OSErrorCode) + else: yield getError(wiInterrupted, dir) else: var d = opendir(dir) if d == nil: - yield(pcFile, dir, osLastError()) + case errno + of ENOENT: yield openDir(odNotFound, dir) + of ENOTDIR: yield openDir(odNotDir, dir) + of EACCES: yield openDir(odAccessDenied, dir) + else: yield openDir(odUnknownError, dir) else: defer: discard closedir(d) + yield openDir(odOpenOk, dir) while true: + errno = 0 var x = readdir(d) - if x == nil: break + if x == nil: + if errno != 0: yield getError(wiInterrupted, dir) + break when defined(nimNoArrayToCstringConversion): var y = $cstring(addr x.d_name) else: @@ -2060,12 +2107,19 @@ iterator walkDirErr*(dir: string; relative=false): when defined(linux) or defined(macosx) or defined(bsd) or defined(genode) or defined(nintendoswitch): if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: k = pcDir - errno = 0 - if x.d_type == DT_LNK: + if x.d_type == DT_DIR: + k = pcDir + yield noErrors(k, y) + elif x.d_type == DT_LNK: + errno = 0 if dirExists(path): k = pcLinkToDir else: k = pcLinkToFile - yield (k, y, osLastError()) # check error in dirExists + if errno == 0: # check error in getSymlinkFileKind + yield noErrors(k, y) + else: + yield getError(wiEntryBad, y) + else: + yield noErrors(k, y) continue errno = 0 @@ -2074,7 +2128,18 @@ iterator walkDirErr*(dir: string; relative=false): k = pcDir elif S_ISLNK(s.st_mode): k = getSymlinkFileKind(path) - yield (k, y, osLastError()) # check error in getSymlinkFileKind + if errno == 0: # check error in getSymlinkFileKind + yield noErrors(k, y) + else: + yield getError(wiEntryBad, y) + +proc tryOpenDir*(dir: string): openDirStatus {. + tags: [ReadDirEffect], raises: [], since:(1,1) .} = + # TODO + for step in tryWalkDir(dir): + case step.kind + of wiOpenDir: return step.openStatus + else: break # can not happen iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. tags: [ReadDirEffect].} = @@ -2101,9 +2166,12 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: ## dirA/fileA1.txt ## dirA/fileA2.txt ## - ## Error checking is minimal and aimed at dropping: if `dir` is not an open - ## directory the iterator yields nothing; all closed symlinks inside `dir` - ## are silently omitted. + ## Error checking is aimed at dropping bad entries: if path `dir` is not + ## found or is a file or is an access-denied directory then the iterator + ## yields nothing; all closed symlinks inside `dir` are silently omitted. + ## However, OSException is raised when opening directory resulted to + ## an unrecognized error or the directory handle was invalidated during + ## the traversal. ## ## See also: ## * `walkDirErr iterator <#walkDirErr.i,string>`_ @@ -2111,9 +2179,14 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: ## * `walkFiles iterator <#walkFiles.i,string>`_ ## * `walkDirs iterator <#walkDirs.i,string>`_ ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - for kind, path, error in walkDirErr(dir, relative): - if error == OSErrorCode(0): - yield (kind, path) + for step in tryWalkDir(dir, relative): + case step.kind + of wiOpenDir: + if step.openStatus == odUnknownError: raiseOSError(step.code) + else: discard + of wiEntryOk: yield (step.entryType, step.path) + of wiEntryBad: discard + of wiInterrupted: raiseOSError(step.code) iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}, diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 649358cb6f9ac..1ced2f5fb6051 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -704,6 +704,7 @@ const ERROR_LOCK_VIOLATION* = 33 ERROR_HANDLE_EOF* = 38 ERROR_BAD_ARGUMENTS* = 165 + ERROR_DIRECTORY* = 267 proc duplicateHandle*(hSourceProcessHandle: Handle, hSourceHandle: Handle, hTargetProcessHandle: Handle, From b3b468f4b666b432232e99dd3defa5dfd6856f41 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 19 Jan 2020 19:38:21 +0300 Subject: [PATCH 03/26] fix for --newruntime --- lib/pure/os.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a6065c8621cf7..e648a401bdee7 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2035,11 +2035,11 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. ## of wiEntryBad: echo "bad symlink " & path & " with error " & ## osErrorMsg(info.code) - proc noErrors(pc: PathComponent, path: string): WalkStep = + proc noErrors(pc: PathComponent, path: string): owned(WalkStep) = WalkStep(kind: wiEntryOk, entryType: pc, code: OSErrorCode(0), path: path) - proc openDir(s: openDirStatus, path: string): WalkStep = + proc openDir(s: openDirStatus, path: string): owned(WalkStep) = WalkStep(kind: wiOpenDir, openStatus: s, code: osLastError(), path: path) - proc getError(k: WalkStepKind, path: string): WalkStep = + proc getError(k: WalkStepKind, path: string): owned(WalkStep) = WalkStep(kind: k, code: osLastError(), path: path) when nimvm: @@ -2184,7 +2184,7 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: of wiOpenDir: if step.openStatus == odUnknownError: raiseOSError(step.code) else: discard - of wiEntryOk: yield (step.entryType, step.path) + of wiEntryOk: yield (step.entryType, move(step.path)) of wiEntryBad: discard of wiInterrupted: raiseOSError(step.code) From b06b5124f8641b9199440aa64279c4f3cf1abfca Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 20 Jan 2020 21:20:01 +0300 Subject: [PATCH 04/26] try to fix: cannot evaluate at compile time: ENOENT --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index e648a401bdee7..4ffec6b048ba3 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2021,7 +2021,7 @@ type WalkStep = ref object discard iterator tryWalkDir*(dir: string; relative=false): WalkStep {. - tags: [ReadDirEffect], raises: [].} = + tags: [ReadDirEffect], raises: [], noNimScript .} = ## The same as `walkDir iterator <#walkDir.i,string>`_ but with full error ## checking: when an error at getting information about some path happened ## then yields the path and the error code (`kind` may be wrong in this case). From 6a2e9a0824821e668c31ae0a750a3151df99dcfa Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 20 Jan 2020 22:44:19 +0300 Subject: [PATCH 05/26] stylistic changes & comment update --- lib/pure/os.nim | 111 ++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 50 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 4ffec6b048ba3..8b77f643c5588 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2003,65 +2003,72 @@ type openDirStatus* = enum odUnknownError type WalkStepKind* = enum - wiOpenDir, ## attempt to open directory - wiEntryOk, ## the entry is OK - wiEntryBad ## (Posix-specific) dangling symlink, so unclear if it + wsOpenDir, ## attempt to open directory + wsEntryOk, ## the entry is OK + wsEntryBad ## (Posix-specific) dangling symlink, so unclear if it ## points to a file or directory - wiInterrupted ## the directory handle got invalidated during reading + wsInterrupted ## the directory handle got invalidated during reading type WalkStep = ref object code*: OSErrorCode path*: string case kind*: WalkStepKind - of wiOpenDir: + of wsOpenDir: openStatus*: openDirStatus - of wiEntryOk: + of wsEntryOk: entryType*: PathComponent - of wiEntryBad, wiInterrupted: + of wsEntryBad, wsInterrupted: discard iterator tryWalkDir*(dir: string; relative=false): WalkStep {. - tags: [ReadDirEffect], raises: [], noNimScript .} = - ## The same as `walkDir iterator <#walkDir.i,string>`_ but with full error - ## checking: when an error at getting information about some path happened - ## then yields the path and the error code (`kind` may be wrong in this case). + tags: [ReadDirEffect], raises: [], noNimScript.} = + ## An analogue of `walkDir iterator <#walkDir.i,string>`_ with full error + ## checking. Yields "steps" of walking through `dir`, each step contains its path `path`, OS error code `code` and additional info. At first step yields info with the status of opening `dir` as a directory, tag `wsOpenDir`. If it's OK, + ## the subsequent steps will yield the directory entries with file/directory type as info, tag `wsEntryOk`. Dangling symlinks on posix are reported as wsEntryBad. If (rarely) walking terminates with an error, tag is `wsInterrupted`. ## ## .. code-block:: Nim - ## for info, path in tryWalkDir("dirA"): - ## case info.kind - ## of wiEntryOk: echo "found " & path & " of kind " & $info.entryType - ## of wiOpenError: echo "directory " & dir & " could not be opened " & - ## $info.openError & " / " & osErrorMsg(info.code) - ## of wiEntryBad: echo "bad symlink " & path & " with error " & - ## osErrorMsg(info.code) - - proc noErrors(pc: PathComponent, path: string): owned(WalkStep) = - WalkStep(kind: wiEntryOk, entryType: pc, code: OSErrorCode(0), path: path) - proc openDir(s: openDirStatus, path: string): owned(WalkStep) = - WalkStep(kind: wiOpenDir, openStatus: s, code: osLastError(), path: path) - proc getError(k: WalkStepKind, path: string): owned(WalkStep) = + ## for step in tryWalkDir("dirA"): + ## let err = " (" & $step.code & " - " & osErrorMsg(step.code) & ")" + ## case step.kind + ## of wsOpenDir: + ## case step.openStatus: + ## of odOpenOk: echo dir & " - directory is opened OK" + ## of odNotDir: echo dir & " is not a directory" & err + ## of odAccessDenied: echo dir & " - Access Denied" & err + ## of odNotFound: echo dir & " - no such path" & err + ## of odUnknownError: echo dir & "unknown error - that's bad!" & err + ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType + ## of wsEntryBad: echo step.path & " is broken symlink " & err + ## of wsInterrupted: + ## echo dir & " traversal interrupted, really bad! " & err + + proc sNoErrors(pc: PathComponent, path: string): owned(WalkStep) = + WalkStep(kind: wsEntryOk, entryType: pc, code: OSErrorCode(0), path: path) + proc sOpenDir(s: openDirStatus, path: string): owned(WalkStep) = + WalkStep(kind: wsOpenDir, openStatus: s, code: osLastError(), path: path) + proc sGetError(k: WalkStepKind, path: string): owned(WalkStep) = WalkStep(kind: k, code: osLastError(), path: path) when nimvm: for k, v in items(staticWalkDir(dir, relative)): - yield noErrors(k, v) + yield sNoErrors(k, v) else: when weirdTarget: for k, v in items(staticWalkDir(dir, relative)): - yield noErrors(k, v) + yield sNoErrors(k, v) elif defined(windows): var f: WIN32_FIND_DATA var h = findFirstFile(dir / "*", f) if h == -1: let errCode = getLastError() case errCode - of ERROR_PATH_NOT_FOUND: yield openDir(odNotFound, dir) - of ERROR_DIRECTORY: yield openDir(odNotDir, dir) - of ERROR_ACCESS_DENIED: yield openDir(odAccessDenied, dir) - else: yield openDir(odUnknownError, dir) + of ERROR_PATH_NOT_FOUND: yield sOpenDir(odNotFound, dir) + of ERROR_DIRECTORY: yield sOpenDir(odNotDir, dir) + of ERROR_ACCESS_DENIED: yield sOpenDir(odAccessDenied, dir) + else: yield sOpenDir(odUnknownError, dir) else: defer: findClose(h) - yield openDir(odOpenOk, dir) + yield sOpenDir(odOpenOk, dir) while true: var k = pcFile if not skipFindData(f): @@ -2071,27 +2078,31 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. k = succ(k) let xx = if relative: extractFilename(getFilename(f)) else: dir / extractFilename(getFilename(f)) - yield noErrors(k, xx) + yield sNoErrors(k, xx) if findNextFile(h, f) == 0'i32: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break - else: yield getError(wiInterrupted, dir) + else: yield sGetError(wsInterrupted, dir) else: + var init = false + #while true: + # if not init + var d = opendir(dir) if d == nil: case errno - of ENOENT: yield openDir(odNotFound, dir) - of ENOTDIR: yield openDir(odNotDir, dir) - of EACCES: yield openDir(odAccessDenied, dir) - else: yield openDir(odUnknownError, dir) + of ENOENT: yield sOpenDir(odNotFound, dir) + of ENOTDIR: yield sOpenDir(odNotDir, dir) + of EACCES: yield sOpenDir(odAccessDenied, dir) + else: yield sOpenDir(odUnknownError, dir) else: defer: discard closedir(d) - yield openDir(odOpenOk, dir) + yield sOpenDir(odOpenOk, dir) while true: errno = 0 var x = readdir(d) if x == nil: - if errno != 0: yield getError(wiInterrupted, dir) + if errno != 0: yield sGetError(wsInterrupted, dir) break when defined(nimNoArrayToCstringConversion): var y = $cstring(addr x.d_name) @@ -2109,17 +2120,17 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. if x.d_type != DT_UNKNOWN: if x.d_type == DT_DIR: k = pcDir - yield noErrors(k, y) + yield sNoErrors(k, y) elif x.d_type == DT_LNK: errno = 0 if dirExists(path): k = pcLinkToDir else: k = pcLinkToFile if errno == 0: # check error in getSymlinkFileKind - yield noErrors(k, y) + yield sNoErrors(k, y) else: - yield getError(wiEntryBad, y) + yield sGetError(wsEntryBad, y) else: - yield noErrors(k, y) + yield sNoErrors(k, y) continue errno = 0 @@ -2129,16 +2140,16 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. elif S_ISLNK(s.st_mode): k = getSymlinkFileKind(path) if errno == 0: # check error in getSymlinkFileKind - yield noErrors(k, y) + yield sNoErrors(k, y) else: - yield getError(wiEntryBad, y) + yield sGetError(wsEntryBad, y) proc tryOpenDir*(dir: string): openDirStatus {. tags: [ReadDirEffect], raises: [], since:(1,1) .} = # TODO for step in tryWalkDir(dir): case step.kind - of wiOpenDir: return step.openStatus + of wsOpenDir: return step.openStatus else: break # can not happen iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. @@ -2181,12 +2192,12 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: ## * `walkDirRec iterator <#walkDirRec.i,string>`_ for step in tryWalkDir(dir, relative): case step.kind - of wiOpenDir: + of wsOpenDir: if step.openStatus == odUnknownError: raiseOSError(step.code) else: discard - of wiEntryOk: yield (step.entryType, move(step.path)) - of wiEntryBad: discard - of wiInterrupted: raiseOSError(step.code) + of wsEntryOk: yield (step.entryType, move(step.path)) + of wsEntryBad: discard + of wsInterrupted: raiseOSError(step.code) iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}, From 682f264f3e585dadfed6df2cf614554389f9ba04 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 20 Jan 2020 22:50:28 +0300 Subject: [PATCH 06/26] fix FreeBSD build --- lib/pure/os.nim | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 8b77f643c5588..00df26944c71a 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2085,15 +2085,12 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. else: yield sGetError(wsInterrupted, dir) else: var init = false - #while true: - # if not init var d = opendir(dir) if d == nil: - case errno - of ENOENT: yield sOpenDir(odNotFound, dir) - of ENOTDIR: yield sOpenDir(odNotDir, dir) - of EACCES: yield sOpenDir(odAccessDenied, dir) + if errno == ENOENT: yield sOpenDir(odNotFound, dir) + elif errno == ENOTDIR: yield sOpenDir(odNotDir, dir) + elif errno == EACCES: yield sOpenDir(odAccessDenied, dir) else: yield sOpenDir(odUnknownError, dir) else: defer: discard closedir(d) From 042cbf7bab731e64cfa03ce5d7381143a0c73ade Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 20 Jan 2020 23:56:22 +0300 Subject: [PATCH 07/26] rewrite posix part to use only 1 yield --- lib/pure/os.nim | 75 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 00df26944c71a..91431725e4168 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2084,28 +2084,50 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. if errCode == ERROR_NO_MORE_FILES: break else: yield sGetError(wsInterrupted, dir) else: - var init = false + var init: bool + var lastIter: bool + var skipYield: bool + var res: WalkStep + var d: ptr DIR - var d = opendir(dir) - if d == nil: - if errno == ENOENT: yield sOpenDir(odNotFound, dir) - elif errno == ENOTDIR: yield sOpenDir(odNotDir, dir) - elif errno == EACCES: yield sOpenDir(odAccessDenied, dir) - else: yield sOpenDir(odUnknownError, dir) - else: - defer: discard closedir(d) - yield sOpenDir(odOpenOk, dir) - while true: + while true: + if init and not skipYield: + yield res + if lastIter: + break + skipYield = false + if not init: + init = true + d = opendir(dir) + if d == nil: + res = + if errno == ENOENT: sOpenDir(odNotFound, dir) + elif errno == ENOTDIR: sOpenDir(odNotDir, dir) + elif errno == EACCES: sOpenDir(odAccessDenied, dir) + else: sOpenDir(odUnknownError, dir) + lastIter = true + continue + else: + res = sOpenDir(odOpenOk, dir) + continue + else: errno = 0 var x = readdir(d) if x == nil: - if errno != 0: yield sGetError(wsInterrupted, dir) - break + if errno == 0: + break # normal end, yielding nothing + else: + res = sGetError(wsInterrupted, dir) + lastIter = true + continue when defined(nimNoArrayToCstringConversion): var y = $cstring(addr x.d_name) else: var y = $x.d_name.cstring - if y != "." and y != "..": + if y == "." or y == "..": + skipYield = true + continue + else: var s: Stat let path = dir / y if not relative: @@ -2117,19 +2139,24 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. if x.d_type != DT_UNKNOWN: if x.d_type == DT_DIR: k = pcDir - yield sNoErrors(k, y) + res = sNoErrors(k, y) + continue elif x.d_type == DT_LNK: errno = 0 if dirExists(path): k = pcLinkToDir else: k = pcLinkToFile - if errno == 0: # check error in getSymlinkFileKind - yield sNoErrors(k, y) + if errno == 0: # check error in dirExists + res = sNoErrors(k, y) + continue else: - yield sGetError(wsEntryBad, y) + res = sGetError(wsEntryBad, y) + continue else: - yield sNoErrors(k, y) - continue + res = sNoErrors(k, y) + continue + # special case of DT_UNKNOWN: some filesystems can't detect that + # entry is a directory and leave its d_type as 0=DT_UNKNOWN errno = 0 if lstat(path, s) < 0'i32: break if S_ISDIR(s.st_mode): @@ -2137,9 +2164,13 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. elif S_ISLNK(s.st_mode): k = getSymlinkFileKind(path) if errno == 0: # check error in getSymlinkFileKind - yield sNoErrors(k, y) + res = sNoErrors(k, y) + continue else: - yield sGetError(wsEntryBad, y) + res = sGetError(wsEntryBad, y) + continue + if d != nil: + discard closedir(d) proc tryOpenDir*(dir: string): openDirStatus {. tags: [ReadDirEffect], raises: [], since:(1,1) .} = From 245d214982c520b1b2c20b06beecf4e2614aeacf Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 21 Jan 2020 00:11:35 +0300 Subject: [PATCH 08/26] fix build error --- lib/pure/os.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 91431725e4168..5743cf878be64 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1995,7 +1995,7 @@ proc staticWalkDir(dir: string; relative: bool): seq[ tuple[kind: PathComponent, path: string]] = discard -type openDirStatus* = enum +type OpenDirStatus* = enum odOpenOk, odNotDir, odAccessDenied, @@ -2014,7 +2014,7 @@ type WalkStep = ref object path*: string case kind*: WalkStepKind of wsOpenDir: - openStatus*: openDirStatus + openStatus*: OpenDirStatus of wsEntryOk: entryType*: PathComponent of wsEntryBad, wsInterrupted: @@ -2044,7 +2044,7 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. proc sNoErrors(pc: PathComponent, path: string): owned(WalkStep) = WalkStep(kind: wsEntryOk, entryType: pc, code: OSErrorCode(0), path: path) - proc sOpenDir(s: openDirStatus, path: string): owned(WalkStep) = + proc sOpenDir(s: OpenDirStatus, path: string): owned(WalkStep) = WalkStep(kind: wsOpenDir, openStatus: s, code: osLastError(), path: path) proc sGetError(k: WalkStepKind, path: string): owned(WalkStep) = WalkStep(kind: k, code: osLastError(), path: path) @@ -2172,7 +2172,7 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. if d != nil: discard closedir(d) -proc tryOpenDir*(dir: string): openDirStatus {. +proc tryOpenDir*(dir: string): OpenDirStatus {. tags: [ReadDirEffect], raises: [], since:(1,1) .} = # TODO for step in tryWalkDir(dir): From d867f1c6b3299775482c1a9169e1c5e0e510c807 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 21 Jan 2020 21:48:32 +0300 Subject: [PATCH 09/26] remove back noNimScript --- lib/pure/os.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 5743cf878be64..b5d3ffabb6afb 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2021,7 +2021,7 @@ type WalkStep = ref object discard iterator tryWalkDir*(dir: string; relative=false): WalkStep {. - tags: [ReadDirEffect], raises: [], noNimScript.} = + tags: [ReadDirEffect], raises: [].} = ## An analogue of `walkDir iterator <#walkDir.i,string>`_ with full error ## checking. Yields "steps" of walking through `dir`, each step contains its path `path`, OS error code `code` and additional info. At first step yields info with the status of opening `dir` as a directory, tag `wsOpenDir`. If it's OK, ## the subsequent steps will yield the directory entries with file/directory type as info, tag `wsEntryOk`. Dangling symlinks on posix are reported as wsEntryBad. If (rarely) walking terminates with an error, tag is `wsInterrupted`. @@ -2173,7 +2173,7 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. discard closedir(d) proc tryOpenDir*(dir: string): OpenDirStatus {. - tags: [ReadDirEffect], raises: [], since:(1,1) .} = + tags: [ReadDirEffect], raises: [], since:(1,1).} = # TODO for step in tryWalkDir(dir): case step.kind From 0616ee91fde3629dd1e643369a183df2176ba0cb Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 23 Jan 2020 00:17:21 +0300 Subject: [PATCH 10/26] finished 1-yield conversion + a fix for lstat --- lib/pure/os.nim | 128 +++++++++++++++++++++++++++--------------------- 1 file changed, 72 insertions(+), 56 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index b5d3ffabb6afb..7609ce88e0864 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2005,7 +2005,7 @@ type OpenDirStatus* = enum type WalkStepKind* = enum wsOpenDir, ## attempt to open directory wsEntryOk, ## the entry is OK - wsEntryBad ## (Posix-specific) dangling symlink, so unclear if it + wsEntryBad ## usually a dangling symlink on posix, so unclear if it ## points to a file or directory wsInterrupted ## the directory handle got invalidated during reading @@ -2045,71 +2045,88 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. proc sNoErrors(pc: PathComponent, path: string): owned(WalkStep) = WalkStep(kind: wsEntryOk, entryType: pc, code: OSErrorCode(0), path: path) proc sOpenDir(s: OpenDirStatus, path: string): owned(WalkStep) = - WalkStep(kind: wsOpenDir, openStatus: s, code: osLastError(), path: path) + let code = if s == odOpenOk: OSErrorCode(0) else: osLastError() + WalkStep(kind: wsOpenDir, openStatus: s, code: code, path: path) proc sGetError(k: WalkStepKind, path: string): owned(WalkStep) = WalkStep(kind: k, code: osLastError(), path: path) + var res: WalkStep when nimvm: for k, v in items(staticWalkDir(dir, relative)): yield sNoErrors(k, v) else: + var yieldAllowed: bool + var lastIter = false when weirdTarget: for k, v in items(staticWalkDir(dir, relative)): yield sNoErrors(k, v) elif defined(windows): + var nStep = 1 + var h: Handle = -1 + defer: + if h != -1: findClose(h) var f: WIN32_FIND_DATA - var h = findFirstFile(dir / "*", f) - if h == -1: - let errCode = getLastError() - case errCode - of ERROR_PATH_NOT_FOUND: yield sOpenDir(odNotFound, dir) - of ERROR_DIRECTORY: yield sOpenDir(odNotDir, dir) - of ERROR_ACCESS_DENIED: yield sOpenDir(odAccessDenied, dir) - else: yield sOpenDir(odUnknownError, dir) - else: - defer: findClose(h) - yield sOpenDir(odOpenOk, dir) - while true: + template resolveFile() = var k = pcFile - if not skipFindData(f): + yieldAllowed = not skipFindData(f) + if yieldAllowed: if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: k = pcDir if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: k = succ(k) let xx = if relative: extractFilename(getFilename(f)) else: dir / extractFilename(getFilename(f)) - yield sNoErrors(k, xx) + res = sNoErrors(k, xx) + while not lastIter: + if nStep == 1: # try to open a first file, then yield directory status + h = findFirstFile(dir / "*", f) + yieldAllowed = true + if h == -1: + lastIter = true + let errCode = getLastError() + res = + case errCode + of ERROR_PATH_NOT_FOUND: sOpenDir(odNotFound, dir) + of ERROR_DIRECTORY: sOpenDir(odNotDir, dir) + of ERROR_ACCESS_DENIED: sOpenDir(odAccessDenied, dir) + else: sOpenDir(odUnknownError, dir) + else: + res = sOpenDir(odOpenOk, dir) + elif nStep == 2: # return the first found file + resolveFile() + else: if findNextFile(h, f) == 0'i32: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break - else: yield sGetError(wsInterrupted, dir) - else: - var init: bool - var lastIter: bool - var skipYield: bool - var res: WalkStep - var d: ptr DIR - - while true: - if init and not skipYield: + else: + lastIter = true + yieldAllowed = true + res = sGetError(wsInterrupted, dir) + else: + resolveFile() + nStep += 1 + if yieldAllowed: yield res - if lastIter: - break - skipYield = false + else: + var init = false + var d: ptr DIR = nil + defer: + if d != nil: + discard closedir(d) + while not lastIter: + yieldAllowed = true if not init: init = true d = opendir(dir) if d == nil: + lastIter = true res = if errno == ENOENT: sOpenDir(odNotFound, dir) elif errno == ENOTDIR: sOpenDir(odNotDir, dir) elif errno == EACCES: sOpenDir(odAccessDenied, dir) else: sOpenDir(odUnknownError, dir) - lastIter = true - continue else: res = sOpenDir(odOpenOk, dir) - continue else: errno = 0 var x = readdir(d) @@ -2117,16 +2134,14 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. if errno == 0: break # normal end, yielding nothing else: - res = sGetError(wsInterrupted, dir) lastIter = true - continue + res = sGetError(wsInterrupted, dir) when defined(nimNoArrayToCstringConversion): var y = $cstring(addr x.d_name) else: var y = $x.d_name.cstring if y == "." or y == "..": - skipYield = true - continue + yieldAllowed = false else: var s: Stat let path = dir / y @@ -2134,43 +2149,44 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. y = path var k = pcFile + var use_lstat = false when defined(linux) or defined(macosx) or defined(bsd) or defined(genode) or defined(nintendoswitch): if x.d_type != DT_UNKNOWN: if x.d_type == DT_DIR: k = pcDir res = sNoErrors(k, y) - continue elif x.d_type == DT_LNK: errno = 0 if dirExists(path): k = pcLinkToDir else: k = pcLinkToFile if errno == 0: # check error in dirExists res = sNoErrors(k, y) - continue else: res = sGetError(wsEntryBad, y) - continue else: res = sNoErrors(k, y) - continue - - # special case of DT_UNKNOWN: some filesystems can't detect that - # entry is a directory and leave its d_type as 0=DT_UNKNOWN - errno = 0 - if lstat(path, s) < 0'i32: break - if S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - if errno == 0: # check error in getSymlinkFileKind - res = sNoErrors(k, y) - continue + use_lstat = false + else: + use_lstat = true else: - res = sGetError(wsEntryBad, y) - continue - if d != nil: - discard closedir(d) + use_lstat = true + if use_lstat: + # special case of DT_UNKNOWN: some filesystems can't detect that + # entry is a directory and keep its d_type as 0=DT_UNKNOWN + if lstat(path, s) < 0'i32: + res = sGetError(wsEntryBad, y) + else: + errno = 0 + if S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + k = getSymlinkFileKind(path) + if errno == 0: # check error in getSymlinkFileKind + res = sNoErrors(k, y) + else: + res = sGetError(wsEntryBad, y) + if yieldAllowed: yield res proc tryOpenDir*(dir: string): OpenDirStatus {. tags: [ReadDirEffect], raises: [], since:(1,1).} = From 456cd521bec9e97894b5d04ffe4d641cd56a45cb Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 23 Jan 2020 23:30:35 +0300 Subject: [PATCH 11/26] fix an edgecase --- lib/pure/os.nim | 87 +++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 7609ce88e0864..fce2bccb8d026 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2136,56 +2136,57 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. else: lastIter = true res = sGetError(wsInterrupted, dir) - when defined(nimNoArrayToCstringConversion): - var y = $cstring(addr x.d_name) else: - var y = $x.d_name.cstring - if y == "." or y == "..": - yieldAllowed = false - else: - var s: Stat - let path = dir / y - if not relative: - y = path - var k = pcFile - - var use_lstat = false - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: - k = pcDir - res = sNoErrors(k, y) - elif x.d_type == DT_LNK: - errno = 0 - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - if errno == 0: # check error in dirExists + when defined(nimNoArrayToCstringConversion): + var y = $cstring(addr x.d_name) + else: + var y = $x.d_name.cstring + if y == "." or y == "..": + yieldAllowed = false + else: + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + var use_lstat = false + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + if x.d_type != DT_UNKNOWN: + if x.d_type == DT_DIR: + k = pcDir res = sNoErrors(k, y) + elif x.d_type == DT_LNK: + errno = 0 + if dirExists(path): k = pcLinkToDir + else: k = pcLinkToFile + if errno == 0: # check error in dirExists + res = sNoErrors(k, y) + else: + res = sGetError(wsEntryBad, y) else: - res = sGetError(wsEntryBad, y) + res = sNoErrors(k, y) + use_lstat = false else: - res = sNoErrors(k, y) - use_lstat = false + use_lstat = true else: use_lstat = true - else: - use_lstat = true - if use_lstat: - # special case of DT_UNKNOWN: some filesystems can't detect that - # entry is a directory and keep its d_type as 0=DT_UNKNOWN - if lstat(path, s) < 0'i32: - res = sGetError(wsEntryBad, y) - else: - errno = 0 - if S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - if errno == 0: # check error in getSymlinkFileKind - res = sNoErrors(k, y) - else: + if use_lstat: + # special case of DT_UNKNOWN: some filesystems can't detect that + # entry is a directory and keep its d_type as 0=DT_UNKNOWN + if lstat(path, s) < 0'i32: res = sGetError(wsEntryBad, y) + else: + errno = 0 + if S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + k = getSymlinkFileKind(path) + if errno == 0: # check error in getSymlinkFileKind + res = sNoErrors(k, y) + else: + res = sGetError(wsEntryBad, y) if yieldAllowed: yield res proc tryOpenDir*(dir: string): OpenDirStatus {. From 58123f7cc23952c771c6b55ce0003405ea4608bc Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 23 Jan 2020 23:53:54 +0300 Subject: [PATCH 12/26] fix useLstat & stylistic changes --- lib/pure/os.nim | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index fce2bccb8d026..3f91898795a3d 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2005,7 +2005,7 @@ type OpenDirStatus* = enum type WalkStepKind* = enum wsOpenDir, ## attempt to open directory wsEntryOk, ## the entry is OK - wsEntryBad ## usually a dangling symlink on posix, so unclear if it + wsEntryBad ## usually a dangling symlink on posix, when it's unclear if it ## points to a file or directory wsInterrupted ## the directory handle got invalidated during reading @@ -2022,9 +2022,17 @@ type WalkStep = ref object iterator tryWalkDir*(dir: string; relative=false): WalkStep {. tags: [ReadDirEffect], raises: [].} = - ## An analogue of `walkDir iterator <#walkDir.i,string>`_ with full error - ## checking. Yields "steps" of walking through `dir`, each step contains its path `path`, OS error code `code` and additional info. At first step yields info with the status of opening `dir` as a directory, tag `wsOpenDir`. If it's OK, - ## the subsequent steps will yield the directory entries with file/directory type as info, tag `wsEntryOk`. Dangling symlinks on posix are reported as wsEntryBad. If (rarely) walking terminates with an error, tag is `wsInterrupted`. + ## A version of `walkDir iterator <#walkDir.i,string>`_ with more + ## thorough error checking. + ## Yields *steps* of walking through `dir`, each step contains + ## its path ``path``, OS error code ``code`` and additional tag ``kind``. + ## At first step it yields info ``openStatus`` of opening + ## status `dir` as a directory, tag ``wsOpenDir``. + ## If it's OK, the subsequent steps will yield the directory entries + ## with component type as info ``entryType``, tag ``wsEntryOk``. + ## Dangling symlinks on posix are reported as ``wsEntryBad``. + ## If (rarely) walking terminates with an error (such as an I/O error), + ## tag is `wsInterrupted`. ## ## .. code-block:: Nim ## for step in tryWalkDir("dirA"): @@ -2037,7 +2045,7 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. ## of odAccessDenied: echo dir & " - Access Denied" & err ## of odNotFound: echo dir & " - no such path" & err ## of odUnknownError: echo dir & "unknown error - that's bad!" & err - ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType + ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType ## of wsEntryBad: echo step.path & " is broken symlink " & err ## of wsInterrupted: ## echo dir & " traversal interrupted, really bad! " & err @@ -2150,7 +2158,7 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. y = path var k = pcFile - var use_lstat = false + var useLstat = false when defined(linux) or defined(macosx) or defined(bsd) or defined(genode) or defined(nintendoswitch): if x.d_type != DT_UNKNOWN: @@ -2167,12 +2175,12 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. res = sGetError(wsEntryBad, y) else: res = sNoErrors(k, y) - use_lstat = false + useLstat = false else: - use_lstat = true + useLstat = true else: - use_lstat = true - if use_lstat: + useLstat = true + if useLstat: # special case of DT_UNKNOWN: some filesystems can't detect that # entry is a directory and keep its d_type as 0=DT_UNKNOWN if lstat(path, s) < 0'i32: From e1ea37266b937291b3bb51525ebb92cf47af47f1 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 25 Jan 2020 17:49:10 +0300 Subject: [PATCH 13/26] modify posix part to use block & break block --- lib/pure/os.nim | 138 ++++++++++++++++++++++-------------------------- 1 file changed, 62 insertions(+), 76 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 3f91898795a3d..44aa6acfc3bc7 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2005,7 +2005,7 @@ type OpenDirStatus* = enum type WalkStepKind* = enum wsOpenDir, ## attempt to open directory wsEntryOk, ## the entry is OK - wsEntryBad ## usually a dangling symlink on posix, when it's unclear if it + wsEntryBad ## a broken symlink on posix, where it's unclear if it ## points to a file or directory wsInterrupted ## the directory handle got invalidated during reading @@ -2023,7 +2023,7 @@ type WalkStep = ref object iterator tryWalkDir*(dir: string; relative=false): WalkStep {. tags: [ReadDirEffect], raises: [].} = ## A version of `walkDir iterator <#walkDir.i,string>`_ with more - ## thorough error checking. + ## thorough error checking. It never raises an exception. ## Yields *steps* of walking through `dir`, each step contains ## its path ``path``, OS error code ``code`` and additional tag ``kind``. ## At first step it yields info ``openStatus`` of opening @@ -2121,89 +2121,75 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. defer: if d != nil: discard closedir(d) - while not lastIter: - yieldAllowed = true - if not init: - init = true - d = opendir(dir) - if d == nil: - lastIter = true - res = - if errno == ENOENT: sOpenDir(odNotFound, dir) - elif errno == ENOTDIR: sOpenDir(odNotDir, dir) - elif errno == EACCES: sOpenDir(odAccessDenied, dir) - else: sOpenDir(odUnknownError, dir) - else: - res = sOpenDir(odOpenOk, dir) - else: - errno = 0 - var x = readdir(d) - if x == nil: - if errno == 0: - break # normal end, yielding nothing - else: - lastIter = true - res = sGetError(wsInterrupted, dir) - else: + block loop: + while not lastIter: + block beforeYield: + if not init: + init = true + d = opendir(dir) + if d == nil: + lastIter = true + res = + if errno == ENOENT: sOpenDir(odNotFound, dir) + elif errno == ENOTDIR: sOpenDir(odNotDir, dir) + elif errno == EACCES: sOpenDir(odAccessDenied, dir) + else: sOpenDir(odUnknownError, dir) + else: + res = sOpenDir(odOpenOk, dir) + break beforeYield + errno = 0 + var x = readdir(d) + if x == nil: + if errno == 0: + break loop # normal end, yielding nothing + else: + lastIter = true + res = sGetError(wsInterrupted, dir) + break beforeYield when defined(nimNoArrayToCstringConversion): var y = $cstring(addr x.d_name) else: var y = $x.d_name.cstring if y == "." or y == "..": - yieldAllowed = false - else: - var s: Stat - let path = dir / y - if not relative: - y = path - var k = pcFile - - var useLstat = false - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: - k = pcDir - res = sNoErrors(k, y) - elif x.d_type == DT_LNK: - errno = 0 - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - if errno == 0: # check error in dirExists - res = sNoErrors(k, y) - else: - res = sGetError(wsEntryBad, y) - else: - res = sNoErrors(k, y) - useLstat = false - else: - useLstat = true - else: - useLstat = true - if useLstat: - # special case of DT_UNKNOWN: some filesystems can't detect that - # entry is a directory and keep its d_type as 0=DT_UNKNOWN - if lstat(path, s) < 0'i32: - res = sGetError(wsEntryBad, y) - else: + continue + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + if x.d_type != DT_UNKNOWN: + if x.d_type == DT_DIR: + k = pcDir + res = sNoErrors(k, y) + elif x.d_type == DT_LNK: errno = 0 - if S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - if errno == 0: # check error in getSymlinkFileKind + if dirExists(path): k = pcLinkToDir + else: k = pcLinkToFile + if errno == 0: # check error in dirExists res = sNoErrors(k, y) else: res = sGetError(wsEntryBad, y) - if yieldAllowed: yield res - -proc tryOpenDir*(dir: string): OpenDirStatus {. - tags: [ReadDirEffect], raises: [], since:(1,1).} = - # TODO - for step in tryWalkDir(dir): - case step.kind - of wsOpenDir: return step.openStatus - else: break # can not happen + else: + res = sNoErrors(k, y) + break beforeYield + # case of d_type==DT_UNKNOWN: some filesystems don't report + # the entry type; one should fallback to lstat + if lstat(path, s) < 0'i32: + res = sGetError(wsEntryBad, y) + else: + errno = 0 + if S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + k = getSymlinkFileKind(path) + if errno == 0: # check error in getSymlinkFileKind + res = sNoErrors(k, y) + else: + res = sGetError(wsEntryBad, y) + yield res iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. tags: [ReadDirEffect].} = From 2264e4289fe48fe6d6eccb83573325487b3b7f5b Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 28 Jan 2020 00:47:38 +0300 Subject: [PATCH 14/26] restore walkDir, added annotations & changelog --- changelog.md | 1 + lib/pure/os.nim | 423 +++++++++++++++++++++++++++--------------------- 2 files changed, 242 insertions(+), 182 deletions(-) diff --git a/changelog.md b/changelog.md index ce47249a8259e..c5fd777b6b1c2 100644 --- a/changelog.md +++ b/changelog.md @@ -80,6 +80,7 @@ - `htmlgen` adds [MathML](https://wikipedia.org/wiki/MathML) support (ISO 40314). - `macros.eqIdent` is now invariant to export markers and backtick quotes. +- Added `os.tryWalkDir` iterator to traverse directories with error checking. - `htmlgen.html` allows `lang` on the `` tag and common valid attributes. - `macros.basename` and `basename=` got support for `PragmaExpr`, so that an expression like `MyEnum {.pure.}` is handled correctly. diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 44aa6acfc3bc7..321d3724800ee 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1995,6 +1995,98 @@ proc staticWalkDir(dir: string; relative: bool): seq[ tuple[kind: PathComponent, path: string]] = discard +iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. + tags: [ReadDirEffect].} = + ## Walks over the directory `dir` and yields for each directory or file in + ## `dir`. The component type and full path for each item are returned. + ## + ## Walking is not recursive. If ``relative`` is true (default: false) + ## the resulting path is shortened to be relative to ``dir``. + ## Example: This directory structure:: + ## dirA / dirB / fileB1.txt + ## / dirC + ## / fileA1.txt + ## / fileA2.txt + ## + ## and this code: + ## + ## .. code-block:: Nim + ## for kind, path in walkDir("dirA"): + ## echo(path) + ## + ## produce this output (but not necessarily in this order!):: + ## dirA/dirB + ## dirA/dirC + ## dirA/fileA1.txt + ## dirA/fileA2.txt + ## + ## See also: + ## * `walkPattern iterator <#walkPattern.i,string>`_ + ## * `walkFiles iterator <#walkFiles.i,string>`_ + ## * `walkDirs iterator <#walkDirs.i,string>`_ + ## * `walkDirRec iterator <#walkDirRec.i,string>`_ + + when nimvm: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + else: + when weirdTarget: + for k, v in items(staticWalkDir(dir, relative)): + yield (k, v) + elif defined(windows): + var f: WIN32_FIND_DATA + var h = findFirstFile(dir / "*", f) + if h != -1: + defer: findClose(h) + while true: + var k = pcFile + if not skipFindData(f): + if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(f)) + else: dir / extractFilename(getFilename(f)) + yield (k, xx) + if findNextFile(h, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break + else: raiseOSError(errCode.OSErrorCode) + else: + var d = opendir(dir) + if d != nil: + defer: discard closedir(d) + while true: + var x = readdir(d) + if x == nil: break + when defined(nimNoArrayToCstringConversion): + var y = $cstring(addr x.d_name) + else: + var y = $x.d_name.cstring + if y != "." and y != "..": + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + if x.d_type != DT_UNKNOWN: + if x.d_type == DT_DIR: k = pcDir + if x.d_type == DT_LNK: + if dirExists(path): k = pcLinkToDir + else: k = pcLinkToFile + yield (k, y) + continue + + if lstat(path, s) < 0'i32: break + if S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + k = getSymlinkFileKind(path) + yield (k, y) + type OpenDirStatus* = enum odOpenOk, odNotDir, @@ -2010,8 +2102,8 @@ type WalkStepKind* = enum wsInterrupted ## the directory handle got invalidated during reading type WalkStep = ref object - code*: OSErrorCode path*: string + code*: OSErrorCode case kind*: WalkStepKind of wsOpenDir: openStatus*: OpenDirStatus @@ -2020,8 +2112,17 @@ type WalkStep = ref object of wsEntryBad, wsInterrupted: discard +proc `$`*(s: WalkStep): string = + let beg = + case s.kind + of wsOpenDir: "(wsOpenDir | " & $s.openStatus + of wsEntryOk: "(wsEntryOk | '" & s.path & "', " & $s.entryType + of wsEntryBad: "(wsEntryBad | '" & s.path & "'" + of wsInterrupted: "(wsInterrupted | '" & s.path & "'" + result = beg & ", err:" & $s.code & ")" + iterator tryWalkDir*(dir: string; relative=false): WalkStep {. - tags: [ReadDirEffect], raises: [].} = + tags: [ReadDirEffect], raises: [], since: (1,1), noNimScript.} = ## A version of `walkDir iterator <#walkDir.i,string>`_ with more ## thorough error checking. It never raises an exception. ## Yields *steps* of walking through `dir`, each step contains @@ -2034,21 +2135,31 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. ## If (rarely) walking terminates with an error (such as an I/O error), ## tag is `wsInterrupted`. ## + ## Example of usage with full error checking: + ## .. code-block:: Nim + ## for step in tryWalkDir("dirA"): + ## let err = " (" & $step.code & " - " & osErrorMsg(step.code) & ")" + ## case step.kind + ## of wsOpenDir: + ## case step.openStatus: + ## of odOpenOk: echo dir & " - directory is opened OK" + ## of odNotDir: echo dir & " is not a directory" & err + ## of odAccessDenied: echo dir & " - Access Denied" & err + ## of odNotFound: echo dir & " - no such path" & err + ## of odUnknownError: echo dir & "unknown error - that's bad!" & err + ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType + ## of wsEntryBad: echo step.path & " is broken symlink " & err + ## of wsInterrupted: + ## echo dir & " traversal interrupted, really bad! " & err + ## + ## For comparison, walkDir iterator may be implemented with tryWalkDir: ## .. code-block:: Nim - ## for step in tryWalkDir("dirA"): - ## let err = " (" & $step.code & " - " & osErrorMsg(step.code) & ")" - ## case step.kind - ## of wsOpenDir: - ## case step.openStatus: - ## of odOpenOk: echo dir & " - directory is opened OK" - ## of odNotDir: echo dir & " is not a directory" & err - ## of odAccessDenied: echo dir & " - Access Denied" & err - ## of odNotFound: echo dir & " - no such path" & err - ## of odUnknownError: echo dir & "unknown error - that's bad!" & err - ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType - ## of wsEntryBad: echo step.path & " is broken symlink " & err - ## of wsInterrupted: - ## echo dir & " traversal interrupted, really bad! " & err + ## for step in tryWalkDir(dir, relative): + ## case step.kind + ## of wsOpenDir: discard + ## of wsEntryOk: yield (step.entryType, move(step.path)) + ## of wsEntryBad: yield (pcLinkToFile, move(step.path)) + ## of wsInterrupted: raiseOSError(step.code) proc sNoErrors(pc: PathComponent, path: string): owned(WalkStep) = WalkStep(kind: wsEntryOk, entryType: pc, code: OSErrorCode(0), path: path) @@ -2059,184 +2170,132 @@ iterator tryWalkDir*(dir: string; relative=false): WalkStep {. WalkStep(kind: k, code: osLastError(), path: path) var res: WalkStep - when nimvm: - for k, v in items(staticWalkDir(dir, relative)): - yield sNoErrors(k, v) - else: + var lastIter = false + when defined(windows): var yieldAllowed: bool - var lastIter = false - when weirdTarget: - for k, v in items(staticWalkDir(dir, relative)): - yield sNoErrors(k, v) - elif defined(windows): - var nStep = 1 - var h: Handle = -1 - defer: - if h != -1: findClose(h) - var f: WIN32_FIND_DATA - template resolveFile() = - var k = pcFile - yieldAllowed = not skipFindData(f) - if yieldAllowed: - if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: - k = pcDir - if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) - let xx = if relative: extractFilename(getFilename(f)) - else: dir / extractFilename(getFilename(f)) - res = sNoErrors(k, xx) - while not lastIter: - if nStep == 1: # try to open a first file, then yield directory status - h = findFirstFile(dir / "*", f) - yieldAllowed = true - if h == -1: - lastIter = true - let errCode = getLastError() - res = - case errCode - of ERROR_PATH_NOT_FOUND: sOpenDir(odNotFound, dir) - of ERROR_DIRECTORY: sOpenDir(odNotDir, dir) - of ERROR_ACCESS_DENIED: sOpenDir(odAccessDenied, dir) - else: sOpenDir(odUnknownError, dir) + var nStep = 1 + var h: Handle = -1 + defer: + if h != -1: findClose(h) + var f: WIN32_FIND_DATA + template resolveFile() = + var k = pcFile + yieldAllowed = not skipFindData(f) + if yieldAllowed: + if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(f)) + else: dir / extractFilename(getFilename(f)) + res = sNoErrors(k, xx) + while not lastIter: + if nStep == 1: # try to open a first file, then yield directory status + h = findFirstFile(dir / "*", f) + yieldAllowed = true + if h == -1: + lastIter = true + let errCode = getLastError() + res = + case errCode + of ERROR_PATH_NOT_FOUND: sOpenDir(odNotFound, dir) + of ERROR_DIRECTORY: sOpenDir(odNotDir, dir) + of ERROR_ACCESS_DENIED: sOpenDir(odAccessDenied, dir) + else: sOpenDir(odUnknownError, dir) + else: + res = sOpenDir(odOpenOk, dir) + elif nStep == 2: # return the first found file + resolveFile() + else: + if findNextFile(h, f) == 0'i32: + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: break else: - res = sOpenDir(odOpenOk, dir) - elif nStep == 2: # return the first found file - resolveFile() + lastIter = true + yieldAllowed = true + res = sGetError(wsInterrupted, dir) else: - if findNextFile(h, f) == 0'i32: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break + resolveFile() + nStep += 1 + if yieldAllowed: + yield res + else: + var init = false + var d: ptr DIR = nil + defer: + if d != nil: + discard closedir(d) + block loop: + while not lastIter: + block beforeYield: + if not init: + init = true + d = opendir(dir) + if d == nil: + lastIter = true + res = + if errno == ENOENT: sOpenDir(odNotFound, dir) + elif errno == ENOTDIR: sOpenDir(odNotDir, dir) + elif errno == EACCES: sOpenDir(odAccessDenied, dir) + else: sOpenDir(odUnknownError, dir) + else: + res = sOpenDir(odOpenOk, dir) + break beforeYield + let errnoSave = errno + defer: errno = errnoSave + errno = 0 + var x = readdir(d) + if x == nil: + if errno == 0: + break loop # normal end, yielding nothing else: lastIter = true - yieldAllowed = true res = sGetError(wsInterrupted, dir) + break beforeYield + when defined(nimNoArrayToCstringConversion): + var y = $cstring(addr x.d_name) else: - resolveFile() - nStep += 1 - if yieldAllowed: - yield res - else: - var init = false - var d: ptr DIR = nil - defer: - if d != nil: - discard closedir(d) - block loop: - while not lastIter: - block beforeYield: - if not init: - init = true - d = opendir(dir) - if d == nil: - lastIter = true - res = - if errno == ENOENT: sOpenDir(odNotFound, dir) - elif errno == ENOTDIR: sOpenDir(odNotDir, dir) - elif errno == EACCES: sOpenDir(odAccessDenied, dir) - else: sOpenDir(odUnknownError, dir) + var y = $x.d_name.cstring + if y == "." or y == "..": + continue + var s: Stat + let path = dir / y + if not relative: + y = path + var k = pcFile + + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + if x.d_type != DT_UNKNOWN: + if x.d_type == DT_DIR: + k = pcDir + res = sNoErrors(k, y) + elif x.d_type == DT_LNK: + errno = 0 + if dirExists(path): k = pcLinkToDir + else: k = pcLinkToFile + if errno == 0: # check error in dirExists + res = sNoErrors(k, y) + else: + res = sGetError(wsEntryBad, y) else: - res = sOpenDir(odOpenOk, dir) + res = sNoErrors(k, y) break beforeYield + # case of d_type==DT_UNKNOWN: some filesystems don't report + # the entry type; one should fallback to lstat + if lstat(path, s) < 0'i32: + res = sGetError(wsEntryBad, y) + else: errno = 0 - var x = readdir(d) - if x == nil: - if errno == 0: - break loop # normal end, yielding nothing - else: - lastIter = true - res = sGetError(wsInterrupted, dir) - break beforeYield - when defined(nimNoArrayToCstringConversion): - var y = $cstring(addr x.d_name) + if S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + k = getSymlinkFileKind(path) + if errno == 0: # check error in getSymlinkFileKind + res = sNoErrors(k, y) else: - var y = $x.d_name.cstring - if y == "." or y == "..": - continue - var s: Stat - let path = dir / y - if not relative: - y = path - var k = pcFile - - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: - k = pcDir - res = sNoErrors(k, y) - elif x.d_type == DT_LNK: - errno = 0 - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - if errno == 0: # check error in dirExists - res = sNoErrors(k, y) - else: - res = sGetError(wsEntryBad, y) - else: - res = sNoErrors(k, y) - break beforeYield - # case of d_type==DT_UNKNOWN: some filesystems don't report - # the entry type; one should fallback to lstat - if lstat(path, s) < 0'i32: res = sGetError(wsEntryBad, y) - else: - errno = 0 - if S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - if errno == 0: # check error in getSymlinkFileKind - res = sNoErrors(k, y) - else: - res = sGetError(wsEntryBad, y) - yield res - -iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: string] {. - tags: [ReadDirEffect].} = - ## Walks over the directory `dir` and yields for each directory or file in - ## `dir`. The component type and full path for each item are returned. - ## - ## Walking is not recursive. If ``relative`` is true (default: false) - ## the resulting path is shortened to be relative to ``dir``. - ## Example: This directory structure:: - ## dirA / dirB / fileB1.txt - ## / dirC - ## / fileA1.txt - ## / fileA2.txt - ## - ## and this code: - ## - ## .. code-block:: Nim - ## for kind, path in walkDir("dirA"): - ## echo(path) - ## - ## produce this output (but not necessarily in this order!):: - ## dirA/dirB - ## dirA/dirC - ## dirA/fileA1.txt - ## dirA/fileA2.txt - ## - ## Error checking is aimed at dropping bad entries: if path `dir` is not - ## found or is a file or is an access-denied directory then the iterator - ## yields nothing; all closed symlinks inside `dir` are silently omitted. - ## However, OSException is raised when opening directory resulted to - ## an unrecognized error or the directory handle was invalidated during - ## the traversal. - ## - ## See also: - ## * `walkDirErr iterator <#walkDirErr.i,string>`_ - ## * `walkPattern iterator <#walkPattern.i,string>`_ - ## * `walkFiles iterator <#walkFiles.i,string>`_ - ## * `walkDirs iterator <#walkDirs.i,string>`_ - ## * `walkDirRec iterator <#walkDirRec.i,string>`_ - for step in tryWalkDir(dir, relative): - case step.kind - of wsOpenDir: - if step.openStatus == odUnknownError: raiseOSError(step.code) - else: discard - of wsEntryOk: yield (step.entryType, move(step.path)) - of wsEntryBad: discard - of wsInterrupted: raiseOSError(step.code) + yield res iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}, From a6edbca2811b4a8b4a1ca9a5c7f4a321ad9786bc Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Mon, 3 Feb 2020 23:27:49 +0300 Subject: [PATCH 15/26] rewrote according to comments --- lib/pure/os.nim | 385 +++++++++++++++++++++++++----------------------- 1 file changed, 197 insertions(+), 188 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 321d3724800ee..e9c8fc22fa177 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2020,7 +2020,14 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: ## dirA/fileA1.txt ## dirA/fileA2.txt ## + ## Error checking is aimed at silent dropping: if path `dir` is not + ## found then the iterator will yield nothing; the same goes if `dir` is a + ## file or is an access-denied directory; all broken symlinks inside `dir` get + ## default type ``pcLinkToFile``. However, OSException will be raised if + ## the directory handle is invalidated during walking. + ## ## See also: + ## * `tryWalkDir iterator <#tryWalkDir.i,string>`_ ## * `walkPattern iterator <#walkPattern.i,string>`_ ## * `walkFiles iterator <#walkFiles.i,string>`_ ## * `walkDirs iterator <#walkDirs.i,string>`_ @@ -2087,215 +2094,217 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: k = getSymlinkFileKind(path) yield (k, y) -type OpenDirStatus* = enum - odOpenOk, - odNotDir, - odAccessDenied, - odNotFound, - odUnknownError - -type WalkStepKind* = enum - wsOpenDir, ## attempt to open directory - wsEntryOk, ## the entry is OK - wsEntryBad ## a broken symlink on posix, where it's unclear if it - ## points to a file or directory - wsInterrupted ## the directory handle got invalidated during reading - -type WalkStep = ref object - path*: string - code*: OSErrorCode - case kind*: WalkStepKind - of wsOpenDir: - openStatus*: OpenDirStatus - of wsEntryOk: - entryType*: PathComponent - of wsEntryBad, wsInterrupted: - discard +type + OpenDirStatus* = enum ## enumeration specifying status of opening a directory + odUnknownError, ## unrecognized error + odOpenOk, ## the directory was opened OK + odNotDir, ## error, not a directory: a file with the same path exists + odAccessDenied, ## permission denied / access denied error + odNotFound ## no such path + WalkStepKind* = enum ## status of the step of walking through directory + wsOpenDir, ## step for attempt to open directory + wsEntryOk, ## step when entry is OK + wsEntryBad ## a broken symlink on posix, where it's unclear if it + ## points to a file or directory + wsInterrupted ## the directory handle got invalidated during reading + WalkStep = object ## step of walking through directory + path*: string ## the path that this step applies to + code*: OSErrorCode ## OS error code for the step + case kind*: WalkStepKind ## discriminator + of wsOpenDir: + openStatus*: OpenDirStatus ## status of opening the directory + of wsEntryOk: + entryType*: PathComponent ## path component + of wsEntryBad, wsInterrupted: + discard proc `$`*(s: WalkStep): string = - let beg = - case s.kind - of wsOpenDir: "(wsOpenDir | " & $s.openStatus - of wsEntryOk: "(wsEntryOk | '" & s.path & "', " & $s.entryType - of wsEntryBad: "(wsEntryBad | '" & s.path & "'" - of wsInterrupted: "(wsInterrupted | '" & s.path & "'" - result = beg & ", err:" & $s.code & ")" - -iterator tryWalkDir*(dir: string; relative=false): WalkStep {. - tags: [ReadDirEffect], raises: [], since: (1,1), noNimScript.} = - ## A version of `walkDir iterator <#walkDir.i,string>`_ with more - ## thorough error checking. It never raises an exception. + let err = if s.code == OSErrorCode(0): "" + else: " ($1 - $2)" % [$s.code, osErrorMsg(s.code)] + case s.kind + of wsOpenDir: "(wsOpenDir | $1 '$2'$3)" % [$s.openStatus, s.path, err] + of wsEntryOk: "(wsEntryOk | $1 '$2')" % [$s.entryType, s.path] + of wsEntryBad: "(wsEntryBad | '$1'$2)" % [s.path, err] + of wsInterrupted: "(wsInterrupted | '$1'$2)" % [s.path, err] + +iterator tryWalkDir*(dir: string, relative=false): WalkStep = + ## Walks over the directory `dir` and yields for each directory or file + ## in `dir`, non-recursively. It's a version of + ## `walkDir iterator <#walkDir.i,string>`_ with more thorough error + ## checking. It never raises an exception. + ## ## Yields *steps* of walking through `dir`, each step contains - ## its path ``path``, OS error code ``code`` and additional tag ``kind``. - ## At first step it yields info ``openStatus`` of opening - ## status `dir` as a directory, tag ``wsOpenDir``. - ## If it's OK, the subsequent steps will yield the directory entries - ## with component type as info ``entryType``, tag ``wsEntryOk``. - ## Dangling symlinks on posix are reported as ``wsEntryBad``. - ## If (rarely) walking terminates with an error (such as an I/O error), - ## tag is `wsInterrupted`. - ## - ## Example of usage with full error checking: + ## its path ``path``, OS error code ``code`` and additional tag ``kind``: + ## + ## - ``kind=wsOpenDir``: yielded once after opening, contains ``openStatus`` + ## of type ``OpenDirStatus`` + ## - ``kind=wsEntryOk``: signifies normal entries with + ## path component ``entryType`` + ## - ``kind=wsEntryBad``: broken symlink on posix without path component + ## - ``kind=wsInterrupted``: signifies that a rare OS error happenned and + ## the walking was terminated + ## + ## An example of usage with full error checking: ## .. code-block:: Nim ## for step in tryWalkDir("dirA"): - ## let err = " (" & $step.code & " - " & osErrorMsg(step.code) & ")" - ## case step.kind - ## of wsOpenDir: - ## case step.openStatus: - ## of odOpenOk: echo dir & " - directory is opened OK" - ## of odNotDir: echo dir & " is not a directory" & err - ## of odAccessDenied: echo dir & " - Access Denied" & err - ## of odNotFound: echo dir & " - no such path" & err - ## of odUnknownError: echo dir & "unknown error - that's bad!" & err - ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType - ## of wsEntryBad: echo step.path & " is broken symlink " & err - ## of wsInterrupted: - ## echo dir & " traversal interrupted, really bad! " & err + ## let err = " (" & $step.code & " - " & osErrorMsg(step.code) & ")" + ## case step.kind + ## of wsOpenDir: + ## case step.openStatus: + ## of odOpenOk: echo "directory is opened OK" + ## of odNotDir: echo "is not a directory" & err + ## of odAccessDenied: echo "Access Denied" & err + ## of odNotFound: echo "no such path" & err + ## of odUnknownError: echo "unknown error - that's bad." & err + ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType + ## of wsEntryBad: echo step.path & " is broken symlink " & err + ## of wsInterrupted: echo " traversal interrupted, really bad. " & err ## ## For comparison, walkDir iterator may be implemented with tryWalkDir: ## .. code-block:: Nim - ## for step in tryWalkDir(dir, relative): - ## case step.kind - ## of wsOpenDir: discard - ## of wsEntryOk: yield (step.entryType, move(step.path)) - ## of wsEntryBad: yield (pcLinkToFile, move(step.path)) - ## of wsInterrupted: raiseOSError(step.code) - - proc sNoErrors(pc: PathComponent, path: string): owned(WalkStep) = - WalkStep(kind: wsEntryOk, entryType: pc, code: OSErrorCode(0), path: path) - proc sOpenDir(s: OpenDirStatus, path: string): owned(WalkStep) = + ## iterator myWalkDir(path: string, relative: bool): + ## tuple[kind: PathComponent, path: string] = + ## for step in tryWalkDir(path, relative): + ## case step.kind + ## of wsOpenDir: discard + ## of wsEntryOk: yield (step.entryType, step.path) + ## of wsEntryBad: yield (pcLinkToFile, step.path) + ## of wsInterrupted: raiseOSError(step.code) + + + var step: WalkStep + var skip = false + proc sOpenDir(s: OpenDirStatus, path: string): WalkStep = let code = if s == odOpenOk: OSErrorCode(0) else: osLastError() - WalkStep(kind: wsOpenDir, openStatus: s, code: code, path: path) - proc sGetError(k: WalkStepKind, path: string): owned(WalkStep) = + result = WalkStep(kind: wsOpenDir, openStatus: s, code: code, path: path) + proc sNoErrors(pc: PathComponent, path: string): WalkStep = + WalkStep(kind: wsEntryOk, entryType: pc, code: OSErrorCode(0), path: path) + proc sGetError(k: WalkStepKind, path: string): WalkStep = WalkStep(kind: k, code: osLastError(), path: path) - var res: WalkStep - var lastIter = false when defined(windows): - var yieldAllowed: bool - var nStep = 1 - var h: Handle = -1 - defer: - if h != -1: findClose(h) + template resolveFile(fdat: WIN32_FIND_DATA, relative: bool): WalkStep = + var k: PathComponent + if (fdat.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: + k = pcDir + if (fdat.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + k = succ(k) + let xx = if relative: extractFilename(getFilename(fdat)) + else: dir / extractFilename(getFilename(fdat)) + sNoErrors(k, xx) + var f: WIN32_FIND_DATA - template resolveFile() = - var k = pcFile - yieldAllowed = not skipFindData(f) - if yieldAllowed: - if (f.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: - k = pcDir - if (f.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) - let xx = if relative: extractFilename(getFilename(f)) - else: dir / extractFilename(getFilename(f)) - res = sNoErrors(k, xx) - while not lastIter: - if nStep == 1: # try to open a first file, then yield directory status - h = findFirstFile(dir / "*", f) - yieldAllowed = true - if h == -1: - lastIter = true - let errCode = getLastError() - res = - case errCode - of ERROR_PATH_NOT_FOUND: sOpenDir(odNotFound, dir) - of ERROR_DIRECTORY: sOpenDir(odNotDir, dir) - of ERROR_ACCESS_DENIED: sOpenDir(odAccessDenied, dir) - else: sOpenDir(odUnknownError, dir) - else: - res = sOpenDir(odOpenOk, dir) - elif nStep == 2: # return the first found file - resolveFile() - else: + var h: Handle = findFirstFile(dir / "*", f) + defer: + if h != -1: + findClose(h) + let status = + if h == -1: + case getLastError() + of ERROR_PATH_NOT_FOUND: odNotFound + of ERROR_DIRECTORY: odNotDir + of ERROR_ACCESS_DENIED: odAccessDenied + else: odUnknownError + else: odOpenOk + step = sOpenDir(status, dir) + + var firstStep = true + while true: + if not skip: + yield step + if h == -1 or step.kind == wsInterrupted: + break + if firstStep: # use file obtained by findFirstFile + firstStep = false + else: # get next file if findNextFile(h, f) == 0'i32: let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: break + if errCode == ERROR_NO_MORE_FILES: + break # normal end, yielding nothing else: - lastIter = true - yieldAllowed = true - res = sGetError(wsInterrupted, dir) - else: - resolveFile() - nStep += 1 - if yieldAllowed: - yield res + skip = false + step = sGetError(wsInterrupted, dir) + continue + skip = skipFindData(f) + if not skip: + step = resolveFile(f, relative) + else: - var init = false - var d: ptr DIR = nil - defer: - if d != nil: - discard closedir(d) - block loop: - while not lastIter: - block beforeYield: - if not init: - init = true - d = opendir(dir) - if d == nil: - lastIter = true - res = - if errno == ENOENT: sOpenDir(odNotFound, dir) - elif errno == ENOTDIR: sOpenDir(odNotDir, dir) - elif errno == EACCES: sOpenDir(odAccessDenied, dir) - else: sOpenDir(odUnknownError, dir) - else: - res = sOpenDir(odOpenOk, dir) - break beforeYield - let errnoSave = errno - defer: errno = errnoSave - errno = 0 - var x = readdir(d) - if x == nil: + proc resolveFile(x: ptr Dirent, path: string, y: string): WalkStep = + var k = pcFile + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + if x.d_type != DT_UNKNOWN: + if x.d_type == DT_DIR: + k = pcDir + return sNoErrors(k, y) + elif x.d_type == DT_LNK: + errno = 0 + if dirExists(path): k = pcLinkToDir + else: k = pcLinkToFile + # check error in dirExists if errno == 0: - break loop # normal end, yielding nothing + return sNoErrors(k, y) else: - lastIter = true - res = sGetError(wsInterrupted, dir) - break beforeYield - when defined(nimNoArrayToCstringConversion): - var y = $cstring(addr x.d_name) + return sGetError(wsEntryBad, y) else: - var y = $x.d_name.cstring - if y == "." or y == "..": - continue - var s: Stat - let path = dir / y - if not relative: - y = path - var k = pcFile + return sNoErrors(k, y) + # case of d_type==DT_UNKNOWN: some filesystems don't report + # the entry type; one should fallback to lstat + var s: Stat + if lstat(path, s) < 0'i32: + return sGetError(wsEntryBad, y) + else: + errno = 0 + if S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + k = getSymlinkFileKind(path) + # check error in getSymlinkFileKind + if errno == 0: + return sNoErrors(k, y) + else: + return sGetError(wsEntryBad, y) - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: - k = pcDir - res = sNoErrors(k, y) - elif x.d_type == DT_LNK: - errno = 0 - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - if errno == 0: # check error in dirExists - res = sNoErrors(k, y) - else: - res = sGetError(wsEntryBad, y) - else: - res = sNoErrors(k, y) - break beforeYield - # case of d_type==DT_UNKNOWN: some filesystems don't report - # the entry type; one should fallback to lstat - if lstat(path, s) < 0'i32: - res = sGetError(wsEntryBad, y) - else: - errno = 0 - if S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - if errno == 0: # check error in getSymlinkFileKind - res = sNoErrors(k, y) - else: - res = sGetError(wsEntryBad, y) - yield res + var d: ptr DIR = opendir(dir) + defer: + if d != nil: + discard closedir(d) + let status = + if d == nil: + if errno == ENOENT: odNotFound + elif errno == ENOTDIR: odNotDir + elif errno == EACCES: odAccessDenied + else: odUnknownError + else: odOpenOk + step = sOpenDir(status, dir) + + while true: + if not skip: + yield step + if d == nil or step.kind == wsInterrupted: + break + let errnoSave = errno + defer: errno = errnoSave + errno = 0 + var x = readdir(d) + if x == nil: + if errno == 0: + break # normal end, yielding nothing + else: + skip = false + step = sGetError(wsInterrupted, dir) + continue + when defined(nimNoArrayToCstringConversion): + var y = $cstring(addr x.d_name) + else: + var y = $x.d_name.cstring + skip = (y == "." or y == "..") + if not skip: + let path = dir / y + if not relative: + y = path + step = resolveFile(x, path, y) iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}, From 98782088ac94f1508c28e70e8743656e2f1c9892 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 4 Feb 2020 00:44:53 +0300 Subject: [PATCH 16/26] forgot pragmas --- lib/pure/os.nim | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index e9c8fc22fa177..6ee40e3e6b2bf 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2127,7 +2127,8 @@ proc `$`*(s: WalkStep): string = of wsEntryBad: "(wsEntryBad | '$1'$2)" % [s.path, err] of wsInterrupted: "(wsInterrupted | '$1'$2)" % [s.path, err] -iterator tryWalkDir*(dir: string, relative=false): WalkStep = +iterator tryWalkDir*(dir: string, relative=false): WalkStep {. + tags: [ReadDirEffect], raises: [], noNimScript.} = ## Walks over the directory `dir` and yields for each directory or file ## in `dir`, non-recursively. It's a version of ## `walkDir iterator <#walkDir.i,string>`_ with more thorough error @@ -2138,11 +2139,12 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep = ## ## - ``kind=wsOpenDir``: yielded once after opening, contains ``openStatus`` ## of type ``OpenDirStatus`` - ## - ``kind=wsEntryOk``: signifies normal entries with - ## path component ``entryType`` - ## - ``kind=wsEntryBad``: broken symlink on posix without path component - ## - ``kind=wsInterrupted``: signifies that a rare OS error happenned and - ## the walking was terminated + ## - then it yields zero or more entries, each can be of the following kind: + ## - ``kind=wsEntryOk``: signifies normal entries with + ## path component ``entryType`` + ## - ``kind=wsEntryBad``: broken symlink (without path component) + ## - ``kind=wsInterrupted``: signifies that a rare OS error happenned and + ## the walking was terminated ## ## An example of usage with full error checking: ## .. code-block:: Nim From dc3b1bf6f13be505f0e07015ffd4e30c0f8a0164 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Tue, 4 Feb 2020 23:56:40 +0300 Subject: [PATCH 17/26] forgot 'since' --- lib/pure/os.nim | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 6ee40e3e6b2bf..66c2865cbf060 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2128,17 +2128,17 @@ proc `$`*(s: WalkStep): string = of wsInterrupted: "(wsInterrupted | '$1'$2)" % [s.path, err] iterator tryWalkDir*(dir: string, relative=false): WalkStep {. - tags: [ReadDirEffect], raises: [], noNimScript.} = - ## Walks over the directory `dir` and yields for each directory or file - ## in `dir`, non-recursively. It's a version of + tags: [ReadDirEffect], raises: [], noNimScript, since: (1, 1).} = + ## Walks over the directory `dir` and yields *steps* for each + ## directory or file in `dir`, non-recursively. It's a version of ## `walkDir iterator <#walkDir.i,string>`_ with more thorough error - ## checking. It never raises an exception. + ## checking. Never raises an exception. ## - ## Yields *steps* of walking through `dir`, each step contains - ## its path ``path``, OS error code ``code`` and additional tag ``kind``: + ## Each step is object variant ``WalkStep``, which contains corresponding + ## path ``path``, OS error code ``code`` and discriminator ``kind``: ## - ## - ``kind=wsOpenDir``: yielded once after opening, contains ``openStatus`` - ## of type ``OpenDirStatus`` + ## - ``kind=wsOpenDir`` is yielded once after opening, contains ``openStatus`` + ## of type ``OpenDirStatus`` ## - then it yields zero or more entries, each can be of the following kind: ## - ``kind=wsEntryOk``: signifies normal entries with ## path component ``entryType`` @@ -2146,7 +2146,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. ## - ``kind=wsInterrupted``: signifies that a rare OS error happenned and ## the walking was terminated ## - ## An example of usage with full error checking: + ## An example of usage with full error logging: ## .. code-block:: Nim ## for step in tryWalkDir("dirA"): ## let err = " (" & $step.code & " - " & osErrorMsg(step.code) & ")" @@ -2173,7 +2173,6 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. ## of wsEntryBad: yield (pcLinkToFile, step.path) ## of wsInterrupted: raiseOSError(step.code) - var step: WalkStep var skip = false proc sOpenDir(s: OpenDirStatus, path: string): WalkStep = From f9b11f42605a5f3e84983e279a32aa0651232296 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 5 Feb 2020 00:19:13 +0300 Subject: [PATCH 18/26] export WalkStep --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 66c2865cbf060..b3b160f4fdd1d 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2107,7 +2107,7 @@ type wsEntryBad ## a broken symlink on posix, where it's unclear if it ## points to a file or directory wsInterrupted ## the directory handle got invalidated during reading - WalkStep = object ## step of walking through directory + WalkStep* = object ## step of walking through directory path*: string ## the path that this step applies to code*: OSErrorCode ## OS error code for the step case kind*: WalkStepKind ## discriminator From 2d3fb2eb7b094134e522d2bd476294aa81203894 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 5 Feb 2020 01:47:47 +0300 Subject: [PATCH 19/26] consistent handling of relative=true for wsOpenDir --- lib/pure/os.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index b3b160f4fdd1d..1e2ce763144cf 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2139,6 +2139,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. ## ## - ``kind=wsOpenDir`` is yielded once after opening, contains ``openStatus`` ## of type ``OpenDirStatus`` + ## (when `relative=true` the path is '.') ## - then it yields zero or more entries, each can be of the following kind: ## - ``kind=wsEntryOk``: signifies normal entries with ## path component ``entryType`` @@ -2207,7 +2208,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. of ERROR_ACCESS_DENIED: odAccessDenied else: odUnknownError else: odOpenOk - step = sOpenDir(status, dir) + step = sOpenDir(status, if relative: "." else: dir) var firstStep = true while true: @@ -2278,7 +2279,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. elif errno == EACCES: odAccessDenied else: odUnknownError else: odOpenOk - step = sOpenDir(status, dir) + step = sOpenDir(status, if relative: "." else: dir) while true: if not skip: From 60fdc4d094b6171961429f727b2a3e8d7e0785e1 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 5 Feb 2020 02:04:21 +0300 Subject: [PATCH 20/26] the same for wsInterrupted --- lib/pure/os.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 1e2ce763144cf..c991cdac7fc51 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2176,6 +2176,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. var step: WalkStep var skip = false + let outDir = if relative: "." else: dir proc sOpenDir(s: OpenDirStatus, path: string): WalkStep = let code = if s == odOpenOk: OSErrorCode(0) else: osLastError() result = WalkStep(kind: wsOpenDir, openStatus: s, code: code, path: path) @@ -2208,7 +2209,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. of ERROR_ACCESS_DENIED: odAccessDenied else: odUnknownError else: odOpenOk - step = sOpenDir(status, if relative: "." else: dir) + step = sOpenDir(status, outDir) var firstStep = true while true: @@ -2225,7 +2226,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. break # normal end, yielding nothing else: skip = false - step = sGetError(wsInterrupted, dir) + step = sGetError(wsInterrupted, outDir) continue skip = skipFindData(f) if not skip: @@ -2279,7 +2280,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. elif errno == EACCES: odAccessDenied else: odUnknownError else: odOpenOk - step = sOpenDir(status, if relative: "." else: dir) + step = sOpenDir(status, outDir) while true: if not skip: @@ -2295,7 +2296,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. break # normal end, yielding nothing else: skip = false - step = sGetError(wsInterrupted, dir) + step = sGetError(wsInterrupted, outDir) continue when defined(nimNoArrayToCstringConversion): var y = $cstring(addr x.d_name) From f62a48006e787d6504a6b341d8675d4eec3ede8a Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 8 Feb 2020 17:55:10 +0300 Subject: [PATCH 21/26] address review comments --- lib/pure/os.nim | 170 +++++++++++++++++++++++------------------------- 1 file changed, 80 insertions(+), 90 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index c991cdac7fc51..1a304536aa1ae 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2095,38 +2095,32 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: yield (k, y) type - OpenDirStatus* = enum ## enumeration specifying status of opening a directory - odUnknownError, ## unrecognized error - odOpenOk, ## the directory was opened OK - odNotDir, ## error, not a directory: a file with the same path exists - odAccessDenied, ## permission denied / access denied error - odNotFound ## no such path - WalkStepKind* = enum ## status of the step of walking through directory - wsOpenDir, ## step for attempt to open directory - wsEntryOk, ## step when entry is OK - wsEntryBad ## a broken symlink on posix, where it's unclear if it - ## points to a file or directory - wsInterrupted ## the directory handle got invalidated during reading + WalkStepKind* = enum ## status of the step of walking through directory. + ## Due to differences between operating systems, + ## opening directory status may not have exactly + ## the same meaning. + wsDirFailure, ## opening directory error: unrecognized OS-specific + wsDirNotFound, ## opening directory error: no such path + wsDirNoAccess, ## opening directory error: access denied a.k.a. + ## permission denied error for the specified path + ## (may have happened at its parent directory on posix) + wsNotDir, ## opening directory error: not a directory + ## (a normal file with the same path exists) + wsDirOpened, ## opening directory OK, the directory can be read + wsEntryOk, ## entry OK: path component is right + wsEntryBad ## entry error: a broken symlink (on posix), where it's + ## unclear if it points to a file or directory + wsInterrupted ## reading directory entries was interrupted WalkStep* = object ## step of walking through directory path*: string ## the path that this step applies to code*: OSErrorCode ## OS error code for the step case kind*: WalkStepKind ## discriminator - of wsOpenDir: - openStatus*: OpenDirStatus ## status of opening the directory of wsEntryOk: entryType*: PathComponent ## path component - of wsEntryBad, wsInterrupted: + of wsDirFailure, wsDirNoAccess, wsDirNotFound, wsNotDir, + wsDirOpened, wsEntryBad, wsInterrupted: discard -proc `$`*(s: WalkStep): string = - let err = if s.code == OSErrorCode(0): "" - else: " ($1 - $2)" % [$s.code, osErrorMsg(s.code)] - case s.kind - of wsOpenDir: "(wsOpenDir | $1 '$2'$3)" % [$s.openStatus, s.path, err] - of wsEntryOk: "(wsEntryOk | $1 '$2')" % [$s.entryType, s.path] - of wsEntryBad: "(wsEntryBad | '$1'$2)" % [s.path, err] - of wsInterrupted: "(wsInterrupted | '$1'$2)" % [s.path, err] - iterator tryWalkDir*(dir: string, relative=false): WalkStep {. tags: [ReadDirEffect], raises: [], noNimScript, since: (1, 1).} = ## Walks over the directory `dir` and yields *steps* for each @@ -2137,39 +2131,33 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. ## Each step is object variant ``WalkStep``, which contains corresponding ## path ``path``, OS error code ``code`` and discriminator ``kind``: ## - ## - ``kind=wsOpenDir`` is yielded once after opening, contains ``openStatus`` - ## of type ``OpenDirStatus`` - ## (when `relative=true` the path is '.') + ## - it yields open directory status once after opening + ## (when `relative=true` the path is '.'), + ## ``kind`` is one of ``wsDirOpened``, ``wsDirNoAccess``, + ## ``wsDirNotFound``, ``wsNotDir``, ``wsDirFailure`` ## - then it yields zero or more entries, each can be of the following kind: ## - ``kind=wsEntryOk``: signifies normal entries with - ## path component ``entryType`` + ## path component ``entryType`` ## - ``kind=wsEntryBad``: broken symlink (without path component) - ## - ``kind=wsInterrupted``: signifies that a rare OS error happenned and - ## the walking was terminated + ## - ``kind=wsInterrupted``: signifies that a rare OS-specific I/O error + ## happenned and the walking was terminated. ## - ## An example of usage with full error logging: + ## An example of usage with just minimal error logging: ## .. code-block:: Nim ## for step in tryWalkDir("dirA"): - ## let err = " (" & $step.code & " - " & osErrorMsg(step.code) & ")" ## case step.kind - ## of wsOpenDir: - ## case step.openStatus: - ## of odOpenOk: echo "directory is opened OK" - ## of odNotDir: echo "is not a directory" & err - ## of odAccessDenied: echo "Access Denied" & err - ## of odNotFound: echo "no such path" & err - ## of odUnknownError: echo "unknown error - that's bad." & err + ## of wsDirOpened: discard ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType - ## of wsEntryBad: echo step.path & " is broken symlink " & err - ## of wsInterrupted: echo " traversal interrupted, really bad. " & err + ## else: echo "got error " & osErrorMsg(step.code) & " on " & step.path ## - ## For comparison, walkDir iterator may be implemented with tryWalkDir: + ## Iterator ``walkDir`` itself may be implemented using tryWalkDir: ## .. code-block:: Nim ## iterator myWalkDir(path: string, relative: bool): - ## tuple[kind: PathComponent, path: string] = + ## tuple[kind: PathComponent, path: string] = ## for step in tryWalkDir(path, relative): ## case step.kind - ## of wsOpenDir: discard + ## of wsDirOpened, wsDirNotFound, + ## wsDirNoAccess, wsDirNoAccess, wsNotDir: discard ## of wsEntryOk: yield (step.entryType, step.path) ## of wsEntryBad: yield (pcLinkToFile, step.path) ## of wsInterrupted: raiseOSError(step.code) @@ -2177,12 +2165,12 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. var step: WalkStep var skip = false let outDir = if relative: "." else: dir - proc sOpenDir(s: OpenDirStatus, path: string): WalkStep = - let code = if s == odOpenOk: OSErrorCode(0) else: osLastError() - result = WalkStep(kind: wsOpenDir, openStatus: s, code: code, path: path) - proc sNoErrors(pc: PathComponent, path: string): WalkStep = + proc openStatus(s: WalkStepKind, path: string): WalkStep = + let code = if s == wsDirOpened: OSErrorCode(0) else: osLastError() + result = WalkStep(kind: s, code: code, path: path) + proc entryOk(pc: PathComponent, path: string): WalkStep = WalkStep(kind: wsEntryOk, entryType: pc, code: OSErrorCode(0), path: path) - proc sGetError(k: WalkStepKind, path: string): WalkStep = + proc entryError(k: WalkStepKind, path: string): WalkStep = WalkStep(kind: k, code: osLastError(), path: path) when defined(windows): @@ -2194,7 +2182,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. k = succ(k) let xx = if relative: extractFilename(getFilename(fdat)) else: dir / extractFilename(getFilename(fdat)) - sNoErrors(k, xx) + entryOk(k, xx) var f: WIN32_FIND_DATA var h: Handle = findFirstFile(dir / "*", f) @@ -2202,14 +2190,15 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. if h != -1: findClose(h) let status = - if h == -1: + if h != -1: + wsDirOpened + else: case getLastError() - of ERROR_PATH_NOT_FOUND: odNotFound - of ERROR_DIRECTORY: odNotDir - of ERROR_ACCESS_DENIED: odAccessDenied - else: odUnknownError - else: odOpenOk - step = sOpenDir(status, outDir) + of ERROR_PATH_NOT_FOUND: wsDirNotFound + of ERROR_DIRECTORY: wsNotDir + of ERROR_ACCESS_DENIED: wsDirNoAccess + else: wsDirFailure + step = openStatus(status, outDir) var firstStep = true while true: @@ -2219,19 +2208,21 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. break if firstStep: # use file obtained by findFirstFile firstStep = false + skip = skipFindData(f) + if not skip: + step = resolveFile(f, relative) else: # get next file - if findNextFile(h, f) == 0'i32: + if findNextFile(h, f) != 0'i32: + skip = skipFindData(f) + if not skip: + step = resolveFile(f, relative) + else: let errCode = getLastError() if errCode == ERROR_NO_MORE_FILES: break # normal end, yielding nothing else: skip = false - step = sGetError(wsInterrupted, outDir) - continue - skip = skipFindData(f) - if not skip: - step = resolveFile(f, relative) - + step = entryError(wsInterrupted, outDir) else: proc resolveFile(x: ptr Dirent, path: string, y: string): WalkStep = var k = pcFile @@ -2240,23 +2231,23 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. if x.d_type != DT_UNKNOWN: if x.d_type == DT_DIR: k = pcDir - return sNoErrors(k, y) + return entryOk(k, y) elif x.d_type == DT_LNK: errno = 0 if dirExists(path): k = pcLinkToDir else: k = pcLinkToFile # check error in dirExists if errno == 0: - return sNoErrors(k, y) + return entryOk(k, y) else: - return sGetError(wsEntryBad, y) + return entryError(wsEntryBad, y) else: - return sNoErrors(k, y) + return entryOk(k, y) # case of d_type==DT_UNKNOWN: some filesystems don't report # the entry type; one should fallback to lstat var s: Stat if lstat(path, s) < 0'i32: - return sGetError(wsEntryBad, y) + return entryError(wsEntryBad, y) else: errno = 0 if S_ISDIR(s.st_mode): @@ -2265,22 +2256,23 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. k = getSymlinkFileKind(path) # check error in getSymlinkFileKind if errno == 0: - return sNoErrors(k, y) + return entryOk(k, y) else: - return sGetError(wsEntryBad, y) + return entryError(wsEntryBad, y) var d: ptr DIR = opendir(dir) defer: if d != nil: discard closedir(d) let status = - if d == nil: - if errno == ENOENT: odNotFound - elif errno == ENOTDIR: odNotDir - elif errno == EACCES: odAccessDenied - else: odUnknownError - else: odOpenOk - step = sOpenDir(status, outDir) + if d != nil: + wsDirOpened + else: + if errno == ENOENT: wsDirNotFound + elif errno == ENOTDIR: wsNotDir + elif errno == EACCES: wsDirNoAccess + else: wsDirFailure + step = openStatus(status, outDir) while true: if not skip: @@ -2288,26 +2280,24 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. if d == nil or step.kind == wsInterrupted: break let errnoSave = errno - defer: errno = errnoSave errno = 0 var x = readdir(d) if x == nil: if errno == 0: + errno = errnoSave break # normal end, yielding nothing else: skip = false - step = sGetError(wsInterrupted, outDir) - continue - when defined(nimNoArrayToCstringConversion): - var y = $cstring(addr x.d_name) + step = entryError(wsInterrupted, outDir) else: - var y = $x.d_name.cstring - skip = (y == "." or y == "..") - if not skip: - let path = dir / y - if not relative: - y = path - step = resolveFile(x, path, y) + var y = $cstring(addr x.d_name) + skip = (y == "." or y == "..") + if not skip: + let path = dir / y + if not relative: + y = path + step = resolveFile(x, path, y) + errno = errnoSave iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}, From 00b3cf89d3e8af751be5a19c94c5e874dc7724ae Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 13 Feb 2020 01:09:37 +0300 Subject: [PATCH 22/26] returned back to more golang style --- lib/pure/os.nim | 152 +++++++++++++++++++++++++----------------------- 1 file changed, 79 insertions(+), 73 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 1a304536aa1ae..1c7f5518b3797 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2095,86 +2095,92 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: yield (k, y) type - WalkStepKind* = enum ## status of the step of walking through directory. - ## Due to differences between operating systems, - ## opening directory status may not have exactly - ## the same meaning. - wsDirFailure, ## opening directory error: unrecognized OS-specific - wsDirNotFound, ## opening directory error: no such path - wsDirNoAccess, ## opening directory error: access denied a.k.a. - ## permission denied error for the specified path - ## (may have happened at its parent directory on posix) - wsNotDir, ## opening directory error: not a directory - ## (a normal file with the same path exists) - wsDirOpened, ## opening directory OK, the directory can be read - wsEntryOk, ## entry OK: path component is right - wsEntryBad ## entry error: a broken symlink (on posix), where it's - ## unclear if it points to a file or directory - wsInterrupted ## reading directory entries was interrupted - WalkStep* = object ## step of walking through directory - path*: string ## the path that this step applies to - code*: OSErrorCode ## OS error code for the step - case kind*: WalkStepKind ## discriminator - of wsEntryOk: - entryType*: PathComponent ## path component - of wsDirFailure, wsDirNoAccess, wsDirNotFound, wsNotDir, - wsDirOpened, wsEntryBad, wsInterrupted: - discard - -iterator tryWalkDir*(dir: string, relative=false): WalkStep {. + WalkStatus* = enum ## status of the step of walking through directory; + ## due to differences between operating systems, + ## opening directory status may not have exactly + ## the same meaning. + wsOpenUnknown, ## opening directory error: unrecognized OS-specific one + wsOpenOk, ## opening directory OK, the directory can be read + wsOpenNotFound, ## opening directory error: no such path + wsOpenNoAccess, ## opening directory error: access denied a.k.a. + ## permission denied error for the specified path + ## (may have happened at its parent directory on posix) + wsOpenNotDir, ## opening directory error: not a directory + ## (a normal file with the same path exists) + wsEntryOk, ## entry OK: path component is correct + wsEntryBad ## entry error: a broken symlink (on posix), where it's + ## unclear if it points to a file or directory + wsInterrupted ## reading directory entries was interrupted + +iterator tryWalkDir*(dir: string, relative=false): + tuple[status: WalkStatus, kind: PathComponent, path: string, + code: OSErrorCode] {. tags: [ReadDirEffect], raises: [], noNimScript, since: (1, 1).} = ## Walks over the directory `dir` and yields *steps* for each ## directory or file in `dir`, non-recursively. It's a version of ## `walkDir iterator <#walkDir.i,string>`_ with more thorough error ## checking. Never raises an exception. ## - ## Each step is object variant ``WalkStep``, which contains corresponding - ## path ``path``, OS error code ``code`` and discriminator ``kind``: + ## Each step is a tuple, which contains ``status: WalkStatus``, + ## path ``path``, entry type ``kind: PathCompenent``, OS error code ``code``. ## - ## - it yields open directory status once after opening + ## - it yields open directory status **once** after opening ## (when `relative=true` the path is '.'), - ## ``kind`` is one of ``wsDirOpened``, ``wsDirNoAccess``, - ## ``wsDirNotFound``, ``wsNotDir``, ``wsDirFailure`` - ## - then it yields zero or more entries, each can be of the following kind: - ## - ``kind=wsEntryOk``: signifies normal entries with - ## path component ``entryType`` - ## - ``kind=wsEntryBad``: broken symlink (without path component) - ## - ``kind=wsInterrupted``: signifies that a rare OS-specific I/O error + ## ``status`` is one of ``wsOpenOk``, ``wsOpenNoAccess``, + ## ``wsOpenNotFound``, ``wsOpenNotDir``, ``wsOpenUnknown`` + ## - then it yields zero or more entries, each can be of the following type: + ## - ``status=wsEntryOk``: signifies normal entries with + ## path component ``kind`` + ## - ``status=wsEntryBad``: broken symlink (without path component) + ## - ``status=wsInterrupted``: signifies that a rare OS-specific I/O error ## happenned and the walking was terminated. ## - ## An example of usage with just minimal error logging: + ## Path component ``kind`` is only meaningful when ``status=wsEntryOk``. + ## + ## An example of usage with just a minimal error logging: + ## .. code-block:: Nim + ## for status, kind, path, code in tryWalkDir("dirA"): + ## case status + ## of wsOpenOk: discard + ## of wsEntryOk: echo path & " is entry of kind " & $kind + ## else: echo "got error " & osErrorMsg(code) & " on " & path + ## + ## To just check whether the directory can be opened or not : ## .. code-block:: Nim - ## for step in tryWalkDir("dirA"): - ## case step.kind - ## of wsDirOpened: discard - ## of wsEntryOk: echo step.path & " is entry of kind " & $step.entryType - ## else: echo "got error " & osErrorMsg(step.code) & " on " & step.path + ## proc tryOpenDir(dir: string): WalkStatus = + ## for status, _, _, _ in tryWalkDir(dir): + ## case status + ## of wsOpenOk, wsOpenUnknown, wsOpenNoAccess, wsOpenNotFound, wsOpenNotDir: + ## return status + ## else: continue # can not happen + ## echo "can be opened: ", tryOpenDir("dirA") == wsOpenOk ## ## Iterator ``walkDir`` itself may be implemented using tryWalkDir: ## .. code-block:: Nim ## iterator myWalkDir(path: string, relative: bool): ## tuple[kind: PathComponent, path: string] = - ## for step in tryWalkDir(path, relative): - ## case step.kind - ## of wsDirOpened, wsDirNotFound, - ## wsDirNoAccess, wsDirNoAccess, wsNotDir: discard - ## of wsEntryOk: yield (step.entryType, step.path) - ## of wsEntryBad: yield (pcLinkToFile, step.path) - ## of wsInterrupted: raiseOSError(step.code) - - var step: WalkStep + ## for (status, kind, path, code) in tryWalkDir(path, relative): + ## case status + ## of wsOpenOk, wsOpenNotFound, + ## wsOpenUnknown, wsOpenNoAccess, wsOpenNotDir: discard + ## of wsEntryOk: yield (kind, path) + ## of wsEntryBad: yield (pcLinkToFile, path) + ## of wsInterrupted: raiseOSError(code) + + var step: tuple[status: WalkStatus, kind: PathComponent, + path: string, code: OSErrorCode] var skip = false let outDir = if relative: "." else: dir - proc openStatus(s: WalkStepKind, path: string): WalkStep = - let code = if s == wsDirOpened: OSErrorCode(0) else: osLastError() - result = WalkStep(kind: s, code: code, path: path) - proc entryOk(pc: PathComponent, path: string): WalkStep = - WalkStep(kind: wsEntryOk, entryType: pc, code: OSErrorCode(0), path: path) - proc entryError(k: WalkStepKind, path: string): WalkStep = - WalkStep(kind: k, code: osLastError(), path: path) + template openStatus(s, p): auto = + let code = if s == wsOpenOk: OSErrorCode(0) else: osLastError() + (status: s, kind: pcDir, path: p, code: code) + template entryOk(pc, p): auto = + (status: wsEntryOk, kind: pc, path: p, code: OSErrorCode(0)) + template entryError(s, p): auto = + (status: s, kind: pcLinkToFile, path: p, code: osLastError()) when defined(windows): - template resolveFile(fdat: WIN32_FIND_DATA, relative: bool): WalkStep = + template resolveFile(fdat: WIN32_FIND_DATA, relative: bool): auto = var k: PathComponent if (fdat.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: k = pcDir @@ -2191,20 +2197,20 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. findClose(h) let status = if h != -1: - wsDirOpened + wsOpenOk else: case getLastError() - of ERROR_PATH_NOT_FOUND: wsDirNotFound - of ERROR_DIRECTORY: wsNotDir - of ERROR_ACCESS_DENIED: wsDirNoAccess - else: wsDirFailure + of ERROR_PATH_NOT_FOUND: wsOpenNotFound + of ERROR_DIRECTORY: wsOpenNotDir + of ERROR_ACCESS_DENIED: wsOpenNoAccess + else: wsOpenUnknown step = openStatus(status, outDir) var firstStep = true while true: if not skip: yield step - if h == -1 or step.kind == wsInterrupted: + if h == -1 or step.status == wsInterrupted: break if firstStep: # use file obtained by findFirstFile firstStep = false @@ -2224,7 +2230,7 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. skip = false step = entryError(wsInterrupted, outDir) else: - proc resolveFile(x: ptr Dirent, path: string, y: string): WalkStep = + proc resolveFile(x: ptr Dirent, path: string, y: string): auto = var k = pcFile when defined(linux) or defined(macosx) or defined(bsd) or defined(genode) or defined(nintendoswitch): @@ -2266,18 +2272,18 @@ iterator tryWalkDir*(dir: string, relative=false): WalkStep {. discard closedir(d) let status = if d != nil: - wsDirOpened + wsOpenOk else: - if errno == ENOENT: wsDirNotFound - elif errno == ENOTDIR: wsNotDir - elif errno == EACCES: wsDirNoAccess - else: wsDirFailure + if errno == ENOENT: wsOpenNotFound + elif errno == ENOTDIR: wsOpenNotDir + elif errno == EACCES: wsOpenNoAccess + else: wsOpenUnknown step = openStatus(status, outDir) while true: if not skip: yield step - if d == nil or step.kind == wsInterrupted: + if d == nil or step.status == wsInterrupted: break let errnoSave = errno errno = 0 From 854b00782948f55e020763d6f17f14f0c30cb407 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sun, 16 Feb 2020 02:33:10 +0300 Subject: [PATCH 23/26] handle special files --- lib/pure/os.nim | 203 +++++++++++++++++++++------------------- lib/windows/winlean.nim | 4 +- 2 files changed, 111 insertions(+), 96 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 1c7f5518b3797..4568a21b20367 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2020,11 +2020,11 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: ## dirA/fileA1.txt ## dirA/fileA2.txt ## - ## Error checking is aimed at silent dropping: if path `dir` is not - ## found then the iterator will yield nothing; the same goes if `dir` is a - ## file or is an access-denied directory; all broken symlinks inside `dir` get - ## default type ``pcLinkToFile``. However, OSException will be raised if - ## the directory handle is invalidated during walking. + ## Error checking is aimed at silent dropping: if path `dir` is missing + ## (or its access is denied) then the iterator will yield nothing; + ## all broken symlinks inside `dir` get default type ``pcLinkToFile``. + ## However, OSException may be raised if the directory handle is + ## invalidated during walking. ## ## See also: ## * `tryWalkDir iterator <#tryWalkDir.i,string>`_ @@ -2100,15 +2100,17 @@ type ## opening directory status may not have exactly ## the same meaning. wsOpenUnknown, ## opening directory error: unrecognized OS-specific one - wsOpenOk, ## opening directory OK, the directory can be read - wsOpenNotFound, ## opening directory error: no such path + wsOpenNotFound, ## opening directory error: no such path or path is invalid wsOpenNoAccess, ## opening directory error: access denied a.k.a. ## permission denied error for the specified path ## (may have happened at its parent directory on posix) wsOpenNotDir, ## opening directory error: not a directory ## (a normal file with the same path exists) + wsOpenOk, ## opening directory OK, the directory can be read wsEntryOk, ## entry OK: path component is correct - wsEntryBad ## entry error: a broken symlink (on posix), where it's + wsEntrySpecial ## special OS-specific entry: non-data file like + ## domain socket, Posix pipe, etc + wsEntryBad, ## entry error: a broken symlink (on posix), where it's ## unclear if it points to a file or directory wsInterrupted ## reading directory entries was interrupted @@ -2131,11 +2133,14 @@ iterator tryWalkDir*(dir: string, relative=false): ## - then it yields zero or more entries, each can be of the following type: ## - ``status=wsEntryOk``: signifies normal entries with ## path component ``kind`` + ## - ``status=wsEntrySpecial``: signifies special (non-data) files with + ## path component ``kind`` ## - ``status=wsEntryBad``: broken symlink (without path component) ## - ``status=wsInterrupted``: signifies that a rare OS-specific I/O error ## happenned and the walking was terminated. ## - ## Path component ``kind`` is only meaningful when ``status=wsEntryOk``. + ## Path component ``kind`` value is reliable only when ``status=wsEntryOk`` + ## or ``wsEntrySpecial``. ## ## An example of usage with just a minimal error logging: ## .. code-block:: Nim @@ -2143,6 +2148,7 @@ iterator tryWalkDir*(dir: string, relative=false): ## case status ## of wsOpenOk: discard ## of wsEntryOk: echo path & " is entry of kind " & $kind + ## of wsEntrySpecial: echo path & " is a special file " & $kind ## else: echo "got error " & osErrorMsg(code) & " on " & path ## ## To just check whether the directory can be opened or not : @@ -2159,11 +2165,11 @@ iterator tryWalkDir*(dir: string, relative=false): ## .. code-block:: Nim ## iterator myWalkDir(path: string, relative: bool): ## tuple[kind: PathComponent, path: string] = - ## for (status, kind, path, code) in tryWalkDir(path, relative): + ## for status, kind, path, code in tryWalkDir(path, relative): ## case status ## of wsOpenOk, wsOpenNotFound, ## wsOpenUnknown, wsOpenNoAccess, wsOpenNotDir: discard - ## of wsEntryOk: yield (kind, path) + ## of wsEntryOk, wsEntrySpecial: yield (kind, path) ## of wsEntryBad: yield (pcLinkToFile, path) ## of wsInterrupted: raiseOSError(code) @@ -2176,100 +2182,104 @@ iterator tryWalkDir*(dir: string, relative=false): (status: s, kind: pcDir, path: p, code: code) template entryOk(pc, p): auto = (status: wsEntryOk, kind: pc, path: p, code: OSErrorCode(0)) + template entrySpecial(pc, p): auto = + (status: wsEntrySpecial, kind: pc, path: p, code: OSErrorCode(0)) template entryError(s, p): auto = (status: s, kind: pcLinkToFile, path: p, code: osLastError()) when defined(windows): - template resolveFile(fdat: WIN32_FIND_DATA, relative: bool): auto = + template resolvePath(fdat: WIN32_FIND_DATA, relative: bool): type(step) = var k: PathComponent if (fdat.dwFileAttributes and FILE_ATTRIBUTE_DIRECTORY) != 0'i32: k = pcDir - if (fdat.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: - k = succ(k) let xx = if relative: extractFilename(getFilename(fdat)) else: dir / extractFilename(getFilename(fdat)) - entryOk(k, xx) + if (fdat.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32: + if (fdat.dwReserved0 and REPARSE_TAG_NAME_SURROGATE) != 0'u32: + k = succ(k) # it's a "surrogate", that is symlink (or junction) + entryOk(k, xx) + else: # some strange reparse point + entrySpecial(k, xx) + else: + entryOk(k, xx) var f: WIN32_FIND_DATA var h: Handle = findFirstFile(dir / "*", f) - defer: - if h != -1: - findClose(h) let status = if h != -1: wsOpenOk else: case getLastError() - of ERROR_PATH_NOT_FOUND: wsOpenNotFound + of ERROR_PATH_NOT_FOUND, ERROR_INVALID_NAME: wsOpenNotFound of ERROR_DIRECTORY: wsOpenNotDir of ERROR_ACCESS_DENIED: wsOpenNoAccess else: wsOpenUnknown step = openStatus(status, outDir) + var firstFile = true - var firstStep = true - while true: - if not skip: - yield step - if h == -1 or step.status == wsInterrupted: - break - if firstStep: # use file obtained by findFirstFile - firstStep = false - skip = skipFindData(f) + try: + while true: if not skip: - step = resolveFile(f, relative) - else: # get next file - if findNextFile(h, f) != 0'i32: + yield step + if h == -1 or step.status == wsInterrupted: + break + if firstFile: # use file obtained by findFirstFile skip = skipFindData(f) if not skip: - step = resolveFile(f, relative) - else: - let errCode = getLastError() - if errCode == ERROR_NO_MORE_FILES: - break # normal end, yielding nothing + step = resolvePath(f, relative) + firstFile = false + else: # load next file + if findNextFile(h, f) != 0'i32: + skip = skipFindData(f) + if not skip: + step = resolvePath(f, relative) else: - skip = false - step = entryError(wsInterrupted, outDir) - else: - proc resolveFile(x: ptr Dirent, path: string, y: string): auto = - var k = pcFile - when defined(linux) or defined(macosx) or - defined(bsd) or defined(genode) or defined(nintendoswitch): - if x.d_type != DT_UNKNOWN: - if x.d_type == DT_DIR: - k = pcDir - return entryOk(k, y) - elif x.d_type == DT_LNK: - errno = 0 - if dirExists(path): k = pcLinkToDir - else: k = pcLinkToFile - # check error in dirExists - if errno == 0: - return entryOk(k, y) + let errCode = getLastError() + if errCode == ERROR_NO_MORE_FILES: + break # normal end, yielding nothing else: - return entryError(wsEntryBad, y) - else: - return entryOk(k, y) - # case of d_type==DT_UNKNOWN: some filesystems don't report - # the entry type; one should fallback to lstat + skip = false + step = entryError(wsInterrupted, outDir) + finally: + if h != -1: + findClose(h) + + else: + template resolveSymlink(p, y: string): type(step) = + var s: Stat + if stat(p, s) >= 0'i32: + if S_ISDIR(s.st_mode): entryOk(pcLinkToDir, y) + elif S_ISREG(s.st_mode): entryOk(pcLinkToFile, y) + else: entrySpecial(pcLinkToFile, y) + else: entryError(wsEntryBad, y) + template resolvePathLstat(path, y: string): type(step) = var s: Stat if lstat(path, s) < 0'i32: - return entryError(wsEntryBad, y) + entryError(wsEntryBad, y) else: - errno = 0 - if S_ISDIR(s.st_mode): - k = pcDir - elif S_ISLNK(s.st_mode): - k = getSymlinkFileKind(path) - # check error in getSymlinkFileKind - if errno == 0: - return entryOk(k, y) + if S_ISREG(s.st_mode): entryOk(pcFile, y) + elif S_ISDIR(s.st_mode): entryOk(pcDir, y) + elif S_ISLNK(s.st_mode): resolveSymlink(path, y) + else: entrySpecial(pcFile, y) + template resolvePath(ent: ptr Dirent; dir, entName: string; + rel: bool): type(step) = + let path = dir / entName + let y = if rel: entName else: path + # fall back to pure-posix stat/lstat when d_type is not available or in + # case of d_type==DT_UNKNOWN (some filesystems can't report entry type) + when defined(linux) or defined(macosx) or + defined(bsd) or defined(genode) or defined(nintendoswitch): + if ent.d_type != DT_UNKNOWN: + if ent.d_type == DT_REG: entryOk(pcFile, y) + elif ent.d_type == DT_DIR: entryOk(pcDir, y) + elif ent.d_type == DT_LNK: resolveSymlink(path, y) + else: entrySpecial(pcFile, y) else: - return entryError(wsEntryBad, y) + resolvePathLstat(path, y) + else: + resolvePathLstat(path, y) var d: ptr DIR = opendir(dir) - defer: - if d != nil: - discard closedir(d) let status = if d != nil: wsOpenOk @@ -2279,31 +2289,34 @@ iterator tryWalkDir*(dir: string, relative=false): elif errno == EACCES: wsOpenNoAccess else: wsOpenUnknown step = openStatus(status, outDir) + var name: string + var ent: ptr Dirent - while true: - if not skip: - yield step - if d == nil or step.status == wsInterrupted: - break - let errnoSave = errno - errno = 0 - var x = readdir(d) - if x == nil: - if errno == 0: - errno = errnoSave - break # normal end, yielding nothing - else: - skip = false - step = entryError(wsInterrupted, outDir) - else: - var y = $cstring(addr x.d_name) - skip = (y == "." or y == "..") + try: + while true: if not skip: - let path = dir / y - if not relative: - y = path - step = resolveFile(x, path, y) - errno = errnoSave + yield step + if d == nil or step.status == wsInterrupted: + break + let errnoSave = errno + errno = 0 + ent = readdir(d) + if ent == nil: + if errno == 0: + errno = errnoSave + break # normal end, yielding nothing + else: + skip = false + step = entryError(wsInterrupted, outDir) + errno = errnoSave + else: + name = $cstring(addr ent.d_name) + skip = (name == "." or name == "..") + if not skip: + step = resolvePath(ent, dir, name, relative) + finally: + if d != nil: + discard closedir(d) iterator walkDirRec*(dir: string, yieldFilter = {pcFile}, followFilter = {pcDir}, diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index 1ced2f5fb6051..0b04a9ce27dbf 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -312,6 +312,7 @@ const MOVEFILE_FAIL_IF_NOT_TRACKABLE* = 0x20'i32 MOVEFILE_REPLACE_EXISTING* = 0x1'i32 MOVEFILE_WRITE_THROUGH* = 0x8'i32 + REPARSE_TAG_NAME_SURROGATE* = 0x20000000'u32 type WIN32_FIND_DATA* {.pure.} = object @@ -321,7 +322,7 @@ type ftLastWriteTime*: FILETIME nFileSizeHigh*: int32 nFileSizeLow*: int32 - dwReserved0: int32 + dwReserved0*: uint32 dwReserved1: int32 cFileName*: array[0..(MAX_PATH) - 1, WinChar] cAlternateFileName*: array[0..13, WinChar] @@ -703,6 +704,7 @@ const ERROR_NO_MORE_FILES* = 18 ERROR_LOCK_VIOLATION* = 33 ERROR_HANDLE_EOF* = 38 + ERROR_INVALID_NAME* = 123 ERROR_BAD_ARGUMENTS* = 165 ERROR_DIRECTORY* = 267 From 60b6847c7f33d7ddb39ee8b782f35e19da900e6c Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Wed, 19 Feb 2020 00:48:44 +0300 Subject: [PATCH 24/26] improved a pair of edge cases --- lib/pure/os.nim | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 4568a21b20367..12a7abba33c6c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2100,16 +2100,19 @@ type ## opening directory status may not have exactly ## the same meaning. wsOpenUnknown, ## opening directory error: unrecognized OS-specific one - wsOpenNotFound, ## opening directory error: no such path or path is invalid + wsOpenNotFound, ## opening directory error: no such path wsOpenNoAccess, ## opening directory error: access denied a.k.a. ## permission denied error for the specified path ## (may have happened at its parent directory on posix) wsOpenNotDir, ## opening directory error: not a directory - ## (a normal file with the same path exists) + ## (a normal file with the same path exists or path is a + ## loop of symlinks) + wsOpenBadPath, ## opening directory error: path is invalid (too long or, + ## in Windows, contains illegal characters) wsOpenOk, ## opening directory OK, the directory can be read wsEntryOk, ## entry OK: path component is correct wsEntrySpecial ## special OS-specific entry: non-data file like - ## domain socket, Posix pipe, etc + ## Posix domain sockets, FIFOs, etc wsEntryBad, ## entry error: a broken symlink (on posix), where it's ## unclear if it points to a file or directory wsInterrupted ## reading directory entries was interrupted @@ -2198,8 +2201,8 @@ iterator tryWalkDir*(dir: string, relative=false): if (fdat.dwReserved0 and REPARSE_TAG_NAME_SURROGATE) != 0'u32: k = succ(k) # it's a "surrogate", that is symlink (or junction) entryOk(k, xx) - else: # some strange reparse point - entrySpecial(k, xx) + else: # some strange reparse point, considering it a normal file + entryOk(k, xx) else: entryOk(k, xx) @@ -2210,7 +2213,8 @@ iterator tryWalkDir*(dir: string, relative=false): wsOpenOk else: case getLastError() - of ERROR_PATH_NOT_FOUND, ERROR_INVALID_NAME: wsOpenNotFound + of ERROR_PATH_NOT_FOUND: wsOpenNotFound + of ERROR_INVALID_NAME: wsOpenBadPath of ERROR_DIRECTORY: wsOpenNotDir of ERROR_ACCESS_DENIED: wsOpenNoAccess else: wsOpenUnknown @@ -2285,8 +2289,9 @@ iterator tryWalkDir*(dir: string, relative=false): wsOpenOk else: if errno == ENOENT: wsOpenNotFound - elif errno == ENOTDIR: wsOpenNotDir + elif errno == ENOTDIR or errno == ELOOP: wsOpenNotDir elif errno == EACCES: wsOpenNoAccess + elif errno == ENAMETOOLONG: wsOpenBadPath else: wsOpenUnknown step = openStatus(status, outDir) var name: string From 3c584bdba78c98fca2436bbce8d88ad48c8f1d78 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Thu, 20 Feb 2020 00:53:50 +0300 Subject: [PATCH 25/26] improve documentation --- lib/pure/os.nim | 66 ++++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 12a7abba33c6c..ed16cfaca2e1c 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -2095,27 +2095,29 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: yield (k, y) type - WalkStatus* = enum ## status of the step of walking through directory; - ## due to differences between operating systems, - ## opening directory status may not have exactly - ## the same meaning. - wsOpenUnknown, ## opening directory error: unrecognized OS-specific one - wsOpenNotFound, ## opening directory error: no such path - wsOpenNoAccess, ## opening directory error: access denied a.k.a. + WalkStatus* = enum ## status of a step got from walking through directory. + ## Due to differences between operating systems, + ## this status may not have exactly the same meaning. + ## It's yielded by + ## `tryWalkDir iterator <#tryWalkDir.i,string>`. + wsOpenUnknown, ## open directory error: unrecognized OS-specific one + wsOpenNotFound, ## open directory error: no such path + wsOpenNoAccess, ## open directory error: access denied a.k.a. ## permission denied error for the specified path - ## (may have happened at its parent directory on posix) - wsOpenNotDir, ## opening directory error: not a directory + ## (it may have happened at its parent directory on posix) + wsOpenNotDir, ## open directory error: not a directory ## (a normal file with the same path exists or path is a ## loop of symlinks) - wsOpenBadPath, ## opening directory error: path is invalid (too long or, - ## in Windows, contains illegal characters) - wsOpenOk, ## opening directory OK, the directory can be read - wsEntryOk, ## entry OK: path component is correct - wsEntrySpecial ## special OS-specific entry: non-data file like - ## Posix domain sockets, FIFOs, etc - wsEntryBad, ## entry error: a broken symlink (on posix), where it's - ## unclear if it points to a file or directory - wsInterrupted ## reading directory entries was interrupted + wsOpenBadPath, ## open directory error: path is invalid (too long or, + ## in Windows, it contains illegal characters) + wsOpenOk, ## open directory OK: the directory can be read + wsEntryOk, ## get entry OK: path component is correct + wsEntrySpecial ## get entry OK: OS-specific entry + ## ("special" or "device" file, not a normal data file) + ## like Posix domain sockets, FIFOs, etc + wsEntryBad, ## get entry error: its path is a broken symlink (on Posix), + ## where it's unclear if it points to a file or directory + wsInterrupted ## walking was interrupted while getting the next entry iterator tryWalkDir*(dir: string, relative=false): tuple[status: WalkStatus, kind: PathComponent, path: string, @@ -2124,16 +2126,18 @@ iterator tryWalkDir*(dir: string, relative=false): ## Walks over the directory `dir` and yields *steps* for each ## directory or file in `dir`, non-recursively. It's a version of ## `walkDir iterator <#walkDir.i,string>`_ with more thorough error - ## checking. Never raises an exception. + ## checking and reporting of "special" files or "defice" files. + ## Never raises an exception. ## - ## Each step is a tuple, which contains ``status: WalkStatus``, - ## path ``path``, entry type ``kind: PathCompenent``, OS error code ``code``. + ## Each step is a tuple containing ``status: WalkStatus``, path ``path``, + ## entry type ``kind: PathComponent``, OS error code ``code``. ## ## - it yields open directory status **once** after opening ## (when `relative=true` the path is '.'), ## ``status`` is one of ``wsOpenOk``, ``wsOpenNoAccess``, ## ``wsOpenNotFound``, ``wsOpenNotDir``, ``wsOpenUnknown`` - ## - then it yields zero or more entries, each can be of the following type: + ## - then it yields zero or more entries, + ## each can be one of the following type: ## - ``status=wsEntryOk``: signifies normal entries with ## path component ``kind`` ## - ``status=wsEntrySpecial``: signifies special (non-data) files with @@ -2145,8 +2149,10 @@ iterator tryWalkDir*(dir: string, relative=false): ## Path component ``kind`` value is reliable only when ``status=wsEntryOk`` ## or ``wsEntrySpecial``. ## - ## An example of usage with just a minimal error logging: + ## **Examples:** + ## ## .. code-block:: Nim + ## # An example of usage with just a minimal error logging: ## for status, kind, path, code in tryWalkDir("dirA"): ## case status ## of wsOpenOk: discard @@ -2154,24 +2160,22 @@ iterator tryWalkDir*(dir: string, relative=false): ## of wsEntrySpecial: echo path & " is a special file " & $kind ## else: echo "got error " & osErrorMsg(code) & " on " & path ## - ## To just check whether the directory can be opened or not : - ## .. code-block:: Nim + ## # To just check whether the directory can be opened or not: ## proc tryOpenDir(dir: string): WalkStatus = ## for status, _, _, _ in tryWalkDir(dir): ## case status - ## of wsOpenOk, wsOpenUnknown, wsOpenNoAccess, wsOpenNotFound, wsOpenNotDir: - ## return status + ## of wsOpenOk, wsOpenUnknown, wsOpenNotFound, + ## wsOpenNoAccess, wsOpenNotDir, wsOpenBadPath: return status ## else: continue # can not happen ## echo "can be opened: ", tryOpenDir("dirA") == wsOpenOk ## - ## Iterator ``walkDir`` itself may be implemented using tryWalkDir: - ## .. code-block:: Nim + ## # Iterator walkDir itself may be implemented using tryWalkDir: ## iterator myWalkDir(path: string, relative: bool): ## tuple[kind: PathComponent, path: string] = ## for status, kind, path, code in tryWalkDir(path, relative): ## case status - ## of wsOpenOk, wsOpenNotFound, - ## wsOpenUnknown, wsOpenNoAccess, wsOpenNotDir: discard + ## of wsOpenOk, wsOpenUnknown, wsOpenNotFound, + ## wsOpenNoAccess, wsOpenNotDir, wsOpenBadPath: discard ## of wsEntryOk, wsEntrySpecial: yield (kind, path) ## of wsEntryBad: yield (pcLinkToFile, path) ## of wsInterrupted: raiseOSError(code) From 24b818352fe0582558698c2f3145d9848821a793 Mon Sep 17 00:00:00 2001 From: Andrey Makarov Date: Sat, 22 Feb 2020 00:30:30 +0300 Subject: [PATCH 26/26] add test --- tests/stdlib/tos.nim | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index 03fc1f1e9c967..6e3a04170948b 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -25,7 +25,7 @@ Raises """ # test os path creation, iteration, and deletion -import os, strutils, pathnorm +import os, strutils, pathnorm, algorithm block fileOperations: let files = @["these.txt", "are.x", "testing.r", "files.q"] @@ -166,10 +166,41 @@ block walkDirRec: when not defined(windows): block walkDirRelative: createDir("walkdir_test") - createSymlink(".", "walkdir_test/c") + createSymlink(".", "walkdir_test/c_goodlink") for k, p in walkDir("walkdir_test", true): doAssert k == pcLinkToDir + + var dir: seq[(WalkStatus, PathComponent, string, OSErrorCode)] + createDir("walkdir_test/d_dir") + createSymlink("walkdir_test/non_existent", "walkdir_test/e_broken") + open("walkdir_test/f_file.txt", fmWrite).close() + for step in tryWalkDir("walkdir_test", true): + dir.add(step) + proc myCmp(x, y: (WalkStatus, PathComponent, string, OSErrorCode)): int = + system.cmp(x[2], y[2]) # sort by entry name, self "." will be the first + dir.sort(myCmp) + doAssert dir.len == 5 + # open step + doAssert dir[0][0] == wsOpenOk + doAssert dir[0][2] == "." + doAssert dir[0][3] == OSErrorCode(0) + # c_goodlink + doAssert dir[1] == (wsEntryOk, pcLinkToDir, "c_goodlink", OSErrorCode(0)) + # d_dir + doAssert dir[2] == (wsEntryOk, pcDir, "d_dir", OSErrorCode(0)) + # e_broken + doAssert dir[3][0] == wsEntryBad + doAssert dir[3][2] == "e_broken" + # f_file.txt + doAssert dir[4] == (wsEntryOk, pcFile, "f_file.txt", OSErrorCode(0)) removeDir("walkdir_test") + # after remove tryWalkDir returns error: + dir = @[] + for step in tryWalkDir("walkdir_test", true): + dir.add(step) + doAssert dir.len == 1 + doAssert dir[0][0] == wsOpenNotFound + doAssert dir[0][2] == "." block normalizedPath: doAssert normalizedPath("") == ""