From 92621c84203f567644b3886abc3d6c80fb4af1be Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 13 Jan 2021 20:16:34 +0200 Subject: [PATCH] stdlib/os: Add followSymlinks = true to copy/move functions Sometimes there is a need to copy symlinks as they are. This commit adds optional argument `followSymlinks = true` to all copy/move functions in os module. Note: Inspired by https://docs.python.org/3/library/shutil.html#shutil.copy --- lib/pure/os.nim | 155 ++++++++++++++++++++++--------------------- tests/stdlib/tos.nim | 10 +++ 2 files changed, 91 insertions(+), 74 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 77499deea2fa0..0e673ad1b9b4d 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1177,7 +1177,7 @@ const ## On Windows ``["exe", "cmd", "bat"]``, on Posix ``[""]``. when defined(windows): ["exe", "cmd", "bat"] else: [""] -proc findExe*(exe: string, followSymlinks: bool = true; +proc findExe*(exe: string, followSymlinks = true; extensions: openArray[string]=ExeExts): string {. tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect], noNimJs.} = ## Searches for `exe` in the current working directory and then @@ -1647,8 +1647,11 @@ proc setFilePermissions*(filename: string, permissions: set[FilePermission]) {. var res2 = setFileAttributesA(filename, res) if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions)) -proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = +proc createSymlink*(src, dest: string) {.tags: [ReadDirEffect, WriteIOEffect], noWeirdTarget.} +proc expandSymlink*(symlinkPath: string): string {.tags: [ReadIOEffect], noWeirdTarget.} + +proc copyFile*(source, dest: string, followSymlinks = true) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist. ## ## If this fails, `OSError` is raised. @@ -1668,11 +1671,11 @@ proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", ## will be preserved and the content overwritten. ## ## See also: - ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyDir proc <#copyDir,string,string,bool>`_ ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `moveFile proc <#moveFile,string,string,bool>`_ when defined(Windows): when useWinUnicode: @@ -1682,34 +1685,37 @@ proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", else: if copyFileA(source, dest, 0'i32) == 0'i32: raiseOSError(osLastError(), $(source, dest)) else: - # generic version of copyFile which works for any platform: - const bufSize = 8000 # better for memory manager - var d, s: File - if not open(s, source): raiseOSError(osLastError(), source) - if not open(d, dest, fmWrite): + if not followSymlinks and source.checkSymlink: + createSymlink(expandSymlink(source), dest) + else: + # generic version of copyFile which works for any platform: + const bufSize = 8000 # better for memory manager + var d, s: File + if not open(s, source): raiseOSError(osLastError(), source) + if not open(d, dest, fmWrite): + close(s) + raiseOSError(osLastError(), dest) + var buf = alloc(bufSize) + while true: + var bytesread = readBuffer(s, buf, bufSize) + if bytesread > 0: + var byteswritten = writeBuffer(d, buf, bytesread) + if bytesread != byteswritten: + dealloc(buf) + close(s) + close(d) + raiseOSError(osLastError(), dest) + if bytesread != bufSize: break + dealloc(buf) close(s) - raiseOSError(osLastError(), dest) - var buf = alloc(bufSize) - while true: - var bytesread = readBuffer(s, buf, bufSize) - if bytesread > 0: - var byteswritten = writeBuffer(d, buf, bytesread) - if bytesread != byteswritten: - dealloc(buf) - close(s) - close(d) - raiseOSError(osLastError(), dest) - if bytesread != bufSize: break - dealloc(buf) - close(s) - flushFile(d) - close(d) - -proc copyFileToDir*(source, dir: string) {.noWeirdTarget, since: (1,3,7).} = + flushFile(d) + close(d) + +proc copyFileToDir*(source, dir: string, followSymlinks = true) {.noWeirdTarget, since: (1,3,7).} = ## Copies a file `source` into directory `dir`, which must exist. if dir.len == 0: # treating "" as "." is error prone raise newException(ValueError, "dest is empty") - copyFile(source, dir / source.lastPathPart) + copyFile(source, dir / source.lastPathPart, followSymlinks = followSymlinks) when not declared(ENOENT) and not defined(Windows): when NoFakeVars: @@ -1739,10 +1745,10 @@ proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirE ## On Windows, ignores the read-only attribute. ## ## See also: - ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyFile proc <#copyFile,string,string,bool>`_ ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `moveFile proc <#moveFile,string,string,bool>`_ result = true when defined(Windows): when useWinUnicode: @@ -1772,10 +1778,10 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n ## ## See also: ## * `removeDir proc <#removeDir,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyFile proc <#copyFile,string,string,bool>`_ ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ + ## * `moveFile proc <#moveFile,string,string,bool>`_ if not tryRemoveFile(file): raiseOSError(osLastError(), file) @@ -1802,8 +1808,8 @@ proc tryMoveFSObject(source, dest: string): bool {.noWeirdTarget.} = raiseOSError(err, $(source, dest, strerror(errno))) return true -proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", - tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = +proc moveFile*(source, dest: string, followSymlinks = true) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} = ## Moves a file from `source` to `dest`. ## ## If this fails, `OSError` is raised. @@ -1812,8 +1818,8 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", ## Can be used to `rename files`:idx:. ## ## See also: - ## * `moveDir proc <#moveDir,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ + ## * `moveDir proc <#moveDir,string,string,bool>`_ + ## * `copyFile proc <#copyFile,string,string,bool>`_ ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ ## * `removeFile proc <#removeFile,string>`_ ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ @@ -1821,7 +1827,7 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", if not tryMoveFSObject(source, dest): when not defined(windows): # Fallback to copy & del - copyFile(source, dest) + copyFile(source, dest, followSymlinks = followSymlinks) try: removeFile(source) except: @@ -2223,9 +2229,9 @@ proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [ ## * `removeFile proc <#removeFile,string>`_ ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `copyDir proc <#copyDir,string,string,bool>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_ + ## * `moveDir proc <#moveDir,string,string,bool>`_ for kind, path in walkDir(dir, checkDir = checkDir): case kind of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path) @@ -2290,9 +2296,9 @@ proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1", ## See also: ## * `removeDir proc <#removeDir,string>`_ ## * `createDir proc <#createDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `copyDir proc <#copyDir,string,string,bool>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_ + ## * `moveDir proc <#moveDir,string,string,bool>`_ result = not rawCreateDir(dir) if result: # path already exists - need to check that it is indeed a directory @@ -2312,9 +2318,9 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1", ## See also: ## * `removeDir proc <#removeDir,string>`_ ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `copyDir proc <#copyDir,string,string,bool>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_ + ## * `moveDir proc <#moveDir,string,string,bool>`_ var omitNext = false when doslikeFileSystem: omitNext = isAbsolute(dir) @@ -2330,8 +2336,8 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1", dir[^1] notin {DirSep, AltSep}: discard existsOrCreateDir(dir) -proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", - tags: [WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = +proc copyDir*(source, dest: string, followSymlinks = true) {.rtl, extern: "nos$1", + tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], benign, noWeirdTarget.} = ## Copies a directory from `source` to `dest`. ## ## If this fails, `OSError` is raised. @@ -2341,46 +2347,47 @@ proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", ## ## On other platforms created files and directories will inherit the ## default permissions of a newly created file/directory for the user. - ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_ ## to preserve attributes recursively on these platforms. ## ## See also: - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - ## * `copyFile proc <#copyFile,string,string>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_ + ## * `copyFile proc <#copyFile,string,string,bool>`_ ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ ## * `removeDir proc <#removeDir,string>`_ ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ ## * `createDir proc <#createDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `moveDir proc <#moveDir,string,string,bool>`_ createDir(dest) for kind, path in walkDir(source): var noSource = splitPath(path).tail case kind of pcFile: - copyFile(path, dest / noSource) + copyFile(path, dest / noSource, followSymlinks = followSymlinks) of pcDir: - copyDir(path, dest / noSource) + copyDir(path, dest / noSource, followSymlinks = followSymlinks) else: discard -proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} = +proc moveDir*(source, dest: string, followSymlinks = true) {.tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], + noWeirdTarget.} = ## Moves a directory from `source` to `dest`. ## ## If this fails, `OSError` is raised. ## ## See also: - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ + ## * `moveFile proc <#moveFile,string,string,bool>`_ + ## * `copyDir proc <#copyDir,string,string,bool>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_ ## * `removeDir proc <#removeDir,string>`_ ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ ## * `createDir proc <#createDir,string>`_ if not tryMoveFSObject(source, dest): when not defined(windows): # Fallback to copy & del - copyDir(source, dest) + copyDir(source, dest, followSymlinks = followSymlinks) removeDir(source) -proc createSymlink*(src, dest: string) {.noWeirdTarget.} = +proc createSymlink*(src, dest: string) {.tags: [ReadDirEffect, WriteIOEffect], noWeirdTarget.} = ## Create a symbolic link at `dest` which points to the item specified ## by `src`. On most operating systems, will fail if a link already exists. ## @@ -2431,7 +2438,7 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} = raiseOSError(osLastError(), $(src, dest)) proc copyFileWithPermissions*(source, dest: string, - ignorePermissionErrors = true) {.noWeirdTarget.} = + ignorePermissionErrors = true, followSymlinks = true) {.noWeirdTarget.} = ## Copies a file from `source` to `dest` preserving file permissions. ## ## This is a wrapper proc around `copyFile <#copyFile,string,string>`_, @@ -2450,12 +2457,12 @@ proc copyFileWithPermissions*(source, dest: string, ## ## See also: ## * `copyFile proc <#copyFile,string,string>`_ - ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyDir proc <#copyDir,string,string,bool>`_ ## * `tryRemoveFile proc <#tryRemoveFile,string>`_ ## * `removeFile proc <#removeFile,string>`_ - ## * `moveFile proc <#moveFile,string,string>`_ - ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_ - copyFile(source, dest) + ## * `moveFile proc <#moveFile,string,string,bool>`_ + ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_ + copyFile(source, dest, followSymlinks = followSymlinks) when not defined(Windows): try: setFilePermissions(dest, getFilePermissions(source)) @@ -2464,17 +2471,17 @@ proc copyFileWithPermissions*(source, dest: string, raise proc copyDirWithPermissions*(source, dest: string, - ignorePermissionErrors = true) {.rtl, extern: "nos$1", + ignorePermissionErrors = true, followSymlinks = true) {.rtl, extern: "nos$1", tags: [WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} = ## Copies a directory from `source` to `dest` preserving file permissions. ## ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir - ## <#copyDir,string,string>`_ and `copyFileWithPermissions + ## <#copyDir,string,string,bool>`_ and `copyFileWithPermissions ## <#copyFileWithPermissions,string,string>`_ procs ## on non-Windows platforms. ## ## On Windows this proc is just a wrapper for `copyDir proc - ## <#copyDir,string,string>`_ since that proc already copies attributes. + ## <#copyDir,string,string,bool>`_ since that proc already copies attributes. ## ## On non-Windows systems permissions are copied after the file or directory ## itself has been copied, which won't happen atomically and could lead to a @@ -2483,11 +2490,11 @@ proc copyDirWithPermissions*(source, dest: string, ## `OSError`. ## ## See also: - ## * `copyDir proc <#copyDir,string,string>`_ + ## * `copyDir proc <#copyDir,string,string,bool>`_ ## * `copyFile proc <#copyFile,string,string>`_ ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_ ## * `removeDir proc <#removeDir,string>`_ - ## * `moveDir proc <#moveDir,string,string>`_ + ## * `moveDir proc <#moveDir,string,string,bool>`_ ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_ ## * `createDir proc <#createDir,string>`_ createDir(dest) @@ -2501,9 +2508,9 @@ proc copyDirWithPermissions*(source, dest: string, var noSource = splitPath(path).tail case kind of pcFile: - copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors) + copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, followSymlinks = followSymlinks) of pcDir: - copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors) + copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors, followSymlinks = followSymlinks) else: discard proc inclFilePermissions*(filename: string, @@ -2524,7 +2531,7 @@ proc exclFilePermissions*(filename: string, ## setFilePermissions(filename, getFilePermissions(filename)-permissions) setFilePermissions(filename, getFilePermissions(filename)-permissions) -proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} = +proc expandSymlink*(symlinkPath: string): string {.tags: [ReadIOEffect], noWeirdTarget.} = ## Returns a string representing the path to which the symbolic link points. ## ## On Windows this is a noop, ``symlinkPath`` is simply returned. diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index c053c16f281ba..70ac87924be6a 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -54,6 +54,16 @@ block fileOperations: doAssert not dirExists(dname/sub) removeFile(dname/fname) removeFile(dname/fname2) + when not defined(windows): + let brokenSymlink = dname/"D20210101T191320_BROKEN_SYMLINK" + let brokenSymlinkDesc = "D20210101T191320_I_DO_NOT_EXIST" + createSymlink(brokenSymlinkDesc, brokenSymlink) + let brokenSymlinkCopy = brokenSymlink & "_COPY" + doAssertRaises(OSError): copyFile(brokenSymlink, brokenSymlinkCopy) + copyFile(brokenSymlink, brokenSymlinkCopy, followSymlinks = false) + doAssert expandSymlink(brokenSymlinkCopy) == brokenSymlinkDesc + removeFile(brokenSymlink) + removeFile(brokenSymlinkCopy) # Test creating files and dirs for dir in dirs: