stdlib/os: Add followSymlinks = true to copy/move functions

Sometimes there is a need to copy symlinks as they are. This commit adds
optional argument `followSymlinks = true` to all copy/move functions in
os module.

Note:
Inspired by https://docs.python.org/3/library/shutil.html#shutil.copy

lib/pure/os.nim

@@ -1177,7 +1177,7 @@ const
     ## On Windows ``["exe", "cmd", "bat"]``, on Posix ``[""]``.
     when defined(windows): ["exe", "cmd", "bat"] else: [""]
 
-proc findExe*(exe: string, followSymlinks: bool = true;
+proc findExe*(exe: string, followSymlinks = true;
               extensions: openArray[string]=ExeExts): string {.
   tags: [ReadDirEffect, ReadEnvEffect, ReadIOEffect], noNimJs.} =
   ## Searches for `exe` in the current working directory and then
@@ -1647,8 +1647,11 @@ proc setFilePermissions*(filename: string, permissions: set[FilePermission]) {.
       var res2 = setFileAttributesA(filename, res)
     if res2 == - 1'i32: raiseOSError(osLastError(), $(filename, permissions))
 
-proc copyFile*(source, dest: string) {.rtl, extern: "nos$1",
-  tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} =
+proc createSymlink*(src, dest: string) {.tags: [ReadDirEffect, WriteIOEffect], noWeirdTarget.}
+proc expandSymlink*(symlinkPath: string): string {.tags: [ReadIOEffect], noWeirdTarget.}
+
+proc copyFile*(source, dest: string, followSymlinks = true) {.rtl, extern: "nos$1",
+  tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} =
   ## Copies a file from `source` to `dest`, where `dest.parentDir` must exist.
   ##
   ## If this fails, `OSError` is raised.
@@ -1668,11 +1671,11 @@ proc copyFile*(source, dest: string) {.rtl, extern: "nos$1",
   ## will be preserved and the content overwritten.
   ##
   ## See also:
-  ## * `copyDir proc <#copyDir,string,string>`_
+  ## * `copyDir proc <#copyDir,string,string,bool>`_
   ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_
   ## * `tryRemoveFile proc <#tryRemoveFile,string>`_
   ## * `removeFile proc <#removeFile,string>`_
-  ## * `moveFile proc <#moveFile,string,string>`_
+  ## * `moveFile proc <#moveFile,string,string,bool>`_
 
   when defined(Windows):
     when useWinUnicode:
@@ -1682,34 +1685,37 @@ proc copyFile*(source, dest: string) {.rtl, extern: "nos$1",
     else:
       if copyFileA(source, dest, 0'i32) == 0'i32: raiseOSError(osLastError(), $(source, dest))
   else:
-    # generic version of copyFile which works for any platform:
-    const bufSize = 8000 # better for memory manager
-    var d, s: File
-    if not open(s, source): raiseOSError(osLastError(), source)
-    if not open(d, dest, fmWrite):
+    if not followSymlinks and source.checkSymlink:
+      createSymlink(expandSymlink(source), dest)
+    else:
+      # generic version of copyFile which works for any platform:
+      const bufSize = 8000 # better for memory manager
+      var d, s: File
+      if not open(s, source): raiseOSError(osLastError(), source)
+      if not open(d, dest, fmWrite):
+        close(s)
+        raiseOSError(osLastError(), dest)
+      var buf = alloc(bufSize)
+      while true:
+        var bytesread = readBuffer(s, buf, bufSize)
+        if bytesread > 0:
+          var byteswritten = writeBuffer(d, buf, bytesread)
+          if bytesread != byteswritten:
+            dealloc(buf)
+            close(s)
+            close(d)
+            raiseOSError(osLastError(), dest)
+        if bytesread != bufSize: break
+      dealloc(buf)
       close(s)
-      raiseOSError(osLastError(), dest)
-    var buf = alloc(bufSize)
-    while true:
-      var bytesread = readBuffer(s, buf, bufSize)
-      if bytesread > 0:
-        var byteswritten = writeBuffer(d, buf, bytesread)
-        if bytesread != byteswritten:
-          dealloc(buf)
-          close(s)
-          close(d)
-          raiseOSError(osLastError(), dest)
-      if bytesread != bufSize: break
-    dealloc(buf)
-    close(s)
-    flushFile(d)
-    close(d)
-
-proc copyFileToDir*(source, dir: string) {.noWeirdTarget, since: (1,3,7).} =
+      flushFile(d)
+      close(d)
+
+proc copyFileToDir*(source, dir: string, followSymlinks = true) {.noWeirdTarget, since: (1,3,7).} =
   ## Copies a file `source` into directory `dir`, which must exist.
   if dir.len == 0: # treating "" as "." is error prone
     raise newException(ValueError, "dest is empty")
-  copyFile(source, dir / source.lastPathPart)
+  copyFile(source, dir / source.lastPathPart, followSymlinks = followSymlinks)
 
 when not declared(ENOENT) and not defined(Windows):
   when NoFakeVars:
@@ -1739,10 +1745,10 @@ proc tryRemoveFile*(file: string): bool {.rtl, extern: "nos$1", tags: [WriteDirE
   ## On Windows, ignores the read-only attribute.
   ##
   ## See also:
-  ## * `copyFile proc <#copyFile,string,string>`_
+  ## * `copyFile proc <#copyFile,string,string,bool>`_
   ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_
   ## * `removeFile proc <#removeFile,string>`_
-  ## * `moveFile proc <#moveFile,string,string>`_
+  ## * `moveFile proc <#moveFile,string,string,bool>`_
   result = true
   when defined(Windows):
     when useWinUnicode:
@@ -1772,10 +1778,10 @@ proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect], n
   ##
   ## See also:
   ## * `removeDir proc <#removeDir,string>`_
-  ## * `copyFile proc <#copyFile,string,string>`_
+  ## * `copyFile proc <#copyFile,string,string,bool>`_
   ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_
   ## * `tryRemoveFile proc <#tryRemoveFile,string>`_
-  ## * `moveFile proc <#moveFile,string,string>`_
+  ## * `moveFile proc <#moveFile,string,string,bool>`_
   if not tryRemoveFile(file):
     raiseOSError(osLastError(), file)
 
@@ -1802,8 +1808,8 @@ proc tryMoveFSObject(source, dest: string): bool {.noWeirdTarget.} =
         raiseOSError(err, $(source, dest, strerror(errno)))
   return true
 
-proc moveFile*(source, dest: string) {.rtl, extern: "nos$1",
-  tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} =
+proc moveFile*(source, dest: string, followSymlinks = true) {.rtl, extern: "nos$1",
+  tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], noWeirdTarget.} =
   ## Moves a file from `source` to `dest`.
   ##
   ## If this fails, `OSError` is raised.
@@ -1812,16 +1818,16 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1",
   ## Can be used to `rename files`:idx:.
   ##
   ## See also:
-  ## * `moveDir proc <#moveDir,string,string>`_
-  ## * `copyFile proc <#copyFile,string,string>`_
+  ## * `moveDir proc <#moveDir,string,string,bool>`_
+  ## * `copyFile proc <#copyFile,string,string,bool>`_
   ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_
   ## * `removeFile proc <#removeFile,string>`_
   ## * `tryRemoveFile proc <#tryRemoveFile,string>`_
 
   if not tryMoveFSObject(source, dest):
     when not defined(windows):
       # Fallback to copy & del
-      copyFile(source, dest)
+      copyFile(source, dest, followSymlinks = followSymlinks)
       try:
         removeFile(source)
       except:
@@ -2223,9 +2229,9 @@ proc removeDir*(dir: string, checkDir = false) {.rtl, extern: "nos$1", tags: [
   ## * `removeFile proc <#removeFile,string>`_
   ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_
   ## * `createDir proc <#createDir,string>`_
-  ## * `copyDir proc <#copyDir,string,string>`_
-  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_
-  ## * `moveDir proc <#moveDir,string,string>`_
+  ## * `copyDir proc <#copyDir,string,string,bool>`_
+  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_
+  ## * `moveDir proc <#moveDir,string,string,bool>`_
   for kind, path in walkDir(dir, checkDir = checkDir):
     case kind
     of pcFile, pcLinkToFile, pcLinkToDir: removeFile(path)
@@ -2290,9 +2296,9 @@ proc existsOrCreateDir*(dir: string): bool {.rtl, extern: "nos$1",
   ## See also:
   ## * `removeDir proc <#removeDir,string>`_
   ## * `createDir proc <#createDir,string>`_
-  ## * `copyDir proc <#copyDir,string,string>`_
-  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_
-  ## * `moveDir proc <#moveDir,string,string>`_
+  ## * `copyDir proc <#copyDir,string,string,bool>`_
+  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_
+  ## * `moveDir proc <#moveDir,string,string,bool>`_
   result = not rawCreateDir(dir)
   if result:
     # path already exists - need to check that it is indeed a directory
@@ -2312,9 +2318,9 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1",
   ## See also:
   ## * `removeDir proc <#removeDir,string>`_
   ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_
-  ## * `copyDir proc <#copyDir,string,string>`_
-  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_
-  ## * `moveDir proc <#moveDir,string,string>`_
+  ## * `copyDir proc <#copyDir,string,string,bool>`_
+  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_
+  ## * `moveDir proc <#moveDir,string,string,bool>`_
   var omitNext = false
   when doslikeFileSystem:
     omitNext = isAbsolute(dir)
@@ -2330,8 +2336,8 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1",
      dir[^1] notin {DirSep, AltSep}:
     discard existsOrCreateDir(dir)
 
-proc copyDir*(source, dest: string) {.rtl, extern: "nos$1",
-  tags: [WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} =
+proc copyDir*(source, dest: string, followSymlinks = true) {.rtl, extern: "nos$1",
+  tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect], benign, noWeirdTarget.} =
   ## Copies a directory from `source` to `dest`.
   ##
   ## If this fails, `OSError` is raised.
@@ -2341,46 +2347,47 @@ proc copyDir*(source, dest: string) {.rtl, extern: "nos$1",
   ##
   ## On other platforms created files and directories will inherit the
   ## default permissions of a newly created file/directory for the user.
-  ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_
+  ## Use `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_
   ## to preserve attributes recursively on these platforms.
   ##
   ## See also:
-  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_
-  ## * `copyFile proc <#copyFile,string,string>`_
+  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_
+  ## * `copyFile proc <#copyFile,string,string,bool>`_
   ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_
   ## * `removeDir proc <#removeDir,string>`_
   ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_
   ## * `createDir proc <#createDir,string>`_
-  ## * `moveDir proc <#moveDir,string,string>`_
+  ## * `moveDir proc <#moveDir,string,string,bool>`_
   createDir(dest)
   for kind, path in walkDir(source):
     var noSource = splitPath(path).tail
     case kind
     of pcFile:
-      copyFile(path, dest / noSource)
+      copyFile(path, dest / noSource, followSymlinks = followSymlinks)
     of pcDir:
-      copyDir(path, dest / noSource)
+      copyDir(path, dest / noSource, followSymlinks = followSymlinks)
     else: discard
 
-proc moveDir*(source, dest: string) {.tags: [ReadIOEffect, WriteIOEffect], noWeirdTarget.} =
+proc moveDir*(source, dest: string, followSymlinks = true) {.tags: [ReadDirEffect, ReadIOEffect, WriteIOEffect],
+  noWeirdTarget.} =
   ## Moves a directory from `source` to `dest`.
   ##
   ## If this fails, `OSError` is raised.
   ##
   ## See also:
-  ## * `moveFile proc <#moveFile,string,string>`_
-  ## * `copyDir proc <#copyDir,string,string>`_
-  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_
+  ## * `moveFile proc <#moveFile,string,string,bool>`_
+  ## * `copyDir proc <#copyDir,string,string,bool>`_
+  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_
   ## * `removeDir proc <#removeDir,string>`_
   ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_
   ## * `createDir proc <#createDir,string>`_
   if not tryMoveFSObject(source, dest):
     when not defined(windows):
       # Fallback to copy & del
-      copyDir(source, dest)
+      copyDir(source, dest, followSymlinks = followSymlinks)
       removeDir(source)
 
-proc createSymlink*(src, dest: string) {.noWeirdTarget.} =
+proc createSymlink*(src, dest: string) {.tags: [ReadDirEffect, WriteIOEffect], noWeirdTarget.} =
   ## Create a symbolic link at `dest` which points to the item specified
   ## by `src`. On most operating systems, will fail if a link already exists.
   ##
@@ -2431,7 +2438,7 @@ proc createHardlink*(src, dest: string) {.noWeirdTarget.} =
       raiseOSError(osLastError(), $(src, dest))
 
 proc copyFileWithPermissions*(source, dest: string,
-                              ignorePermissionErrors = true) {.noWeirdTarget.} =
+                              ignorePermissionErrors = true, followSymlinks = true) {.noWeirdTarget.} =
   ## Copies a file from `source` to `dest` preserving file permissions.
   ##
   ## This is a wrapper proc around `copyFile <#copyFile,string,string>`_,
@@ -2450,12 +2457,12 @@ proc copyFileWithPermissions*(source, dest: string,
   ##
   ## See also:
   ## * `copyFile proc <#copyFile,string,string>`_
-  ## * `copyDir proc <#copyDir,string,string>`_
+  ## * `copyDir proc <#copyDir,string,string,bool>`_
   ## * `tryRemoveFile proc <#tryRemoveFile,string>`_
   ## * `removeFile proc <#removeFile,string>`_
-  ## * `moveFile proc <#moveFile,string,string>`_
-  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string>`_
-  copyFile(source, dest)
+  ## * `moveFile proc <#moveFile,string,string,bool>`_
+  ## * `copyDirWithPermissions proc <#copyDirWithPermissions,string,string,bool>`_
+  copyFile(source, dest, followSymlinks = followSymlinks)
   when not defined(Windows):
     try:
       setFilePermissions(dest, getFilePermissions(source))
@@ -2464,17 +2471,17 @@ proc copyFileWithPermissions*(source, dest: string,
         raise
 
 proc copyDirWithPermissions*(source, dest: string,
-    ignorePermissionErrors = true) {.rtl, extern: "nos$1",
+    ignorePermissionErrors = true, followSymlinks = true) {.rtl, extern: "nos$1",
     tags: [WriteIOEffect, ReadIOEffect], benign, noWeirdTarget.} =
   ## Copies a directory from `source` to `dest` preserving file permissions.
   ##
   ## If this fails, `OSError` is raised. This is a wrapper proc around `copyDir
-  ## <#copyDir,string,string>`_ and `copyFileWithPermissions
+  ## <#copyDir,string,string,bool>`_ and `copyFileWithPermissions
   ## <#copyFileWithPermissions,string,string>`_ procs
   ## on non-Windows platforms.
   ##
   ## On Windows this proc is just a wrapper for `copyDir proc
-  ## <#copyDir,string,string>`_ since that proc already copies attributes.
+  ## <#copyDir,string,string,bool>`_ since that proc already copies attributes.
   ##
   ## On non-Windows systems permissions are copied after the file or directory
   ## itself has been copied, which won't happen atomically and could lead to a
@@ -2483,11 +2490,11 @@ proc copyDirWithPermissions*(source, dest: string,
   ## `OSError`.
   ##
   ## See also:
-  ## * `copyDir proc <#copyDir,string,string>`_
+  ## * `copyDir proc <#copyDir,string,string,bool>`_
   ## * `copyFile proc <#copyFile,string,string>`_
   ## * `copyFileWithPermissions proc <#copyFileWithPermissions,string,string>`_
   ## * `removeDir proc <#removeDir,string>`_
-  ## * `moveDir proc <#moveDir,string,string>`_
+  ## * `moveDir proc <#moveDir,string,string,bool>`_
   ## * `existsOrCreateDir proc <#existsOrCreateDir,string>`_
   ## * `createDir proc <#createDir,string>`_
   createDir(dest)
@@ -2501,9 +2508,9 @@ proc copyDirWithPermissions*(source, dest: string,
     var noSource = splitPath(path).tail
     case kind
     of pcFile:
-      copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors)
+      copyFileWithPermissions(path, dest / noSource, ignorePermissionErrors, followSymlinks = followSymlinks)
     of pcDir:
-      copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors)
+      copyDirWithPermissions(path, dest / noSource, ignorePermissionErrors, followSymlinks = followSymlinks)
     else: discard
 
 proc inclFilePermissions*(filename: string,
@@ -2524,7 +2531,7 @@ proc exclFilePermissions*(filename: string,
   ##   setFilePermissions(filename, getFilePermissions(filename)-permissions)
   setFilePermissions(filename, getFilePermissions(filename)-permissions)
 
-proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} =
+proc expandSymlink*(symlinkPath: string): string {.tags: [ReadIOEffect], noWeirdTarget.} =
   ## Returns a string representing the path to which the symbolic link points.
   ##
   ## On Windows this is a noop, ``symlinkPath`` is simply returned.

tests/stdlib/tos.nim

@@ -54,6 +54,16 @@ block fileOperations:
     doAssert not dirExists(dname/sub)
     removeFile(dname/fname)
     removeFile(dname/fname2)
+    when not defined(windows):
+      let brokenSymlink = dname/"D20210101T191320_BROKEN_SYMLINK"
+      let brokenSymlinkDesc = "D20210101T191320_I_DO_NOT_EXIST"
+      createSymlink(brokenSymlinkDesc, brokenSymlink)
+      let brokenSymlinkCopy = brokenSymlink & "_COPY"
+      doAssertRaises(OSError): copyFile(brokenSymlink, brokenSymlinkCopy)
+      copyFile(brokenSymlink, brokenSymlinkCopy, followSymlinks = false)
+      doAssert expandSymlink(brokenSymlinkCopy) == brokenSymlinkDesc
+      removeFile(brokenSymlink)
+      removeFile(brokenSymlinkCopy)
 
   # Test creating files and dirs
   for dir in dirs: