how can i download a directory from a server via SFTP/SCP to a zip file

[insinfo]

I don't know how to do this using SSH.NET

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

using Renci.SshNet;
using Renci.SshNet.Sftp;

namespace ConsoleApp1
{
    class Program
    {
        static void Main(string[] args)
        {

            var connectionInfo = new ConnectionInfo("192.168.133.13", "user",
                                        new PasswordAuthenticationMethod("isaque.neves", "pass"));
            using (var client = new SftpClient(connectionInfo))
            {
                client.Connect();
                DownloadDirectory(client,"/var/www/teste", @"C:\MyCsharpProjects\fsbackup\download");
             
            }

            Console.WriteLine("end");
        }

        public static void DownloadDirectory( SftpClient sftpClient, string sourceRemotePath, string destLocalPath)
        {
            Directory.CreateDirectory(destLocalPath);
            IEnumerable<SftpFile> files = sftpClient.ListDirectory(sourceRemotePath);
            foreach (SftpFile file in files)
            {
                if ((file.Name != ".") && (file.Name != ".."))
                {
                    string sourceFilePath = sourceRemotePath + "/" + file.Name;
                    string destFilePath = Path.Combine(destLocalPath, file.Name);
                    if (file.IsDirectory)
                    {
                        DownloadDirectory(sftpClient, sourceFilePath, destFilePath);
                    }
                    else
                    {
                        using (Stream fileStream = File.Create(destFilePath))
                        {
                            sftpClient.DownloadFile(sourceFilePath, fileStream);
                        }
                    }
                }
            }
        }
    }
}

I already did it in rust, but I don't know how to do it using SSH.NET

pub fn filetransfer_recv_dir_as_zip(&mut self, dir_to_copy: &FsDirectory, local_path: &Path, output_file_name: &str) -> FileTransferResult<()> {
        let start =  Instant::now();
        //obter lista de diretorio recursivamente
        match self.client.list_dir_recursively(&dir_to_copy.abs_path.clone()) {
            Ok(it) => {
                let duration = start.elapsed();
                info!("Time elapsed in list_dir_recursively {:?}", duration);

                let dst_file_writer = File::create(local_path.join(output_file_name)).unwrap();
                let mut zip_writer = zip::ZipWriter::new(dst_file_writer);

                let mut buffer = Vec::new();

                for entry in it {
                    let options = zip::write::FileOptions::default()
                        .compression_method(zip::CompressionMethod::Stored)
                        .unix_permissions(0o755);

                    match entry {
                        FsEntry::File(remote) => {
                            let path = &remote.abs_path;
                            //remove a parte inicial do caminho
                            let name = path.strip_prefix(&dir_to_copy.abs_path).unwrap();
                            //debug!("source: {}",name.display());

                            match self.client.recv_file(&remote) {
                                Ok(mut rhnd) => {
                                    #[allow(deprecated)]
                                    match zip_writer.start_file_from_path(name, options) {
                                        Ok(re) => {}
                                        Err(err) => {
                                            self.log(
                                                LogLevel::Error,
                                                format!(
                                                    "Error on zip_writer.start_file_from_path : {:?}",
                                                    err
                                                ),
                                            );
                                            return Err(FileTransferError::new(FileTransferErrorType::ZipCompressError));
                                        }
                                    }

                                    match  rhnd.read_to_end(&mut buffer) {
                                        Ok(re) => {}
                                        Err(err) => {
                                            self.log(
                                                LogLevel::Error,
                                                format!(
                                                    "Error on read_to_end : {}",
                                                    err
                                                ),
                                            );
                                            return Err(FileTransferError::new(FileTransferErrorType::ProtocolError));
                                        }
                                    }

                                    match  zip_writer.write_all(&*buffer) {
                                        Ok(re) => {}
                                        Err(err) => {
                                            self.log(
                                                LogLevel::Error,
                                                format!(
                                                    "Error zip_writer.write_all : {}",
                                                    err
                                                ),
                                            );
                                            return Err(FileTransferError::new(FileTransferErrorType::ZipCompressError));
                                        }
                                    }


                                    buffer.clear();
                                }
                                Err(err) if err.kind() == FileTransferErrorType::UnsupportedFeature => {
                                    error!("FileTransferErrorType::UnsupportedFeature");
                                }
                                Err(err) => {
                                    error!("FileTransferError {:?}",err);
                                }
                            }
                        }
                        FsEntry::Directory(dir) => {}
                    }
                }
                zip_writer.finish().unwrap();

                let duration = start.elapsed();
                info!("Time elapsed filetransfer_recv_dir_as_zip {:?}", duration);
                Ok(())
            }
            Err(err) => {
                self.log(
                    LogLevel::Error,
                    format!(
                        "Failed list dir recursively \"{}\": {}",
                        dir_to_copy.abs_path.display(),
                        err
                    ),
                );
                Err(err)
            }
        }
    }

[insinfo]

I made this code below but I don't know if it's the best approach

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Ionic.Zip;
using Renci.SshNet;
using Renci.SshNet.Sftp;

namespace ConsoleApp1
{
    class Program
    {
        static void Main(string[] args)
        {

            var connectionInfo = new ConnectionInfo("192.168.133.13", "user",
                                        new PasswordAuthenticationMethod("isaque.neves", "pass"));
            using (var client = new SftpClient(connectionInfo))
            {
                client.Connect();
                DownloadDirectoryAsZip(client,"/var/www/teste", @"C:\MyCsharpProjects\fsbackup\download.zip");               
            }
            Console.WriteLine("end");
        }
       
        public static void DownloadDirectoryAsZip(SftpClient sftpClient, string sourceRemotePath, string destLocalPath)
        {
            ZipFile zip = new ZipFile();          
            DownloadDirectoryAsZipRec(zip, sftpClient, sourceRemotePath);
            zip.Save(destLocalPath);
            zip.Dispose();

         }
        private static void DownloadDirectoryAsZipRec(ZipFile zip,SftpClient sftpClient, string sourceRemotePath)
        {
            
        
            IEnumerable<SftpFile> files = sftpClient.ListDirectory(sourceRemotePath);
            foreach (SftpFile file in files)
            {
                if ((file.Name != ".") && (file.Name != ".."))
                {
                    string sourceFilePath = sourceRemotePath + "/" + file.Name;           
                    if (file.IsDirectory)
                    {
                        DownloadDirectoryAsZipRec(zip,sftpClient, sourceFilePath);
                    }
                    else
                    {
                        var memoryStream = new MemoryStream();                                               
                        sftpClient.DownloadFile(sourceFilePath, memoryStream);                                                
                        zip.AddEntry(sourceFilePath, memoryStream);                        
                    }
                }
            }
        }


    }
}

[IgorMilavec]

I have two suggestions:

• Use built-in System.IO.Compression instead of taking an external dependency
• I don't know what size of files you have, but using MemoryStream might bite you some day. At last you should be using it. If possible, file some way to stream directly from SSH stream into ZIP entry. Example in https://docs.microsoft.com/en-us/dotnet/api/system.io.compression.ziparchiveentry?view=net-6.0 might prove useful.

[insinfo]

@IgorMilavec
I implemented a code following your suggestion to use System.IO.Compression , but it is using more than 3GB of memory to copy the files, or I made a mistake or there is a memory leak bug.

static void Main(string[] args)
        {
            var watch = new System.Diagnostics.Stopwatch();
            watch.Start();

            var connectionInfo = new ConnectionInfo("192.168.2.77", "root",
                                        new PasswordAuthenticationMethod("root", "pass"));
            var client = new SftpClient(connectionInfo);            
            client.Connect();          
            DownloadDirectoryAsZip2(client, "/var/www/html", @"C:\MyCsharpProjects\fsbackup\download.zip");
            client.Dispose();  

            Console.WriteLine("end Download");
            watch.Stop();
            Console.WriteLine($"Execution Time: {watch.ElapsedMilliseconds} ms");
        }
 public static void DownloadDirectoryAsZip2(SftpClient sftpClient, string sourceRemotePath, string destLocalPath)
        {
            using (var zipFile = new FileStream(destLocalPath, FileMode.OpenOrCreate))
            {
                using (var archive = new ZipArchive(zipFile, ZipArchiveMode.Update))
                {
                    DownloadDirectoryAsZipRec2(archive, sftpClient, sourceRemotePath);                    
                }
            }                
        }
        private static void DownloadDirectoryAsZipRec2(ZipArchive archive, SftpClient sftpClient, string sourceRemotePath)
        {
            try
            {
                var files = sftpClient.ListDirectory(sourceRemotePath);
                foreach (var file in files)
                {
                    if ((file.Name != ".") && (file.Name != ".."))
                    {
                        var sourceFilePath = sourceRemotePath + "/" + file.Name;
                      
                        if (file.IsDirectory)
                        {
                            DownloadDirectoryAsZipRec2(archive, sftpClient, sourceFilePath);
                        }
                        else
                        {                           
                            ZipArchiveEntry entry = archive.CreateEntry(sourceFilePath);
                            using (Stream stream = entry.Open()) { 
                                try
                                {
                                    sftpClient.DownloadFile(sourceFilePath, stream);
                                }
                                catch (Exception e)
                                {
                                    Console.WriteLine($"Download File failed: {sourceFilePath} | Error:{e}");
                                }
                                
                            }
                        }
                    }
                }
            }catch (Exception e)
            {
                Console.WriteLine($"Download Directory failed: {sourceRemotePath} | Error:{e}");
            }
        }
    }

[IgorMilavec]

Your code looks OK at first glance.
Why don't you have a look at what is consuming the memory? https://docs.microsoft.com/en-us/visualstudio/profiling/memory-usage-without-debugging2?view=vs-2022

[insinfo]

@IgorMilavec
This is really a memory leak, give the team a thumbs up to fix this issue.
dotnet/runtime#1544

Is there a convenient and easy way to show directory download progress with SSH.NET?

It would be nice if SSH.NET already provides the download directory to zip with a progress callback.

[IgorMilavec]

You can provide a callback to DownloadFile and it will notify you about the progress.
Download to ZIP is really an applicative/business domain concern and as such I think it does not belong to a library.

[insinfo]

@IgorMilavec
How can I know the size of a directory in bytes before downloading it, so I can calculate the progress ?
I implemented code for this in dart a while ago for SCP, but I believe SSH.NET should have this right?

 ///return total size in bytes of Directory
  int getSizeOfDirectory(
      Pointer<ssh_session_struct> session, String remoteDirectoryPath,
      {bool isThrowException = true}) {

    var cmdToGetTotaSize =
        "find $remoteDirectoryPath -type f -print0 | xargs -0 stat --format=%s | awk '{s+=\$1} END {print s}'";
    String? cmdRes;
    try {
      cmdRes = execCommandSync(session, cmdToGetTotaSize);
      if (cmdRes.trim().isEmpty) {
        //check is symbolic link    
        cmdRes = execCommandSync(session, 'file $remoteDirectoryPath');
     //is it a symbolic link?
        if (cmdRes.contains('symbolic')) {
        // get the real path of the symbolic link
          var cmdRe =
              execCommandSync(session, 'readlink -f $remoteDirectoryPath');
          if (cmdRe.trim().isNotEmpty) {
            cmdRes = execCommandSync(session,
                "find ${cmdRe.replaceAll(RegExp(r'\n'), '')} -type f -print0 | xargs -0 stat --format=%s | awk '{s+=\$1} END {print s}'");
          }
        }
      }
      //for old debian like debian  6
      //old debian bug 1.57903e+10
      if (cmdRes.contains('e+')) {
        cmdRes = execCommandSync(session,
            'find $remoteDirectoryPath -type f -print0 | xargs -0 stat --format=%s |  paste -sd+ - | bc -l');
      } else if (int.tryParse(cmdRes) == null) {
     
        cmdRes = execCommandSync(session,
            'find $remoteDirectoryPath -type f -print0 | du -scb -L --apparent-size --files0-from=- | tail -n 1');
        return int.parse(cmdRes.split(' ').first.trim());
      }

      return int.parse(cmdRes);
    } catch (e) {
      print(
          'getSizeOfDirectory: $e \r\n cmdToGetTotaSize: $cmdToGetTotaSize \r\n cmdRes: $cmdRes');
      if (isThrowException) {
        throw LibsshGetFileSizeException(
            'Unable to get the size of a directory in bytes, \r\n cmd: $cmdToGetTotaSize cmdResult: $cmdRes');
      }
    }
    return 0;
  }

code from libssh_binding
I believe that at least there should be a DownloadDirectory method in the API to download a directory via SFTP with a callback