paramiko hangs on get after ownloading 20 MB of file

Question

I am in need of python sftp client to download files from a sftp server. I started to use Paramiko. Small files in KB works well but however when I try to download 600 MB of file, it hangs indefinitely after downloading 20 MB of file. Unable to figure out what the issue is. Increasing the window size did not solve either. Any help would be much appreciated!

host = config.getsafe(section, "host")
username = config.getsafe(section, "username")
port = config.getsafe(section, "port")
remote_dir = config.getsafe(section, "remote_dir")
download_dir = config.getsafe(section, "download_dir")
archive_dir = config.getsafe(section, "archive_dir") if config.has_option(section, "archive_dir") else \
    None
password = config.getsafe(section, "password") if config.has_option(section, "password") else None
file_pattern = config.getsafe(section, "file_pattern") if config.has_option(section, "file_pattern") \
    else "*"
passphrase = config.getsafe(section, "passphrase") if config.has_option(section, "passphrase") else None
gnupg_home = config.getsafe(section, "gnupg_home") if config.has_option(section, "gnupg_home") else None

ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host, port=int(port), username=username, password=password)

sftp = ssh.open_sftp()
sftp.sshclient = ssh

sftp.get("/SFTP/PL_DEV/test.dat", "C:/import/download/test.dat")

Answer 1

I did two things to solve a similar problem:

1. increase window size – you say you tried that too; for me, this helped to get from a few ten MBs to half a GB but no further

2. effectively disable rekeying – this might have security implications, but helped me to get files over a GB from a weird windows sftp server

   with paramiko.Transport((_SFTP['host'], 22)) as transport:
       # SFTP FIXES
       transport.default_window_size=paramiko.common.MAX_WINDOW_SIZE
       transport.packetizer.REKEY_BYTES = pow(2, 40)  # 1TB max, this is a security degradation!
       transport.packetizer.REKEY_PACKETS = pow(2, 40)  # 1TB max, this is a security degradation!
       # / SFTP FIXES
   
       transport.connect(username=_SFTP['user'], password=_SFTP['password'])
           with paramiko.SFTPClient.from_transport(transport) as sftp:
               listdir = sftp.listdir()
               # ...
               sftp.get(remotepath=filename, localpath=localpath)
   

Answer 2

Increasing default_max_packet_size and default_window_size as follows worked for me:

client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.load_system_host_keys()

client.connect(hostname, username=username, password=password, port=port)
tr = client.get_transport()
tr.default_max_packet_size = 100000000
tr.default_window_size = 100000000
sftp = client.open_sftp()
sftp.get(remote_file, local_filepath)

client.close()

Answer 3

Using the latest paramiko 2.4.2, I have a similar issue. In my case, our vendor switched their SFTP provider from Globalscape (SSH-2.0-1.82_sshlib Globalscape) to Cerberus (SSH-2.0-CerberusFTPServer_10.0) a few days ago. Ever since then, paramiko has been unable to download a ~450MB file.

Here is the symptom: The downloading speed is extremely slow. After downloading 20~30MB, it always errors out with: Server connection dropped msg.

Here is the log (Globalscape) - successful download:

"paramiko.transport", "DEBUG", "starting thread (client mode): 0x160096d8"
"paramiko.transport", "DEBUG", "Local version/idstring: SSH-2.0-paramiko_2.4.1"
"paramiko.transport", "DEBUG", "Remote version/idstring: SSH-2.0-1.82_sshlib Globalscape"
"paramiko.transport", "INFO", "Connected (version 2.0, client 1.82_sshlib)"
"paramiko.transport", "DEBUG", "kex algos:['diffie-hellman-group14-sha1', 'diffie-hellman-group-exchange-sha1', 'diffie-hellman-group1-sha1'] server key:['ssh-rsa'] client encrypt:['twofish256-cbc', 'twofish-cbc', 'twofish128-cbc', 'blowfish-cbc', '3des-cbc', 'arcfour', 'cast128-cbc', 'aes256-cbc', 'aes128-cbc', 'aes256-ctr', 'aes128-ctr'] server encrypt:['twofish256-cbc', 'twofish-cbc', 'twofish128-cbc', 'blowfish-cbc', '3des-cbc', 'arcfour', 'cast128-cbc', 'aes256-cbc', 'aes128-cbc', 'aes256-ctr', 'aes128-ctr'] client mac:['hmac-sha1', 'hmac-md5', 'hmac-sha1-96', 'hmac-md5-96'] server mac:['hmac-sha1', 'hmac-md5', 'hmac-sha1-96', 'hmac-md5-96'] client compress:['zlib', 'none'] server compress:['zlib', 'none'] client lang:[''] server lang:[''] kex follows?False"
"paramiko.transport", "DEBUG", "HostKey agreed: ssh-rsa"
"paramiko.transport", "DEBUG", "Cipher agreed: aes128-ctr"
"paramiko.transport", "DEBUG", "MAC agreed: hmac-sha1"
"paramiko.transport", "DEBUG", "Compression agreed: none"
"paramiko.transport", "DEBUG", "Got server p (2048 bits)"
"paramiko.transport", "DEBUG", "kex engine KexGex specified hash_algo <built-in function openssl_sha1>"
"paramiko.transport", "DEBUG", "Switch to new keys ..."
"paramiko.transport", "DEBUG", "Attempting public-key auth..."
"paramiko.transport", "DEBUG", "userauth is OK"
"paramiko.transport", "INFO", "Auth banner: b'Welcome to the our Secure FTP Server'"
"paramiko.transport", "INFO", "Authentication (publickey) successful!"
"paramiko.transport", "DEBUG", "[chan 0] Max packet in: 32768 bytes"
"paramiko.transport", "DEBUG", "[chan 0] Max packet out: 35840 bytes"
"paramiko.transport", "DEBUG", "Secsh channel 0 opened."
"paramiko.transport", "DEBUG", "[chan 0] Sesch channel 0 request ok"
"paramiko.transport.sftp", "INFO", "[chan 0] Opened sftp connection (server version 3)"
"paramiko.transport.sftp", "DEBUG", "[chan 0] stat(b'data.csv')"
"paramiko.transport.sftp", "DEBUG", "[chan 0] open(b'data.csv', 'rb')"
"paramiko.transport.sftp", "DEBUG", "[chan 0] open(b'data.csv', 'rb') -> 31"
"paramiko.transport.sftp", "DEBUG", "[chan 0] close(31)"
"paramiko.transport.sftp", "INFO", "[chan 0] sftp session closed."
"paramiko.transport", "DEBUG", "[chan 0] EOF sent (0)"
"paramiko.transport", "DEBUG", "EOF in transport thread"

Here is the log (Cerberus) - failed download:

"paramiko.transport", "DEBUG", "starting thread (client mode): 0x119706d8"
"paramiko.transport", "DEBUG", "Local version/idstring: SSH-2.0-paramiko_2.4.1"
"paramiko.transport", "DEBUG", "Remote version/idstring: SSH-2.0-CerberusFTPServer_10.0"
"paramiko.transport", "INFO", "Connected (version 2.0, client CerberusFTPServer_10.0)"
"paramiko.transport", "DEBUG", "kex algos:['ecdh-sha2-nistp256', 'ecdh-sha2-nistp384', 'ecdh-sha2-nistp521', 'diffie-hellman-group-exchange-sha256', 'diffie-hellman-group-exchange-sha1', 'diffie-hellman-group14-sha1', 'diffie-hellman-group1-sha1'] server key:['ssh-rsa'] client encrypt:['aes128-ctr', 'aes128-cbc', 'aes192-ctr', 'aes192-cbc', 'aes256-ctr', 'aes256-cbc', '3des-cbc'] server encrypt:['aes128-ctr', 'aes128-cbc', 'aes192-ctr', 'aes192-cbc', 'aes256-ctr', 'aes256-cbc', '3des-cbc'] client mac:['hmac-sha1', 'hmac-sha1-96', 'hmac-sha2-256', 'hmac-sha2-256-96', 'hmac-sha2-512', 'hmac-sha2-512-96', 'hmac-ripemd160', '[email protected]', 'hmac-md5'] server mac:['hmac-sha1', 'hmac-sha1-96', 'hmac-sha2-256', 'hmac-sha2-256-96', 'hmac-sha2-512', 'hmac-sha2-512-96', 'hmac-ripemd160', '[email protected]', 'hmac-md5'] client compress:['none'] server compress:['none'] client lang:['en-US'] server lang:['en-US'] kex follows?False"
"paramiko.transport", "DEBUG", "Kex agreed: ecdh-sha2-nistp256"
"paramiko.transport", "DEBUG", "HostKey agreed: ssh-rsa"
"paramiko.transport", "DEBUG", "Cipher agreed: aes128-ctr"
"paramiko.transport", "DEBUG", "MAC agreed: hmac-sha2-256"
"paramiko.transport", "DEBUG", "Compression agreed: none"
"paramiko.transport", "DEBUG", "kex engine KexNistp256 specified hash_algo <built-in function openssl_sha256>"
"paramiko.transport", "DEBUG", "Switch to new keys ..."
"paramiko.transport", "DEBUG", "Attempting public-key auth..."
"paramiko.transport", "DEBUG", "userauth is OK"
"paramiko.transport", "INFO", "Authentication (publickey) successful!"
"paramiko.transport", "DEBUG", "[chan 0] Max packet in: 32768 bytes"
"paramiko.transport", "DEBUG", "[chan 0] Max packet out: 32768 bytes"
"paramiko.transport", "DEBUG", "Secsh channel 0 opened."
"paramiko.transport", "DEBUG", "[chan 0] Sesch channel 0 request ok"
"paramiko.transport.sftp", "INFO", "[chan 0] Opened sftp connection (server version 3)"
"paramiko.transport.sftp", "DEBUG", "[chan 0] stat(b'data.csv')"
"paramiko.transport.sftp", "DEBUG", "[chan 0] open(b'data.csv', 'rb')"
"paramiko.transport.sftp", "DEBUG", "[chan 0] open(b'data.csv', 'rb') -> 7b45394343333830462d383832352d343436342d393831302d4444373838314237303433367d"
"paramiko.transport", "DEBUG", "EOF in transport thread"

Adding

transport.default_window_size = paramiko.common.MAX_WINDOW_SIZE

works for me (at least for now). Not sure what would happen if the file size increases from ~450MB to >>0.5GB.

Answer 4

I have been beating my head against this issue for some time, after pulling in about four different suggestions and mixing them together here is the method that I got to work for me:

First (connect to your sftp and loop through the list of files that match your request):

def getzipfiles(directory):
    # configuration file collection used to build my 
    # custom classlib.dataconnection json files
    configfilename = [fname for fname in configfiles if 'verifty_get' in fname]
    sftp_get = Configs.get_sftp_settings(configfilename[0])

    print("got here")
    try:
        cnopts = pysftp.CnOpts()
        cnopts.hostkeys = None #debug in dev (set your hostkeys!!!)
        sftpconn_get = pysftp.Connection(sftp_get.hostname,
                                         username=sftp_get.username,
                                         password=sftp_get.password,
                                         cnopts=cnopts)

        filelist = sftpconn_get.listdir()
        sftpconn_get.close()

        for filename in filelist:
            matchval = re.search(r'D*********_(?P<date>\d{8})_(?P<time>(\d{2}-?){3}.\d{1,8}).zip', filename, re.I)
            if matchval:
                getlargezipfiles(directory, filename)

    except:
        e = sys.exc_info()
        sftp_exception = e
        print("SFTP listdir failed, exception: {}".format(e))

Second (pass in the directory you want to save the file in, and the filename)

def getlargezipfiles(directory, filename):
    configfilename = [fname for fname in configfiles if 'verifty_get' in fname]
    sftp_get = Configs.get_sftp_settings(configfilename[0])

    MAX_RETRIES = 2

    port = 22
    sftp_file = filename
    local_file = "{}{}".format(directory,filename)
    ssh_conn = sftp_client = None
    start_time = time.time()

    for retry in range(MAX_RETRIES):
        try:
            ssh_conn = paramiko.Transport((sftp_get.hostname, port))
            ssh_conn.packetizer.REKEY_BYTES = pow(2, 40)  # 1TB max, this is a security degradation!
            ssh_conn.packetizer.REKEY_PACKETS = pow(2, 40)  # 1TB max, this is a security degradation!
            ssh_conn.default_window_size = paramiko.common.MAX_WINDOW_SIZE
            ssh_conn.connect(username=sftp_get.username, password=sftp_get.password)
            sftp_client = paramiko.SFTPClient.from_transport(ssh_conn)
            filesize = sftp_client.stat(sftp_file).st_size
            sftp_client.get_channel().in_window_size = 2097152
            sftp_client.get_channel().out_window_size = 2097152
            sftp_client.get_channel().in_max_packet_size = 2097152
            sftp_client.get_channel().out_max_packet_size = 2097152

            print("Getting {} size [{}] at {}".format(sftp_file, filesize, datetime.now()))
            sftp_client.get(sftp_file, local_file)
            
            break
        except (EOFError, paramiko.ssh_exception.SSHException, OSError) as x:
            retry += 1
            print("%s %s - > retrying %s..." % (type(x), x, retry))
            time.sleep(abs(retry) * 10)
            # back off in steps of 10, 20.. seconds
        finally:
            if hasattr(sftp_client, "close") and callable(sftp_client.close):
                sftp_client.close()
            if hasattr(ssh_conn, "close") and callable(ssh_conn.close):
                ssh_conn.close()

    print("Loading File %s Took %d seconds " % (sftp_file, time.time() - start_time))