How to collect, store and organize output with network and threading?

Question

Need your help and suggestion, I'm collecting data from networking devices and I'm storing it to dictionary "output_dict={}" at the same time I'm going to use this dictionary to organize the data, however I'm stuck collecting all data when using multi threading.

There are 2 issues right now.

1. There are instances that I'm able to get the data from device 1 and 2. Although there are instances that I getting only 1 device. :(

Result 1:
['192.168.2.111', '192.168.2.113']      <-- List of devices
Processing:  192.168.2.111
[ 192.168.2.111 ] Loging into IP
Processing:  192.168.2.113
[ 192.168.2.113 ] Loging into IP
[ 192.168.2.111 ] Logged out
[ 192.168.2.113 ] Logged out
{2: {'MGMT IP': '192.168.2.113', 'OUTPUT': '*00:20:45.015 UTC Thu May 6 2021'}} <-- only device2

Result 2:
['192.168.2.111', '192.168.2.113']
Processing:  192.168.2.111
[ 192.168.2.111 ] Loging into IP
Processing:  192.168.2.113
[ 192.168.2.113 ] Loging into IP
[ 192.168.2.111 ] Logged out
[ 192.168.2.113 ] Logged out
{1: {'MGMT IP': '192.168.2.111', 'OUTPUT': '\nhostname R1'}, 2: {'MGMT IP': '192.168.2.113', 'OUTPUT': '*00:23:19.967 UTC Thu May 6

2021'}} <-- Both device1 & 2 are present.

2. I'm trying to append the key value pair to dictionary but it overides the data.

['192.168.2.111', '192.168.2.113']
Processing:  192.168.2.111
[ 192.168.2.111 ] Loging into IP
Processing:  192.168.2.113
[ 192.168.2.113 ] Loging into IP
[ 192.168.2.111 ] Logged out
[ 192.168.2.113 ] Logged out
{'MGMT IP': '192.168.2.113', 'OUTPUT': '*00:55:21.055 UTC Thu May 6 2021'} <-- Overides devices 1

My target is to have something like this. So I can easily organize the data. [enter image description here][1] [1]: https://i.stack.imgur.com/EL5om.png

cli output: 
x = {'MGMT IP': '192.168.2.111', 'OUTPUT': 'output1'}{'MGMT IP': '192.168.2.111', 'OUTPUT': 'output2'}{'MGMT IP': '192.168.2.113',

'output1'}{'MGMT IP': '192.168.2.113', 'output2'}

How can achieve that when using multithreading and dictionary. Thanks. Here's my code.

# ~~~~~~~~
# Import modules
# ~~~~~~~~
import subprocess, logging, re, getpass,time
from netmiko import ConnectHandler
from netmiko.ssh_exception import NetMikoTimeoutException, NetMikoAuthenticationException
from paramiko.ssh_exception import SSHException
import threading
from queue import Queue
##from threading import Thread

logging.basicConfig(level=logging.INFO,
                    format="{asctime} {levelname:<8} {message}",
                    style='{',
                    filename='%slog' % __file__[:-2],
                    filemode='a'
                    )
##logging.info("this is a message")

# ~~~~~~~~
# Define functions
# ~~~~~~~~
dev_list = []
success_dev = []
failed_dev = []
output_dict = {}

def ping_ip(ip):
    global gips
    rstlongstr = ''
    (output,error) = subprocess.Popen((['ping', ip, '-c', '2']), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True).communicate()
    if 'bytes from' in output:
        dev_list.append(ip)
        logging.info(output)
##        print (output)
        reg_rttavg = re.findall(r'(?=rtt.*=\s[\d.]+\/([\d.]+))',output)
        strpreg_rttavg = [test2.strip() for test2 in reg_rttavg]
        str_rttavg = '\n'.join(strpreg_rttavg)
        reg_ploss = re.findall(r'[\d%.]+(?= packet loss)',output)
        strpreg_ploss = [test2.strip() for test2 in reg_ploss]
        str_ploss = '\n'.join(strpreg_ploss)
        return "[ " + ip + " ] Reachable - rtt(avg): %s ploss: %s" % (str_rttavg,str_ploss)
    elif 'Host Unreachable' in output:
        return "[ " + ip + " ] Unreachable"
    else:
        return "[ " + ip + " ] Unreachable"
    thread_q.put

def seq_sendcmd(device_conn,ip_a):
    global success_dev, failed_dev,output_dict
    FinalResult = ''
    sh_commands  = ['show run | i hostname', 'show clock']
    for cmd in sh_commands:
        result_out = device_conn.send_command(cmd)
        output_dict = {}
        output_dict['MGMT IP'] = ip_a
        output_dict['OUTPUT'] = result_out
        FinalResult = FinalResult + ('*****%s\n\n%s\n\n' %(cmd,result_out))
    device_conn.disconnect()
    print ('[ ' + ip_a + ' ] Logged out')
    return FinalResult

def connect_now(ip_a,username,password,enable_s):
    global success_dev, failed_dev
    print ('[ ' + ip_a + ' ] Loging into IP')
    
    try:
        
        print ("X is: ",x)
        device_conn = ConnectHandler(device_type='cisco_ios_ssh', ip=ip_a,
                                             username=username, password=password,
                                             secret=enable_s, verbose=False)
        output = seq_sendcmd(device_conn,ip_a)
        output_q.put(output)
        
    except Exception as e: print(e) 

#~~~~~~~~~~~~~~
#MAIN
#~~~~~~~~~~~~~~
##ipadd = ['192.168.2.111',"192.168.2.113", '9.9.9.222','192.168.2.112',"192.168.2.66",]
ipadd = ['192.168.2.111',"192.168.2.113"]
thread_q = Queue()
for x in ipadd:
    m_thread = threading.Thread(target=ping_ip, args=(x,))
    m_thread.start()
    time.sleep(3)
m_thread.join()    
print (dev_list)   

Results = []
uname = "test"
pswd = "test"
ena = "test"

if uname == "" or pswd == "" or ena == "":
    print('username, password or enable_secret cannot be empty. Please enter valid credentials!!!')
    logging.info("username, password or enable_secret cannot be empty. Please enter valid credentials!!!")
    sys.exit()

logging.info("List of reachable device: " + str(dev_list))
start = time.time()
output_q = Queue()
threads = []
x = 0
for ip in dev_list:
    x+=1
    print ("Processing: ",ip)
    t = threading.Thread(target=connect_now, args=(ip,uname,pswd,ena))
    threads.append(t)
    t.start()
    time.sleep(3)
for t in threads:
  t.join()
##print (output_q.get())
print (output_dict)
end = time.time()
total = int(end - start)
print("\n Elapsd Time: " + str(total) + " Sec\n")

Answer 1

If your problem is to resolve overriding problem you can use an algorithm as following so that you should be able to save your IP and Output pair without overriding:

IP = ["192.168.1.1", "192.168.2.1", "192.168.1.1"]
Output = ["output1", "output2", "output3"]

result_list = []
result2 = {}

i = 0

while i < len(IP):
    result2 = {IP[i], Output[i]}
    result_list.append(result2)
    i = i + 1

print(result_list)

You can find the result below: