I have a dataframe that has 15,000 host IP addresses (IP v4 and IP v6) I am trying to check which of these hosts are up(running) by pinging the hosts.
I have the following code that I have written
def ping_host(hostname):
# hostname = "10.0.0.10 #example
print(hostname)
if hostname != hostname:
return "No Hostname"
response = os.system("ping -c 1 " + hostname + " > /dev/null 2>&1")
# and then check the response...
if response == 0:
print(hostname, 'is up!')
return "Up"
else:
print(hostname, 'is down!')
return "Down"
df['Host_Status'] = df['IP Addresses'].apply(ping_host)
This takes for ever to complete. Is there a better/faster way to do this?
I did try -
df['Host_Status'] = df['IP Addresses'].swifter.apply(ping_host)
but even that didn't increase the speed by much.
Edit 1-
I let this run using multi-threading for 5 hours using 32/64/256 threads(I think only 32 threads were working at any given point) but at the end of the script
from multiprocessing.pool import ThreadPool
def ping_host(hostname):
# hostname = "10.0.0.10 #example
print(hostname)
if hostname != hostname:
return "No Hostname"
response = os.system("ping -c 1 " + hostname + " > /dev/null 2>&1")
# and then check the response...
if response == 0:
print(hostname, 'is up!')
return "Up"
else:
print(hostname, 'is down!')
return "Down"
# you can fiddle with pool size
with ThreadPool(processes=32) as t:
df['Host_Status'] = t.map(ping_host, df['IP Addresses'])
df.to_excel('output.xlsx')
there when I tried to export it I got the following error -
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 926, in _bootstrap_inner
self.run()
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_comm/pydev_transport.py", line 64, in _read_forever
self._read_and_dispatch_next_frame()
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_comm/pydev_transport.py", line 37, in _read_and_dispatch_next_frame
direction, frame = self._read_frame()
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_comm/pydev_transport.py", line 45, in _read_frame
buff = readall(self._socket.recv, 4)
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_comm/pydev_io.py", line 110, in readall
chunk = read_fn(sz - have)
TimeoutError: [Errno 60] Operation timed out
Traceback (most recent call last):
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_bundle/pydev_code_executor.py", line 112, in add_exec
self.finish_exec(more)
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_bundle/pydev_console_utils.py", line 210, in finish_exec
return server.notifyFinished(more)
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_comm/pydev_transport.py", line 226, in _req
return super(TSyncClient, self)._req(_api, *args, **kwargs)
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/third_party/thriftpy/_shaded_thriftpy/thrift.py", line 160, in _req
return self._recv(_api)
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/third_party/thriftpy/_shaded_thriftpy/thrift.py", line 172, in _recv
fname, mtype, rseqid = self._iprot.read_message_begin()
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/third_party/thriftpy/_shaded_thriftpy/protocol/binary.py", line 372, in read_message_begin
self.trans, strict=self.strict_read)
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/third_party/thriftpy/_shaded_thriftpy/protocol/binary.py", line 164, in read_message_begin
sz = unpack_i32(inbuf.read(4))
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/third_party/thriftpy/_shaded_thriftpy/transport/__init__.py", line 32, in read
return readall(self._read, sz)
File "/Applications/PyCharm.app/Contents/plugins/python/helpers/third_party/thriftpy/_shaded_thriftpy/transport/__init__.py", line 20, in readall
"End of file reading from transport")
_shaded_thriftpy.transport.TTransportException: TTransportException(type=4, message='End of file reading from transport')
Try 2 Asyncio-
After reading the dataframe I tried -
async def async_ping(host):
proc = await asyncio.create_subprocess_shell(
f"/sbin/ping -c 1 {host} > /dev/null 2>&1",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
status = await proc.wait()
if status == 0:
return 'Alive'
else:
return 'Timeout'
async def async_main(hosts):
tasks1 = deque()
for host in hosts:
tasks1.append(asyncio.create_task(async_ping(host)))
return (t1 for t1 in await asyncio.gather(*tasks1))
start = time.perf_counter()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
resp = loop.run_until_complete(async_main(df['IP Addresses'].to_list()))
loop.close()
finish = time.perf_counter()
df['Status'] = list(resp)
print(df)
print(f'Runtime: {round(finish-start,4)} seconds')
I ran into Blocking error -
Traceback (most recent call last):
File "~path/venv/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3331, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-6-2646fe9bd357>", line 26, in <module>
resp = loop.run_until_complete(async_main(df['IP Addresses'].to_list()))
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py", line 587, in run_until_complete
return future.result()
File "<ipython-input-6-2646fe9bd357>", line 20, in async_main
return (t1 for t1 in await asyncio.gather(*tasks1))
File "<ipython-input-6-2646fe9bd357>", line 5, in async_ping
stderr=asyncio.subprocess.PIPE
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/subprocess.py", line 202, in create_subprocess_shell
stderr=stderr, **kwds)
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py", line 1514, in subprocess_shell
protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/unix_events.py", line 190, in _make_subprocess_transport
**kwargs)
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_subprocess.py", line 37, in __init__
stderr=stderr, bufsize=bufsize, **kwargs)
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/unix_events.py", line 775, in _start
universal_newlines=False, bufsize=bufsize, **kwargs)
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/subprocess.py", line 800, in __init__
restore_signals, start_new_session)
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/subprocess.py", line 1482, in _execute_child
restore_signals, start_new_session, preexec_fn)
BlockingIOError: [Errno 35] Resource temporarily unavailable
When I saw this question I wanted to try to make something that used the asyncio module to run pings concurrently. The below script runs my test IP addresses in ~7 seconds. When I run the same test IP address list synchronously it takes ~127 seconds. I used python version 3.8.2 and Windows 10 OS. Maybe it'll work for you.
import asyncio
import time
from collections import deque
import pandas as pd
async def async_ping(host, semaphore):
async with semaphore:
for _ in range(5):
proc = await asyncio.create_subprocess_shell(
f'C:\\Windows\\System32\\ping {host} -n 1 -w 1 -l 1',
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
status = await proc.wait()
if status == 0:
return 'Alive'
return 'Timeout'
async def async_main(hosts, limit):
semaphore = asyncio.Semaphore(limit)
tasks1 = deque()
for host in hosts:
tasks1.append(asyncio.create_task(
async_ping(host, semaphore))
)
return (t1 for t1 in await asyncio.gather(*tasks1))
host_df = pd.read_csv('ping_ip_dest.csv')
# set concurrent task limit
limit = 256
start = time.perf_counter()
loop = asyncio.get_event_loop()
asyncio.set_event_loop(loop)
resp = loop.run_until_complete(async_main(host_df['IP'].to_list(), limit))
loop.close()
finish = time.perf_counter()
host_df['Status'] = list(resp)
print(host_df)
print(f'Runtime: {round(finish-start,4)} seconds')
OUTPUT:
0 N. Virginia (Virginia, USA) 23.235.60.92 Alive
1 Dallas (Texas, USA) 69.162.81.155 Alive
2 Denver (Colorado, USA) 192.199.248.75 Alive
3 Miami (Florida, USA) 162.254.206.227 Alive
4 Minneapolis (Minnesota, USA) 207.250.234.100 Alive
5 Montreal (Canada) 184.107.126.165 Alive
6 New York (New York, USA) 206.71.50.230 Alive
7 San Francisco (California, USA) 65.49.22.66 Alive
8 Seattle (Washington, USA) 23.81.0.59 Alive
9 Washington DC (Virginia, USA) 207.228.238.7 Alive
10 Buenos Aires (Argentina) 131.255.7.26 Alive
11 Amsterdam (Netherlands) 95.142.107.181 Alive
12 Copenhagen (Denmark) 185.206.224.67 Alive
13 Frankfurt (Germany) 195.201.213.247 Alive
14 London (United Kingdom) 5.152.197.179 Alive
15 Madrid (Spain) 195.12.50.155 Alive
16 Paris (France) 51.158.22.211 Alive
17 Warsaw (Poland) 46.248.187.100 Alive
18 Johannesburg (South Africa) 197.221.23.194 Alive
19 Beijing (China) 47.94.129.116 Alive
20 Hong Kong (China) 103.1.14.238 Alive
21 Mumbai (India) 103.120.178.71 Alive
22 Shanghai (China) 106.14.156.213 Alive
23 Tokyo (Japan) 110.50.243.6 Alive
24 Brisbane 223.252.19.130 Alive
25 Sydney 101.0.86.43 Alive
26 Tel-Aviv (Israel) 185.229.226.83 Alive
27 Test 47.94.129.115 Timeout
Runtime: 3.1945 seconds
You can do it with a thread pool.
Updated for parameter to timeout ping after 1 second, chunksize 1, more threads and skipping intermediate shell - worked for a small number of IP addresses on my local machine duplicated to 14000 total. Wait time and chunksize may be the most important. By default, ping times out after 10 seconds and threadpool runs 14000/threadcount pings before returning to the main thread. Considering a timeout was hit when testing this code, the updates should be a good improvement.
from multiprocessing.pool import ThreadPool
import subprocess as subp
def ping_host(hostname):
# hostname = "10.0.0.10 #example
print(hostname)
if hostname != hostname:
return "No Hostname"
response = subp.run(["ping","-c", "1", "-W", "1", hostname], stdout=subp.DEVNULL,
stderr=subp.DEVNULL).returncode
# and then check the response...
if response == 0:
print(hostname, 'is up!')
return "Up"
else:
print(hostname, 'is down!')
return "Down"
# you can fiddle with pool size
with ThreadPool(processes=128) as t:
df['Host_Status'] = t.map(ping_host, df['IP Addresses'],
chunksize=1)
Related
I have a working script which invokes multiple blocking read methods in different threads and whenever it gets data in the blocking method it puts the data in queue that will be read by task.
import asyncio
import atexit
import concurrent.futures
import threading
def blocking_read(x, loop):
while True:
print(f"blocking_read {x}")
time.sleep(1)
# implementation is wait for data and put into the queue
class Recorder():
def __init__(self):
self.count = 0
def run_reader_threads(self, thread_pool, event):
loop = asyncio.get_running_loop()
threads = []
for rx in range(2):
threads.append(loop.run_in_executor(thread_pool, blocking_read, rx, loop))
return threads
def wiatforever():
while True:
time.sleep(1)
reader_futures = []
executor = concurrent.futures.ThreadPoolExecutor()
event = threading.Event()
def main():
async def doit():
recorder_app = Recorder()
global reader_futures
reader_futures = recorder_app.run_reader_threads(executor, event)
#reader_handlers = recorder_app.create_handlers()
await wiatforever()
try:
print("RUN DO IT")
asyncio.run(doit())
except KeyboardInterrupt:
# cancel all futures but geeting exception on cancelling 1st future
reader_futures[0].cancel()
pass
#--------------------------------------------------------------------------------------------
if __name__ == '__main__':
main()
The problem is whenever I want to stop the script gracefully by cancelling all futures I am getting exception that RuntimeError: Event loop is closed
OUTPUT
RUN DO IT
blocking_read 0
blocking_read 1
blocking_read 1
blocking_read 0
blocking_read 0
blocking_read 1
^CTraceback (most recent call last):
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 45, in main
asyncio.run(doit())
File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 629, in run_until_complete
self.run_forever()
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 596, in run_forever
self._run_once()
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 1890, in _run_once
handle._run()
File "/usr/local/lib/python3.9/asyncio/events.py", line 80, in _run
self._context.run(self._callback, *self._args)
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 41, in doit
await wiatforever()
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 27, in wiatforever
time.sleep(1)
KeyboardInterrupt
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 53, in <module>
main()
File "/home/devuser/Desktop/rnr_crash/2trial.py", line 48, in main
reader_futures[0].cancel()
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 746, in call_soon
self._check_closed()
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 510, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
blocking_read 0
blocking_read 1
blocking_read 0
blocking_read 1
blocking_read 0
blocking_read 1
^CException ignored in: <module 'threading' from '/usr/local/lib/python3.9/threading.py'>
Traceback (most recent call last):
File "/usr/local/lib/python3.9/threading.py", line 1411, in _shutdown
atexit_call()
File "/usr/local/lib/python3.9/concurrent/futures/thread.py", line 31, in _python_exit
t.join()
File "/usr/local/lib/python3.9/threading.py", line 1029, in join
self._wait_for_tstate_lock()
File "/usr/local/lib/python3.9/threading.py", line 1045, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt:
so can anyone help me to find out the issue here and exit the script correctly without any exception.
I have a long running python script for a weather station. The script captures data from the weather sensors and uploads them to a mySql database as well as weather underground.
The issue I am having is that the script will run for multiple days and then crashes without a traceback. I have been troubleshooting this for weeks to no avail. I was initially running the script in IDLE. I have also ran it in the terminal. The latest update that I "think" is the real issue is that I ran the script in Visual Studio Code and when it crashed it posted: munmap_chunk() invalid pointer in the terminal.
My understanding is that munmap_chunk is a C issue so it is most likely being generated by a module that I import. I also understand that is is thrown when a pointer is passed to free() that hasn't been obtained through malloc().
With that said, I don't know how to narrow down where the issue is actually being generated from so I can potentially correct it.
Prior to seeing the munmap_chunk error I had thought maybe it was a memory leak somewhere. I manually called the garbage collector which in almost all cases you shouldn't do. That allowed for the script to run longer before the it finally crashed again.
The main thing I'm trying to determine is how to zero in on where the issue is coming from since the script just crashes and I don't get any traceback information or exception being thrown.
#!/usr/bin/python3
from gpiozero import Button
import time
import math
import Wind_Direction
import statistics
import Database
import requests
import sys
import logging
import board
from adafruit_bme280 import basic as adafruit_bme280
logging.basicConfig(format='%(asctime)s **%(levelname)4s** - %(module)s: %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', filename='/home/testa/testb/testc.txt', filemode = 'w', level=logging.DEBUG)
logging.debug("The program has started")
print ("\nWeather_Station_BYO: The program has started")
i2c = board.I2C() # uses board.SCL and board.SDA
# ------- WU URL and credentials ---------
WUurl = "https://weatherstation.wunderground.com/weatherstation/updateweatherstation.php?"
WUid = "TESTEST" # add your WS ID
WUpwd = "TESTEST" # add your password
WUcreds = "ID=" + WUid + "&PASSWORD="+ WUpwd
wind_count = 0 # Count how many half-rotations of the anemometer
radius_in = 3.54 # Radius of your anemometer in inches (9 cm)
wind_interval = 5 # How often (secs) to report speed
interval = 300 # 5*60 seconds = 5 minutes = 300 seconds
inches_in_mile = 63360
SECS_IN_AN_HOUR = 3600
ADJUSTMENT = 1.18 # Anemometer adjustment
BUCKET_SIZE = 0.011 # Rain gauge tips once when (.2794 mm = .011 in) of rain falls
rain_count = 0
store_speeds = []
store_directions = []
def spin():
"""Counts the number of every half-rotation of the anenmometer and returns that count"""
global wind_count
wind_count = wind_count + 1 # Every half-rotation, add 1 to count
# print("spin" + str(wind_count))
def calculate_speed(time_sec):
"""Uses the number of half-rotations stored in the global variable wind_count and the time used to count the rotations and generates a final wind speed"""
global wind_count
circumference_in = (2 * math.pi) * radius_in
rotations = wind_count / 2.0
dist_in = (circumference_in * rotations) / inches_in_mile # Calculate distance travelled by a cup in inches
mile_per_sec = dist_in / time_sec
mph = mile_per_sec * SECS_IN_AN_HOUR
final_speed = mph * ADJUSTMENT
# print("final speed" + str(final_speed))
return final_speed
def bucket_tipped():
"""Counts the number of times the rain gauge bucket filled and tipped"""
global rain_count
rain_count = rain_count + 1
# print(rain_count * BUCKET_SIZE)
def reset_rainfall():
"""Resets the global rain_count variable set in the bucket_tipped function"""
global rain_count
rain_count = 0
def reset_wind():
"""Resets the global wind_count variable set in the spin function"""
global wind_count
wind_count = 0
def read_all():
"""Reads bme280 (temp, humididty, pressure) values converts them to proper units and calculates the dewpoint"""
bme280 = adafruit_bme280.Adafruit_BME280_I2C(i2c)
bme280.sea_level_pressure = 1013.25 # change this to match the location's pressure (hPa) at sea level
humidity = bme280.humidity # % Relative humidity
pressure = bme280.pressure * 100 / 3386 # Pascals converted to inHg
ambient_temperature = bme280.temperature * 9/5 + 32 # Convert to fahrenheit rounded
b = 17.62
c = 243.12
gamma = (b * bme280.temperature /(c + bme280.temperature)) + math.log(bme280.humidity / 100.0)
dewpoint = (c * gamma) / (b - gamma)
return humidity, pressure, ambient_temperature, dewpoint
wind_speed_sensor = Button(5)
wind_speed_sensor.when_pressed = spin # Whenever the wind sensor magnet closes the reed switch then add 1 to the number of rotations
# temp_probe = ds18b20_therm.DS18B20()
rain_sensor = Button(6)
rain_sensor.when_pressed = bucket_tipped
db = Database.weather_database()
i = 1
while db.wxlogin() != True:
time.sleep(30)
i = i + 1
if (i>5):
logging.debug("Could not establish a connection to the mysql database and program will be terminated.")
time.sleep(5)
sys.exit("Could not establish connection") # Exit the script after i amount of connection attempts
logging.debug("Initial connection to database was successful!")
print("\nWeather_Station_BYO: Initial connection to database was successful!")
# Loop to measure wind speed at 300-second intervals
while True:
try:
start_time = time.time()
while time.time() - start_time <= interval:
# print("In loop 1")
wind_start_time = time.time()
reset_wind()
while time.time() - wind_start_time <= wind_interval:
# print("In loop 2")
store_directions.append(Wind_Direction.get_value(wind_interval)) # wind_interval is how long to take measuremennts for wind direction
final_speed = calculate_speed(wind_interval) # Add this speed to the list & wind_interval is how long to take measurements for wind speed
store_speeds.append(final_speed)
logger = logging.getLogger(__name__)
logger.debug("Wind speed and direction obtained")
# print("Out of both loops")
# print("stored directions " + str(store_directions))
# print("stored speeds " + str(store_speeds))
wind_dir = Wind_Direction.get_average(store_directions)
wind_gust = max(store_speeds)
wind_speed = statistics.mean(store_speeds)
rainfall = rain_count * BUCKET_SIZE
reset_rainfall()
store_speeds = []
store_directions = []
humidity, pressure, ambient_temp, dewpoint = read_all()
logger.debug("Humidity, pressure, ambient_temp and dewpoint obtained")
# print(wind_dir)
if 348.75 <= wind_dir or wind_dir <= 11.25:
wind_direction = 'N'
elif 11.26 <= wind_dir <= 33.75:
wind_direction = 'NNE'
elif 33.76 <= wind_dir <= 56.25:
wind_direction = 'NE'
elif 56.26 <= wind_dir <= 78.75:
wind_direction = 'ENE'
elif 78.76 <= wind_dir <= 101.25:
wind_direction = 'E'
elif 101.25 <= wind_dir <= 123.75:
wind_direction = 'ESE'
elif 123.76 <= wind_dir <= 146.25:
wind_direction = 'SE'
elif 146.26 <= wind_dir <= 168.75:
wind_direction = 'SSE'
elif 168.76 <= wind_dir <= 191.25:
wind_direction = 'S'
elif 191.26 <= wind_dir <= 213.75:
wind_direction = 'SSW'
elif 213.76 <= wind_dir <= 236.25:
wind_direction = 'SW'
elif 226.26 <= wind_dir <= 258.75:
wind_direction = 'WSW'
elif 258.76 <= wind_dir <= 281.25:
wind_direction = 'W'
elif 281.26 <= wind_dir <= 303.75:
wind_direction = 'WNW'
elif 303.76 <= wind_dir <= 326.25:
wind_direction = 'NW'
elif 326.25 <= wind_dir <= 348.75:
wind_direction = 'NNW'
else: wind_direction = ''
#print('',wind_direction, 'Wind Direction \n',round(wind_speed,1), 'Wind Speed (mph) \n', round(wind_gust,1),'Wind Gust (mph) \n', round(rainfall,1), 'Rainfall (in) \n',
# round(humidity,1), '% Relative Humidity \n', round(dewpoint,1), 'Dew Point F \n', round(pressure,2), 'Barometric Pressure (inHg) \n', round(ambient_temp,1), 'Temperature F \n')
i = 1
while db.check_connection() != True:
time.sleep(30)
i = i + 1
if (i>5):
logger.debug("Connection to the mysql database was disconnected and connection could not be restablished. Program will be terminated.")
time.sleep(5)
sys.exit("Could not re-establish connection") # Exit the script after i amount of connection attempts
db.wxlogin()
db.insert(1,round(ambient_temp,1), 0, 0, round(pressure,2), round(humidity,1), round(dewpoint,1), wind_direction, round(wind_speed,1), round(wind_gust,1), round(rainfall,1)) # 10 items
# Weatherunderground
# print("Uploading to Weather Underground")
f_date = "&dateutc=now"
f_humid = "&humidity=" + str(humidity) # % Relative humidity
f_wspeed = "&windspeedmph=" + str(wind_speed) # mph
f_gust = "&windgustmph=" + str(wind_gust) # mph
f_airtemp = "&tempf=" + str(ambient_temp) # degrees F
f_rain = "&rainin=" + str(rainfall) # inches
f_press = "&baromin=" + str(pressure) # inches
# f_groundtemp = "&soiltempf=" + str(ground_temp_f) # degrees F
f_winddir = "&winddir=" + str(wind_dir) # degrees
f_action = "&action=updateraw"
try:
r = requests.get(WUurl+WUcreds+f_date+f_humid+f_wspeed+f_winddir+f_gust+f_airtemp+f_rain+f_press+f_action, timeout = 30)
print("\nReceived " + str(r.status_code) + " " + str(r.text) + " from WU")
except requests.ConnectionError as e:
logger.debug("HTTP Error - Connection Error. Make sure you are connected to the internet. Technical details given below.\n")
logger.debug(str(e))
print("\nHTTP Error - Connection Error. Make sure you are connected to internet. Technical details given below.\n")
print(str(e))
continue
except requests.Timeout as e:
logger.debug("HTTP Error - Timeout Error")
logger.debug(str(e))
print("\nHTTP Error - Timeout Error")
print(str(e))
continue
except requests.RequestException as e:
logger.debug("HTTP Error - General Error")
logger.debug(str(e))
print("\nHTTP Error - General Error")
print(str(e))
continue
except KeyboardInterrupt:
print("\nSomeone closed the program")
if (str(r.status_code) != '200'):
logger.debug("Upload to Weather Underground was not sucessful. HTTP Response Code: " + str(r.status_code) + " " + str(r.text))
except Exception:
logger.exception("Unhandled exception in the script!", exc_info = True)
Traceback from faulthandler in VSCode:
munmap_chunk(): invalid pointer
Fatal Python error: Aborted
Thread 0xb0fa5440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/usr/lib/python3/dist-packages/gpiozero/mixins.py", line 539 in held
File "/usr/lib/python3.7/threading.py", line 865 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py", line 1054 in __call__
Thread 0xb27ff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/usr/lib/python3/dist-packages/gpiozero/mixins.py", line 543 in held
File "/usr/lib/python3.7/threading.py", line 865 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydev_bundle/pydev_monkey.py", line 1054 in __call__
Thread 0xb31ff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/pydevd.py", line 246 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb3bff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/pydevd.py", line 200 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb45ff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 296 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_timeout.py", line 43 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb4fff440 (most recent call first):
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py", line 219 in _read_line
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py", line 237 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb599b440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/queue.py", line 179 in get
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py", line 382 in _on_run
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_daemon_thread.py", line 46 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Current thread 0xb6f34b40 (most recent call first):
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 85 in _words_to_int
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 220 in _read
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 157 in value
File "/home/pi/Liberty Ridge WX Reduced/Wind_Direction.py", line 97 in get_value
File "/home/pi/Liberty Ridge WX Reduced/Liberty_Ridge_WX.py", line 199 in <module>
File "/usr/lib/python3.7/runpy.py", line 85 in _run_code
File "/usr/lib/python3.7/runpy.py", line 96 in _run_module_code
File "/usr/lib/python3.7/runpy.py", line 263 in run_path
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/../debugpy/server/cli.py", line 285 in run_file
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/../debugpy/server/cli.py", line 444 in main
File "/home/pi/.vscode/extensions/ms-python.python-2021.12.1559732655/pythonFiles/lib/python/debugpy/__main__.py", line 45 in <module>
File "/usr/lib/python3.7/runpy.py", line 85 in _run_code
File "/usr/lib/python3.7/runpy.py", line 193 in _run_module_as_main
Traceback from faulthandler in terminal:
munmap_chunk(): invalid pointer
Fatal Python error: Aborted
Thread 0xb44ff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/usr/lib/python3/dist-packages/gpiozero/mixins.py", line 539 in held
File "/usr/lib/python3.7/threading.py", line 865 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Thread 0xb4eff440 (most recent call first):
File "/usr/lib/python3.7/threading.py", line 300 in wait
File "/usr/lib/python3.7/threading.py", line 552 in wait
File "/usr/lib/python3/dist-packages/gpiozero/mixins.py", line 539 in held
File "/usr/lib/python3.7/threading.py", line 865 in run
File "/usr/lib/python3.7/threading.py", line 917 in _bootstrap_inner
File "/usr/lib/python3.7/threading.py", line 885 in _bootstrap
Current thread 0xb6f53b40 (most recent call first):
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 85 in _words_to_int
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 220 in _read
File "/usr/lib/python3/dist-packages/gpiozero/spi_devices.py", line 157 in value
File "/home/pi/Liberty Ridge WX Reduced/Wind_Direction.py", line 97 in get_value
File "/home/pi/Liberty Ridge WX Reduced/Liberty_Ridge_WX.py", line 189 in <module>
Aborted
I'm running very similar code on my Raspberry Pi weather station, and also get the munmap_chunk() error after a couple of days, running either from a command line or from Thonny.
Like you, I get no information, apart from the munmap_chunk() line. That makes me think the error is not in Python, as that would surely provide trace information. So, maybe it's from the libraries Python uses, or from the part of the operating system that does memory management.
Here's what I believe to be the source code for where the error happens, and the comments are interesting...
static void
munmap_chunk (mchunkptr p)
{
INTERNAL_SIZE_T size = chunksize (p);
assert (chunk_is_mmapped (p));
/* Do nothing if the chunk is a faked mmapped chunk in the dumped
main arena. We never free this memory. */
if (DUMPED_MAIN_ARENA_CHUNK (p))
return;
uintptr_t block = (uintptr_t) p - prev_size (p);
size_t total_size = prev_size (p) + size;
/* Unfortunately we have to do the compilers job by hand here. Normally
we would test BLOCK and TOTAL-SIZE separately for compliance with the
page size. But gcc does not recognize the optimization possibility
(in the moment at least) so we combine the two values into one before
the bit test. */
if (__builtin_expect (((block | total_size) & (GLRO (dl_pagesize) - 1)) != 0, 0))
malloc_printerr ("munmap_chunk(): invalid pointer");
atomic_decrement (&mp_.n_mmaps);
atomic_add (&mp_.mmapped_mem, -total_size);
/* If munmap failed the process virtual memory address space is in a
bad shape. Just leave the block hanging around, the process will
terminate shortly anyway since not much can be done. */
__munmap ((char *) block, total_size);
}
In the end, I decided a better thing to do was restart the program once a day, but I didn't want to use crontab, rc.local or similar. There's actually a way for a Python program to restart itself! I found it here https://blog.petrzemek.net/2014/03/23/restarting-a-python-script-within-itself/
Here's my test program...
#!/usr/bin/env/python3
import sys
import os
import time
import datetime as dt
began = dt.datetime.now()
print("Began at ", str(began))
ending = began + dt.timedelta(minutes = 2)
print("Ending at ", str(ending))
while True:
time.sleep(10)
present = dt.datetime.now()
print(" It's ", str(present))
if present > ending:
print("Time's up!")
os.execv(sys.executable, ['python3'] + sys.argv)
quit()
I think amending your weather station code to restart itself daily, before the munmap_chunk() error happens is likely to be easier than finding out why it happens.
Update:
That's what I did, adding this code before the main loop...
ending = dt.datetime.now() + dt.timedelta(hours = 24)
...and this at the end of the main loop
present = dt.datetime.now()
if present > ending:
print("Time for my 24 hour restart!")
os.execv(sys.executable, ['python3'] + sys.argv)
quit()
The program restarts every day (approximately), and has so far run for five days without the munmap_chunk() error.
I would like to take advantage of the InfiniBand network to connect Dask Client and the workers and scheduler (especially between the clients and workers -not necessary with GPUs- as I scatter some data directly to workers).
I am using the CLI to launch the scheduler in one separate node, and the workers in N nodes and I have M Clients. Apparently adding --interface ib0 is not enough. And adding --protocol ucx:// generates this error :
0: cpu-bind=MASK - node124, task 0 0 [21536]: mask 0xffffffffff set
0: distributed.scheduler - INFO - -----------------------------------------------
0: distributed.scheduler - INFO - -----------------------------------------------
0: distributed.scheduler - INFO - Clear task state
0: Traceback (most recent call last):
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/cli/dask_scheduler.py", line 217, in main
0: loop.run_sync(run)
0: File ".conda/envs/Env/lib/python3.8/site-packages/tornado/ioloop.py", line 532, in run_sync
0: return future_cell[0].result()
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/cli/dask_scheduler.py", line 213, in run
0: await scheduler
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/core.py", line 305, in _
0: await self.start()
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/scheduler.py", line 1477, in start
0: await self.listen(
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/core.py", line 430, in listen
0: listener = await listen(
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/comm/core.py", line 209, in _
0: await self.start()
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/comm/ucx.py", line 385, in start
0: init_once()
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/comm/ucx.py", line 56, in init_once
0: import ucp as _ucp
0: ModuleNotFoundError: No module named 'ucp'
0:
0: During handling of the above exception, another exception occurred:
0:
0: Traceback (most recent call last):
0: File ".conda/envs/Env/bin/dask-scheduler", line 11, in <module>
0: sys.exit(go())
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/cli/dask_scheduler.py", line 226, in go
0: main()
0: File ".conda/envs/Env/lib/python3.8/site-packages/click/core.py", line 829, in __call__
0: return self.main(*args, **kwargs)
0: File ".conda/envs/Env/lib/python3.8/site-packages/click/core.py", line 782, in main
0: rv = self.invoke(ctx)
0: File ".conda/envs/Env/python3.8/site-packages/click/core.py", line 1066, in invoke
0: return ctx.invoke(self.callback, **ctx.params)
0: File ".conda/envs/Env/python3.8/site-packages/click/core.py", line 610, in invoke
0: return callback(*args, **kwargs)
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/cli/dask_scheduler.py", line 221, in main
0: logger.info("End scheduler at %r", scheduler.address)
0: File ".conda/envs/Env/lib/python3.8/site-packages/distributed/core.py", line 389, in address
0: raise ValueError("cannot get address of non-running Server")
0: ValueError: cannot get address of non-running Server
Do I need to install UCX-Py first in my system, then use --protocol ucx:// and it's enough to take advantage of IB? if not how can I enable IB or IB through UCX?
Is there any other way to use the infiniband for communication in Dask.distributed ?
If I use UXC and the IB, does this imply that my clients also communicate over it, or just the workers and the scheduler do?
Thank you :)
I have an issue in Python 3.7.3 where my multiprocessing operation (using Queue, Pool, and apply_async) deadlocks when handling large computational tasks.
For small computations, this multiprocessing task works just fine. However, when dealing with larger processes, the multiprocessing task stops, or deadlocks, altogether without exiting the process! I read that this will happen if you "grow your queue without bounds, and you are joining up to a subprocess that is waiting for room in the queue [...] your main process is stalled waiting for that one to complete, and it never will." (Process.join() and queue don't work with large numbers)
I am having trouble converting this concept into code. I would greatly appreciate guidance on refactoring the code I have written below:
import multiprocessing as mp
def listener(q, d): # task to queue information into a manager dictionary
while True:
item_to_write = q.get()
if item_to_write == 'kill':
break
foo = d['region']
foo.add(item_to_write)
d['region'] = foo # add items and set to manager dictionary
def main():
manager = mp.Manager()
q = manager.Queue()
d = manager.dict()
d['region'] = set()
pool = mp.Pool(mp.cpu_count() + 2)
watcher = pool.apply_async(listener, (q, d))
jobs = []
for i in range(24):
job = pool.apply_async(execute_search, (q, d)) # task for multiprocessing
jobs.append(job)
for job in jobs:
job.get() # begin multiprocessing task
q.put('kill') # kill multiprocessing task (view listener function)
pool.close()
pool.join()
print('process complete')
if __name__ == '__main__':
main()
Ultimately, I would like to prevent deadlocking altogether to facilitate a multiprocessing task that could operate indefinitely until completion.
BELOW IS THE TRACEBACK WHEN EXITING DEADLOCK IN BASH
^CTraceback (most recent call last):
File "multithread_search_cl_gamma.py", line 260, in <module>
main(GEOTAG)
File "multithread_search_cl_gamma.py", line 248, in main
job.get()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/pool.py", line 651, in get
Process ForkPoolWorker-28:
Process ForkPoolWorker-31:
Process ForkPoolWorker-30:
Process ForkPoolWorker-27:
Process ForkPoolWorker-29:
Process ForkPoolWorker-26:
self.wait(timeout)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/pool.py", line 648, in wait
Traceback (most recent call last):
Traceback (most recent call last):
Traceback (most recent call last):
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/pool.py", line 110, in worker
task = get()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/pool.py", line 110, in worker
task = get()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/queues.py", line 351, in get
with self._rlock:
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/queues.py", line 351, in get
self._event.wait(timeout)
File "/Users/Ira/anaconda3/lib/python3.7/threading.py", line 552, in wait
Traceback (most recent call last):
Traceback (most recent call last):
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/pool.py", line 110, in worker
task = get()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/queues.py", line 352, in get
res = self._reader.recv_bytes()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/connection.py", line 216, in recv_bytes
buf = self._recv_bytes(maxlength)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/connection.py", line 407, in _recv_bytes
buf = self._recv(4)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/connection.py", line 379, in _recv
chunk = read(handle, remaining)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/pool.py", line 110, in worker
task = get()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/queues.py", line 351, in get
with self._rlock:
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/synchronize.py", line 95, in __enter__
return self._semlock.__enter__()
KeyboardInterrupt
KeyboardInterrupt
signaled = self._cond.wait(timeout)
File "/Users/Ira/anaconda3/lib/python3.7/threading.py", line 296, in wait
waiter.acquire()
KeyboardInterrupt
with self._rlock:
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/synchronize.py", line 95, in __enter__
return self._semlock.__enter__()
KeyboardInterrupt
Traceback (most recent call last):
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/pool.py", line 110, in worker
task = get()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/queues.py", line 351, in get
with self._rlock:
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/synchronize.py", line 95, in __enter__
return self._semlock.__enter__()
KeyboardInterrupt
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/synchronize.py", line 95, in __enter__
return self._semlock.__enter__()
KeyboardInterrupt
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/pool.py", line 110, in worker
task = get()
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/queues.py", line 351, in get
with self._rlock:
File "/Users/Ira/anaconda3/lib/python3.7/multiprocessing/synchronize.py", line 95, in __enter__
return self._semlock.__enter__()
KeyboardInterrupt
Below is the updated script:
import multiprocessing as mp
import queue
def listener(q, d, stop_event):
while not stop_event.is_set():
try:
while True:
item_to_write = q.get(False)
if item_to_write == 'kill':
break
foo = d['region']
foo.add(item_to_write)
d['region'] = foo
except queue.Empty:
pass
time.sleep(0.5)
if not q.empty():
continue
def main():
manager = mp.Manager()
stop_event = manager.Event()
q = manager.Queue()
d = manager.dict()
d['region'] = set()
pool = mp.get_context("spawn").Pool(mp.cpu_count() + 2)
watcher = pool.apply_async(listener, (q, d, stop_event))
stop_event.set()
jobs = []
for i in range(24):
job = pool.apply_async(execute_search, (q, d))
jobs.append(job)
for job in jobs:
job.get()
q.put('kill')
pool.close()
pool.join()
print('process complete')
if __name__ == '__main__':
main()
UPDATE::
execute_command executes several processes necessary for search, so I put in code for where q.put() lies.
Alone, the script will take > 72 hrs to finish. Each multiprocess never completes the entire task, rather they work individually and reference a manager.dict() to avoid repeating tasks. These tasks work until every tuple in the manager.dict() has been processed.
def area(self, tup, housing_dict, q):
state, reg, sub_reg = tup[0], tup[1], tup[2]
for cat in housing_dict:
"""
computationally expensive, takes > 72 hours
for a list of 512 tup(s)
"""
result = self.search_geotag(
state, reg, cat, area=sub_reg
)
q.put(tup)
The q.put(tup) is ultimately placed in the listener function to add tup to the manager.dict()
Since listener and execute_search are sharing the same queue object, there could be race,
where execute_search gets 'kill' from queue before listener does, thus listener will stuck in blocking get() forever, since there are no more new items.
For that case you can use Event object to signal all processes to stop:
import multiprocessing as mp
import queue
def listener(q, d, stop_event):
while not stop_event.is_set():
try:
item_to_write = q.get(timeout=0.1)
foo = d['region']
foo.add(item_to_write)
d['region'] = foo
except queue.Empty:
pass
print("Listener process stopped")
def main():
manager = mp.Manager()
stop_event = manager.Event()
q = manager.Queue()
d = manager.dict()
d['region'] = set()
pool = mp.get_context("spawn").Pool(mp.cpu_count() + 2)
watcher = pool.apply_async(listener, (q, d, stop_event))
stop_event.set()
jobs = []
for i in range(24):
job = pool.apply_async(execute_search, (q, d))
jobs.append(job)
try:
for job in jobs:
job.get(300) #get the result or throws a timeout exception after 300 seconds
except multiprocessing.TimeoutError:
pool.terminate()
stop_event.set() # stop listener process
print('process complete')
if __name__ == '__main__':
main()
I have the following piece of code in which I am trying to fetch mails from different directories of a mailbox concurrently. However, it is showing the folloing problem. Have attached the shortened stacktrace.
import multiprocessing as mlp
import imaplib as impl
def somefunc(dirc):
mail.select(dirc)
result, data = mail.uid("search", None, "All")
uids = data[0].split()
print dirc
for mail_id in uids:
result, data = mail.uid("fetch", mail_id, "(RFC822)")
if __name__ == '__main__':
mail = impl.IMAP4_SSL("somedomain")
mail.login("username","password")
jobs = []
p1 = mlp.Process(target = somefunc, args = ("INBOX",))
jobs.append(p1)
p1.start()
p2 = mlp.Process(target = somefunc, args = ("Sent",))
jobs.append(p2)
p2.start()
for i in jobs:
i.join()
It throws error:
Process Process-2:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
Process Process-1:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
........
File "/usr/lib/python2.7/imaplib.py", line 859, in _command
raise self.abort('socket error: %s' % val)
abort: socket error: [Errno 32] Broken pipe
typ, dat = self._simple_command(name, mailbox)
........
error: [Errno 104] Connection reset by peer
Is it not possible to do imap connection concurrently???
Thanks... :)
My guess is that you are having problems because you are attempting to share the mail socket connection object across processes. Instead, try having each process create its connection:
import multiprocessing as mlp
import imaplib as impl
def somefunc(domain, name, password, dirc):
mail = impl.IMAP4_SSL(domain)
mail.login(name, password)
mail.select(dirc)
result, data = mail.uid("search", None, "All")
uids = data[0].split()
print dirc
for mail_id in uids:
result, data = mail.uid("fetch", mail_id, "(RFC822)")
if __name__ == '__main__':
jobs = []
for box in ("INBOX", "Sent"):
p = mlp.Process(
target = somefunc,
args = ("somedomain", "username", "password", box)
)
jobs.append(p)
p.start()
for i in jobs:
i.join()