I been trying to figure out this for ages, but wondering if anyone might know how I can pass the s variable to the pool without making it into an argument?
import ctypes
import multiprocessing as mp
import os
def worker1(n):
k = n*3
print(k)
print(s)
# print(ctypes_int.value)
if __name__ == '__main__':
# global s
somelist = [1,2,3]
# ctypes_int = mp.Value(ctypes.c_wchar_p , "hi")
s = "TESTING"
# worker1(1)
p = mp.Pool(1)
p.map(worker1,somelist)
This is the error I am getting:
3
6
9
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Program Files\Python\Python37\lib\multiprocessing\pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "C:\Program Files\Python\Python37\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "C:\Users\light\AppData\Local\Temp\tempCodeRunnerFile.python", line 10, in worker1
print(s)
NameError: name 's' is not defined
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\light\AppData\Local\Temp\tempCodeRunnerFile.python", line 21, in <module>
p.map(worker1,somelist)
File "C:\Program Files\Python\Python37\lib\multiprocessing\pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Program Files\Python\Python37\lib\multiprocessing\pool.py", line 657, in get
raise self._value
NameError: name 's' is not defined
You can pass your variable along with each item in somelist:
import multiprocessing as mp
def worker1(p):
n,s = p
k = n*3
print(k)
print(s)
if __name__ == '__main__':
somelist = [1,2,3]
s = "TESTING"
p = mp.Pool(1)
p.map(worker1,[(n,s) for n in somelist])
The parameter (n,s) gets passed as p and I unpack it into n,s.
Related
I create a PyTable object W_hat where processes should share and save the results their instead of returning them.
from multiprocessing import Lock
from multiprocessing import Pool
import tables as tb
def parallel_l21(labels, X, lam, g, W_hat):
g_indxs = np.where(labels == g)[0]
tmp = rfs(X[g_indxs, 1:].T, X[:, :-1].T, gamma=lam, verbose=False).T
tmp[abs(tmp) <= 1e-6] = 0
with lock:
W_hat[:, g_indxs] = np.array(tmp)
def init_child(lock_):
global lock
lock = lock_
#Previous code is omitted.
n_ = X_test.shape[0]
tb.file._open_files.close_all()
f = tb.open_file(path_name + 'dot' + sub_num + str(lam) + '.h5', 'w')
filters = tb.Filters(complevel=5, complib='blosc')
W_hat = f.create_carray(f.root, 'data', tb.Float32Atom(), shape=(n_, n_), filters=filters)
W_hats = []
for i in np.unique(labels):
W_hats.append(W_hat)
lock = Lock()
with Pool(processes=cpu_count, initializer=init_child, initargs=(lock,)) as pool:
print(pool)
pool.starmap(parallel_l21, zip(repeat(labels), repeat(X), repeat(lam), np.unique(labels), W_hats))
Now, when running into starmap, this error shows up:
Traceback (most recent call last):
File "/Applications/PyCharm CE 2.app/Contents/plugins/python-ce/helpers/pydev/_pydevd_bundle/pydevd_exec2.py", line 3, in Exec
exec(exp, global_vars, local_vars)
File "<input>", line 1, in <module>
File "/usr/local/Cellar/python#3.8/3.8.6_1/Frameworks/Python.framework/Versions/3.8/lib/python3.8/multiprocessing/pool.py", line 372, in starmap
return self._map_async(func, iterable, starmapstar, chunksize).get()
File "/usr/local/Cellar/python#3.8/3.8.6_1/Frameworks/Python.framework/Versions/3.8/lib/python3.8/multiprocessing/pool.py", line 771, in get
raise self._value
File "/usr/local/Cellar/python#3.8/3.8.6_1/Frameworks/Python.framework/Versions/3.8/lib/python3.8/multiprocessing/pool.py", line 537, in _handle_tasks
put(task)
File "/usr/local/Cellar/python#3.8/3.8.6_1/Frameworks/Python.framework/Versions/3.8/lib/python3.8/multiprocessing/connection.py", line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "/usr/local/Cellar/python#3.8/3.8.6_1/Frameworks/Python.framework/Versions/3.8/lib/python3.8/multiprocessing/reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
File "stringsource", line 2, in tables.hdf5extension.Array.__reduce_cython__
TypeError: self.dims,self.dims_chunk,self.maxdims cannot be converted to a Python object for pickling
Note: I thought that the code works fine on Python 3.6.8 but it turns out that it is not the case.
>>> import concurrent.futures
>>> from collections import namedtuple
>>> #1. Initialise namedtuple here
>>> # tm = namedtuple("tm", ["pk"])
>>> class T:
... #2. Initialise named tuple here
... #tm = namedtuple("tm", ["pk"])
... def __init__(self):
... #3: Initialise named tuple here
... tm = namedtuple("tm", ["pk"])
... self.x = {'key': [tm('value')]}
... def test1(self):
... with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor:
... results = executor.map(self.test, ["key"])
... return results
... def test(self, s):
... print(self.x[s])
...
>>> t = T().test1()
This gets stuck here.
^CTraceback (most recent call last):
File "<stdin>", line 1, in <module>
Process ForkProcess-1:
File "<stdin>", line 10, in test1
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/_base.py", line 623, in __exit__
self.shutdown(wait=True)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/process.py", line 681, in shutdown
self._queue_management_thread.join()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 1044, in join
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/process.py", line 233, in _process_worker
call_item = call_queue.get(block=True)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/queues.py", line 94, in get
res = self._recv_bytes()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/connection.py", line 216, in recv_bytes
buf = self._recv_bytes(maxlength)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/connection.py", line 407, in _recv_bytes
buf = self._recv(4)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/connection.py", line 379, in _recv
chunk = read(handle, remaining)
KeyboardInterrupt
self._wait_for_tstate_lock()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 1060, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt
If I initialise the named tuple outside of the class (in #1), in that case, this works fine. Could someone please let me know what is the issue if I initialise as per #2 or #3 ?
You're not changing where you initialize the namedtuple. You're changing where you create the namedtuple class.
When you create a namedtuple class named "x" in module "y" with collections.namedtuple, its __module__ is set to 'y' and its __qualname__ is set to 'x'. Pickling and unpickling relies on this class actually being available in the y.x location indicated by these attributes, but in cases 2 and 3 of your example, it's not.
Python can't pickle the namedtuple, which breaks inter-process communication with the workers. Executing self.test in a worker process relies on pickling self.test and unpickling a copy of it in the worker process, and that can't happen if self.x is an instance of a class that can't be pickled.
I'm trying to write a program to take screen cap using a global hot key. Below is the corresponding code:
from datetime import datetime
import os
from pynput import keyboard
import pyautogui
import pathlib
def on_activate():
today = datetime.now()
d = today.strftime("%Y-%m-%d-%H-%M-%S")
myScreenshot = pyautogui.screenshot(region=(200,200, 1720, 800))
time_stamp = '{:%y%d%m}'.format(today)
fpath = pathlib.Path("C:","Users","Desktop","TestScreenCap", time_stamp)
if not os.path.exists(fpath):
os.makedirs(fpath)
myScreenshot.save(pathlib.Path(fpath,d,".jpg"))
def for_canonical(f):
return lambda k: f(l.canonical(k))
hotkey = keyboard.HotKey(
keyboard.HotKey.parse('<ctrl>+<alt>+h'),
on_activate)
with keyboard.Listener(
on_press=for_canonical(hotkey.press),
on_release=for_canonical(hotkey.release)) as l:
l.join()
Error Message:
Unhandled exception in listener callback
Traceback (most recent call last):
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\PIL\Image.py", line 2138, in save format = EXTENSION[ext]
KeyError: ''
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\_util\__init__.py", line 211, in inner
return f(self, *args, **kwargs)
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\keyboard\_win32.py", line 280, in _process self.on_press(key)
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\_util\__init__.py", line 127, in inner
if f(*args) is False:
File "C:/Users/PycharmProjects/untitled1/123.py", line 18, in <lambda>
return lambda k: f(l.canonical(k))
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\keyboard\__init__.py", line 182, in press
self._on_activate()
File "C:/Users/PycharmProjects/untitled1/123.py", line 15, in on_activate
myScreenshot.save(pathlib.Path(fpath,d,".jpg"))
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\PIL\Image.py", line 2140, in save
raise ValueError("unknown file extension: {}".format(ext)) from e
ValueError: unknown file extension:
Traceback (most recent call last):
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\PIL\Image.py", line 2138, in save
format = EXTENSION[ext]
KeyError: ''
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/PycharmProjects/untitled1/123.py", line 26, in <module>
l.join()
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\_util\__init__.py", line 259, in join
six.reraise(exc_type, exc_value, exc_traceback)
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\six.py", line 702, in reraise
raise value.with_traceback(tb)
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\_util\__init__.py", line 211, in inner
return f(self, *args, **kwargs)
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\keyboard\_win32.py", line 280, in _process
self.on_press(key)
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\_util\__init__.py", line 127, in inner
if f(*args) is False:
File "C:/Users/PycharmProjects/untitled1/123.py", line 18, in <lambda>
return lambda k: f(l.canonical(k))
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\pynput\keyboard\__init__.py", line 182, in press
self._on_activate()
File "C:/Users/PycharmProjects/untitled1/123.py", line 15, in on_activate
myScreenshot.save(pathlib.Path(fpath,d,".jpg"))
File "C:\Users\PycharmProjects\untitled1\venv\lib\site-packages\PIL\Image.py", line 2140, in save
raise ValueError("unknown file extension: {}".format(ext)) from e
ValueError: unknown file extension:
I do not know how to save the image to the folder. I expect the code below will not work this:
myScreenshot.save(pathlib.Path(fpath,d,".jpg"))
Changing the function to on_activate with a simple action such as:
Print("Hello")
The hotkey script works perfectly.
I am new to programming, so any help is appreciated!
Here the fixed code:
from datetime import datetime
import os
from pynput import keyboard
import pyautogui
import pathlib
def on_activate():
today = datetime.now()
d = today.strftime("%Y-%m-%d-%H-%M-%S")
myScreenshot = pyautogui.screenshot(region=(200,200, 1720, 800))
time_stamp = '{:%y%d%m}'.format(today)
fpath = pathlib.Path("C:\\","Users","Desktop","TestScreenCap", time_stamp)
if not os.path.exists(fpath):
os.makedirs(fpath)
myScreenshot.save(pathlib.Path(fpath,d + ".jpg"))
def for_canonical(f):
return lambda k: f(l.canonical(k))
hotkey = keyboard.HotKey(
keyboard.HotKey.parse('<ctrl>+<alt>+h'),
on_activate)
with keyboard.Listener(
on_press=for_canonical(hotkey.press),
on_release=for_canonical(hotkey.release)) as l:
l.join()
The problem was that you were using:
myScreenshot.save(pathlib.Path(fpath,d,".jpg"))
The function pathlib.Path(fpath,d,".jpg") join the string adding the \ to the end of each, so the path where you are trying to save the file is:
C:Users\Desktop\TestScreenCap\200309\2020-09-03-16-50-20\.jpg
As you can see the filename is the extension, that throw the exception.
Just replace with this: myScreenshot.save(pathlib.Path(fpath,d + ".jpg"))
Another problem on the specified path is that C: is interpreted as Drive, so when you save the function, is saved to the path where your program is currently running, not at the real specified path. To fix this part, just replace C: with C:\\.
Reading your code I think you would like to save the screenshot in the current user Desktop, if so replace the line: fpath = pathlib.Path("C:\\","Users","Desktop","TestScreenCap", time_stamp) with fpath = pathlib.Path(os.path.join(os.path.join(os.environ['USERPROFILE']), 'Desktop'), time_stamp) and just add on the top the import os. With this you don't need to care about the right generation for the path.
I'm trying to compute a feature for every vertex in my graph using gremlinpython. It's too slow to sequentially iterate over every single vertex. While batching could help to provide a speedup, I thought first I'd try parallizing the query.
Broadly, 1. get the full set of vertices, 2. split them over num_cores=x, 3. iterate over each sub-vertex set in parallel.
But I'm getting the error "OSError: [Errno 9] Bad file descriptor". The below code is my latest attempt at solving this.
import multiprocessing
from gremlin_python.structure.graph import Graph
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
from gremlin_python.process.traversal import lt
def create_traversal_object():
graph = Graph()
g = graph.traversal().withRemote(DriverRemoteConnection('ws://localhost:8182/gremlin', 'g'))
return g
g = create_traversal_object()
num_cores = 1
vertex_lsts = np.array_split(g.V().limit(30).id().toList(), num_cores)
class FeatureClass():
def __init__(self, g, vertex_list):
self.g = g
self.vertex_list = vertex_list
def orchestrator(self):
for vertex_id in self.vertex_list:
self.compute_number_of_names(float(vertex_id))
def get_names(self, vertex_id):
return self.g.V(vertex_id).inE().values('benef_nm').dedup().toList()
class Simulation(multiprocessing.Process):
def __init__(self, id, worker, *args, **kwargs):
# must call this before anything else
multiprocessing.Process.__init__(self)
self.id = id
self.worker = worker
self.args = args
self.kwargs = kwargs
sys.stdout.write('[%d] created\n' % (self.id))
def run(self):
sys.stdout.write('[%d] running ... process id: %s\n' % (self.id, os.getpid()))
self.worker.orchestrator()
sys.stdout.write('[%d] completed\n' % (self.id))
list_of_objects = [FeatureClass(create_traversal_object(), vertex_lst) for vertex_lst in vertex_lsts]
list_of_sim = [Simulation(id=k, worker=obj) for k, obj in enumerate(list_of_objects)]
for sim in list_of_sim:
sim.start()
Here's the full stack-trace, looks like it's an issue with tornado, which gremlinpython uses.
Process Simulation-1:
Traceback (most recent call last):
File "/Users/greatora/anaconda3/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "<ipython-input-4-b3177477fabe>", line 42, in run
self.worker.orchestrator()
File "<ipython-input-4-b3177477fabe>", line 23, in orchestrator
self.compute_number_of_names(float(vertex_id))
File "<ipython-input-4-b3177477fabe>", line 26, in compute_number_of_names
print(self.g.V(vertex_id).inE().values('benef_nm').dedup().count().next())
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/process/traversal.py", line 88, in next
return self.__next__()
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/process/traversal.py", line 47, in __next__
self.traversal_strategies.apply_strategies(self)
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/process/traversal.py", line 512, in apply_strategies
traversal_strategy.apply(traversal)
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/driver/remote_connection.py", line 148, in apply
remote_traversal = self.remote_connection.submit(traversal.bytecode)
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/driver/driver_remote_connection.py", line 53, in submit
result_set = self._client.submit(bytecode)
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/driver/client.py", line 108, in submit
return self.submitAsync(message, bindings=bindings).result()
File "/Users/greatora/anaconda3/lib/python3.6/concurrent/futures/_base.py", line 432, in result
return self.__get_result()
File "/Users/greatora/anaconda3/lib/python3.6/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/driver/connection.py", line 63, in cb
f.result()
File "/Users/greatora/anaconda3/lib/python3.6/concurrent/futures/_base.py", line 425, in result
return self.__get_result()
File "/Users/greatora/anaconda3/lib/python3.6/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
File "/Users/greatora/anaconda3/lib/python3.6/concurrent/futures/thread.py", line 56, in run
result = self.fn(*self.args, **self.kwargs)
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/driver/protocol.py", line 74, in write
self._transport.write(message)
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/gremlin_python/driver/tornado/transport.py", line 37, in write
lambda: self._ws.write_message(message, binary=True))
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 453, in run_sync
self.start()
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 863, in start
event_pairs = self._impl.poll(poll_timeout)
File "/Users/greatora/anaconda3/lib/python3.6/site-packages/tornado/platform/kqueue.py", line 66, in poll
kevents = self._kqueue.control(None, 1000, timeout)
OSError: [Errno 9] Bad file descriptor
I'm using Pythton3.7, gremlinpython==3.4.6, MacOS.
I'm still not entirely sure what the issue was, but this works.
import multiprocessing
from multiprocessing import Pool
import itertools
def graph_function(vertex_id_list):
graph = Graph()
g = graph.traversal().withRemote(DriverRemoteConnection('ws://localhost:8182/gremlin', 'g'))
res = []
for vertex_id in vertex_id_list:
res.append(g.V(str(vertex_id)).inE().values('benef_nm').dedup().toList())
return res
num_cores = 4
vertex_lst = g.V().limit(30).id().toList()
vertex_lsts = np.array_split(vertex_lst, num_cores)
with Pool(processes=num_cores) as pool:
results = pool.map(graph_function, vertex_lsts)
results = [*itertools.chain.from_iterable(results)]
I have a python object - list of dictionaries which I want to fill with key-value pairs in each of those dicts but simultaneously using multiple processors and using the multiprocessing module in python. For that purpose I am using the Manager module for storing that python object. Here is the following code:
from pylab import *
from numpy.random import *
import multiprocessing
import threading
import random
def tasks_start(id, global_lists):
counter_lock = threading.Lock()
with counter_lock:
num = int(10*random.random())
global_lists[num] = {'1':'Random'}
print("Id: ", id)
print(global_lists[0])
if __name__ == '__main__':
numProcessors = 6
pool = multiprocessing.Pool(numProcessors)
global_list = multiprocessing.Manager().list(range(100))
for idx in range(100):
global_list[idx] = multiprocessing.Manager().dict()
tasks = []
for id in range(10):
tasks.append((id, global_list))
pool.starmap(tasks_start, tasks)
pool.close()
pool.join()
So what I am doing here is creating a list of dictionaries stored as global_list and then calling the tasks_start() method 10 times using the python's starmap() module (just so that I can later extend to multiple arguments) to fill the list of dictionaries. As a simple test case, I just use the random generator to randomly pick up one dictionary among the lists everytime and fill it with some value. When I run the program, the following error occurs:
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "/usr/lib/python3.4/multiprocessing/pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "/usr/lib/python3.4/multiprocessing/pool.py", line 47, in starmapstar
return list(itertools.starmap(args[0], args[1]))
File "/home/cysis/inhibition_soum/motif_temporal_patterns/code_versions/2016/09/09_08/parallel_test/test_error_manager.py", line 14, in tasks_start
print(global_lists[0])
File "<string>", line 2, in __getitem__
File "/usr/lib/python3.4/multiprocessing/managers.py", line 732, in _callmethod
kind, result = conn.recv()
File "/usr/lib/python3.4/multiprocessing/connection.py", line 251, in recv
return ForkingPickler.loads(buf.getbuffer())
File "/usr/lib/python3.4/multiprocessing/managers.py", line 852, in RebuildProxy
return func(token, serializer, incref=incref, **kwds)
File "/usr/lib/python3.4/multiprocessing/managers.py", line 706, in __init__
self._incref()
File "/usr/lib/python3.4/multiprocessing/managers.py", line 756, in _incref
conn = self._Client(self._token.address, authkey=self._authkey)
File "/usr/lib/python3.4/multiprocessing/connection.py", line 495, in Client
c = SocketClient(address)
File "/usr/lib/python3.4/multiprocessing/connection.py", line 624, in SocketClient
s.connect(address)
FileNotFoundError: [Errno 2] No such file or directory
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/cysis/inhibition_soum/motif_temporal_patterns/code_versions/2016/09/09_08/parallel_test/test_error_manager.py", line 29, in <module>
pool.starmap(tasks_start, tasks)
File "/usr/lib/python3.4/multiprocessing/pool.py", line 268, in starmap
return self._map_async(func, iterable, starmapstar, chunksize).get()
File "/usr/lib/python3.4/multiprocessing/pool.py", line 599, in get
raise self._value
FileNotFoundError: [Errno 2] No such file or directory
In my opinion before the last print(global_lists[0) is executed, the Manager exits and therefore is not able to find global_lists[0]. Can anybody shed some light on this sort of stuff?