Develop Reference

Python Process cannot pickle

Code:
from aiohttp import web
from aiortc.mediastreams import MediaStreamTrack
from aiortc import RTCPeerConnection, RTCSessionDescription
from aiortc.contrib.media import MediaPlayer
import asyncio
import json
import os
from multiprocessing import Process, freeze_support
from queue import Queue
import sys
import threading
from time import sleep
import fractions
import time
class RadioServer(Process):
def __init__(self,q):
super().__init__()
self.q = q
self.ROOT = os.path.dirname(__file__)
self.pcs = []
self.channels = []
self.stream_offers = []
self.requests = []
def run(self):
self.app = web.Application()
self.app.on_shutdown.append(self.on_shutdown)
self.app.router.add_get("/", self.index)
self.app.router.add_get("/radio.js", self.javascript)
self.app.router.add_get("/jquery-3.5.1.min.js", self.jquery)
self.app.router.add_post("/offer", self.offer)
threading.Thread(target=self.fill_the_queues).start()
web.run_app(self.app, access_log=None, host="192.168.1.20", port="8080", ssl_context=None)
def fill_the_queues(self):
while(True):
frame = self.q.get()
for stream_offer in self.stream_offers:
stream_offer.q.put(frame)
async def index(self,request):
content = open(os.path.join(self.ROOT, "index.html"), encoding="utf8").read()
return web.Response(content_type="text/html", text=content)
async def javascript(self,request):
content = open(os.path.join(self.ROOT, "radio.js"), encoding="utf8").read()
return web.Response(content_type="application/javascript", text=content)
async def jquery(self,request):
content = open(os.path.join(self.ROOT, "jquery-3.5.1.min.js"), encoding="utf8").read()
return web.Response(content_type="application/javascript", text=content)
async def offer(self,request):
params = await request.json()
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
pc = RTCPeerConnection()
self.pcs.append(pc)
self.requests.append(request)
# prepare epalxeis media
self.stream_offers.append(CustomRadioStream())
pc.addTrack(self.stream_offers[-1])
#pc.on("iceconnectionstatechange")
async def on_iceconnectionstatechange():
if pc.iceConnectionState == "failed":
self.pcs.remove(pc)
self.requests.remove(request)
print(str(request.remote)+" disconnected from radio server")
print("Current peer connections:"+str(len(self.pcs)))
# handle offer
await pc.setRemoteDescription(offer)
# send answer
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
return web.Response(content_type="application/json",text=json.dumps({"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}))
async def on_shutdown(self,app):
# close peer connections
if self.pcs:
coros = [pc.close() for pc in self.pcs]
await asyncio.gather(*coros)
self.pcs = []
self.channels = []
self.stream_offers = []
"""
some other classes here such as CustomRadioStream and RadioOutputStream
"""
if __name__ == "__main__":
freeze_support()
q = Queue()
custom_server_child_process = RadioServer(q)
custom_server_child_process.start()
Error
Traceback (most recent call last):
File "123.py", line 106, in <module>
custom_server_child_process.start()
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/process.py", line 121, i
n start
self._popen = self._Popen(self)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/context.py", line 224, i
n _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/context.py", line 327, i
n _Popen
return Popen(process_obj)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/popen_spawn_win32.py", l
ine 93, in __init__
reduction.dump(process_obj, to_child)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/reduction.py", line 60,
in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle '_thread.lock' object
What I am doing wrong?
If I call the run function (instead of start) directly, then there is no problem, but i want to use processing for this class.
Edit: Ok with multiprocessing.Queue works fine but now with similar code there is this error:
$ python "Papinhio_player.py"
Traceback (most recent call last):
File "Papinhio_player.py", line 3078, in <module>
program = PapinhioPlayerCode()
File "Papinhio_player.py", line 250, in __init__
self.manage_decks_instance = Manage_Decks(self)
File "C:\python\scripts\Papinhio player\src\main\python_files/manage_decks.py"
, line 356, in __init__
self.custom_server_child_process.start()
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/process.py", line 121, i
n start
self._popen = self._Popen(self)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/context.py", line 224, i
n _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/context.py", line 327, i
n _Popen
return Popen(process_obj)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/popen_spawn_win32.py", l
ine 93, in __init__
reduction.dump(process_obj, to_child)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/reduction.py", line 60,
in dump
ForkingPickler(file, protocol).dump(obj)
File "stringsource", line 2, in av.audio.codeccontext.AudioCodecContext.__redu
ce_cython__
TypeError: self.parser,self.ptr cannot be converted to a Python object for pickl
ing
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/spawn.py", line 116, in
spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:/msys64/mingw64/lib/python3.8/multiprocessing/spawn.py", line 126, in
_main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input

Some objects cannot be serialized then unserialized.
The stack trace you posted mentions :
TypeError: cannot pickle '_thread.lock' object
a lock, which holds a state in memory and gives guarantees that no other process can own the same lock at the same moment, is typically a very bad candidate for this operation -- what should be created when you deserialize it ?
To fix this : choose a way to select the relevant fields of the object you want to serialize, and pickle/unpickle that part.

Can't use namedtuple with concurrent.futures? [duplicate]

>>> import concurrent.futures
>>> from collections import namedtuple
>>> #1. Initialise namedtuple here
>>> # tm = namedtuple("tm", ["pk"])
>>> class T:
... #2. Initialise named tuple here
... #tm = namedtuple("tm", ["pk"])
... def __init__(self):
... #3: Initialise named tuple here
... tm = namedtuple("tm", ["pk"])
... self.x = {'key': [tm('value')]}
... def test1(self):
... with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor:
... results = executor.map(self.test, ["key"])
... return results
... def test(self, s):
... print(self.x[s])
...
>>> t = T().test1()
This gets stuck here.
^CTraceback (most recent call last):
File "<stdin>", line 1, in <module>
Process ForkProcess-1:
File "<stdin>", line 10, in test1
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/_base.py", line 623, in __exit__
self.shutdown(wait=True)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/process.py", line 681, in shutdown
self._queue_management_thread.join()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 1044, in join
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/concurrent/futures/process.py", line 233, in _process_worker
call_item = call_queue.get(block=True)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/queues.py", line 94, in get
res = self._recv_bytes()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/connection.py", line 216, in recv_bytes
buf = self._recv_bytes(maxlength)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/connection.py", line 407, in _recv_bytes
buf = self._recv(4)
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/connection.py", line 379, in _recv
chunk = read(handle, remaining)
KeyboardInterrupt
self._wait_for_tstate_lock()
File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/threading.py", line 1060, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt
If I initialise the named tuple outside of the class (in #1), in that case, this works fine. Could someone please let me know what is the issue if I initialise as per #2 or #3 ?

You're not changing where you initialize the namedtuple. You're changing where you create the namedtuple class.
When you create a namedtuple class named "x" in module "y" with collections.namedtuple, its __module__ is set to 'y' and its __qualname__ is set to 'x'. Pickling and unpickling relies on this class actually being available in the y.x location indicated by these attributes, but in cases 2 and 3 of your example, it's not.
Python can't pickle the namedtuple, which breaks inter-process communication with the workers. Executing self.test in a worker process relies on pickling self.test and unpickling a copy of it in the worker process, and that can't happen if self.x is an instance of a class that can't be pickled.

How do I put an existing object into shared memory?

I've managed to create an object that can exist in shared memory with BaseManager and NamespaceProxy, however all the examples I've seen require me to create the object with the proxy. For example:
class Foo:
def __init__(self):
self._a = 1000
def get_a(self):
return self._a
class SharedFoo(NamespaceProxy):
_exposed_ = ('__getattribute__', '__getattr__', '__setattr__', '__init__', 'get_a')
def get_a(self):
callmethod = object.__getattribute__(self, '_callmethod')
return callmethod('get_a', ())
class FooManager(BaseManager):
pass
def test():
FooManager.register('Foo', Foo, SharedFoo)
with FooManager() as manager:
ls = []
t = time.time()
for i in range(100):
ls.append(manager.Foo())
print(time.time() - t)
which prints out:
0.44 (and some other numbers that i ommitted)
Since I would likely create millions of Foo objects, this is too slow for the task. I tried to make it faster like this:
def do_stuff(obj):
obj.ls[4].set_a(300)
def test():
FooManager.register('Foo', Foo, SharedFoo)
with FooManager() as manager:
if manager._Server != None:
manager._Server.mutex = NoLock()
ls = []
t = time.time()
for i in range(100000):
ls.append(Foo())
foos = manager.Foo()
foos.ls = mp.Manager().list(ls)
print(time.time() - t)
processes = [Process(target=do_stuff, args = (foos,)) for _ in range(3)]
for process in processes:
process.start()
for process in processes:
process.join()
print(foos.ls[4].get_a())
which gave me this error:
Traceback (most recent call last):
File "/path/to/lib/python3.7/multiprocessing/managers.py", line 788, in _callmethod
conn = self._tls.connection
AttributeError: 'ForkAwareLocal' object has no attribute 'connection'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/path/to/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "/path/to/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "/path/to/myproject/test.py", line 121, in do_stuff
obj.ls[4].set_a(300)
File "/path/to/lib/python3.7/multiprocessing/managers.py", line 1099, in __getattr__
return callmethod('__getattribute__', (key,))
File "/path/to/lib/python3.7/multiprocessing/managers.py", line 792, in _callmethod
self._connect()
File "/path/to/lib/python3.7/multiprocessing/managers.py", line 779, in _connect
conn = self._Client(self._token.address, authkey=self._authkey)
File "/path/to/lib/python3.7/multiprocessing/connection.py", line 492, in Client
c = SocketClient(address)
File "/path/to/lib/python3.7/multiprocessing/connection.py", line 619, in SocketClient
s.connect(address)
FileNotFoundError: [Errno 2] No such file or directory
Is what I'm trying to do possible? If so what should I use (not looking for a complete solution, just some ressources on how to do it)? I'm using Python 3.7 on Linux if that's relevant
Thanks
Edit: is this feasible with mmap, or am I going in a completely wrong direction? It looks promising but the documentations seems to say that it's more for files (again not looking for a complete solution, just if mmap would work with custom objects)

Getting results from query with alchimia library

Im trying to use alchimia for get asynchronous API for DB. Trying to make a simple request to DB, like that:
def authorization(self, data):
"""
Checking user with DB
"""
def __gotResult(user):
yield engine.execute(sqlalchemy.select([Users]).where(Users.name == user))
result = __gotResult(data['user'])
log.msg("[AUTH] User=%s trying to auth..." % data['user'])
data, result_msg = commands.AUTH(result, data)
log.msg(result_msg)
return data
And cant understand - what i doing wrong? Maybe issue in option for engine (where reactor=[])?
Source code:
import sys
from json import dumps, loads
import sqlalchemy
from twisted.internet import reactor, ssl
from twisted.python import log, logfile
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.twisted.websocket import WebSocketServerFactory, WebSocketServerProtocol, listenWS
import commands
from db.tables import Users
from alchimia import TWISTED_STRATEGY
log_file = logfile.LogFile("service.log", ".")
log.startLogging(log_file)
engine = sqlalchemy.create_engine('postgresql://test:test#localhost/testdb', pool_size=20, max_overflow=0,strategy=TWISTED_STRATEGY, reactor=[])
class DFSServerProtocol(WebSocketServerProtocol):
commands = commands.commands_user
def __init__(self):
self.commands_handlers = self.__initHandlersUser()
def __initHandlersUser(self):
handlers = commands.commands_handlers_server
handlers['AUTH'] = self.authorization
handlers['READ'] = None
handlers['WRTE'] = None
handlers['DELT'] = None
handlers['RNME'] = None
handlers['SYNC'] = None
handlers['LIST'] = None
return handlers
def authorization(self, data):
"""
Checking user with DB
"""
def __gotResult(user):
yield engine.execute(sqlalchemy.select([Users]).where(Users.name == data['user']))
result = __gotResult(data['user'])
log.msg("[AUTH] User=%s trying to auth..." % data['user'])
data, result_msg = commands.AUTH(result, data)
log.msg(result_msg)
return data
def onMessage(self, payload, isBinary):
json_data = loads(payload)
json_auth = json_data['auth']
json_cmd = json_data['cmd']
if json_auth == False:
if json_cmd == 'AUTH':
json_data = self.commands_handlers['AUTH'](json_data)
# for authorized users
else:
if json_cmd in commands.commands_user.keys():
if self.commands_handlers[json_cmd] is not None:
json_data = self.commands_handlers[json_cmd](json_data)
else:
json_data['error'] = '%s command is not already realized...' % json_cmd
else:
json_data['auth'] = False
json_data['error'] = 'This command is not supported on server...'
response = dumps(json_data)
self.sendMessage(str(response))
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == 'debug':
log.startLogging(sys.stdout)
debug = True
else:
debug = False
contextFactory = ssl.DefaultOpenSSLContextFactory('keys/server.key', 'keys/server.crt')
factory = WebSocketServerFactory("wss://localhost:9000", debug = debug, debugCodePaths = debug)
factory.protocol = DFSServerProtocol
factory.setProtocolOptions(allowHixie76 = True)
listenWS(factory, contextFactory)
webdir = File("./web/")
webdir.contentTypes['.crt'] = 'application/x-x509-ca-cert'
web = Site(webdir)
reactor.listenSSL(8080, web, contextFactory)
#reactor.listenTCP(8080, web)
reactor.run()
Traceback:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/twisted/python/log.py", line 88, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/usr/local/lib/python2.7/dist-packages/twisted/python/log.py", line 73, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/local/lib/python2.7/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/local/lib/python2.7/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
--- <exception caught here> ---
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite
why = selectable.doRead()
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 215, in doRead
return self._dataReceived(data)
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 221, in _dataReceived
rval = self.protocol.dataReceived(data)
File "/usr/local/lib/python2.7/dist-packages/twisted/protocols/tls.py", line 419, in dataReceived
self._flushReceiveBIO()
File "/usr/local/lib/python2.7/dist-packages/twisted/protocols/tls.py", line 389, in _flushReceiveBIO
ProtocolWrapper.dataReceived(self, bytes)
File "/usr/local/lib/python2.7/dist-packages/twisted/protocols/policies.py", line 120, in dataReceived
self.wrappedProtocol.dataReceived(data)
File "/usr/local/lib/python2.7/dist-packages/autobahn/twisted/websocket.py", line 78, in dataReceived
self._dataReceived(data)
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1270, in _dataReceived
self.consumeData()
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1286, in consumeData
while self.processData() and self.state != WebSocketProtocol.STATE_CLOSED:
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1445, in processData
return self.processDataHybi()
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1758, in processDataHybi
fr = self.onFrameEnd()
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 1887, in onFrameEnd
self._onMessageEnd()
File "/usr/local/lib/python2.7/dist-packages/autobahn/twisted/websocket.py", line 107, in _onMessageEnd
self.onMessageEnd()
File "/usr/local/lib/python2.7/dist-packages/autobahn/websocket/protocol.py", line 734, in onMessageEnd
self._onMessage(payload, self.message_is_binary)
File "/usr/local/lib/python2.7/dist-packages/autobahn/twisted/websocket.py", line 110, in _onMessage
self.onMessage(payload, isBinary)
File "server.py", line 84, in onMessage
json_data = self.commands_handlers['AUTH'](json_data)
File "server.py", line 68, in authorization
data, result_msg = commands.AUTH(result, data)
File "/home/relrin/code/Helenae/helenae/commands.py", line 68, in AUTH
if result['name'] == data['user']:
exceptions.TypeError: 'generator' object has no attribute '__getitem__'

I think you are missing an #inlineCallbacks around __gotResult() That might not help you quite enough, though; since a single statement generator wrapped with inlineCallbacks is sort of pointless. You should get used to working with explicit deferred handling anyway. Lets pull this apart:
def authorization(self, data):
"""
Checking user with DB
"""
# engine.execute already gives us a deferred, will grab on to that.
user = data['user']
result_d = engine.execute(sqlalchemy.select([Users]).where(Users.name == user))
# we don't have the result in authorization,
# we need to wrap any code that works with its result int a callback.
def result_cb(result):
data, result_msg = commands.AUTH(result, data)
return data
result_d.addCallback(result_cb)
# we want to pass the (asynchronous) result out, it's hiding in our deferred;
# so we return *that* instead; callers need to add more callbacks to it.
return result_d
If you insist; we can squish this down into an inline callbacks form:
from twisted.internet.defer import inlineCallbacks, returnValue
#inlineCallbacks
def authorization(self, data):
user = data['user']
result = yield engine.execute(sqlalchemy.select([Users]).where(Users.name == user))
data, result_msg = commands.AUTH(result, data)
yield returnValue(data)
as before, though, authorization() is asynchronous; and must be since engine.execute is async. to use it, you must attach a callback to the deferred it returns (although the caller may also yield it if it is also wrapped in inlineCallbacks

Error in python code

Please see the code and the error in the last.
from brisa.core.reactors import install_default_reactor
reactor = install_default_reactor()
print reactor
import os
from brisa.upnp.device import Device, Service
from brisa.upnp.device.service import StateVariable
class BinaryLight(Device):
def __init__(self):
Device.__init__(self,
'urn:schemas-upnp-org:device:BinaryLight:1',
'Binary Light device')
class SwitchPower(Service):
def __init__(self):
Service.__init__(self,
'SwitchPower',
'urn:schemas-upnp-org:service:SwitchPower:1',
'',
os.getcwd() + '/SwitchPower-scpd.xml')
self.target = False
self.status = False
self.varin = StateVariable(self, "Status",
True, False, "boolean")
self.varin.subscribe_for_update(self.varUpdateCallback)
self.add_state_variable(self.varin)
def varUpdateCallback(self, name, value):
print name, 'was updated to', value
def SetTarget(self, *args, **kwargs):
self.target = kwargs['NewTargetValue']
self.status = self.target
self.set_state_variable('Status', self.target)
print 'Light switched ', {'1': 'on', '0': 'off'}.get(self.target, None)
return {}
def GetTarget(self, *args, **kwargs):
return {'RetTargetValue': self.target}
def soap_GetStatus(self, *args, **kwargs):
return {'ResultStatus': self.status}
if __name__ == '__main__':
device = BinaryLight()
device += SwitchPower()
# Start device
device.start()
# Setup main loop
reactor.add_after_stop_func(device.stop)
reactor.main()
I am getting an error:-
ankit#ubuntu:~/Desktop$ python binary_light.py Could you please tell me where I am doing mistake??
Error:
Traceback (most recent call last):
File "binary_light.py", line 8, in <module>
from brisa.upnp.device import Device, Service
File "/usr/local/lib/python2.6/dist-packages/brisa/upnp/device/__init__.py", line 8, in <module>
from brisa.upnp.device.device import Device
File "/usr/local/lib/python2.6/dist-packages/brisa/upnp/device/device.py", line 10, in <module>
from brisa.core import log, config, webserver, network
File "/usr/local/lib/python2.6/dist-packages/brisa/core/webserver.py", line 39, in <module>
raise RuntimeError('Network is down.')
RuntimeError: Network is down.
NEW ERROR:
Traceback (most recent call last):
File "binary_light.py", line 57, in <module>
device = BinaryLight()
File "binary_light.py", line 21, in __init__
'Binary Light device')
File "/usr/local/lib/python2.6/dist-packages/brisa/upnp/device/device.py", line 83, in __init__
additional_headers=additional_ssdp_headers)
File "/usr/local/lib/python2.6/dist-packages/brisa/upnp/ssdp.py", line 71, in __init__
data_callback=self._datagram_received)
File "/usr/local/lib/python2.6/dist-packages/brisa/core/network_listeners.py", line 188, in __init__
self._create_socket(shared_socket)
File "/usr/local/lib/python2.6/dist-packages/brisa/core/network_listeners.py", line 227, in _create_socket
"Couldn't bind address")
brisa.core.network_listeners.CannotListenError
please let me know where I am doing mistake??

As others have mentioned, that's not an error. However, my guess would be that you have an error and something's going wrong somewhere or you wouldn't have bothered posting here. Try removing the print statement. Then let us know if you get any more "errors".

This is not an error. On line 3 you are printing the object "reactor".

(In response to updated question)
Well, the error suggests it can't find the network.
Looking at the code, trying doing this at the start of your script:
import brisa
brisa.__enable_offline_mode__ = True
from brisa.core.reactors import install_default_reactor