Develop Reference

Pytorch RPC RuntimeError: connect: Resource temporarily unavailable (this error originated at tensorpipe/common/socket.cc:114)

I encountered the following problem, when I set a larger world_size
File "/root/anaconda3/envs/final/lib/python3.9/site-packages/torch/distributed/rpc/__init__.py", line 195, in init_rpc
_init_rpc_backend(backend, store, name, rank, world_size, rpc_backend_options)
File "/root/anaconda3/envs/final/lib/python3.9/site-packages/torch/distributed/rpc/__init__.py", line 229, in _init_rpc_backend
rpc_agent = backend_registry.init_backend(
File "/root/anaconda3/envs/final/lib/python3.9/site-packages/torch/distributed/rpc/backend_registry.py", line 106, in init_backend
return backend.value.init_backend_handler(*args, **kwargs)
File "/root/anaconda3/envs/final/lib/python3.9/site-packages/torch/distributed/rpc/backend_registry.py", line 315, in _tensorpipe_init_backend_handler
api._all_gather(None, timeout=rpc_constants.DEFAULT_RPC_TIMEOUT_SEC)
File "/root/anaconda3/envs/final/lib/python3.9/site-packages/torch/distributed/rpc/api.py", line 77, in wrapper
return func(*args, **kwargs)
File "/root/anaconda3/envs/final/lib/python3.9/site-packages/torch/distributed/rpc/api.py", line 204, in _all_gather
rpc_sync(
File "/root/anaconda3/envs/final/lib/python3.9/site-packages/torch/distributed/rpc/api.py", line 77, in wrapper
return func(*args, **kwargs)
File "/root/anaconda3/envs/final/lib/python3.9/site-packages/torch/distributed/rpc/api.py", line 765, in rpc_sync
return fut.wait()
RuntimeError: connect: Resource temporarily unavailable (this error originated at tensorpipe/common/socket.cc:114)
And this is my code:
import os
import time
from entity.Server import Server
import torch.multiprocessing as mp
from torch.distributed import rpc
from utils.options import args_parser
import torch.distributed as dist
SERVER_NAME = "Server"
CLIENT_NAME = "Client{}"
os.environ['MASTER_ADDR'] = 'localhost'
os.environ['MASTER_PORT'] = '29500'
def run(rank, args):
if rank == 0:
rpc.init_rpc(SERVER_NAME, rank=rank, world_size=args.world_size)
# server = Server(args)
else:
rpc.init_rpc(CLIENT_NAME.format(rank), rank=rank, world_size=args.world_size)
rpc.shutdown()
if __name__ == "__main__":
args = args_parser()
mp.spawn(
run,
args=(args, ),
nprocs=args.world_size,
join=True
)
When I set the world_size value above 25, the code fails.
I think this may be a problem with the linux system configuration, does anyone know how to configure or fix this?

How to make telepot bot read the message each time a message is received?

I am trying to make a quiz bot for training Irregular English Verbs.
The bot should work in a following way:
It sends a message with a line of the poem with an irregular verb in Russian language.
When a man responds it with a message, it should compare a text of a message with the right answer.
If the answer is correct, bot should send a message and than give another question.
After 5 attempts it should give the final result.
I made a QuizStarter object that gives the 'Hello' message. And then closes the process.
class QuizStarter(telepot.helper.ChatHandler):
def __init__(self, *args, **kwargs):
super(QuizStarter, self).__init__(*args, **kwargs)
def on_chat_message(self, msg):
content_type, chat_type, chat_id = telepot.glance(msg)
keyboard = InlineKeyboardMarkup(inline_keyboard=[
[InlineKeyboardButton(text='Random Verb', callback_data='Random Verb')],
])
self.sender.sendMessage('Hello. Choose the game below:', reply_markup=keyboard)
self.close()
After that I created Quizzer object that plays the main thing
class Quizzer(telepot.helper.ChatHandler):
def __init__(self, *args, **kwargs):
super(Quizzer, self).__init__(*args, **kwargs)
self.attempts = 5
self.score = 0
self._answer = None
self.guess = None
def on_callback_query(self, msg):
query_id, from_id, query_data = telepot.glance(msg, flavor='callback_query')
print(query_id, from_id, query_data)
self.show_question()
def show_question(self):
with open('IrregularVerbs.txt', 'r', encoding="utf8") as f:
IrrVerbsStory = f.read()
f.close()
IrrVerbsDict = {}
ReText = re.compile('[a-zA-z]+-[a-zA-z]+-[a-zA-z]+')
result1 = ReText.findall(IrrVerbsStory)
IrrVerbsStory1 = IrrVerbsStory.split(sep='\n')
for i in result1:
for n in IrrVerbsStory1:
if i in n:
IrrVerbsDict[i] = n
IrrVerbsStory1.remove(n)
break
randomDictLine = random.choice(list(IrrVerbsDict.values()))
for x in IrrVerbsDict.keys():
if x in randomDictLine:
time.sleep(1)
self._answer = x.lower()
resultLine = randomDictLine.replace(x, '()')
self.sender.sendMessage(text=f'Please write a nessessary verbs with dashes: {resultLine}\n')
def on_chat_message(self, msg):
self.guess = msg['text']
if self.guess.lower().lstrip() == self._answer.lower().lstrip():
self.score +=1
self.sender.sendMessage(id, f'Good Job! Your score is {self.score}')
else:
self.sender.sendMessage(id, f'Wrong! The right answer is {self._answer} \n')
But whenever I am trying to send a message, it poses an error:
Traceback (most recent call last):
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\telepot\delegate.py", line 265, in wait_loop
j.on_message(msg)
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\telepot\helper.py", line 689, in augmented
return handler(msg)
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\telepot\helper.py", line 1067, in on_message
self._router.route(msg)
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\telepot\helper.py", line 1041, in route
return fn(msg, *args, **kwargs)
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\telepot\helper.py", line 1050, in <lambda>
'callback_query': lambda msg: self.on_callback_query(msg),
File "c:\Users\Growing\Desktop\PythonScripts\Telebot Bot.py", line 38, in on_callback_query
self.show_question()
File "c:\Users\Growing\Desktop\PythonScripts\Telebot Bot.py", line 62, in show_question
self.sender.sendMessage(text=f'Please write a nessessary verbs with dashes: {resultLine}\n')
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\telepot\__init__.py", line 513, in sendMessage
return self._api_request('sendMessage', _rectify(p))
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\telepot\__init__.py", line 491, in _api_request
return api.request((self._token, method, params, files), **kwargs)
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\telepot\api.py", line 154, in request
r = fn(*args, **kwargs) # `fn` must be thread-safe
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\request.py", line 155, in request_encode_body
body, content_type = encode_multipart_formdata(
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\filepost.py", line 78, in encode_multipart_formdata
for field in iter_field_objects(fields):
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\filepost.py", line 42, in iter_field_objects
yield RequestField.from_tuples(*field)
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\fields.py", line 182, in from_tuples
content_type = guess_content_type(filename)
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\fields.py", line 20, in guess_content_type
return mimetypes.guess_type(filename)[0] or default
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\mimetypes.py", line 309, in guess_type
return _db.guess_type(url, strict)
File "C:\Users\Growing\AppData\Local\Programs\Python\Python310\lib\mimetypes.py", line 122, in guess_type
url = os.fspath(url)
TypeError: expected str, bytes or os.PathLike object, not int
ERROR:root:on_close() called due to TypeError: expected str, bytes or os.PathLike object, not int
I can't get where the mistake is and why I'm getting an error.

Sometimes empty connection string from `getenv` in module

We have a db_connection.py module which looks like below.
from os import getenv
from collections import OrderedDict
import asyncpg
from tornado import gen
from util.utils import custom_exception
import time
db_name = getenv("DB_NAME", "XXX")
db_user = getenv("DB_USER", "YYY")
db_password = getenv("DB_PASSWORD", "ZZZ")
db_host = getenv("DB_HOST", "localhost")
db_port = getenv("DB_PORT", "5432")
db_args = dict(user=db_user, password=db_password,
database=db_name, host=db_host, port=db_port)
class db_connection(object):
def __init__(self, **db_args):
self.db_pool = []
self.init(**db_args)
# create a pool ref coroutine where we can get connections from
# needs user, password, database, host, port
#gen.coroutine
def init(self,**db_args):
self.db_pool = yield asyncpg.create_pool(**db_args)
#gen.coroutine
def exit(self):
yield self.db_pool.close()
# run a queer
async def run(self,q):
if not self.db_pool:
self.init() #try again
if not self.db_pool:
raise custom_exception(reason='Database connection error', status_code=500)
async with self.db_pool.acquire() as connection:
ret = await connection.fetch(q)
# format to match the old data types
return [OrderedDict(e) for e in ret]
def __exit__(self, exc_type, exc_val, exc_tb):
self.exit()
self.db_pool = []
In the app, we initialize the db_connection object as follows:
from util.db_connection import db_args, db_connection
(...)
if __name__ == "__main__":
AsyncIOMainLoop().install()
dbc = db_connection(**db_args)
# What we have here is a rather basic tornado app
app = make_app()
app.listen(port=8080)
asyncio.get_event_loop().run_forever()
The problem we saw is that, and this is something I can not explain, it seems that the connection string (db_args) is not always set and sometimes appears to be an empty dictionary; asyncpg then falls back and tries to get a connection string from the linux user associated to the container via os.
Mar 5 16:59:50 ABC: ERROR:
tornado.application:Future <tornado.concurrent.Future object at 0x7eff6005c320>
exception was never retrieved:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 1063, in run
yielded = self.gen.throw(*exc_info)
File "/abc/def/db_connection.py", line 26, in init
self.db_pool = yield asyncpg.create_pool(**db_args)
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 1055, in run
value = future.result()
File "/usr/local/lib/python3.6/site-packages/tornado/concurrent.py", line 238, in result
raise_exc_info(self._exc_info) Mar 5 16:59:50 ABC: File "<string>", line 4, in raise_exc_info
File "/usr/local/lib/python3.6/site-packages/tornado/gen.py", line 307, in wrapper
yielded = next(result)
File "<string>", line 6, in _wrap_awaitable
File "/usr/local/lib/python3.6/site-packages/asyncpg/pool.py", line 356, in _async__init__
await first_ch.connect()
File "/usr/local/lib/python3.6/site-packages/asyncpg/pool.py", line 126, in connect
**self._connect_kwargs)
File "/usr/local/lib/python3.6/site-packages/asyncpg/connection.py", line 1498, in connect
max_cacheable_statement_size=max_cacheable_statement_size)
File "/usr/local/lib/python3.6/site-packages/asyncpg/connect_utils.py", line 296, in _connect
addrs, params, config=_parse_connect_arguments(timeout=timeout, **kwargs)
File "/usr/local/lib/python3.6/site-packages/asyncpg/connect_utils.py", line 242, in _parse_connect_arguments
server_settings=server_settings)
File "/usr/local/lib/python3.6/site-packages/asyncpg/connect_utils.py", line 152, in _parse_connect_dsn_and_args
user=getpass.getuser()
File "/usr/local/lib/python3.6/getpass.py", line 169, in getuser
return pwd.getpwuid(os.getuid())[0]
KeyError: 'getpwuid(): uid not found: 10001

Dynamically setting scrapy request call back

I'm working with scrapy. I want to rotate proxies on a per request basis and get a proxy from an api I have that returns a single proxy. My plan is to make a request to the api, get a proxy, then use it to set the proxy based on :
http://stackoverflow.com/questions/39430454/making-request-to-api-from-within-scrapy-function
I have the following:
class ContactSpider(Spider):
name = "contact"
def parse(self, response):
....
PR = Request(
'my_api'
headers=self.headers,
meta={'newrequest': Request(url_to_scrape, headers=self.headers),},
callback=self.parse_PR
)
yield PR
def parse_PR(self, response):
newrequest = response.meta['newrequest']
proxy_data = response.body
newrequest.meta['proxy'] = 'http://'+proxy_data
newrequest.replace(url = 'http://ipinfo.io/ip') #TESTING
newrequest.replace(callback= self.form_output) #TESTING
yield newrequest
def form_output(self, response):
open_in_browser(response)
but I'm getting:
Traceback (most recent call last):
File "C:\twisted\internet\defer.py", line 1126, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "C:\twisted\python\failure.py", line 389, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "C:\scrapy\core\downloader\middleware.py", line 43, in process_request
defer.returnValue((yield download_func(request=request,spider=spider)))
File "C:\scrapy\utils\defer.py", line 45, in mustbe_deferred
result = f(*args, **kw)
File "C:\scrapy\core\downloader\handlers\__init__.py", line 65, in download_request
return handler.download_request(request, spider)
File "C:\scrapy\core\downloader\handlers\http11.py", line 60, in download_request
return agent.download_request(request)
File "C:\scrapy\core\downloader\handlers\http11.py", line 255, in download_request
agent = self._get_agent(request, timeout)
File "C:\scrapy\core\downloader\handlers\http11.py", line 235, in _get_agent
_, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
File "C:\scrapy\core\downloader\webclient.py", line 37, in _parse
return _parsed_url_args(parsed)
File "C:\scrapy\core\downloader\webclient.py", line 20, in _parsed_url_args
host = b(parsed.hostname)
File "C:\scrapy\core\downloader\webclient.py", line 17, in <lambda>
b = lambda s: to_bytes(s, encoding='ascii')
File "C:\scrapy\utils\python.py", line 117, in to_bytes
'object, got %s' % type(text).__name__)
TypeError: to_bytes must receive a unicode, str or bytes object, got NoneType
what am I doing wrong?

The stacktrace info suggests Scrapy has encountered a request object whose url is None, which is expected to be of string type.
These two lines in your code:
newrequest.replace(url = 'http://ipinfo.io/ip') #TESTING
newrequest.replace(callback= self.form_output) #TESTING
would not work as expected, since method Request.replace returns a new instance instead of modifying the original request in-place.
You would need something like this:
newrequest = newrequest.replace(url = 'http://ipinfo.io/ip') #TESTING
newrequest = newrequest.replace(callback= self.form_output) #TESTING
or simply:
newrequest = newrequest.replace(
url='http://ipinfo.io/ip',
callback=self.form_output
)

exception raised by tornado

I wrote a simple http server with Tornado, but sometimes it raise exception below, self.session.response runs in a threadpool.
code of server:
class AuToServer(object):
module_path = os.path.split(os.path.realpath(__file__))[0] + '/../module/'
def __init__(self, prefix=DEFAULT_PREFIX, work_root=module_path, module=[], **kwargs):
self._prefix = prefix
self._thread_pool = ExecutorDelegate(20)
self._module_manager = ModuleManager(work_root)
self.initilize(module, kwargs)
def initilize(self, modules, kwargs):
self._module_manager.set_args(kwargs)
if not self._module_manager.prepare(modules):
print "initilize module fail. [%s]" % (str(modules))
raise AppInitError("initilize %s fail" % (modules))
def __call__(self, request): # request handler
session = Session(request)
if not session.parse(self._prefix):
print "parse query fail. [%s]" % (session.query)
session.response("url parse fail [%s]\n" % session.query)
return
self._thread_pool.submit(WorkerHander(self._module_manager, session))
def start(self, port=8880):
http_server = tornado.httpserver.HTTPServer(self, max_header_size=128*1024)
http_server.listen(port)
tornado.ioloop.IOLoop.instance().start()
code of WorkerHander:
class WorkerHander(BaseExecutor):
def __init__(self, module_manage, session):
self.module_manage = module_manage
self.session = session
super(WorkerHander, self).__init__(self)
def run(self):
method = self.session.get_command()
if not method:
self.session.response("invalid url [%s]\n" % (self.session.query))
return
context = self.module_manage.run(method, self.session)
try:
if context:
self.session.response(context) # raise exception
else:
self.session.response("None\n")
except Exception, error:
Logger.warning("session response fail. [%s][%s]" % (self.session.query, traceback.format_exc()))
code of session.response
def response(self, context):
if not self.request:
return False
if isinstance(context, unicode):
context = context.encode("utf8")
self.request.write(utf8("%s %d\r\n\r\n%s" % (
HTTP_HEADER,
len(context),
context)))
Logger.debug("QUERY:[cmd:%s][%s] [%s] [%s]" % (
self.command,
time.time() - self.begin_time,
self.request.remote_ip,
self.query))
self.request.finish()
return True
exception:
2016-02-29 16:29:26,852 WARNING: server.py:run:94: session response fail. [/auto?start_timestamp=1456730772&_cmd=get_index_info][Traceback (most recent call last):
File "/home/admin/autobots/lib/python2.7/site-packages/autobase-0.0.1-py2.7.egg/autobase/service/server.py", line 90, in run
self.session.response(context)
File "/home/admin/autobots/lib/python2.7/site-packages/autobase-0.0.1-py2.7.egg/autobase/service/session.py", line 43, in response
self.request.finish()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/httputil.py", line 407, in finish
self.connection.finish()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 459, in finish
self._pending_write.add_done_callback(self._finish_request)
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/concurrent.py", line 243, in add_done_callback
fn(self)
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 491, in _finish_request
self.close()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 297, in close
self.stream.close()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/iostream.py", line 427, in close
self.close_fd()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/iostream.py", line 995, in close_fd
self.socket.close()
AttributeError: 'NoneType' object has no attribute 'close'
]
Do I use Tornado in a wrong way? ask for help. thanks~

Tornado is not thread-safe by design. You can use a thread pool to do your own work, but you must call all Tornado methods (including in this case self.request.write and self.request.finish) from the IOLoop thread.