I have facing some issue with python requests in a Django project. It only occur in 2nd requests.post().
Although It exited with exception TypeError: getresponse() got an unexpected keyword argument 'buffering'.
But after updating urllib3. There is no exception in traceback.
[2021-12-08 15:54:48 +0000] [11410] [CRITICAL] WORKER TIMEOUT (pid:11439)
Traceback (most recent call last):
File "/home/ubuntu/projects/project/api/views/a_view.py", line 765, in create_power_trace
headers=power_trace_headers)
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/requests/api.py", line 117, in post
return request('post', url, data=data, json=json, **kwargs)
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/requests/sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/requests/sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.7/http/client.py", line 1373, in getresponse
response.begin()
File "/usr/local/lib/python3.7/http/client.py", line 319, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.7/http/client.py", line 280, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/local/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/home/ubuntu/projects/venv/lib/python3.7/site-packages/gunicorn/workers/base.py", line 201, in handle_abort
sys.exit(1)
SystemExit: 1
For your information, my code is something like this:-
res1 = requests.post(url1, data=data1)
result = res1.json()
print(result['id']) # successfully prints
data2 = {'id': result['id']}
res2 = requests.post(url2, data=data2) # System exited
print(res2)
Above snippet works fine outside of my project (tested in a different script with same environment same instance).
And of course, no issue found in local.
Actually the issue was not caused by python or django.
Request lost for Nginx misconfiguration. I think my request hit the destination but reponse cannot came back to source as I forgot to add server_name in nginx conf.
Related
I am trying to get the HTML source from a webpage using the following code:
import requests
url = "https://dictionary.cambridge.org/us/dictionary/english-arabic/hi"
r = requests.get(url)
but, I am getting the following error:
Traceback (most recent call last):
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.8/http/client.py", line 1347, in getresponse
response.begin()
File "/usr/lib/python3.8/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.8/http/client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/username/ak_env/lib/python3.8/site-packages/requests/adapters.py", line 439, in send
resp = conn.urlopen(
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/util/retry.py", line 532, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/packages/six.py", line 734, in reraise
raise value.with_traceback(tb)
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/home/username/ak_env/lib/python3.8/site-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.8/http/client.py", line 1347, in getresponse
response.begin()
File "/usr/lib/python3.8/http/client.py", line 307, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.8/http/client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/username/ak_env/lib/python3.8/site-packages/requests/api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "/home/username/ak_env/lib/python3.8/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/home/username/ak_env/lib/python3.8/site-packages/requests/sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "/home/username/ak_env/lib/python3.8/site-packages/requests/sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "/home/username/ak_env/lib/python3.8/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
Is there any way to fix this issue? any other function to get the source code?
I used wget and it works. I may consider using wget to download the page if there is no direct fix or another python function.
You are missing headers to authenticate yourself as a User.
This one will help you:
import requests
url = 'https://dictionary.cambridge.org/us/dictionary/english-arabic/hi'
headers = requests.utils.default_headers()
headers.update(
{
'User-Agent': 'My User Agent 1.0',
}
)
response = requests.get(url, headers=headers)
print(response.text)
It works on my machine. Some more information according to headers:
Sending "User-agent" using Requests library in Python
Using headers with the Python requests library's get method
I have a local machine (local_user#local_machine). And a hadoop file system is present on a different server (some_user#another_server). One of the users in the hadoop server is named target_user. How do I access files present in target_user from local_user#local_machine? More precisely, say there's a file /user/target_user/test.txt present in the HDFS on some_user#another_server. What is the correct file path I should use when accessing /user/target_user/test.txt from local_user#local_machine?
I can access the file in the hdfs itself with hdfs dfs -cat /user/target_user/test.txt. But I can't access the file from my local machine using a python script I have written to read & write from the HDFS (that takes 3 arguments - local file path, remote file path, and read or write), most probably because I am not giving the correct path.
I have tried the following, but none of them work:
$ #local_user#local_machine
$ python3 rw_hdfs.py ./to_local_test.txt /user/target_user/test.txt read
$ python3 rw_hdfs.py ./to_local_test.txt some_user#another_server/user/target_user/test.txt read
The all give the exact same error:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 377, in _make_request
httplib_response = conn.getresponse(buffering=True)
TypeError: getresponse() got an unexpected keyword argument 'buffering'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 560, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 379, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1197, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 279, in _read_status
raise BadStatusLine(line)
http.client.BadStatusLine:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 376, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 610, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 247, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/lib/python3/dist-packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 560, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 379, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1197, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 297, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 279, in _read_status
raise BadStatusLine(line)
requests.packages.urllib3.exceptions.ProtocolError: ('Connection aborted.', BadStatusLine('\x15\x03\x03\x00\x02\x02\n',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "python_hdfs.py", line 63, in <module>
status, name, nnaddress= check_node_status(node)
File "python_hdfs.py", line 18, in check_node_status
request = requests.get("%s/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus"%name,verify=False).json()
File "/usr/lib/python3/dist-packages/requests/api.py", line 67, in get
return request('get', url, params=params, **kwargs)
File "/usr/lib/python3/dist-packages/requests/api.py", line 53, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 468, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 426, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', BadStatusLine('\x15\x03\x03\x00\x02\x02\n',))
More precisely, say there's a file /user/target_user/test.txt present in the HDFS on some_user#another_server
First, HDFS isn't a single directory on one machine. Therefore trying to access it like that doesn't make sense.
Secondly, whatever Python library you're using is trying to communicate over WebHDFS, which you must specifically enable for the cluster.
https://hadoop.apache.org/docs/r2.7.3/hadoop-project-dist/hadoop-hdfs/WebHDFS.html
BadStatusLine in the error might indicate that you're dealing with a Kerberized, secure cluster, so you might need a different way to read files
For example, PySpark or the Ibis project
I ran this script (https://github.com/mikehking/highrise-analysis/blob/master/highrise-analysis.py) to calculate usage by user for the Highrise CRM system (www.highrisehq.com). Recently, the script has been returning this error; but I'm not sure what it is telling me -- can anyone help with where I should be looking to fix this?
Thanks
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 557, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 382, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.4/http/client.py", line 1065, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python3.4/http/client.py", line 1103, in _send_request
self.endheaders(body)
File "/usr/lib/python3.4/http/client.py", line 1061, in endheaders
self._send_output(message_body)
File "/usr/lib/python3.4/http/client.py", line 906, in _send_output
self.send(msg)
File "/usr/lib/python3.4/http/client.py", line 841, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 133, in connect
ssl_version=resolved_ssl_version)
File "/usr/lib/python3/dist-packages/urllib3/util.py", line 619, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "/usr/lib/python3.4/ssl.py", line 364, in wrap_socket
_context=self)
File "/usr/lib/python3.4/ssl.py", line 578, in __init__
self.do_handshake()
File "/usr/lib/python3.4/ssl.py", line 805, in do_handshake
self._sslobj.do_handshake()
ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:598)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 330, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 590, in urlopen
raise SSLError(e)
urllib3.exceptions.SSLError: EOF occurred in violation of protocol (_ssl.c:598)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mikehking/workspace/highrise-analysis/highrise-analysis.py", line 188, in <module>
Create_Notes_Backup(PROD_API_KEY, PROD_API_USR, 'highrise-production-notes.bak', 'highrise-production-users.bak', 'highrise-production-people.bak', 'highrise-production-cases.bak', trailing_days = 365) # Production Environment
File "/home/mikehking/workspace/highrise-analysis/highrise-analysis.py", line 71, in Create_Notes_Backup
tmp_notes = high.get_person_notes(person.highrise_id)
File "/usr/local/lib/python3.4/dist-packages/highton/highton.py", line 436, in get_person_notes
return self._get_notes(subject_id, 'people')
File "/usr/local/lib/python3.4/dist-packages/highton/highton.py", line 433, in _get_notes
highrise_type, subject_id)), Note)
File "/usr/local/lib/python3.4/dist-packages/highton/highton.py", line 115, in _get_data
content = self._get_request(endpoint, params).content
File "/usr/local/lib/python3.4/dist-packages/highton/highton.py", line 44, in _get_request
params=params,
File "/usr/lib/python3/dist-packages/requests/api.py", line 55, in get
return request('get', url, **kwargs)
File "/usr/lib/python3/dist-packages/requests/api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 455, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 558, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 385, in send
raise SSLError(e)
requests.exceptions.SSLError: EOF occurred in violation of protocol (_ssl.c:598)
The last traceback suggests that the script is trying to Create_Notes_Backup, but encounters an error when communicating with the Highrise API.
The error message:
ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:598)
is consistent with a server that accepts your connection but then closes it immediately or prematurely. This might be a firewall issue.
If you have OpenSSL installed on this machine, you can try something like this to check the connection:
openssl s_client -crlf -connect USER.highrisehq.com:443
where USER is your Highrise user. You should normally get a long TLS handshake trace full of technical details, ending with --- and then it waits for your input.
when I make a GET call using requests module I get this error AttributeError: set_default_verify_paths. I am using python 2.6.9 version. when I use cURL command, I get the proper output. Its just not happening in python program. Am I missing any configuration setting/ package.
here is the traceback:
File "<string>", line 1, in <module>
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/api.py", line 69, in get
return request('get', url, params=params, **kwargs)
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/api.py", line 50, in request
response = session.request(method=method, url=url, **kwargs)
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/sessions.py", line 465, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/sessions.py", line 573, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/adapters.py", line 370, in send
timeout=timeout
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/packages/urllib3/connectionpool.py", line 544, in urlopen
body=body, headers=headers)
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/packages/urllib3/connectionpool.py", line 341, in _make_request
self._validate_conn(conn)
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/packages/urllib3/connectionpool.py", line 761, in _validate_conn
conn.connect()
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/packages/urllib3/connection.py", line 238, in connect
ssl_version=resolved_ssl_version)
File "/usr/local/lib64/python2.6/site-packages/requests-2.7.0-py2.6.egg/requests/packages/urllib3/contrib/pyopenssl.py", line 269, in ssl_wrap_socket
ctx.set_default_verify_paths()
AttributeError: set_default_verify_paths
Racking my brain over this. Just signed up here because I was having so much trouble with this. Here is my super simple code:
from jira.client import JIRA
jira_user='username'
jira_pass='password'
jira_server='https://jira-01:8443'
jira_options = {'server': jira_server, 'verify': False}
jira = JIRA(options=jira_options, basic_auth=(jira_user, jira_pass))
When executing, I get this:
Traceback (most recent call last):
File "jira1.py", line 12, in <module>
jira = JIRA(options=jira_options, basic_auth=(jira_user, jira_pass))
File "/usr/lib/python2.6/site-packages/jira/client.py", line 219, in __init__
si = self.server_info()
File "/usr/lib/python2.6/site-packages/jira/client.py", line 1504, in server_info
return self._get_json('serverInfo')
File "/usr/lib/python2.6/site-packages/jira/client.py", line 1894, in _get_json
r = self._session.get(url, params=params)
File "/usr/lib/python2.6/site-packages/jira/resilientsession.py", line 78, in get
return self.__verb('GET', url, **kwargs)
File "/usr/lib/python2.6/site-packages/jira/resilientsession.py", line 63, in __verb
r = method(url, **kwargs)
File "/usr/lib/python2.6/site-packages/requests/sessions.py", line 477, in get
return self.request('GET', url, **kwargs)
File "/usr/lib/python2.6/site-packages/requests/sessions.py", line 465, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python2.6/site-packages/requests/sessions.py", line 573, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python2.6/site-packages/requests/adapters.py", line 370, in send
timeout=timeout
File "/usr/lib/python2.6/site-packages/requests/packages/urllib3/connectionpool.py", line 544, in urlopen
body=body, headers=headers)
File "/usr/lib/python2.6/site-packages/requests/packages/urllib3/connectionpool.py", line 341, in _make_request
self._validate_conn(conn)
File "/usr/lib/python2.6/site-packages/requests/packages/urllib3/connectionpool.py", line 761, in _validate_conn
conn.connect()
File "/usr/lib/python2.6/site-packages/requests/packages/urllib3/connection.py", line 238, in connect
ssl_version=resolved_ssl_version)
File "/usr/lib/python2.6/site-packages/requests/packages/urllib3/contrib/pyopenssl.py", line 279, in ssl_wrap_socket
cnx.set_tlsext_host_name(server_hostname)
AttributeError: '_socketobject' object has no attribute 'set_tlsext_host_name'
The same thing happens if I remove 'verify': False
I know I can access the API and this user has permissions because I can hit it correctly with curl and get the right results.
Any help is appreciated.
Resolved by reinstalling python-requests via yum.