I've been having trouble storing JSON to database with non ASCII characters.
Here's what I have so far. I'm using python requests library.
getJSON = requests.get('https://jsonapi.com/jsonfile')
storeDB = requests.post(
'http://databaseAPI/Database/item',
data='{"payload": %s}' % (getJSON.text),
headers={'Content-Type': 'application/json'}
)
But it's failing with this error because my JSON contains non ASCII character. How can I fix that?
Error -
Traceback (most recent call last):
File "addKonvo.py", line 134, in <module>
headers={'Content-Type': 'application/json'}
File "/usr/local/lib/python3.7/site-packages/requests/api.py", line 119, in post
return request('post', url, data=data, json=json, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 1297, in _send_request
body = _encode(body, 'body')
File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/http/client.py", line 170, in _encode
(name.title(), data[err.start:err.end], name)) from None
UnicodeEncodeError: 'latin-1' codec can't encode character '\u20ac' in position 505: Body ('€') is not valid Latin-1. Use body.encode('utf-8') if you want to send it encoded in UTF-8.```
you need to encode the data in UTF-8 format using getJSON.text.encode('utf8')
Related
I create a Django App and send rest http request to Plaid. if I start up django alone(python manage.py run server), it works fine. But if I use Nginx + Gunicorn + Django, I will get error. The error message is:
File "/var/www/sp_plaid/api/views.py", line 91, in create_link_token
response = client.link_token_create(p_request)
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 769, in __call__
return self.callable(self, *args, **kwargs)
File "/usr/local/lib/python3.9/dist-packages/plaid/api/plaid_api.py", line 6863, in __link_token_create
return self.call_with_http_info(**kwargs)
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 831, in call_with_http_info
return self.api_client.call_api(
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 406, in call_api
return self.__call_api(resource_path, method,
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 193, in __call_api
response_data = self.request(
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 452, in request
return self.rest_client.POST(url,
File "/usr/local/lib/python3.9/dist-packages/plaid/rest.py", line 264, in POST
return self.request("POST", url,
File "/usr/local/lib/python3.9/dist-packages/plaid/rest.py", line 150, in request
r = self.pool_manager.request(
File "/usr/lib/python3/dist-packages/urllib3/request.py", line 78, in request
return self.request_encode_body(
File "/usr/lib/python3/dist-packages/urllib3/request.py", line 170, in request_encode_body
return self.urlopen(method, url, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/poolmanager.py", line 375, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 394, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 234, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.9/http/client.py", line 1279, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.9/http/client.py", line 1320, in _send_request
self.putheader(hdr, value)
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 219, in putheader
_
HTTPConnection.putheader(self, header, *values)
File "/usr/lib/python3.9/http/client.py", line 1256, in putheader
if _is_illegal_header_value(values):
TypeError: expected string or bytes-like object
python version: 3.8, Django:3.2.4.
Anyone can help? Thanks.
I can't request any page. Am I doing something wrong? I've used this code before, so I don't know what's happening. I've tried other ways as well. Seems to just not get a response. Did something change with requests? Or could it be an anti-virus firewall thing?
Any help?
>>> import requests, json
>>> url = 'https://api.chess.com/pub/player/Hikaru/games/archives'
>>> resp= requests.get(url)
Traceback (most recent call last):
File "<pyshell#4>", line 1, in <module>
resp= requests.get(url)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\requests\api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\requests\api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\requests\sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\requests\sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\requests\adapters.py", line 449, in send
timeout=timeout
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\urllib3\connectionpool.py", line 600, in urlopen
chunked=chunked)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\urllib3\connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1229, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1275, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1224, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 1016, in _send_output
self.send(msg)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\http\client.py", line 977, in send
self.sock.sendall(data)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\urllib3\contrib\pyopenssl.py", line 328, in sendall
sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE])
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\urllib3\contrib\pyopenssl.py", line 317, in _send_until_done
return self.connection.send(data)
File "C:\Users\\AppData\Local\Programs\Python\Python37\lib\site-packages\OpenSSL\SSL.py", line 1644, in send
with _from_buffer(buf) as data:
AttributeError: __enter__
import requests, json
url = 'https://api.chess.com/pub/player/Hikaru/games/archives'
resp= requests.get(url)
print(resp)
I am trying to send a request with the header: ":hello". However, the leading colon causes the script to not function properly, and emit this traceback:
Traceback (most recent call last):
(first few lines removed for my privacy)
File "C:\Python27\lib\site-packages\requests\api.py", line 109, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Python27\lib\site-packages\requests\api.py", line 50, in request
response = session.request(method=method, url=url, **kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 468, in request
resp = self.send(prep, **send_kwargs)
File "C:\Python27\lib\site-packages\requests\sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "C:\Python27\lib\site-packages\requests\adapters.py", line 370, in send
timeout=timeout
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 559, in urlopen
body=body, headers=headers)
File "C:\Python27\lib\site-packages\requests\packages\urllib3\connectionpool.py", line 353, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Python27\lib\httplib.py", line 1057, in request
self._send_request(method, url, body, headers)
File "C:\Python27\lib\httplib.py", line 1096, in _send_request
self.putheader(hdr, value)
File "C:\Python27\lib\httplib.py", line 1030, in putheader
raise ValueError('Invalid header name %r' % (header,))
ValueError: Invalid header name ':hello'
Is there a workaround for this? My script is:
import requests
headers = {'user-agent': 'alsotesting', ':hello': 'test'}
requests.post("my server", headers=headers)
As your error says, :header is not a valid HTTP header name (you cannot start headers with ":" - see documentation). You should change
headers = {'user-agent': 'alsotesting', ':hello': 'test'}
to
headers = {'user-agent': 'alsotesting', 'hello': 'test'}
Edit: HTTP/2 uses pseudo-headers fields, which start with a colon (see documentation). Also, as explained here, you may see some headers starting with a colon in Chrome's Developer Tools, which can happen when Chrome is talking to a web server using SPDY - and also HTTP/2 (which is based on SPDY/2), which correspond to pseudo-headers. As stated in documentation, pseudo-header fields are not HTTP header fields.
In conclusion, header fields starting with a colon are not allowed with standard HTTP protocol, so that is why you are getting the Invalid header name error
I'm doing pretty simple thing - get file from client and send it to process to another server:
import requests
import logging
logger = logging.getLogger(__name__)
data = request.environ['wsgi.input'].read()
url = request.cfg.DOC_PARSE_URL
params = {'AddedPath': str(request.cfg.FORM_TEMP_URL+uid+'/'), 'Button': 'Generate'}
logger.warning('url:'+repr(url))
for k in params:
logger.warning('params:'+repr(k)+' '+repr(params[k]))
logger.warning('data:'+repr(type(data))+str(len(data)))
#logger.warning('data:'+data)
files = {'WordFile': ('process.doc', data)}
r = requests.post(url, files=files, data=params, stream=False)
and everything works fine on my own PC and on dev server, but on production server this code crashes with UnicodeDecodeError:
Traceback (most recent call last):
File "/sites/mo/admin/utils/dispatcher.py", line 492, in __call__
response = self.view_map[endpoint](request, **params)
File "/sites/mo/admin/utils/admin_views.py", line 231, in parse_doc
r = requests.post(url, files=files, data=params, stream=False)
File "/sites/mo/admin/../third_party/requests/api.py", line 88, in post
return request('post', url, data=data, **kwargs)
File "/sites/mo/admin/../third_party/requests/api.py", line 44, in request
return session.request(method=method, url=url, **kwargs)
File "/sites/mo/admin/../third_party/requests/sessions.py", line 360, in request
resp = self.send(prep, **send_kwargs)
File "/sites/mo/admin/../third_party/requests/sessions.py", line 463, in send
r = adapter.send(request, **kwargs)
File "/sites/mo/admin/../third_party/requests/adapters.py", line 292, in send
timeout=timeout
File "/sites/mo/admin/../third_party/requests/packages/urllib3/connectionpool.py", line 428, in urlopen
body=body, headers=headers)
File "/sites/mo/admin/../third_party/requests/packages/urllib3/connectionpool.py", line 283, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib64/python2.7/httplib.py", line 946, in request
self._send_request(method, url, body, headers)
File "/usr/lib64/python2.7/httplib.py", line 987, in _send_request
self.endheaders(body)
File "/usr/lib64/python2.7/httplib.py", line 940, in endheaders
self._send_output(message_body)
File "/usr/lib64/python2.7/httplib.py", line 801, in _send_output
msg += message_body
UnicodeDecodeError: 'ascii' codec can't decode byte 0xd0 in position 739: ordinal not in range(128)
All values are bytes string (I inserted logging to ensure) so there shouldn't be any unicode issues. Seems there is some environment influence, but i can't figure it out. Does anybody know what could cause such problem?
I am running into the following error when I try to connect to an API using M2Crypto HTTPSConnection library using the GET method.
Traceback (most recent call last):
File "osg-gridadmin-cert-request", line 665, in <module>
status = check_quota_limit(request_count)
File "osg-gridadmin-cert-request", line 543, in check_quota_limit
conn.request('GET', arguments['quotaurl'], headers)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 941, in request
self._send_request(method, url, body, headers)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 975, in _send_request
self.endheaders(body)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 937, in endheaders
self._send_output(message_body)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 801, in _send_output
self.send(message_body)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 773, in send
self.sock.sendall(data)
File "build/bdist.macosx-10.7-intel/egg/M2Crypto/SSL/Connection.py", line 217, in write
return self._write_nbio(data)
File "build/bdist.macosx-10.7-intel/egg/M2Crypto/SSL/Connection.py", line 202, in _write_nbio
return m2.ssl_write_nbio(self.ssl, data)
TypeError: expected a readable buffer object
I am accessing the API using this code
conn = M2Crypto.httpslib.HTTPSConnection(arguments['host'],
ssl_context=ssl_context)
conn.request('GET', arguments['quotaurl'], headers)
response = conn.getresponse()
Could you please help me with this problem?
See the error message
self._send_request(method, url, body, headers)
This indicates that you are passing "headers" where you are supposed to pass "body".
So, try calling
conn.request('GET', arguments['quotaurl'], None, headers)