I tried this python example from freebase and run it in my windows and ubuntu machine.
http://mql.freebaseapps.com/ch04.html
import sys # Command-line arguments, etc.
import simplejson # JSON encoding.
import urllib # URI encoding.
import urllib2 # High-level URL content fetching.
# These are some constants we'll use.
SERVER = 'api.freebase.com' # Metaweb server
SERVICE = '/api/service/mqlread' # Metaweb service
# Compose our MQL query as a Python data structure.
# The query is an array in case multiple bands share the same name.
band = sys.argv[1] # The desired band, from command line.
query = [{'type': '/music/artist', # Our MQL query in Python.
'name': band, # Place the band in the query.
'album': [{ 'name': None, # None is Python's null.
'release_date': None,
'sort': 'release_date' }]}]
# Put the query in an envelope
envelope = {
'query': query, # The query property specifies the query.
'escape': False # Turns off HTML escaping.
}
# These five lines are the key code for using mqlread
encoded = simplejson.dumps(envelope) # JSON encode the envelope.
params = urllib.urlencode({'query':encoded}) # Escape request parameters.
url ='http://%s%s?%s' % (SERVER,SERVICE,params) # The URL to request.
f = urllib2.urlopen(url) # Open the URL as a file.
response = simplejson.load(f) # Read and JSON parse response.
# Check for errors and exit with a message if the query failed.
if response['code'] != '/api/status/ok': # If not okay...
error = response['messages'][0] # First msg object.
sys.exit('%s: %s' % (error['code'], error['message'])) # Display code,msg.
# No errors, so handle the result
result = response['result'] # Open the response envelope, get result.
# Check the number of matching bands
if len(result) == 0:
sys.exit('Unknown band')
elif len(result) > 1:
print "Warning: multiple bands named " + band + ". Listing first only."
result = result[0] # Get first band from array of matches.
if not result['album']: # Exit if band has no albums
sys.exit(band + ' has no known albums.')
for album in result['album']: # Loop through the result albums.
name = album['name'] # Album name.
date = album['release_date'] # Release date timestamp or null.
if not date: date = ''
else: date = ' [%s]' % date[0:4] # Just the 4-digit year in brackets.
print "%s%s" % (name, date) # Print name and date.
However, it did not work. Could someone explain it why i got this error?
Traceback (most recent call last):
File "mql.py", line 29, in <module>
f = urllib2.urlopen(url) # Open the URL as a file.
File "C:\Python27\lib\urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "C:\Python27\lib\urllib2.py", line 394, in open
response = self._open(req, data)
File "C:\Python27\lib\urllib2.py", line 412, in _open
'_open', req)
File "C:\Python27\lib\urllib2.py", line 372, in _call_chain
result = func(*args)
File "C:\Python27\lib\urllib2.py", line 1199, in http_open
return self.do_open(httplib.HTTPConnection, req)
File "C:\Python27\lib\urllib2.py", line 1168, in do_open
h.request(req.get_method(), req.get_selector(), req.data, headers)
File "C:\Python27\lib\httplib.py", line 955, in request
self._send_request(method, url, body, headers)
File "C:\Python27\lib\httplib.py", line 989, in _send_request
self.endheaders(body)
File "C:\Python27\lib\httplib.py", line 951, in endheaders
self._send_output(message_body)
File "C:\Python27\lib\httplib.py", line 811, in _send_output
self.send(msg)
File "C:\Python27\lib\httplib.py", line 773, in send
self.connect()
File "C:\Python27\lib\httplib.py", line 754, in connect
self.timeout, self.source_address)
File "C:\Python27\lib\socket.py", line 562, in create_connection
sock.connect(sa)
One more thing, I am using python 2.7 and i don't have any special proxy configuration for the internet setting.
api.freebase.com has been retired and the Python client library was never updated to work with the new endpoint.
It would be better if you can enclose your code within a try-except block as below:
try:
#all the logic to create a URL and store in a variable say url for MQLReader service
invoke urllib2.urlopen(url)
#perform other processing.
except urllib2.URLError, e:
print str(e)
This will give you an idea about what is going wrong.
I tried your code sample and I get an error saying that:
A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
Hence catching the exception and examining it will give you the root cause of the problem.
Related
I am trying to send SMS following Python suds procedure but facing some problems related to urllib2
The API URL, username & are provided by mobile service company when i try to execute the code i receive some errors relate to ###urllib2.URLError###
from suds.client import Client
client = Client(url='https://cbs.zong.com.pk/reachcwsv2 /corporatesms.svc?wsdl',
username='9231089894269149',
password='Zo98989ng#123', cache=None)
obj_QuickSMS = client.factory.create('obj_QuickSMS')
obj_QuickSMS.loginId = '91313212304269149'
obj_QuickSMS.loginPassword = 'zqong#435'
obj_QuickSMS.Destination = '923459332997'
obj_QuickSMS.Mask = 'Edex'
obj_QuickSMS.Message = 'HELLO'
obj_QuickSMS.UniCode = '0'
obj_QuickSMS.ShortCodePrefered = 'n'
result = client.service.QuickSMS(obj_QuickSMS)
i am receiving the following error:
File "/usr/lib/python2.7/urllib2.py", line 407, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 1228, in http_open
return self.do_open(httplib.HTTPConnection, req)>
File "/usr/lib/python2.7/urllib2.py", line 1198, in do_open
raise URLError(err)
urllib2.URLError:
You have white space in your url, it should be like this:
client = Client(url='https://cbs.zong.com.pk/reachcwsv2/corporatesms.svc?wsdl',
username='9231089894269149',
password='Zo98989ng#123', cache=None)
EDIT: I've majorly edited the content of this post since the original to specify my problem:
I am writing a program to download webcomics, and I'm getting this weird error when downloading a page of the comic. The code I am running essentially boils down to the following line followed by the error. I do not know what is causing this error, and it is confusing me greatly.
>>> urllib.request.urlopen("http://abominable.cc/post/47699281401")
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python3.4/urllib/request.py", line 161, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.4/urllib/request.py", line 470, in open
response = meth(req, response)
File "/usr/lib/python3.4/urllib/request.py", line 580, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python3.4/urllib/request.py", line 502, in error
result = self._call_chain(*args)
File "/usr/lib/python3.4/urllib/request.py", line 442, in _call_chain
result = func(*args)
File "/usr/lib/python3.4/urllib/request.py", line 685, in http_error_302
return self.parent.open(new, timeout=req.timeout)
File "/usr/lib/python3.4/urllib/request.py", line 464, in open
response = self._open(req, data)
File "/usr/lib/python3.4/urllib/request.py", line 482, in _open
'_open', req)
File "/usr/lib/python3.4/urllib/request.py", line 442, in _call_chain
result = func(*args)
File "/usr/lib/python3.4/urllib/request.py", line 1211, in http_open
return self.do_open(http.client.HTTPConnection, req)
File "/usr/lib/python3.4/urllib/request.py", line 1183, in do_open
h.request(req.get_method(), req.selector, req.data, headers)
File "/usr/lib/python3.4/http/client.py", line 1137, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python3.4/http/client.py", line 1172, in _send_request
self.putrequest(method, url, **skips)
File "/usr/lib/python3.4/http/client.py", line 1014, in putrequest
self._output(request.encode('ascii'))
UnicodeEncodeError: 'ascii' codec can't encode characters in position 37-38: ordinal not in range(128)
The entirety of my program can be found here: https://github.com/nstephenh/pycomic
I was having the same problem. The root cause is that the remote server isn't playing by the rules. HTTP Headers are supposed to be US-ASCII only but apparently the leading http webservers (apache2, nginx) doesn't care and send direct UTF-8 encoded string.
However in http.client the parse_header function fetch the headers as iso-8859, and the default HTTPRedirectHandler in urllib doesn't care to quote the location or URI header, resulting in the aformentioned error.
I was able to 'work around' both thing by overriding the default HTTPRedirectHandler and adding three line to counter the latin1 decoding and add a path quote:
import urllib.request
from urllib.error import HTTPError
from urllib.parse import (
urlparse, quote, urljoin, urlunparse)
class UniRedirectHandler(urllib.request.HTTPRedirectHandler):
# Implementation note: To avoid the server sending us into an
# infinite loop, the request object needs to track what URLs we
# have already seen. Do this by adding a handler-specific
# attribute to the Request object.
def http_error_302(self, req, fp, code, msg, headers):
# Some servers (incorrectly) return multiple Location headers
# (so probably same goes for URI). Use first header.
if "location" in headers:
newurl = headers["location"]
elif "uri" in headers:
newurl = headers["uri"]
else:
return
# fix a possible malformed URL
urlparts = urlparse(newurl)
# For security reasons we don't allow redirection to anything other
# than http, https or ftp.
if urlparts.scheme not in ('http', 'https', 'ftp', ''):
raise HTTPError(
newurl, code,
"%s - Redirection to url '%s' is not allowed" % (msg, newurl),
headers, fp)
if not urlparts.path:
urlparts = list(urlparts)
urlparts[2] = "/"
else:
urlparts = list(urlparts)
# Header should only contain US-ASCII chars, but some servers do send unicode data
# that should be quoted back before reused
# Need to re-encode the string as iso-8859-1 before use of ""quote"" to cancel the effet of parse_header() in http/client.py
urlparts[2] = quote(urlparts[2].encode('iso-8859-1'))
newurl = urlunparse(urlparts)
newurl = urljoin(req.full_url, newurl)
# XXX Probably want to forget about the state of the current
# request, although that might interact poorly with other
# handlers that also use handler-specific request attributes
new = self.redirect_request(req, fp, code, msg, headers, newurl)
if new is None:
return
# loop detection
# .redirect_dict has a key url if url was previously visited.
if hasattr(req, 'redirect_dict'):
visited = new.redirect_dict = req.redirect_dict
if (visited.get(newurl, 0) >= self.max_repeats or
len(visited) >= self.max_redirections):
raise HTTPError(req.full_url, code,
self.inf_msg + msg, headers, fp)
else:
visited = new.redirect_dict = req.redirect_dict = {}
visited[newurl] = visited.get(newurl, 0) + 1
# Don't close the fp until we are sure that we won't use it
# with HTTPError.
fp.read()
fp.close()
return self.parent.open(new, timeout=req.timeout)
http_error_301 = http_error_303 = http_error_307 = http_error_302
[...]
# Change default Redirect Handler in urllib, should be done once at the beginning of the program
opener = urllib.request.build_opener(UniRedirectHandler())
urllib.request.install_opener(opener)
This is python3 code but should be easily adapted for python2 if need be.
Base script:
from sanction import Client
# client_id & client_secret are omitted but are valid
client_pin = input('Enter PIN:')
access_token_url = 'https://api.home.nest.com/oauth2/access_token'
c = Client(
token_endpoint=access_token_url,
client_id=client_id,
client_secret=client_secret)
c.request_token(code = client_pin)
[See edits for history]
Running c.request('/devices') returned:
Traceback (most recent call last):
File "C:\py\nest_testing_sanction.py", line 36, in <module>
c.request("/devices")
File "C:\Python34\lib\site-packages\sanction-0.4.1-py3.4.egg\sanction\__init__.py", line 169, in request
File "C:\Python34\lib\site-packages\sanction-0.4.1-py3.4.egg\sanction\__init__.py", line 211, in transport_query
File "C:\Python34\lib\urllib\request.py", line 258, in __init__
self.full_url = url
File "C:\Python34\lib\urllib\request.py", line 284, in full_url
self._parse()
File "C:\Python34\lib\urllib\request.py", line 313, in _parse
raise ValueError("unknown url type: %r" % self.full_url)
ValueError: unknown url type: 'None/devices?access_token=c.[some long session token]'
Given the output it seems like I need to be putting in a generic URL so I tried c.request('wss://developer-api.nest.com'):
Traceback (most recent call last):
File "C:\py\nest_testing_sanction.py", line 36, in <module>
data = c.request(query_url)
File "C:\Python34\lib\site-packages\sanction-0.4.1-py3.4.egg\sanction\__init__.py", line 171, in request
File "C:\Python34\lib\urllib\request.py", line 153, in urlopen
return opener.open(url, data, timeout)
File "C:\Python34\lib\urllib\request.py", line 455, in open
response = self._open(req, data)
File "C:\Python34\lib\urllib\request.py", line 478, in _open
'unknown_open', req)
File "C:\Python34\lib\urllib\request.py", line 433, in _call_chain
result = func(*args)
File "C:\Python34\lib\urllib\request.py", line 1257, in unknown_open
raise URLError('unknown url type: %s' % type)
urllib.error.URLError: <urlopen error unknown url type: nonewss>
I also tried https as per:
- same result
By contrast, this works (for a firebase.io virtual device):
firebase = firebase.FirebaseApplication('https://nesttest.firebaseio.com', None)
thermostat_result = firebase.get('/devices', 'thermostats')
In Python I would use something like sanction to keep things simple. You should be able to get it to work with the Nest API using code like: (untested, using token flow rather than pin flow)
from sanction.client import Client
# instantiating a client to get the auth URI
c = Client(auth_endpoint="https://home.nest.com/login/oauth2",
client_id=config["nest.client_id"])
# instantiating a client to process OAuth2 response
c = Client(token_endpoint="https://api.home.nest.com/oauth2/access_token",
client_id=config["nest.client_id"],
client_secret=config["nest.client_secret"])
The library is well documented, so you should be able to figure it out from here if something is missing.
This is more of a comment, but the system does not let me comment just yet.
To your question about where to put the web pin simply add code = pin to the request_token call.
c.request_token(code = nest_client_pin)
This still does not fully solve the issue as I can only use a PIN once. After I have used it once, every subsequent call will fail again as you describe. Still researching that.
I have a bunch of files in wav. I made a simple script to convert them to flac so I can use it with the google speech api. Here is the python code:
import urllib2
url = "https://www.google.com/speech-api/v1/recognize?client=chromium&lang=en-US"
audio = open('somefile.flac','rb').read()
headers={'Content-Type': 'audio/x-flac; rate=16000', 'User-Agent':'Mozilla/5.0'}
request = urllib2.Request(url, data=audio, headers=headers)
response = urllib2.urlopen(request)
print response.read()
However I am getting this error:
Traceback (most recent call last):
File "transcribe.py", line 7, in <module>
response = urllib2.urlopen(request)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 392, in open
response = self._open(req, data)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 410, in _open
'_open', req)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 370, in _call_chain
result = func(*args)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1194, in https_open
return self.do_open(httplib.HTTPSConnection, req)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 1161, in do_open
raise URLError(err)
urllib2.URLError: <urlopen error [Errno 32] Broken pipe>
I thought at first that it was because the file is too big. But I recorded myself for 5 seconds and it still does the same.
I dont think google ha released the api yet so it's hard to understand why its failing.
Is there any other good speech-to-text api out there that can be used in either Python or Node?
----- Editing for my attempt with requests:
import json
import requests
url = 'https://www.google.com/speech-api/v1/recognize?client=chromium&lang=en-US'
data = {'file': open('file.flac', 'rb')}
headers = {'Content-Type': 'audio/x-flac; rate=16000', 'User-Agent':'Mozilla/5.0'}
r = requests.post(url, data=data, headers=headers)
# r = requests.post(url, files=data, headers=headers) ## does not work either
# r = requests.post(url, data=open('file.flac', 'rb').read(), headers=headers) ## does not work either
print r.text
Produced the same problem as above.
The API Accepts HTTP POST requests. You're using a HTTP GET Request here. This can be confirmed by loading the URI in your code directly into a browser:
HTTP method GET is not supported by this URL
Error 405
Also, i'd recommend using the requests python library. See http://www.python-requests.org/en/latest/user/quickstart/#post-a-multipart-encoded-file
Lastly, it seems that the API only accepts segments up to 15 seconds long. Perhaps your error is the file is too large? If you can upload an example flac file, perhaps we could diagnose further.
I'm getting an error when trying to open a website url with Python 3.1, urllib & json
urllib.error.URLError:
Here's the code. The first website loads fine. The second one
import json
import urllib.request
import urllib.parse
import util
# This one works fine
response = urllib.request.urlopen('http://python.org/')
html = response.read()
print(html)
# parms - CSV filename, company, ....
p_filename = "c:\\temp\\test.csv"
jg_token = "zzzzzzzzzzzzzzzzzzzzzzzzz"
jg_proto = "https://"
jg_webst = "www.jigsaw.com/rest/"
jg_cmd_searchContact = "searchContact.json"
jg_key_companyName = "companyName"
jg_key_levels = "levels"
jg_key_departments = "departments"
jg_args = {
"token":jg_token,
jg_key_companyName: "Technical Innovations",
jg_key_departments: "HR"
}
jg_url = jg_proto + jg_webst + jg_cmd_searchContact + "?" + urllib.parse.urlencode(jg_args)
# This one generates teh error
result = json.load(urllib.request.urlopen(jg_url))
urllib.error.URLError:
File "c:\dev\xdev\PyJigsaw\searchContact.py", line 46, in
result = json.load(urllib.request.urlopen(jg_url))
File "c:\dev\tdev\Python31\Lib\urllib\request.py", line 121, in urlopen
return _opener.open(url, data, timeout)
File "c:\dev\tdev\Python31\Lib\urllib\request.py", line 349, in open
response = self._open(req, data)
File "c:\dev\tdev\Python31\Lib\urllib\request.py", line 367, in _open
'_open', req)
File "c:\dev\tdev\Python31\Lib\urllib\request.py", line 327, in _call_chain
result = func(*args)
File "c:\dev\tdev\Python31\Lib\urllib\request.py", line 1098, in https_open
return self.do_open(http.client.HTTPSConnection, req)
File "c:\dev\tdev\Python31\Lib\urllib\request.py", line 1075, in do_open
raise URLError(err)
Please edit the title and tags and maybe even the question body:
This has nothing to do with JSON and everything to do with Windows. It's also at a lower level than urllib. (Probably in the SSL code.) Distilled:
Both of the following approaches fail on Python 3.1.2 for Vista, but work fine on Linux (Python 3.1.3)
print( HTTPSConnection(hostname).request('GET',url).getresponse().read() )
print( urllib.request.urlopen('https://'+hostname+url).read() )
Change them to not use SSL, and then they work fine on Windows:
print( HTTPConnection(hostname).request('GET',url).getresponse().read() )
print( urllib.request.urlopen('http://'+hostname+url).read() )
On Vista, I just upgraded from Python 3.1.2 to Python 3.2 and this is no longer a problem.
The following now works just fine:
print( urllib.request.urlopen('https://'+hostname+url).read() )