import urllib.request
import time
import json
import random
QUERY = "http://localhost:8080/query?id={}"
N = 500
def getDataPoint(quote):
stock = quote['stock']
bid_price = float(quote['top_bid']['price'])
ask_price = float(quote['top_ask']['price'])
price = (bid_price + ask_price)/2
return stock, bid_price, ask_price, price
def getRatio(price_a, price_b):
if(price_b==0):
return
return price_a/price_b
if __name__ == "__main__":
for _ in range(N):
quotes = json.loads(urllib.request.urlopen(
QUERY.format(random.random())).read())
prices = {}
for quote in quotes:
stock, bid_price, ask_price, price = getDataPoint(quote)
prices[stock] = price
print ("Quoted %s at (bid:%s, ask:%s, price:%s)" % (stock,
bid_price, ask_price, price))
print ("Ratio %s" % getRatio(prices['ABC'], prices['DEF']))
Traceback (most recent call last):
File "C:/Users/AppData/Local/Programs/Python/Python37/ client.py", line 54, in
quotes= json.loads(urllib.request.urlopen(QUERY.format(random.random())).read())
File "C:\Users\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 531, in open
response = meth(req, response)
File "C:\Users\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 641, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 569, in error
result = self._call_chain(*args)
File "C:\Users\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 503, in _call_chain
result = func(*args)
File "C:\Users\AppData\Local\Programs\Python\Python37\lib\urllib\request.py", line 649, in in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found
I've error with the URL.Did some research and tried to clear still not so sure why the client part throws error while server part works fine.
This is caused due to the firewall of your computer blocking port 8080. Change the port from 8080 to say 8085 in both the client and server files.
In the above code,
Change QUERY = "http://localhost:8080/query?id={}"
To QUERY = "http://localhost:8085/query?id={}"
Similarly, there should be 8080 as the port number in the server file, change it to 8085.
Another solution would be to disable your firewall, which is not recommended.
Simply say, your server application is not running in the said location: http://localhost:8080/query
Related
I'm trying to make a terminal app to crawl a website and return the time of the entered city name. this is my code so far:
import re
import urllib.request
city = input('Enter city name: ')
url = 'https://time.is/'
rawData = urllib.request.urlopen(url).read()
decodedData = rawData.decode('utf-8')
print(decodedData)
after the last line i get this error:
Traceback (most recent call last):
File "<pyshell#13>", line 1, in <module>
rawData = urllib.request.urlopen(url).read()
File "~/Python\Python35-32\lib\urllib\request.py", line 163, in urlopen
return opener.open(url, data, timeout)
File "~/Python\Python35-32\lib\urllib\request.py", line 472, in open
response = meth(req, response)
File "~/Python\Python35-32\lib\urllib\request.py", line 582, in http_response
'http', request, response, code, msg, hdrs)
File "~/Python\Python35-32\lib\urllib\request.py", line 510, in error
return self._call_chain(*args)
File "~/Python\Python35-32\lib\urllib\request.py", line 444, in _call_chain
result = func(*args)
File "~/Python\Python35-32\lib\urllib\request.py", line 590, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 403: Forbidden
why do i get this error? what's wrong?
[EDIT]
the reason is time.is banns requests. Always remember to read terms and conditions when doing web scraping. free APIs can be found to do the same job too.
When this happens, I usually open the debugger and try to find out whats being called when I access the website. It seems like time.is doesn't like having scripts call their website.
A quick search yielded this:
1532027279136 0 161_(UTC,_UTC+00:00) 1532027279104
Time.is is for humans. To use from scripts and apps, please ask about our API. Thank you!
Here are some APIs you could use to build your project. https://www.programmableweb.com/category/time/api
I am using an embedded python (2.4.3) in digi Connectport X4 here is the code to post to an Azure IoT Hub using HTTPS:
import urllib, sys, datetime, time
import urllib2
iot_device_id = 'HTTP_Device'
iot_endpoint = 'https://winiothub.azure-devices.net/devices/' + iot_device_id + '/messages/events?api-version=2016-02-03'
sas_header = {'Authorization': 'SharedAccessSignature sr=winiothub.azure-devices.net%2Fdevices%2FHTTP_Device&sig=o7dndsA%2FJOnkzYRUhqAwMrQXVhOTpIJqJqILyGDdQAc%3D&se=1522414643'}
while True:
#try:
body_data = { 'gateway_serial': '123', 'readingvalue':'66.00', 'date': str(datetime.datetime.now())}
iot_request = urllib2.Request(iot_endpoint, str(body_data), sas_header)
resp = urllib2.urlopen(iot_request)
contents = resp.read()
resp.close()
time.sleep(1)
#except:
# print 'error'
# time.sleep(1)
The code actually post the message to the hub, but is throwing the following error:
Traceback (most recent call last):
File "C:\Users\JeffreyBiesecker\documents\visual studio 2017\Projects\NewGateAzure\NewGateAzure\NewGateAzure2.py", line 14, in ?
urllib2.urlopen(iot_request)
File "C:\Python24\lib\urllib2.py", line 130, in urlopen
return _opener.open(url, data)
File "C:\Python24\lib\urllib2.py", line 364, in open
response = meth(req, response)
File "C:\Python24\lib\urllib2.py", line 471, in http_response
response = self.parent.error(
File "C:\Python24\lib\urllib2.py", line 402, in error
return self._call_chain(*args)
File "C:\Python24\lib\urllib2.py", line 337, in _call_chain
result = func(*args)
File "C:\Python24\lib\urllib2.py", line 480, in http_error_default
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
urllib2.HTTPError: HTTP Error 204: No Content
Press any key to continue . . .
I get an error if running the code in the embedded Digi gateway or if I run in Python in Visual Studio using version 2.4.3.
urllib2.HttpError includes the response code that's received. Because of this, you can catch the exception, test for 204, and continue safely if it is the case. Otherwise, you can handle (or rethrow) the exception.
try:
resp = urllib2.urlopen(iot_request)
except urllib2.HTTPError as e:
if e.code == 204: pass
else: raise
the following script reads a .txt of ticker symbols of companies to download the corresponding financial informations in a .csv . The data is pulled from Yahoo Finance and saved in a local directory.
import urllib.request
import requests
import time
#Define the URL to download the .csv from.
url_begin = "http://real-chart.finance.yahoo.com/table.csv?s="
url_end = "&a=00&b=1&c=1950&d=11&e=31&f=2050&g=d&ignore=.csv"
#Function that reads all available ticker symbols from ticker_daten.txt. This file should be in the same directory as the program.
def readTickers(file):
read_ticker = []
ins = open( file, "r" )
for line in ins:
if line.endswith('\n'):
line=line[:-1]
read_ticker.append(line)
ins.close()
return read_ticker
#File location for tickersymbols to download
tickers = readTickers("C:/Users/Win7ADM/Desktop/Jonas/stock-price-leecher/ticker_daten.txt")
#Loop through list of ticker symbols and download .csv's .
for i in tickers:
#Forge downloadable link.
link_created = url_begin + i + url_end
#Make sure that the link actually leads to a file.
try:
r = requests.head(link_created)
if r.status_code==404:
print(str(r.status_code)+": No page found!")
time.sleep(0.5)
else:
print(link_created)
#Finally download the file, if it does exist.
urllib.request.urlretrieve(link_created, "C:/Users/Win7ADM/Desktop/Jonas/stock-price-leecher/data/"+i+".csv")
time.sleep(0.5)
except requests.ConnectionError:
#A Connection error occurred.
print ("ConnectionError: 404 No page found!")
except requests.HTTPError:
#An HTTP error occurred.
print ("HTTPError!")
except requests.Timeout:
#Connection timed out.
print ("Timeout!")
The Problem: The script crashes randomlyafter loading between 20-1750 .csv's. The crash produces the following output.
Process started >>>
http://real-chart.finance.yahoo.com/table.csv?s=0055.HK&a=00&b=1&c=1950&d=11&e=31&f=2050&g=d&ignore=.csv
http://real-chart.finance.yahoo.com/table.csv?s=0056.HK&a=00&b=1&c=1950&d=11&e=31&f=2050&g=d&ignore=.csv
http://real-chart.finance.yahoo.com/table.csv?s=0057.HK&a=00&b=1&c=1950&d=11&e=31&f=2050&g=d&ignore=.csv
http://real-chart.finance.yahoo.com/table.csv?s=0058.HK&a=00&b=1&c=1950&d=11&e=31&f=2050&g=d&ignore=.csv
Traceback (most recent call last):
File "Stock-Price Leecher.py", line 40, in <module>
urllib.request.urlretrieve(link_created, "C:/Users/Win7ADM/Desktop/Jonas/stock-price-leecher/data/"+i+".csv")
File "c:\Python34\lib\urllib\request.py", line 178, in urlretrieve
with contextlib.closing(urlopen(url, data)) as fp:
File "c:\Python34\lib\urllib\request.py", line 153, in urlopen
return opener.open(url, data, timeout)
File "c:\Python34\lib\urllib\request.py", line 461, in open
response = meth(req, response)
File "c:\Python34\lib\urllib\request.py", line 571, in http_response
'http', request, response, code, msg, hdrs)
File "c:\Python34\lib\urllib\request.py", line 499, in error
return self._call_chain(*args)
File "c:\Python34\lib\urllib\request.py", line 433, in _call_chain
result = func(*args)
File "c:\Python34\lib\urllib\request.py", line 579, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found
<<< Process finished. (Exit code 1)
================ READY ================
Does anyone of you have any Idea why this might happen?
I cannot consistently get JSON from a given url. It works only about 60% of the time
jsonurl = urlopen('http://www.reddit.com/r/funny/hot.json?limit=16')
r_content = json.load(jsonurl)['data']['children']
The program crashes on the second line sometimes, because the info from the url is not retrieved properly for some reason
With some debugging, I found out that I was getting the following error from the first line:
<addinfourl at 4321460952 whose fp = <socket._fileobject object at 0x10185b050>>
This error occurs about 40% of the time, the other 60% of the time, the code works perfectly. What am I doing wrong? How do I make the url opening more consistent?
It is usually not an issue from the client side. Your code is consistent in behavior but the server response can vary.
I ran your code a few times and It does throw up certain issues:
>>> jsonurl = urlopen('http://www.reddit.com/r/funny/hot.json?limit=16')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 126, in urlopen
return _opener.open(url, data, timeout)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 406, in open
response = meth(req, response)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 519, in http_response
'http', request, response, code, msg, hdrs)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 444, in error
return self._call_chain(*args)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 378, in _call_chain
result = func(*args)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 527, in http_error_default
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
urllib2.HTTPError: HTTP Error 429: Unknown
You have to handle cases where server response is anything but HTTP 200. You can wrap your code in a try / except block and you should pass jsonurl to json.loads() only when your request succeeds.
Also urlopen returns a file-like descriptor. Hence if you print jsourl, it simply provides jsonurl.__repr__() value. See below:
>>> jsonurl.__repr__()
'<addinfourl at 4393153672 whose fp = <socket._fileobject object at 0x105978450>>'
You have to look for the following::
>>> jsonurl.getcode()
200
>>>
and only if it 200, should you process the data obtained from the request.
So I was trying to dive into Python because I want to write an Agent for the Plex Media Server. This Agent will access the MyAnimeList.net API with HTTP Authentication (more about that here) my Username and passwords work but I don't have a clue why I still get a 401 error from the server as response.
Here is some code (I'm using python 2.5 because plex said so) :)
import urllib2
username = "someUser"
password = "somePass"
url = "http://myanimelist.net/api/anime/search.xml?q=bleach"
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
top_level_url = "http://myanimelist.net/"
password_mgr.add_password(None, top_level_url, username, password)
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(urllib2.HTTPHandler, handler)
request = urllib2.Request(url)
print request.get_full_url()
f = urllib2.urlopen(request).read()
print(f)
and this is what I get as response
> http://myanimelist.net/api/anime/search.xml?q=bleach Traceback (most
> recent call last): File
> "C:\Users\Daraku\Desktop\MAL.bundle\Contents\Code\__init__.py", line
> 16, in <module>
> f = urllib2.urlopen(request).read() File "C:\Python25\lib\urllib2.py", line 121, in urlopen
> return _opener.open(url, data) File "C:\Python25\lib\urllib2.py", line 380, in open
> response = meth(req, response) File "C:\Python25\lib\urllib2.py", line 491, in http_response
> 'http', request, response, code, msg, hdrs) File "C:\Python25\lib\urllib2.py", line 418, in error
> return self._call_chain(*args) File "C:\Python25\lib\urllib2.py", line 353, in _call_chain
> result = func(*args) File "C:\Python25\lib\urllib2.py", line 499, in http_error_default
> raise HTTPError(req.get_full_url(), code, msg, hdrs, fp) HTTPError: HTTP Error 401: Unauthorized
Bear in mind that this is my first time programming in Python and im kind of confused with many examples in the web because they did something with the urllib2 so that it doesn't exists in python 3.0, i think, anymore
Any ideas? Or any better ways to do this?