I have a script to scrape a site but I keep getting an "urllib.error.HTTPError: HTTP Error 404: Not Found". I have tried adding the user agent to the header and running the script and I still get the same error. Here is my code
from urllib.request import urlopen, Request
from bs4 import BeautifulSoup as soup
import json
atd_url = 'https://courses.lumenlearning.com/catalog/achievingthedream'
#opening up connection and grabbing page
res = Request(atd_url,headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'})
uClient = urlopen(res)
page_html = uClient.read()
uClient.close()
#html parsing
page_soup = soup(page_html, "html.parser")
#grabs info for each textbook
containers = page_soup.findAll("div",{"class":"book-info"})
data = []
for container in containers:
item = {}
item['type'] = "Course"
item['title'] = container.h2.text
item['author'] = container.p.text
item['link'] = container.p.a["href"]
item['source'] = "Achieving the Dream Courses"
item['base_url'] = "https://courses.lumenlearning.com/catalog/achievingthedream"
data.append(item) # add the item to the list
with open("./json/atd-lumen.json", "w") as writeJSON:
json.dump(data, writeJSON, ensure_ascii=False)
Here is the full error message I get every time I run the script
Traceback (most recent call last):
File "atd-lumen.py", line 9, in <module>
uClient = urlopen(res)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 532, in open
response = meth(req, response)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 642, in http_response
'http', request, response, code, msg, hdrs)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 570, in error
return self._call_chain(*args)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 650, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found
Any suggestions on how to fix this issue? It is a valid link when entered into a browser.
Use requests library instead, this works:
import requests
#opening up connection and grabbing page
response = requests.get(atd_url,headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'})
#html parsing
page_soup = soup(response.content, "html.parser")
Related
I am trying to build a python webscraper with beautifulsoup4. If I run the code on my Macbook the script works, but if I let the script run on my homeserver (ubuntu vm) I get the following error msg (see below). I tried a vpn connection and multiple headers without success.
Highly appreciate your feedback on how to get the script working. THANKS!
Here the error msg:
{'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.41 Safari/534.7 ChromePlus/1.5.0.0alpha1'}
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "/usr/lib/python3.10/http/client.py", line 1374, in getresponse
response.begin()
File "/usr/lib/python3.10/http/client.py", line 318, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
[...]
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
[Finished in 15.9s with exit code 1]
Here my code:
from bs4 import BeautifulSoup
import requests
import pyuser_agent
URL = f"https://www.edmunds.com/inventory/srp.html?radius=5000&sort=publishDate%3Adesc&pagenumber=2"
ua = pyuser_agent.UA()
headers = {'User-Agent': ua.random}
print(headers)
response = requests.get(url=URL, headers=headers)
soup = BeautifulSoup(response.text, 'lxml')
overview = soup.find()
print(overview)
I tried multiple headers, but do not get a result
Try to use real web-browser User Agent instead random one from pyuser_agent. For example:
import requests
from bs4 import BeautifulSoup
URL = f"https://www.edmunds.com/inventory/srp.html?radius=5000&sort=publishDate%3Adesc&pagenumber=2"
headers = {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:108.0) Gecko/20100101 Firefox/108.0"}
response = requests.get(url=URL, headers=headers)
soup = BeautifulSoup(response.text, "lxml")
overview = soup.find()
print(overview)
The possible explanation could be that server keeps a list of real-world User Agents and don't serve any page to some non-existent ones.
I'm pretty bad at figuring out the right set of headers and cookies, so in these situations, I often end up resorting to:
either cloudscraper
response = cloudscraper.create_scraper().get(URL)
or HTMLSession - which is particularly nifty in that it also parses the HTML and has some JavaScript support as well
response = HTMLSession().get(URL)
I am trying to make a python3 script that iterates through a list of mods hosted on a shared website and download the latest one. I have gotten stuck on step one, go to the website and get the mod version list. I am trying to use urllib but it is throwing a 403: Forbidden error.
I thought it might be due to this being some sort of anti-scraping rejection from the server and I read that you could get around it via defining the headers. I ran wireshark while using my browser and was able to identify the headers it was sending out:
Host: ocsp.pki.goog\r\n
User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0\r\n
Accept: */*\r\n
Accept-Language: en-US,en;q=0.5\r\n
Accept-Encoding: gzip, deflate\r\n
Content-Type: application/ocsp-request\r\n
Content-Length: 83\r\n
Connection: keep-alive\r\n
\r\n
I believe I was able to define the header correctly, but I had to back two entries out as they gave a 400 error:
from urllib.request import Request, urlopen
count = 0
mods = ['mod1', 'mod2', ...] #this has been created to complete the URL and has been tested to work
#iterate through all mods and download latest version
while mods:
url = 'https://Domain/'+mods[count]
#change the header to the browser I was using at the time of writing the script
req = Request(url)
#req.add_header('Host', 'ocsp.pki.goog\\r\\n') #this reports 400 bad request
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0\\r\\n')
req.add_header('Accept', '*/*\\r\\n')
req.add_header('Accept-Language', 'en-US,en;q=0.5\\r\\n')
req.add_header('Accept-Encoding', 'gzip, deflate\\r\\n')
req.add_header('Content-Type', 'application/ocsp-request\\r\\n')
#req.add_header('Content-Length', '83\\r\\n') #this reports 400 bad request
req.add_header('Connection', 'keep-alive\\r\\n')
html = urlopen(req).read().decode('utf-8')
This still throws a 403: Forbidden error:
Traceback (most recent call last):
File "SCRIPT.py", line 19, in <module>
html = urlopen(req).read().decode('utf-8')
File "/usr/lib/python3.8/urllib/request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.8/urllib/request.py", line 531, in open
response = meth(req, response)
File "/usr/lib/python3.8/urllib/request.py", line 640, in http_response
response = self.parent.error(
File "/usr/lib/python3.8/urllib/request.py", line 569, in error
return self._call_chain(*args)
File "/usr/lib/python3.8/urllib/request.py", line 502, in _call_chain
result = func(*args)
File "/usr/lib/python3.8/urllib/request.py", line 649, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 403: Forbidden
I'm not sure what I'm doing wrong. I assume there is something wrong with the way I've defined my header values, but I am not sure what is wrong with them. Any help would be appreciated.
I've got this code with the purpose of getting the HTML code, and scrape it using bs4.
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup
myUrl = '' #Here goes de the webpage.
# opening up connection and downloadind the page
uClient = uReq(myUrl)
pageHtml = uClient.read()
uClient.close()
#html parse
pageSoup = soup(pageHtml, "html.parser")
print(pageSoup)
However, it does not work, here are the errors shown by the terminal:
Traceback (most recent call last):
File "main.py", line 7, in <module>
uClient = uReq(myUrl)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 531, in open
response = meth(req, response)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 640, in http_response
response = self.parent.error(
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 569, in error
return self._call_chain(*args)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 502, in _call_chain
result = func(*args)
File "C:\ProgramData\Anaconda3\lib\urllib\request.py", line 649, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 403: Forbidden
You are missing some headers that the site may require.
I suggests using requests package instead of urllib, as it's more flexible. See a working example below:
import requests
url = "https://www.idealista.com/areas/alquiler-viviendas/?shape=%28%28wt_%7BF%60m%7Be%40njvAqoaXjzjFhecJ%7BebIfi%7DL%29%29"
querystring = {"shape":"((wt_{F`m{e#njvAqoaXjzjFhecJ{ebIfi}L))"}
payload = ""
headers = {
'authority': "www.idealista.com",
'cache-control': "max-age=0",
'upgrade-insecure-requests': "1",
'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36",
'accept': "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
'sec-fetch-site': "none",
'sec-fetch-mode': "navigate",
'sec-fetch-user': "?1",
'sec-fetch-dest': "document",
'accept-language': "en-US,en;q=0.9"
}
response = requests.request("GET", url, data=payload, headers=headers, params=querystring)
print(response.text)
From there you can parse the body using bs4:
pageSoup = soup(response.text, "html.parser")
However, beware that the site your are trying to scrape may show a CAPTCHA, so you'll probably need to rotate your user-agent header and IP address.
A HTTP 403 error which you have received means that the web server rejected the request for the page made by the script because it did not have permission/credentials to access it.
I can access the page in your example from here, so most likely what happened is that the web server noticed that you were trying to scrape it and banned your IP address from requesting any more pages. Web servers often do this to prevent scrapers from affecting its performance.
The web site explicitly forbids what you are trying to do in their terms here: https://www.idealista.com/ayuda/articulos/legal-statement/?lang=en
So I would suggest you contact the site owner to request an API to use (this probably won't be free though).
Folks
Below program is for finding out the IP address given in the page http://whatismyipaddress.com/
import urllib2
import re
response = urllib2.urlopen('http://whatismyipaddress.com/')
p = response.readlines()
for line in p:
ip = re.findall(r'(\d+.\d+.\d+.\d+)',line)
print ip
But I am not able to trouble shoot the issue as it was giving below error
Traceback (most recent call last):
File "Test.py", line 5, in <module>
response = urllib2.urlopen('http://whatismyipaddress.com/')
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 154, in urlopen
return opener.open(url, data, timeout)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 437, in open
response = meth(req, response)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 550, in http_response
'http', request, response, code, msg, hdrs)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 475, in error
return self._call_chain(*args)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 409, in _call_chain
result = func(*args)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 558, in http_error_default
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
urllib2.HTTPError: HTTP Error 403: Forbidden
anyone have any idea what change is required to remove the errors and get the required output?
The http error code 403 tells you that the server does not want to respond to your request for some reason. In this case, I think it is the user agent of your query (the default used by urllib2).
You can change the user agent:
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
response = opener.open('http://www.whatismyipaddress.com/')
Then your query will work.
But there is no guarantee that this will keep working. The site could decide to block automated queries.
Try this
>>> import urllib2
>>> import re
>>> site= 'http://whatismyipaddress.com/'
>>> hdr = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
... 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
... 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
... 'Accept-Encoding': 'none',
... 'Accept-Language': 'en-US,en;q=0.8',
... 'Connection': 'keep-alive'}
>>> req = urllib2.Request(site, headers=hdr)
>>> response = urllib2.urlopen(req)
>>> p = response.readlines()
>>> for line in p:
... ip = re.findall(r'(\d+.\d+.\d+.\d+)',line)
... print ip
urllib2-httperror-http-error-403-forbidden
You may try the requests package here, instead of the urllib2
it is much easier to use :
import requests
url='http://whereismyip.com'
header = {'user-Agent':'curl/7.21.3'}
r= requests.get(url,header)
you can use curl as the user-Agent
I am trying to eventually make a program parsing the html of a particular website, but I get a bad status line error for the website I'd like to use. This code has worked fine for any other website I've tried. Is this something they are doing intentionally and there is nothing I can do?
My code:
from lxml import html
import requests
webpage = 'http://www.whosampled.com/search/?q=de+la+soul'
page = requests.get(webpage)
tree = html.fromstring(page.text)
The error message I receive:
Traceback (most recent call last):
File "/home/kyle/Documents/web.py", line 6, in <module>
page = requests.get(webpage)
File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 65, in get
return request('get', url, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 49, in request
response = session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 461, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 573, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 415, in send
raise ConnectionError(err, request=request)
ConnectionError: ('Connection aborted.', BadStatusLine("''",))
Provide a User-Agent header and it would work for you:
webpage = 'http://www.whosampled.com/search/?q=de+la+soul'
page = requests.get(webpage,
headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'})
Proof:
>>> from lxml import html
>>> import requests
>>>
>>> webpage = 'http://www.whosampled.com/search/?q=de+la+soul'
>>> page = requests.get(webpage, headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'})
>>> tree = html.fromstring(page.content)
>>> tree.findtext('.//title')
Search Results for "de la soul" | WhoSampled
FYI, it would also work if you switch to https:
>>> webpage = 'https://www.whosampled.com/search/?q=de+la+soul'
>>> page = requests.get(webpage)
>>> tree = html.fromstring(page.content)
>>> tree.findtext('.//title')
'Search Results for "de la soul" | WhoSampled'