Develop Reference

How to get all listings urls from main page with python web scraping

I wrote a code for web scraping, My code is ok just except two issues. From detail page, everything is ok just ISBN NO, and from main page, I need all listing URLs so that my code could scrape date from aa listings. Please guide me how can I fix this issue. Both(main page and details page )URLs are in the code. Thank you!
here is my code:
import requests
from bs4 import BeautifulSoup
import csv
def get_page(url):
response = requests.get(url)
if not response.ok:
print('server responded:', response.status_code)
else:
soup = BeautifulSoup(response.text, 'html.parser') # 1. html , 2. parser
return soup
def get_detail_data(soup):
try:
title = soup.find('span',class_="title product-field",id=False).text
except:
title = 'empty'
print(title)
try:
writer = soup.find('a',class_="contributor-name",id=False).text
except:
writer = 'empty'
print(writer)
try:
original_price = soup.find('div',class_="original-price",id=False).find('span').text
except:
original_price = 'empty'
print(original_price)
try:
active_price = soup.find('div',class_="active-price",id=False).find('span').text
except:
active_price = 'empty'
print(active_price)
try:
img = soup.find('div',class_="image-actions image-container product-type-icon-container book",id=False).find('img').attrs['src']
except:
img = 'empty'
print(img)
try:
isbn = soup.find('div',class_="bookitem-secondary-metadata",id=False).find('li').attrs['ISBN: ']
except:
isbn = 'empty'
print(isbn)
data = {
'title' : title,
'writer' : writer,
'original_price' : original_price,
'active_price' : active_price,
'image' : img,
'isbn' : isbn
}
return data
def get_index_data(soup):
titles_link = soup.find_all('a',class_="body_link_11")
try:
inks = soup.find('div', class_="item-info",id=False).find('p').find('a').get('href')
except:
inks = "empty"
print(inks)
def main():
#detail_page_url = "https://www.kobo.com/ww/en/ebook/mum-dad-1"
mainurl = "https://www.kobo.com/ww/en/list/new-hot-in-fiction/youL53408U25RHrVu3wR5Q"
#get_page(url)
#get_detail_data(get_page(detail_page_url))
get_index_data(get_page(mainurl))
if __name__ == '__main__':
main()

import requests
import re
import json
from bs4 import BeautifulSoup
import csv
def Soup(content):
soup = BeautifulSoup(content, 'html.parser')
return soup
def Main(url):
r = requests.get(url)
soup = Soup(r.content)
scripts = soup.findAll("script", type="application/ld+json",
text=re.compile("data"))
prices = [span.text for span in soup.select(
"p.product-field.price span span") if span.text != "USD"]
with open("data.csv", 'w', newline="") as f:
writer = csv.writer(f)
writer.writerow(["Title", "Writer", "Price", "ISBN", "IMG", "URL"])
for script, price in zip(scripts, prices):
script = json.loads(script.text)
title = script["data"]["name"]
author = script["data"]["author"][0]["name"]
img = f'https:{script["data"]["thumbnailUrl"]}'
isbn = script["data"]["isbn"]
url = script["data"]["url"]
writer.writerow([title, author, price, isbn, img, url])
Main("https://www.kobo.com/ww/en/list/new-hot-in-fiction/youL53408U25RHrVu3wR5Q")
Output: View-Online
Output Sample:

I'm getting KeyError trying to scrape data from website

I wrote a code for data scraping; it works well for some pages, but for some it displays:
KeyError: 'isbn'.
Could you please guide me on how can I solve this issue?
Here is my code:
import requests
import re
import json
from bs4 import BeautifulSoup
import csv
import sys
import codecs
def Soup(content):
soup = BeautifulSoup(content, 'html.parser')
return soup
def Main(url):
r = requests.get(url)
soup = Soup(r.content)
scripts = soup.findAll("script", type="application/ld+json",
text=re.compile("data"))
prices = [span.text for span in soup.select(
"p.product-field.price span span") if span.text != "USD"]
with open("AudioBook/Fiction & Literature/African American.csv", 'a', encoding="utf-8", newline="") as f:
writer = csv.writer(f)
writer.writerow(["Title", "Writer", "Price", "IMG", "URL", "ISBN"])
for script, price in zip(scripts, prices):
script = json.loads(script.text)
title = script["data"]["name"]
author = script["data"]["author"][0]["name"]
img = f'https:{script["data"]["thumbnailUrl"]}'
isbn = script["data"]["isbn"]
url = script["data"]["url"]
writer.writerow([title, author, price, img, url, isbn])
for x in range(1,10):
url = ("https://www.kobo.com/ww/en/audiobooks/contemporary-1?pageNumber=" + str(x))
print("Scrapin page " + str(x) + ".....")
Main(url)

Since audiobooks don't have an ISBN on the listings page, you could prepare for this case with a default value, e.g.:
isbn = script["data"].get("isbn", "")
In this case, if the "isbn" key doesn't exist in script["data"], it will fallback on the value of an empty string.
Alternatively, you could get the book ISBN from the audiobook-specific page (your script["data"]["url"] above), e.g.:
def Main(url):
r = requests.get(url)
soup = Soup(r.content)
scripts = soup.findAll("script", type="application/ld+json",
text=re.compile("data"))
prices = [span.text for span in soup.select(
"p.product-field.price span span") if span.text != "USD"]
with open("AudioBook/Fiction & Literature/African American.csv", 'a', encoding="utf-8", newline="") as f:
writer = csv.writer(f)
writer.writerow(["Title", "Writer", "Price", "IMG", "URL", "ISBN"])
for script, price in zip(scripts, prices):
script = json.loads(script.text)
title = script["data"]["name"]
author = script["data"]["author"][0]["name"]
img = f'https:{script["data"]["thumbnailUrl"]}'
# NEW CODE
url = script["data"]["url"]
if "isbn" in script["data"]:
# ebook listings
isbn = script["data"]["isbn"]
else:
# audiobook listings
r = requests.get(url)
inner_soup = Soup(r.content)
try:
inner_script = json.loads(
inner_soup.find("script", type="application/ld+json",
text=re.compile("workExample")).text)
isbn = inner_script["workExample"]["isbn"]
except AttributeError:
isbn = ""
# END NEW CODE
writer.writerow([title, author, price, img, url, isbn])

ValueError when scraping Tripadvisor for reviews with BeautifulSoup

I am trying to scrape some Tripadvisor reviews as a complete newbie to this.
I'm using code from Susanli2016.
It worked (though, removing the attribute "language") for one link but it doesn't work for any more link (for example.)
I'm receiving the error:
Traceback (most recent call last):
File "<pyshell#37>", line 4, in <module>
items = scrape(url)
File "<pyshell#13>", line 11, in scrape
items = parse(session, url + '?filterLang=' + lang)
File "<pyshell#18>", line 15, in parse
num_reviews = int(num_reviews) # convert text into integer
ValueError: invalid literal for int() with base 10: '5.695'
(where 5,695 is the number of reviews in the page)
I'm attaching the code here in case someone can help me.
Thank you so much!
Silvia
--
Hereby the complete code:
import requests
from bs4 import BeautifulSoup
import csv
import webbrowser
import io
def display(content, filename='output.html'):
with open(filename, 'wb') as f:
f.write(content)
webbrowser.open(filename)
def get_soup(session, url, show=False):
r = session.get(url)
if show:
display(r.content, 'temp.html')
if r.status_code != 200: # not OK
print('[get_soup] status code:', r.status_code)
else:
return BeautifulSoup(r.text, 'html.parser')
def post_soup(session, url, params, show=False):
'''Read HTML from server and convert to Soup'''
r = session.post(url, data=params)
if show:
display(r.content, 'temp.html')
if r.status_code != 200: # not OK
print('[post_soup] status code:', r.status_code)
else:
return BeautifulSoup(r.text, 'html.parser')
def scrape(url, lang='ALL'):
# create session to keep all cookies (etc.) between requests
session = requests.Session()
session.headers.update({
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0',
})
items = parse(session, url + '?filterLang=' + lang)
return items
def parse(session, url):
'''Get number of reviews and start getting subpages with reviews'''
print('[parse] url:', url)
soup = get_soup(session, url)
if not soup:
print('[parse] no soup:', url)
return
num_reviews = soup.find('span', class_='reviews_header_count').text # get text
num_reviews = num_reviews[1:-1]
num_reviews = num_reviews.replace(',', '')
num_reviews = int(num_reviews) # convert text into integer
print('[parse] num_reviews ALL:', num_reviews)
url_template = url.replace('.html', '-or{}.html')
print('[parse] url_template:', url_template)
items = []
offset = 0
while(True):
subpage_url = url_template.format(offset)
subpage_items = parse_reviews(session, subpage_url)
if not subpage_items:
break
items += subpage_items
if len(subpage_items) < 5:
break
offset += 5
return items
def get_reviews_ids(soup):
items = soup.find_all('div', attrs={'data-reviewid': True})
if items:
reviews_ids = [x.attrs['data-reviewid'] for x in items][::2]
print('[get_reviews_ids] data-reviewid:', reviews_ids)
return reviews_ids
def get_more(session, reviews_ids):
url = 'https://www.tripadvisor.com/OverlayWidgetAjax?Mode=EXPANDED_HOTEL_REVIEWS_RESP&metaReferer=Hotel_Review'
payload = {
'reviews': ','.join(reviews_ids), # ie. "577882734,577547902,577300887",
#'contextChoice': 'DETAIL_HR', # ???
'widgetChoice': 'EXPANDED_HOTEL_REVIEW_HSX', # ???
'haveJses': 'earlyRequireDefine,amdearly,global_error,long_lived_global,apg-Hotel_Review,apg-Hotel_Review-in,bootstrap,desktop-rooms-guests-dust-en_US,responsive-calendar-templates-dust-en_US,taevents',
'haveCsses': 'apg-Hotel_Review-in',
'Action': 'install',
}
soup = post_soup(session, url, payload)
return soup
def parse_reviews(session, url):
'''Get all reviews from one page'''
print('[parse_reviews] url:', url)
soup = get_soup(session, url)
if not soup:
print('[parse_reviews] no soup:', url)
return
hotel_name = soup.find('h1', id='HEADING').text
reviews_ids = get_reviews_ids(soup)
if not reviews_ids:
return
soup = get_more(session, reviews_ids)
if not soup:
print('[parse_reviews] no soup:', url)
return
items = []
for idx, review in enumerate(soup.find_all('div', class_='reviewSelector')):
badgets = review.find_all('span', class_='badgetext')
if len(badgets) > 0:
contributions = badgets[0].text
else:
contributions = '0'
if len(badgets) > 1:
helpful_vote = badgets[1].text
else:
helpful_vote = '0'
user_loc = review.select_one('div.userLoc strong')
if user_loc:
user_loc = user_loc.text
else:
user_loc = ''
bubble_rating = review.select_one('span.ui_bubble_rating')['class']
bubble_rating = bubble_rating[1].split('_')[-1]
item = {
'review_body': review.find('p', class_='partial_entry').text,
'review_date': review.find('span', class_='ratingDate')['title'], # 'ratingDate' instead of 'relativeDate'
}
items.append(item)
print('\n--- review ---\n')
for key,val in item.items():
print(' ', key, ':', val)
print()
return items
def write_in_csv(items, filename='results.csv',
headers=['hotel name', 'review title', 'review body',
'review date', 'contributions', 'helpful vote',
'user name' , 'user location', 'rating'],
mode='w'):
print('--- CSV ---')
with io.open(filename, mode, encoding="utf-8") as csvfile:
csv_file = csv.DictWriter(csvfile, headers)
if mode == 'w':
csv_file.writeheader()
csv_file.writerows(items)
DB_COLUMN = 'review_body'
DB_COLUMN1 = 'review_date'
start_urls = [
'https://www.tripadvisor.com/Restaurant_Review-g187823-d2101904-Reviews-Eataly_Genova-Genoa_Italian_Riviera_Liguria.html',
]
headers = [
DB_COLUMN,
DB_COLUMN1,
]
lang = 'it'
for url in start_urls:
# get all reviews for 'url' and 'lang'
items = scrape(url)
if not items:
print('No reviews')
else:
# write in CSV
filename = url.split('Reviews-')[1][:-5]
print('filename:', filename)
write_in_csv(items, filename + '.csv', headers, mode='w')
Thanks to all the commenters. I realized the issue lied in the Italian and US paradigm for writing thousand separators (we use ".", whereas the americans use ",").

You seem to have the following string for number of views 5.695 before trying to type cast it to int with num_reviews = int(num_reviews).
Probably the . in 5.695 is a thousands separator.
So remove the . like this before using int():
num_reviews = num_reviews.replace('.', '')
num_reviews = int(num_reviews)

The error is due to the full stop in the int you are trying to convert. To make sure it works with all typing formats, you need to filter for numerical characters only before converting to int:
num_reviews = soup.find('span', class_='reviews_header_count').text # get text
num_reviews = num_reviews[1:-1]
num_reviews = num_reviews.replace(',', '').replace('.','')
num_reviews = int(num_reviews)
Or more in a more generic way, only include numerical chars in the string num_reviews

You cannot be parsed directly to an integer value, In this case you first convert it into float then if you want convert it as Int.
num_reviews = int(float(num_reviews))

Appending scraped data to CSV file

I have been messing around with python in the past few days and while following Edmund Martin's tutorial I ran into a problem:
I would like to append the name and title that I scraped to a CSV file.
Only problem is the data I scraped does not appear into the file.
Could you explain to me the logic of why only "rank" "description" and "title" is being written onto the CSV file and not the actual data. Also how can I solve that?
Below is the code I have found from the tutorial website with the last three lines that I added:
import requests
from bs4 import BeautifulSoup
import time
import csv
USER_AGENT = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 '
'Safari/537.36'}
def fetch_results(search_term, number_results, language_code):
assert isinstance(search_term, str), 'Search term must be a string'
assert isinstance(number_results, int), 'Number of results must be an integer'
escaped_search_term = search_term.replace(' ', '+')
google_url = 'https://www.google.com/search?q={}&num={}&hl={}'.format(
escaped_search_term, number_results, language_code)
response = requests.get(google_url, headers=USER_AGENT)
response.raise_for_status()
return search_term, response.text
def parse_results(html, keyword):
soup = BeautifulSoup(html, 'html.parser')
found_results = []
rank = 1
result_block = soup.find_all('div', attrs={'class': 'g'})
for result in result_block:
link = result.find('a', href=True)
title = result.find('h3', attrs={'class': 'r'})
description = result.find('span', attrs={'class': 'st'})
if link and title:
link = link['href']
title = title.get_text()
description = description.get_text()
if link != '#':
found_results.append({
'rank': rank,
'title': title,
'description': description
})
rank += 1
return found_results
def scrape_google(search_term, number_results, language_code):
try:
keyword, html = fetch_results(search_term, number_results, language_code)
results = parse_results(html, keyword)
return results
except AssertionError:
raise Exception("Incorrect arguments parsed to function")
except requests.HTTPError:
raise Exception("You appear to have been blocked by Google")
except requests.RequestException:
raise Exception("Appears to be an issue with your connection")
if __name__ == '__main__':
keywords = ['python']
data = []
for keyword in keywords:
try:
results = scrape_google(keyword,2, "en")
for result in results:
data.append(result)
except Exception as e:
print(e)
finally:
time.sleep(1)
print(data)
with open('python_scrape.csv', 'w') as csvFile:
writer = csv.writer(csvFile)
writer.writerows(data)
csvFile.close()import requests
from bs4 import BeautifulSoup
import time
import csv
USER_AGENT = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 '
'Safari/537.36'}
def fetch_results(search_term, number_results, language_code):
assert isinstance(search_term, str), 'Search term must be a string'
assert isinstance(number_results, int), 'Number of results must be an integer'
escaped_search_term = search_term.replace(' ', '+')
google_url = 'https://www.google.com/search?q={}&num={}&hl={}'.format(
escaped_search_term, number_results, language_code)
response = requests.get(google_url, headers=USER_AGENT)
response.raise_for_status()
return search_term, response.text
def parse_results(html, keyword):
soup = BeautifulSoup(html, 'html.parser')
found_results = []
rank = 1
result_block = soup.find_all('div', attrs={'class': 'g'})
for result in result_block:
link = result.find('a', href=True)
title = result.find('h3', attrs={'class': 'r'})
description = result.find('span', attrs={'class': 'st'})
if link and title:
link = link['href']
title = title.get_text()
description = description.get_text()
if link != '#':
found_results.append({
'rank': rank,
'title': title,
'description': description
})
rank += 1
return found_results
def scrape_google(search_term, number_results, language_code):
try:
keyword, html = fetch_results(search_term, number_results, language_code)
results = parse_results(html, keyword)
return results
except AssertionError:
raise Exception("Incorrect arguments parsed to function")
except requests.HTTPError:
raise Exception("You appear to have been blocked by Google")
except requests.RequestException:
raise Exception("Appears to be an issue with your connection")
if __name__ == '__main__':
keywords = ['python']
data = []
for keyword in keywords:
try:
results = scrape_google(keyword,2, "en")
for result in results:
data.append(result)
except Exception as e:
print(e)
finally:
time.sleep(1)
print(data)
with open('python_scrape.csv', 'w') as csvFile:
writer = csv.writer(csvFile)
writer.writerows(data)
csvFile.close()
Thanks for the help!

Because you're using csv.writer.writerows (which ends in 's', rows is plural), rather than writerow, csv writer expects a list of "iterable objects", which it will treat as rows.
Your main() function uses scrape_google() to return a list of dictionaries, which are all like {'rank': rank, 'title': title, 'description': description}.
Python iterates through dictionaries by returning each key, so what writerows is seeing is just the keys "rank", "title", and "description" in each row.
The fastest way to fix what is happening is to add a line
results = [[j[i] for i in j] for j in results]
before your "with open('python_scrape.csv'..." line. This uses list comprehension, which is a good thing to learn about as a new python user.
A better way to fix your code would be to make sure that it is building up a list of lists to be written to the csv instead of a list of dictionaries.

def parse_results(html, keyword):
# code ....
for result in result_block:
link = result.find('a', href=True) # here you get links
title = result.find('h3', attrs={'class': 'r'}) # here you get title
description = result.find('span', attrs={'class': 'st'}) # here you get description
# if you want something to search here
# for example you can print(result) here an see what data have result variable
# and after that parse that data and save in variable for example
# body = result.find('h1', attrs={'class': 'h1'})
if link and title:
link = link['href']
title = title.get_text()
description = description.get_text()
# here we take text from that body
# body = body.get_text()
if link != '#':
found_results.append({
'rank': rank,
'title': title,
'description': description,
# and here we append to list
'body': body
})
rank += 1
return found_results

Scraping reviews from tripadvisor

Suppose I am scraping a reviews from the url
https://www.tripadvisor.com/Hotel_Review-g562819-d289642-Reviews-Hotel_Caserio-Playa_del_Ingles_Maspalomas_Gran_Canaria_Canary_Islands.html
It contents no of pages which contains the reviews which I want to scrape. So how can I scrape the reviews of all the next pages.
I used the below code but still shows only the reviews in first page only!
from bs4 import BeautifulSoup
import requests
URL_BASE = "https://www.tripadvisor.com/Hotel_Review-g562819-d289642-Reviews-Hotel_Caserio-Playa_del_Ingles_Maspalomas_Gran_Canaria_Canary_Islands.html"
MAX_PAGES = 30
counter = 0
for i in range(1, MAX_PAGES):
if i > 1:
url = "%spage/%d/" % (URL_BASE, i)
else:
url = URL_BASE
req = requests.get(url)
statusCode = req.status_code
if statusCode == 200:
html = BeautifulSoup(req.text, "html.parser")
resultsoup = html.find_all('P', {'class': 'partial_entry'})
else:
break
for review in resultsoup:
review_list = review.get_text()
print(review_list)

Based on example for scrapy.
Server adds to url (in any place before .html)
-or5 to get second page,
-or10 to get third page,
etc.
You could even skip words (which are for SEO) and use only
https://www.tripadvisor.com/g562819-d289642-or5.html
https://www.tripadvisor.com/g562819-d289642-or10.html
to get next pages with reviews.
from bs4 import BeautifulSoup
import requests
import re
#import webbrowser
def get_soup(url):
headers = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0'}
r = s.get(url, headers=headers)
#with open('temp.html', 'wb') as f:
# f.write(r.content)
# webbrowser.open('temp.html')
if r.status_code != 200:
print('status code:', r.status_code)
else:
return BeautifulSoup(r.text, 'html.parser')
def parse(url, response):
if not response:
print('no response:', url)
return
# get number of reviews
num_reviews = response.find('span', class_='reviews_header_count').text
num_reviews = num_reviews[1:-1] # remove `( )`
num_reviews = num_reviews.replace(',', '') # remove `,`
num_reviews = int(num_reviews)
print('num_reviews:', num_reviews, type(num_reviews))
# create template for urls to pages with reviews
url = url.replace('.html', '-or{}.html')
print('template:', url)
# load pages with reviews
for offset in range(0, num_reviews, 5):
print('url:', url.format(offset))
url_ = url.format(offset)
parse_reviews(url_, get_soup(url_))
return # for test only - to stop after first page
def parse_reviews(url, response):
print('review:', url)
if not response:
print('no response:', url)
return
# get every review
for idx, review in enumerate(response.find_all('div', class_='review-container')):
item = {
'hotel_name': response.find('h1', class_='heading_title').text,
'review_title': review.find('span', class_='noQuotes').text,
'review_body': review.find('p', class_='partial_entry').text,
'review_date': review.find('span', class_='relativeDate')['title'],#.text,#[idx],
'num_reviews_reviewer': review.find('span', class_='badgetext').text,
'reviewer_name': review.find('span', class_='scrname').text,
'bubble_rating': review.select_one('div.reviewItemInline span.ui_bubble_rating')['class'][1][7:],
}
results.append(item) # <--- add to global list
#~ yield item
for key,val in item.items():
print(key, ':', val)
print('----')
#return # for test only - to stop after first review
# --- main ---
s = requests.Session()
start_urls = [
'https://www.tripadvisor.com/Hotel_Review-g562819-d289642-Reviews-Hotel_Caserio-Playa_del_Ingles_Maspalomas_Gran_Canaria_Canary_Islands.html',
#'https://www.tripadvisor.com/Hotel_Review-g60795-d102542-Reviews-Courtyard_Philadelphia_Airport-Philadelphia_Pennsylvania.html',
#'https://www.tripadvisor.com/Hotel_Review-g60795-d122332-Reviews-The_Ritz_Carlton_Philadelphia-Philadelphia_Pennsylvania.html',
]
results = [] # <--- global list for items
for url in start_urls:
parse(url, get_soup(url))
import pandas as pd
df = pd.DataFrame(results) # <--- convert list to DataFrame
df.to_csv('output.csv') # <--- save in file