I want to edit a Kodi addon that use re.compile to scrape data, and make it use BeautifulSoup4 instead.
The original code is like this:
import urllib, urllib2, re, sys, xbmcplugin, xbmcgui
link = read_url(url)
match = re.compile('<a class="frame[^"]*"'
' href="(http://somelink.com/section/[^"]+)" '
'title="([^"]+)">.*?<img src="([^"]+)".+?Length:([^<]+)',
re.DOTALL).findall(link)
for url, name, thumbnail, length in match:
addDownLink(name + length, url, 2, thumbnail)
The HTML it is scraping is like this:
<div id="content">
<span class="someclass">
<span class="sec">
<a class="frame" href="http://somlink.com/section/name-here" title="name here">
<img src="http://www.somlink.com/thumb/imgsection/thumbnail.jpg" >
</a>
</span>
<h3 class="title">
name here
</h3>
<span class="details"><span class="length">Length: 99:99</span>
</span>
.
.
.
</div>
How do I get all of url (href), name, length and thumbnail using BeautifulSoup4, and add them in addDownLink(name + length, url, 2, thumbnail)?
from bs4 import BeautifulSoup
html = """<div id="content">
<span class="someclass">
<span class="sec">
<a class="frame" href="http://somlink.com/section/name-here" title="name here">
<img src="http://www.somlink.com/thumb/imgsection/thumbnail.jpg" >
</a>
</span>
<h3 class="title">
name here
</h3>
<span class="details"><span class="length">Length: 99:99</span>
</span>
</div>
"""
soup = BeautifulSoup(html, "lxml")
sec = soup.find("span", {"class": "someclass"})
# get a tag with frame class
fr = sec.find("a", {"class": "frame"})
# pull img src and href from the a/frame
url, img = fr["href"], fr.find("img")["src"]
# get h3 with title class and extract the text from the anchor
name = sec.select("h3.title a")[0].text
# "size" is in the span with the details class
size = sec.select("span.details")[0].text.split(None,1)[-1]
print(url, img, name.strip(), size.split(None,1)[1].strip())
Which gives you:
('http://somlink.com/section/name-here', 'http://www.somlink.com/thumb/imgsection/thumbnail.jpg', u'name here', u'99:99')
If you have multiple sections, we just need find_all and to apply the logic to each section:
def secs():
soup = BeautifulSoup(html, "lxml")
sections = soup.find_all("span", {"class": "someclass"})
for sec in sections:
fr = sec.find("a", {"class": "frame"})
url, img = fr["href"], fr.find("img")["src"]
name, size = sec.select("h3.title a")[0].text, sec.select("span.details")[0].text.split(None,1)[-1]
yield url, name, img,size
If you don't know all the class but you know for instance there is one img tag you can call find on the section:
sec.find("img")["src"]
And the same logic applies to the rest.
Related
I stuck in getting all data within span tag. My code gives me only every first value in every a() within the span tag and ignore other values. In my example: (NB I reduced the span contents here, but it lot of inside)
<span class="block-niveaux-sponsors">
<a href="http://www.keolis.com/" id="a47-logo-part-keolis" target="_blank">
<img src="images/visuels_footer/footer/part_keolis.201910210940.jpg"/>
</a>
<div class="clearfix"></div>
</span>
<span class="block-niveaux-sponsors">
<a href="http://www.cg47.fr/" id="a47-logo-part-cg47" target="_blank">
<img src="images/visuels_footer/footer/part_cg47.201910210940.jpg"/>
</a>
<div class="clearfix"></div>
</span>
<span class="block-niveaux-sponsors">
<a href="http://www.errea.it/fr/" id="a47-logo-part-errea" target="_blank">
<img src="images/visuels_footer/footer/part_errea.201910210940.jpg"/>
</a>
<div class="clearfix"></div>
</span>
My code is:
page = urlopen(lien_suagen)
soup = bs(page, 'html.parser')
title_box_agen = soup.find_all('div', attrs={'id':'autres'})
for tag in title_box_agen:
for each_row in tag.find_all('span'):
links = each_row.find('a', href=True)
title = links.get('id')
print(title)
This give me only the first id values in .
I want all id.
You should try:
page = urlopen(lien_suagen)
soup = bs(page, 'html.parser')
title_box_agen = soup.find_all('div', attrs={'id':'autres'})
for tag in title_box_agen:
for each_row in tag.find_all('span'):
links = each_row.find_all('a', href=True)
for link in links:
title = link.get('id')
print(title)
You can get all the link ids for each of the niveux class like this.
(not tested)
page = urlopen(lien_suagen)
soup = bs(page, 'html.parser')
spans_niveux = soup.find_all('span' class_='block-niveaux-sponsors')
for span in spans_niveux:
link = span.find('a', href=True)
id = link.id
print(id)
Here's the HTML code:
<div class="sizeBlock">
<div class="size">
<a class="selectSize" id="44526" data-size-original="36.5">36.5</a>
</div>
<div class="size inactive active">
<a class="selectSize" id="44524" data-size-original="40">40</a>
</div>
<div class="size ">
<a class="selectSize" id="44525" data-size-original="40.5">40.5</a>
</div>
</div>
I want to get the values of the id tag and the data-size-original.
Here's my code:
for sizeBlock in soup.find_all('a', class_="selectSize"):
aid = sizeBlock.get('id')
size = sizeBlock.get('data-size-us')
The problem is that it gets the values of other ids that have the same class "selectSize".
I think this is what you want. You won't have ids and size from data in div class='size inactive active'
for sizeBlock in soup.select('div.size a.selectSize'):
aid = sizeBlock.get('id')
size = sizeBlock.get('data-size-us')
Already answered here How to Beautiful Soup (bs4) match just one, and only one, css class
Use soup.select. Here's a simple test:
from bs4 import BeautifulSoup
html_doc = """<div class="size">
<a class="selectSize otherclass" id="44526" data-ean="0193394075362" " data-tprice="" data-sku="1171177-36.5" data-size-original="36.5">5</a>
</div>"""
soup = BeautifulSoup(html_doc, 'html.parser')
#for sizeBlock in soup.find_all('a', class_= "selectSize"): # this would include the anchor
for sizeBlock in soup.select("a[class='selectSize']"):
aid = sizeBlock.get('id')
size = sizeBlock.get('data-size-original')
print aid, size
I am trying to scan a web page to find the link to a specific product using part of the product name.
The HTML below is the part I am trying to extract information from:
<article class='product' data-json-url='/en/GB/men/products/omia066s188000161001.json' id='product_24793' itemscope='' itemtype='http://schema.org/Product'>
<header>
<h3>OMIA066S188000161001</h3>
</header>
<a itemProp="url" href="/en/GB/men/products/omia066s188000161001"><span content='OFF WHITE Shoes OMIA066S188000161001' itemProp='name' style='display:none'></span>
<span content='OFF WHITE' itemProp='brand' style='display:none'></span>
<span content='OMIA066S188000161001' itemProp='model' style='display:none'></span>
<figure>
<img itemProp="image" alt="OMIA066S188000161001 image" class="top" src="https://cdn.off---white.com/images/156374/product_OMIA066S188000161001_1.jpg?1498806560" />
<figcaption>
<div class='brand-name'>
HIGH 3.0 SNEAKER
</div>
<div class='category-and-season'>
<span class='category'>Shoes</span>
</div>
<div class='price' itemProp='offers' itemscope='' itemtype='http://schema.org/Offer'>
<span content='530.0' itemProp='price'>
<strong>£ 530</strong>
</span>
<span content='GBP' itemProp='priceCurrency'></span>
</div>
<div class='size-box js-size-box'>
<!-- / .available-size -->
<!-- / = render 'availability', product: product -->
<div class='sizes'></div>
</div>
</figcaption>
</figure>
</a></article>
My code is below:
import requests
from bs4 import BeautifulSoup
item_to_find = 'off white shoes'
s = requests.Session()
r = s.get('https://www.off---white.com/en/GB/section/new-arrivals.js')
soup = BeautifulSoup(r.content, 'html.parser')
#find_url = soup.find("a", {"content":item_to_find})['href']
#print(find_url)
How do I filter only the line where 'content' contains item_to_find and then extract the 'href' for that product?
The final output should look like the below:
/en/GB/men/products/omia066s188000161001
Give this a shot.
import requests
from bs4 import BeautifulSoup
item_to_find = 'off white shoes'
s = requests.Session()
r = s.get('https://www.off---white.com/en/GB/section/new-arrivals.js')
soup = BeautifulSoup(r.content, 'html.parser')
links = soup.find_all("a")
for link in links:
if 'OFF WHITE Shoes' in link.encode_contents():
print link.get('href')
Since the "OFF WHITE Shoes" text exists within a span we can use encode_contents() to check all of the mark up within each link. If the text we are searching for exists we get the link by using BeautifulSoups .get method.
More specific answer considering python 3 would be:
import requests
from urllib.parse import urljoin
from bs4 import BeautifulSoup
search_item = 'orange timberland' #make sure the search terms are in small letters (a portion of text will suffice)
URL = 'https://www.off---white.com/en/GB/section/new-arrivals.js'
res = requests.get(URL)
soup = BeautifulSoup(res.text, 'html.parser')
for link in soup.find_all(class_="brand-name"):
if search_item in link.text.lower():
item_name = link.get_text(strip=True)
item_link = urljoin(URL,link.find_parents()[2].get('href'))
print("Name: {}\nLink: {}".format(item_name,item_link))
Output:
Name: ORANGE TIMBERLAND BOOTS
Link: https://www.off---white.com/en/GB/men/products/omia073s184780161900
I'm trying to extract a class tag from an HTML file, but only if it is located before a given stopping point. What I have is:
import requests
from bs4 import BeautifulSoup
page = requests.get("https://mysite")
soup = BeautifulSoup(page.content, 'html.parser')
class_extr = soup.find_all("span", class_="myclass")
This works, but it finds all instances of myclass, and i only want those before the following text shows in the soup:
<h4 class="cat-title" id="55">
Title text N1
<small>
Title text N2.
</small>
</h4>
The thing that makes this block unique are the Title text N lines, especially the Title text N2. line. There are many cat-title tags before, so I can't use that as a stopping condition.
The code surrounding this block looks like this:
...
<div class="myc">
<a class="bbb" href="linkhere_893">
<span class="myclass">Text893</span>
<img data-lazy="https://link893.jpg"/>
</a>
</div>
<div class="myc">
<a class="bbb" href="linkhere_96">
<span class="myclass">Text96</span>
<img data-lazy="https://link96.jpg"/>
</a>
</div>
</div><!-- This closes a list that starts above -->
<h4 class="cat-title" id="55">Title text N1 <small> Title text N2.</small></h4>
<div class="list" id="55">
<div class="myc">
<a class="bbb" href="linkhere_34">
<span class="myclass">Text34</span>
<img data-lazy="https://link34.jpg"/>
</a>
</div>
<div class="myc">
...
continuing both above and below.
How can I do this?
Try using find_all_previous():
import requests
from bs4 import BeautifulSoup
page = requests.get("https://mysite")
soup = BeautifulSoup(page.content, 'html.parser')
stop_at = soup.find("h4", class_="cat-title", id='55') # finds your stop tag
class_extr = stop_at.find_all_previous("span", class_="myclass")
This will stop at the first <h4 class='cat-title', id=55> tag in the event that there are multiple.
Reference: Beautiful Soup Documentation
How about this:
page = requests.get("https://mysite")
# Split your page and unwanted string, then parse with BeautifulSoup
text = page.text.split('Title text N2.')
soup = BeautifulSoup(text[0], 'html.parser')
class_extr = soup.find_all("span", class_="myclass")
You can try something like this:
from bs4 import BeautifulSoup
page = """
<html><body><p>
<span class="myclass">text 1</span>
<span class="myclass">text 2</span>
</p>
<h4 class="cat-title" id="55">
Title text N1
<small>
Title text N2.
</small>
</h4>
<p>
<span class="myclass">text 3</span>
<span class="myclass">text 4</span>
</p>
</body>
</html>
"""
soup = BeautifulSoup(page, 'html.parser')
for i in soup.find_all():
if i.name == 'h4' and i.has_attr('class') and i['class'][0] == 'cat-title' and i.has_attr('id') and i['id'] == '55':
if i.find("small") and i.find("small").text.strip()== "Title text N2.":
break
elif i.name == 'span'and i.has_attr('class') and i['class'][0] == 'myclass':
print (i)
Outputs:
<span class="myclass">text 1</span>
<span class="myclass">text 2</span>
I found that I can extract all the information I want from this HTML. I need to extract title, href abd src from this.
HTML:
<div class="col-md-2 col-sm-2 col-xs-2 home-hot-thumb">
<a itemprop="url" href="/slim?p=3090" class="main">
<img src="/FileUploads/Post/3090.jpg?w=70&h=70&mode=crop" alt="apple" title="apple" />
</a>
</div>
<div class="col-md-2 col-sm-2 col-xs-2 home-hot-thumb">
<a itemprop="url" href="/slim?p=3091" class="main">
<img src="/FileUploads/Post/3091.jpg?w=70&h=70&mode=crop" alt="banana" title="banana" />
</a>
</div>
Code:
import requests
from bs4 import BeautifulSoup
res = requests.get('http://www.cad.com/')
soup = BeautifulSoup(res.text,"lxml")
for a in soup.findAll('div', {"id":"home"}):
for b in a.select(".main"):
print ("http://www.cad.com"+b.get('href'))
print(b.get('title'))
I can successfully get href from this, but since title and src are in another line, I don't know how to extract them. After this, I want to save them in excel, so maybe I need to finish one first then do the second one.
Expected output:
/slim?p=3090
apple
/FileUploads/Post/3091.jpg?w=70&h=70&mode=crop" alt="banana" title="banana
/slim?p=3091
banana
/FileUploads/Post/3091.jpg?w=70&h=70&mode=crop" alt="banana" title="banana
My own solution:
import requests
from bs4 import BeautifulSoup
res = requests.get('http://www.cad.com/')
soup = BeautifulSoup(res.text,"lxml")
for a in soup.findAll('div', {"id":"home"}):
div = a.findAll('div', {"class": "home-hot-thumb"})
for div in div:
title=(div.img.get('title'))
print(title)
href=('http://www.cad.com/'+div.a.get('href'))
print(href)
src=('http://www.cad.com/'+div.img.get('src'))
print(src.replace('?w=70&h=70&mode=crop', ''))