Python Selenium don't get all results - python

Hey Guys it is my first time asking a question here so pls dont be to harsh to me and my english is not the yellow from the egg as well.
i have a problem i tried to webscrape companys but i am able to get all company names but if i try to scrape for the phone number and other data i only get the first 4 results hope someon can help me thanks in advance
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC, wait
from selenium.webdriver.support.expected_conditions import presence_of_element_located
from selenium.webdriver.support.expected_conditions import presence_of_element_located
PATH = "C:/Program Files (x86)/Chromedriver.exe"
driver = webdriver.Chrome(PATH)
driver.get("https://www.gelbeseiten.de/Suche/KFZ/50968")
try:
main = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "gs_treffer"))
)
except:
driver.quit
element = driver.find_element_by_id ('cmpbntyestxt')
element.click()
Firma = driver.find_elements_by_css_selector("h2")
for firma in Firma:
print(firma.text)
PLZ = driver.find_elements_by_class_name("mod-AdresseKompakt__phoneNumber")
for plz in PLZ:
print(plz.text)
driver.quit()


You need to scroll into each element panel, and then you can interact with them.
You can try the below solution :
driver = webdriver.Chrome(driver_path)
driver.maximize_window()
driver.implicitly_wait(50)
driver.get("https://www.gelbeseiten.de/Suche/KFZ/50968")
wait = WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.XPATH, "//span[#id='cmpbntyestxt']/.."))).click()
lngth = len(driver.find_elements(By.XPATH, "//h2[#data-wipe-name='Titel']"))
print(lngth)
j = 0
scrolling = 0
for item in range(lngth):
elements = driver.find_elements(By.XPATH, "//h2[#data-wipe-name='Titel']")
lst = driver.find_elements(By.XPATH, "//div[#id='gs_treffer']/child::article")
driver.execute_script("arguments[0].scrollIntoView(true);", lst[scrolling])
time.sleep(1)
print(elements[j].text)
print(elements[j].find_element(By.XPATH, ".//following-sibling::address/p[2]").text)
j = j + 1
scrolling = scrolling + 1
Imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Output :
"C:\Program Files\Python39\python.exe" C:/Users/***/PycharmProjects/SeleniumSO/Chrome.py
16
Scherschel Andreas
0221 38 59 93
Scherschel Andreas Dipl.-Ing. KFZ-Sachverst.
0221 38 59 93
La Linea Franca GmbH,Volvo Vertragshändler KFZ-Handel
0221 8 00 60-0
Kuehler Willems Services UG
0221 3 76 11 11
Autoservice Baum
0221 38 59 31
Mückl u. Rehse GbR Ingenieurbüro
0221 93 46 49-0
Procar Automobile GmbH & Co. KG - Köln Süd
0221 37 69 80
Allianz Versicherung Wolfgang Georg
0221 3 40 49 51
Pohlen Robert Autoservice
0221 37 84 72
Barmenia Versicherung - Norin Amraei
01522 6 22 70 04
Auto-Kühlerbau
0221 3 76 11 11
Barmenia Versicherung - Anders OHG
0221 9 85 49 30
Barmenia Versicherung - Kevin Wimmer
0176 70 29 27 17
Barmenia Versicherung - Yazdan Izadi
0221 9 85 49 30
GTÜ Prùfstelle Köln Bayenthal
0221 93 46 49-0
Continentale: T & T Versicherungsservice
0221 34 02 91 10
Process finished with exit code 0


You can scrape all the data from the landing page using requests module as well.
import json
import requests
from bs4 import BeautifulSoup
link = 'https://www.gelbeseiten.de/Suche/KFZ/50968'
def get_lazyloaded_details(soup):
all_details = []
for lazyload_data in soup.select("article[data-lazyloaddata]"):
container = json.loads(lazyload_data.get("data-lazyloaddata"))
name = container['name']
for item in container['trefferButtonListList']['trefferButtonListList']:
for elem in item:
if elem['gcLink']['text']=='E-Mail':
email = elem['gcLink']['href'].split("?")[0].split(":")[1]
elif elem['gcLink']['text']=='Anrufen':
phone = elem['gcLink']['href'].split(":")[1]
elif elem['gcLink']['text']=='Mehr Details':
detail_page_link = elem['gcLink']['href']
all_details.append([name,email,phone,detail_page_link])
return all_details
with requests.Session() as s:
s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'
resp = s.get(link)
soup = BeautifulSoup(resp.text,"lxml")
all_details = get_lazyloaded_details(soup)
for item in soup.select("article:not([data-lazyloaddata])"):
name = item.select_one("h2[data-wipe-name]").get_text(strip=True)
email = item.select_one("a.contains-icon-email").get("href").split("?")[0].split(":")[1]
phone = item.select_one("p.mod-AdresseKompakt__phoneNumber").text
detail_page_link = item.select_one("a.contains-icon-details").get("href")
all_details.append([name,email,phone,detail_page_link])
for element in all_details:
print(element)
Output (truncated):
['Scherschel Andreas', 'info#scherschel.com', '0221 38 59 93', 'https://www.gelbeseiten.de/gsbiz/5cac50a0-a60e-49ef-8937-2f0a7bcad241']
['Scherschel Andreas Dipl.-Ing. KFZ-Sachverst.', 'info#scherschel.com', '0221 38 59 93', 'https://www.gelbeseiten.de/gsbiz/5cac50a0-a60e-49ef-8937-2f0a7bcad241']
['La Linea Franca GmbH,Volvo Vertragshändler KFZ-Handel', 'buschdorf#lalinea.de', '0221800600', 'https://www.gelbeseiten.de/gsbiz/1db12570-6d7c-4fe2-ab3d-d0c02e74324a']
['Kuehler Willems Services UG', 'info#kuehler-willems.eu', '02213761111', 'https://www.gelbeseiten.de/gsbiz/54d91f8c-eebe-41e3-962e-acdbfac6be46']

Related

How to scrape all links of products using selenium python?

There is a webpage and 42 products. I would like to get all links of 42 products to scrape individually them. But When I try to get them, I am getting only 16-20 of them.
I used two approaches:
I got page source using Selenium then scraped with BeautifulSoup
I only used selenium driver(css_selector, class_name) to get links.
The link need to scrape: https://thrivecausemetics.com/collections/all?page=4&sort=ss_days_since_published%253Dasc
my 1st approach code:
driver = webdriver.Chrome()
webpage = "https://thrivecausemetics.com/collections/all?page=4&sort=ss_days_since_published%253Dasc"
driver.get(webpage)
time.sleep(15)
page_source = driver.page_source
soup = BeautifulSoup(page_source, 'lxml')
links = [link['href'] for link in soup.find("ul", class_="grid-list").find_all('a', class_='tile-images')]
print(links)
print(len(links))
my 2nd approach
driver = webdriver.Chrome()
webpage = "https://thrivecausemetics.com/collections/all?page=4&sort=ss_days_since_published%253Dasc"
driver.get(webpage)
time.sleep(15)
ul_tag = driver.find_element(By.CSS_SELECTOR, "ul.grid-list")
print(ul_tag)
li_tags = ul_tag.find_elements(By.CSS_SELECTOR, "li.grid-item.is-visible")
# print(li_tags)
print(len(li_tags))
All two approaches are helping to get all links. Using above codes, it is taking only 16 product links.
Any help is appreciate

That data is being pulled from an API endpoint by javascript, once the page loads, so requests cannot see it. The way forward is to scrape the actual API endpoint (you can find it in Dev tools - Network tab). Here is one way to obtain that data:
import requests
import pandas as pd
url = 'https://b7i79y.a.searchspring.io/api/search/search.json?resultsFormat=native&page=1&resultsPerPage=500&sort.ss_days_since_published=asc&siteId=b7i79y'
r = requests.get(url)
df = pd.json_normalize(r.json()['results'])
print(df)
This will display in terminal:
brand collection_id handle id imageUrl intellisuggestData intellisuggestSignature msrp name popularity price product_type_unigram rating ratingCount reviews_total_reviews sku ss_available ss_image_alt ss_inventory_count ss_name_type tags thumbnailImageUrl uid url variant_id variant_mfield_filter_color
0 Bigger Than Beauty Skincare [159254708314, 174020034650, 262184763482, 263320010842] pumpkin-spice-latte-liquid-balm-treatment bed045c1cec90548f830bfa4bc3e2e56 https://cdn.shopify.com/s/files/1/0582/2885/products/PSL_Component_1_medium.jpg?v=1662478574 eJxKMs80t6xkYGAICXM3NDZhMGQwZDBgMLdgSC_KTAEEAAD__1t7Bhw 5a3173ae3360eadabcc446e464c51a6269f0e28ab8d79b2be8b1da2b0f0201da 0 Pumpkin Spice Latte Liquid Balm Lip Treatment™ 10669 26 treatment 4.45424 295 295 TVG134 1 https://cdn.shopify.com/s/files/1/0582/2885/products/PSL_Swatch_New_medium.jpg?v=1662478574 20060 lip treatment [2261, 4522, 50, 800, Benefits:Hydrating, Benefits:Plumping, collection-badge::BACK IN STOCK!, collection::hide-variants, Face, Fill Size:< 1 fl oz, linked::liquid-balm-set, lip plumper, lip plumping, plump, plumper, plumping, recommendation::all-skincare, Skin Concern:Dull and Dry Skin, swatches::show, travel size, Vegan] https://cdn.shopify.com/s/files/1/0582/2885/products/PSL_Component_1_medium.jpg?v=1662478574 4742230212698 https://thrive-causemetics.myshopify.com/products/pumpkin-spice-latte-liquid-balm-treatment [32526428766298] NaN
1 Thrive Causemetics NaN dream-lash-duo 26b794e35fad33ba5496223db9f1bed4 https://cdn.shopify.com/s/files/1/0582/2885/products/Mascara_LashSerum_PDPSets_medium.jpg?v=1659461093 eJxKMs80t6xkYGAICXM3NDZhMGQwYjBgMLdgSC_KTAEEAAD__1uGBh0 12ef5b3a76c62cc8e9d2b0f6f2b2341a3903bbc584f3c347b96f6b9d67f38c05 0 Dream Lash Duo NaN 71 duo NaN NaN NaN NaN 1 https://cdn.shopify.com/s/files/1/0582/2885/products/Mascara_LashSerum_PDPSets_nocopy_medium.jpg?v=1659491015 274 dream lash duo [collection::hide-variants, YBlacklist] https://cdn.shopify.com/s/files/1/0582/2885/products/Mascara_LashSerum_PDPSets_medium.jpg?v=1659461093 6766529675354 https://thrive-causemetics.myshopify.com/products/dream-lash-duo [40035119235162, 40035119267930, 40035119300698] NaN
2 Thrive Causemetics NaN liquid-lash-extensions-lash-serum 096bf1756363b494a31863ae20803818 https://cdn.shopify.com/s/files/1/0582/2885/products/LashSerum_Component_medium.jpg?v=1659566057 eJxKMs80t6xkYGAICXM3MrNgMGQwZjBgMLdgSC_KTAEEAAD__1wOBiY 5fa69eead6ac5da701c5be908298ba006e9490183a18de4e398e7599d0a01eb6 0 Liquid Lash Extensions™ Lash Serum 21949 56 serum 4.075 40 40 TVG268 1 NaN 75132 lash serum [collection-badge::New!] https://cdn.shopify.com/s/files/1/0582/2885/products/LashSerum_Component_medium.jpg?v=1659566057 6729553772634 https://thrive-causemetics.myshopify.com/products/liquid-lash-extensions-lash-serum [39909600854106] NaN
3 Thrive Causemetics [267668095066] brilliant-face-highlighter-skin-perfecting-powder 9ff61df38853620f61d4c39e7363f5a2 https://cdn.shopify.com/s/files/1/0582/2885/products/Brilliant-Face-Highlighter_Component_ToQuyen_medium.jpg?v=1657292791 eJxKMs80t6xkYGAICXM3MjJnMGQwYTBgMLdgSC_KTAEEAAD__1vKBiI 7d651c91af12c272ce4478e268a2764530f5df50b7a4a78817eaeda1251cd85b 0 Brilliant Face Highlighter™ Skin Perfecting Powder 12525 34 highlighter 4.18182 66 66 TVG227 1 https://cdn.shopify.com/s/files/1/0582/2885/products/Brilliant-Face-Highlighter_Component_Shael_medium.jpg?v=1657292793 44920 highlighter [collection-badge::trending, Highlight, Highlighter, Highlighting] https://cdn.shopify.com/s/files/1/0582/2885/products/Brilliant-Face-Highlighter_Component_ToQuyen_medium.jpg?v=1657292791 6729555247194 https://thrive-causemetics.myshopify.com/products/brilliant-face-highlighter-skin-perfecting-powder [39909605703770, 39909605736538, 39909605769306] [gold]
4 Thrive Causemetics NaN brilliant-face-set dab6ca20bb4cf41740cacbbc37fb4f20 https://cdn.shopify.com/s/files/1/0582/2885/products/Highlighter_BEB_Primer_Set_PDP_medium.jpg?v=1657585503 eJxKMs80t6xkYGAICXM3MjJnMGQwZTBgMLdgSC_KTAEEAAD__1vVBiM c0e8eb4abe31f0bd31324450909de60b570dafb73a7e5f6f4c3cb49e93b7a9e4 0 Brilliant Face Set NaN 84 sets NaN NaN NaN NaN 1 https://cdn.shopify.com/s/files/1/0582/2885/products/Highlighter_BEB_Primer_Set_V2_medium.jpg?v=1657585503 3889 brilliant face sets [collection-badge::New!, collection::hide-variants, ST-unpublished] https://cdn.shopify.com/s/files/1/0582/2885/products/Highlighter_BEB_Primer_Set_PDP_medium.jpg?v=1657585503 6765261324378 https://thrive-causemetics.myshopify.com/products/brilliant-face-set [40031327682650, 40031327715418, 40031327748186, 40031327780954, 40031327813722, 40031327846490, 40031327879258, 40031327912026, 40031327944794, 40031327977562, 40031328010330, 40031328043098, 40031328075866, 40031328108634, 40031328141402, 40031328174170, 40031328206938, 40031328239706, 40031328272474, 40031328305242, 40031328338010, 40031328370778, 40031328403546, 40031328436314, 40031328469082, 40031328501850, 40031328534618, 40031328567386, 40031328600154, 40031328632922, 40031328665690, 40031328698458, 40031328731226, 40031328763994, 40031328796762, 40031328829530, 40031328862298, 40031328895066, 40031328927834] NaN
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
73 Thrive Causemetics [27186779, 209907910, 237619142, 333714566, 399044812, 2738815001, 4013293593, 5575639065, 5575770137, 5575802905, 6686801945, 56789368922, 56833736794, 57405112410, 81244520538, 82036260954, 82769608794, 83615547482, 84961689690, 85599420506, 86598778970, 87577591898, 88078483546, 89189417050, 89505661018, 89797230682, 91221393498, 91755741274, 93260021850, 96613498970, 149781872730, 153671794778, 157076848730, 159474384986, 166734495834, 173874675802, 262216908890, 262273564762, 263767261274, 263970160730, 264544747610, 264579055706, 264579121242, 265485779034, 266068197466, 266346889306, 266381164634, 266457251930, 266889232474, 267015094362, 267954094170] triple-threat-color-stick 908b72d51839441d48d27fc251340e88 https://cdn.shopify.com/s/files/1/0582/2885/products/TCCS_Triple_Threat_Color_Stick_Isabella_V2_2db06b39-24da-4e68-8029-dab1f68a985e_medium.jpg?v=1601483873 eJxKMs80t6xkYGAICXM3NDVhMGQwN2EwYDC3YEgvykwBBAAA__9h-AZY 67aceb71cc8a5bfa95da136235f5c18504688f10daed921caa8efe7e75dcfa8d 0 Triple Threat™ Color Stick 35732 36 threat 4.42416 3171 3171 TVG154 1 https://cdn.shopify.com/s/files/1/0582/2885/products/TCCS_Triple_Threat_Color_Stick_Mieko_V2_e92bcbce-3708-4941-a23b-5122e9881820_medium.jpg?v=1601483873 164632 triple threat [Benefits:Hydrating, Benefits:Waterproof, Best Sellers, blush, body, collection-badge::Multi-Use!, Coverage:Buildable, Finish:Dewy, Finish:Shimmer, Formulation:Cream, intl::ca, Lips, Lipstick, recommendation::face, shade-finder::thumbnails, Triple Threat Color Stick, TVG285, TVG286, TVG287, Vegan, YCRF_cheeks] https://cdn.shopify.com/s/files/1/0582/2885/products/TCCS_Triple_Threat_Color_Stick_Isabella_V2_2db06b39-24da-4e68-8029-dab1f68a985e_medium.jpg?v=1601483873 5892103302 https://thrive-causemetics.myshopify.com/products/triple-threat-color-stick [32456620376154, 18635615622, 18635615558, 32456620310618, 32456620408922, 18635615430, 18635615686, 40078997586010, 40078998175834, 40078999191642] [pink, gold, purple, red, peach]
74 Thrive Causemetics [27186779, 209907910, 237619142, 343406086, 383763660, 2738815001, 4013293593, 6686801945, 6845464601, 57475530842, 81244487770, 81951588442, 82036260954, 82769608794, 83476283482, 83810091098, 86001451098, 86594289754, 86765207642, 88078483546, 91221393498, 93260021850, 93929963610, 94846025818, 149781872730, 151323705434, 157076848730, 159474384986, 162671919194, 166112591962, 263766736986, 264805384282, 266185965658, 267195973722] infinity-waterproof-brow-liner 05a0b2ec067e0d40becc91a6d7ff10a9 https://cdn.shopify.com/s/files/1/0582/2885/products/BrowLiner_Component_Christina_medium.jpg?v=1637091941 eJxKMs80t6xkYGAICXM3MLRgMGQwN2UwYDC3YEgvykwBBAAA__9h7QZY c25ffd3a46285a1f90da35783af2fb62d7900453d9fa7fbd7e288f8fd13b9f1d 0 Infinity Waterproof Eyebrow Liner™ 39291 23 liner 4.49396 2235 2235 TVG018 1 https://cdn.shopify.com/s/files/1/0582/2885/products/BrowLiner_Component_Audrey_medium.jpg?v=1637091946 209279 brow liner [Benefits:Waterproof, Coverage:Buildable, default_variant::2, Infinity Waterproof Brow Liner, Ingredients:Shea Butter, intl::ca, recommendation::eyes, shade-finder::thumbnails, Vegan, YCRF_eyes] https://cdn.shopify.com/s/files/1/0582/2885/products/BrowLiner_Component_Christina_medium.jpg?v=1637091941 781737155 https://thrive-causemetics.myshopify.com/products/infinity-waterproof-brow-liner [2199676227, 39591112081498, 2199676163, 35014122444, 39591112343642] [beige, red, brown, black, grey]
75 Thrive Causemetics [27186779, 91101891, 5576228889, 81244487770, 174020034650] gift-card 15c74aab8aa83d300f8c66cdca7c1cb1 https://cdn.shopify.com/s/files/1/0582/2885/products/egift-card_1__2_medium.png?v=1659654650 eJxKMs80t6xkYGAICXM3MLRgMGQwN2MwYDC3YEgvykwBBAAA__9h-AZZ 7ba842c650d93f11c8936dfaf70818667c3b3024b44fc6688ee25874d0ccf019 0 eGift Card NaN 25 card 5 11 11 NaN 1 https://cdn.shopify.com/s/files/1/0582/2885/products/Thrive_PDP_GiftCard_medium.jpg?v=1659654650 -16399 gift card [::hide-dropdown-swatch, collection::hide-variants, Gift Cards, image::no-swap, intl::ca, swag, YBlacklist] https://cdn.shopify.com/s/files/1/0582/2885/products/egift-card_1__2_medium.png?v=1659654650 337553443 https://thrive-causemetics.myshopify.com/products/gift-card [12622098246, 782092871, 12622102150, 782092875] NaN
76 Thrive Causemetics [27186779, 237619142, 343406086, 389141580, 81244520538, 91755741274, 153671794778, 157076848730, 159474384986] jackie 4f5bf8da32c6904051e29c44b49a4516 https://cdn.shopify.com/s/files/1/0582/2885/products/Jackie_Faux_Lashes_1_medium.jpg?v=1582596256 eJxKMs80t6xkYGAICXM3NDdiMGQwN2cwYDC3YEgvykwBBAAA__9iGwZb 3425b4b7cf441485cfbc7cc37da68ae40efbe65e842df16229f0c1c29b172b7d 0 Jackie Faux Lashes™ 150 26 lashes 4.85714 14 14 TVG172 1 https://cdn.shopify.com/s/files/1/0582/2885/products/PDP_lashes_jackie_1024x1024_1_medium.jpg?v=1582596246 827 faux lashes [Faux Lashes, recommendation::eyes, swatches::hide, Vegan, YCRF_eyes] https://cdn.shopify.com/s/files/1/0582/2885/products/Jackie_Faux_Lashes_1_medium.jpg?v=1582596256 334825111 https://thrive-causemetics.myshopify.com/products/jackie [775766255] NaN
77 Thrive Causemetics [27186779, 237619142, 343406086, 389141580, 81244520538, 91755741274, 157076848730, 159474384986] robin cfe488b97e5e61b13c3060260a920885 https://cdn.shopify.com/s/files/1/0582/2885/products/Robin_Faux_Lashes_medium.jpg?v=1582233291 eJxKMs80t6xkYGAICXM3NDdmMGQwt2AwABHpRZkpgAAAAP__YjYGXQ 3b387d1b5edb7855f9d83403ddac5c5559ddac6a8ea440cde30447897896cfa6 0 Robin Faux Lashes™ 130 26 lashes 4.9 10 10 TVG173 1 https://cdn.shopify.com/s/files/1/0582/2885/products/PDP_lashes_robin_1024x1024_7a28a8f4-b602-4049-9480-6eddb8e94944_medium.jpg?v=1582233282 2152 faux lashes [Faux Lashes, recommendation::eyes, swatches::hide, Vegan, YCRF_eyes] https://cdn.shopify.com/s/files/1/0582/2885/products/Robin_Faux_Lashes_medium.jpg?v=1582233291 334825555 https://thrive-causemetics.myshopify.com/products/robin [775768027] NaN
78 rows × 26 columns
The actual XHR request is asking only for 12 products (and then continues to ask for more products, as you scroll the page). I went ahead and asked for 500 products (see url), to make sure I get them all.
Requests documentation:https://requests.readthedocs.io/en/latest/
Also, pandas relevant documentation:
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.json_normalize.html
EDIT: And here is a solution based off Selenium/chromedriver. Setup is for linux/chrome/chromedriver, you can adapt to your own setup - just observe the imports, and the code after defining the browser/driver:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import time as t
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
browser = webdriver.Chrome(service=webdriver_service, options=chrome_options)
actions = ActionChains(browser)
wait = WebDriverWait(browser, 20)
url = 'https://thrivecausemetics.com/collections/all?page=4&sort=ss_days_since_published%253Dasc'
browser.get(url)
try:
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.ID, "onetrust-reject-all-handler"))).click()
print('declined cookies')
except Exception as e:
print('no cookie button!')
t.sleep(2)
try:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'div[class="dialog dialog-email"]'))).find_element(By.CSS_SELECTOR, 'div[class="icon close"]').click()
print('dismissed 10% offer')
except Exception as e:
print('no 10% offer, damn')
try:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'div[class="dialog dialog-country"]'))).find_element(By.CSS_SELECTOR, 'div[class="icon close"]').click()
print('dismissed country popup')
except Exception as e:
print('no country popup')
products = [x.find_element(By.TAG_NAME, 'a') for x in wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'li[class="grid-item is-visible"]'))) if len(x.text) > 3]
print('Total items:', len(products))
for p in products:
print(p.get_attribute('href'))
print('______________')
Result printed in terminal:
declined cookies
dismissed 10% offer
dismissed country popup
Total items: 42
https://thrivecausemetics.com/products/brilliant-eye-brightener
______________
https://thrivecausemetics.com/products/liquid-lash-extensions-mascara
______________
https://thrivecausemetics.com/products/waterproof-eyeliner
______________
https://thrivecausemetics.com/products/sheer-strength-hydrating-lip-tint
______________
https://thrivecausemetics.com/products/infinity-waterproof-eyeshadow-stick
______________
https://thrivecausemetics.com/products/triple-threat-color-stick
______________
https://thrivecausemetics.com/products/infinity-waterproof-brow-liner
[...]
For Selenium documentation, please visit https://www.selenium.dev/documentation/

Try this code:
ul_tag = driver.find_elements(By.CSS_SELECTOR, ".grid-list.text-.align- .grid-item.is-visible .tile-heading-lockup a")
print("Total products: ", len(ul_tag))
for product_link in ul_tag:
print("Product link: ", product_link.get_attribute("href"))
Output:
Total products: 42
Product link: https://thrivecausemetics.com/products/brilliant-eye-brightener
Product link: https://thrivecausemetics.com/products/liquid-lash-extensions-mascara
Product link: https://thrivecausemetics.com/products/waterproof-eyeliner
Product link: https://thrivecausemetics.com/products/sheer-strength-hydrating-lip-tint
Product link: https://thrivecausemetics.com/products/infinity-waterproof-eyeshadow-stick
Product link: https://thrivecausemetics.com/products/triple-threat-color-stick
Product link: https://thrivecausemetics.com/products/infinity-waterproof-brow-liner
Product link: https://thrivecausemetics.com/products/instant-brow-fix-semi-permanent-eyebrow-gel
Product link: https://thrivecausemetics.com/products/liquid-lash-extensions-lash-serum
Product link: https://thrivecausemetics.com/products/buildable-blur-cc-cream-with-spf-35
and so on...

How can I make click over a SVG element?

I'm newbie with web scraping and using selenium I would like to make click over a SVG element to get access to the information showed by a modal window.
Making click over a point or cross in the basketball court of this webpage: https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart You will get a modal window with information like you can see in the next picture:
I have made this development which works find because find the "svg" element printing the values of his attributes "x" and "y":
b = self.driver
b.set_window_size(300, 300)
b.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
periods = b.find_element_by_css_selector('g.team-A').find_element_by_css_selector('g.period-list').find_elements_by_css_selector("g.period-item")
for quarter in periods:
shots = quarter.find_element_by_css_selector("g.shot-list").find_elements_by_css_selector("g.shot-item")
for item in shots:
print(f"x: {item.find_element_by_tag_name('svg').get_attribute('x')} - y: {item.find_element_by_tag_name('svg').get_attribute('y')}")
print("\n")
You can see exit of the code here:
x: 40.5 - y: 99.5
x: 151.5 - y: 211.5
x: 34.5 - y: 125.5
x: 35.5 - y: 121.5
x: 157.5 - y: 204.5
x: 59.5 - y: 122.5
x: 32 - y: 142
x: 40 - y: 121
x: 27.5 - y: 117.5
x: 164 - y: 124
x: 80.5 - y: 7.5
x: 49.5 - y: 111.5
x: 135.5 - y: 42.5
x: 34.5 - y: 67.5
x: 27.5 - y: 117.5
x: 138 - y: 54
x: 22 - y: 140
x: 119.5 - y: 32.5
x: 135.5 - y: 42.5
x: 154.5 - y: 186.5
x: 37.5 - y: 106.5
x: 39 - y: 117
x: 31 - y: 114
x: 40.5 - y: 117.5
x: 22 - y: 5
x: 46.5 - y: 4.5
x: 20 - y: 125
x: 148.5 - y: 197.5
x: 71.5 - y: 169.5
x: 118 - y: 230
x: 30.5 - y: 263.5
x: 25 - y: 124
x: 135.5 - y: 213.5
x: 82.5 - y: 128.5
x: 40 - y: 119
x: 158.5 - y: 131.5
x: 50.5 - y: 174.5
x: 166.5 - y: 82.5
x: 26 - y: 149
x: 36 - y: 133
x: 114.5 - y: 239.5
x: 48 - y: 222
x: 127.5 - y: 226.5
x: 23 - y: 132
x: 110.5 - y: 107.5
x: 114 - y: 138
x: 15 - y: 260
x: 137.5 - y: 131.5
x: 34 - y: 118
x: 75 - y: 65
x: 54.5 - y: 167.5
x: 30.5 - y: 127.5
But, If I try to make click over "svg" component adding this code:
point = item.find_element_by_tag_name('svg')
point.click()
Finally, my code will be:
b = self.driver
b.set_window_size(300, 300)
b.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
periods = b.find_element_by_css_selector('g.team-A').find_element_by_css_selector('g.period-list').find_elements_by_css_selector("g.period-item")
for quarter in periods:
shots = quarter.find_element_by_css_selector("g.shot-list").find_elements_by_css_selector("g.shot-item")
for item in shots:
print(f"x: {item.find_element_by_tag_name('svg').get_attribute('x')} - y: {item.find_element_by_tag_name('svg').get_attribute('y')}")
point = item.find_element_by_tag_name('svg')
point.click()
print("\n")
I've got this error:
selenium.common.exceptions.WebDriverException: Message: unknown error: Element <svg class="shot-miss icon icon-miss clickable" x="40.5" y="99.5" width="16" height="16" viewBox="0 0 30 30" title="">...</svg> is not clickable at point (44, 165). Other element would receive the click: <p class="cc_message">...</p>
(Session info: chrome=72.0.3626.121)
(Driver info: chromedriver=2.44.609551 (5d576e9a44fe4c5b6a07e568f1ebc753f1214634),platform=Linux 5.4.0-88-generic x86_64)
How is that possible? What am I doing wrong? How can I get the content of the modal window?
Edit I (solution provided by #Prophet):
Now, my code is:
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
b = self.driver
wait = WebDriverWait(b, 20)
b.set_window_size(1920, 1080)
b.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a.cc_btn_accept_all"))).click()
b.execute_script("window.scrollTo(0, document.body.scrollHeight);")
periods = b.find_element_by_css_selector('g.team-A').find_element_by_css_selector('g.period-list').find_elements_by_css_selector("g.period-item")
for quarter in periods:
shots = quarter.find_element_by_css_selector("g.shot-list").find_elements_by_css_selector("g.shot-item")
for item in shots:
print(f"x: {item.find_element_by_tag_name('svg').get_attribute('x')} - y: {item.find_element_by_tag_name('svg').get_attribute('y')}")
point = item.find_element_by_tag_name('svg')
point.click()
print("\n")
And I've got this error:
selenium.common.exceptions.WebDriverException: Message: unknown error: Element <svg class="shot-miss icon icon-miss clickable" x="151.5" y="211.5" width="16" height="16" viewBox="0 0 30 30" title="">...</svg> is not clickable at point (805, 312). Other element would receive the click: <th class="player"></th>
(Session info: chrome=72.0.3626.121)
(Driver info: chromedriver=2.44.609551 (5d576e9a44fe4c5b6a07e568f1ebc753f1214634),platform=Linux 5.4.0-88-generic x86_64)
Edit II (solution provided by #cruisepandey:
I have edited my code and now I've got this code:
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
b = self.driver
b.set_window_size(1920, 1080)
b.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
b.execute_script("window.scrollTo(0, document.body.scrollHeight);")
periods = b.find_element_by_css_selector('g.team-A').find_element_by_css_selector('g.period-list').find_elements_by_css_selector("g.period-item")
WebDriverWait(b, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[name()='svg']"))).click()
for quarter in periods:
shots = quarter.find_element_by_css_selector("g.shot-list").find_elements_by_css_selector("g.shot-item")
for item in shots:
print(f"x: {item.find_element_by_tag_name('svg').get_attribute('x')} - y: {item.find_element_by_tag_name('svg').get_attribute('y')}")
point = item.find_element_by_xpath("//*[name()='svg']")
point.click()
print("\n")
But, It doesn't work :( I've got this error:
Traceback (most recent call last):
File "/home/josecarlos/Workspace/python/basketmetrics/test/test_shot_chart_get_data.py", line 27, in test_something
WebDriverWait(b, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[name()='svg']"))).click()
File "/home/josecarlos/Workspace/python/basketmetrics/venv/python/lib/python3.8/site-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
Edit III:
I have tried to access to the "svg" tag through his absoloute XPath route. Using the tools for developers of Firefox or Chrome, we can get this absolute Xpath. In my case, I have got the XPath of the element in blue:
This is the route:
/html/body/div[3]/div[3]/div/section/div[2]/div/div/ul[2]/li[6]/div/div[1]/div/div[2]/div[2]/div[2]/svg/g[1]/g/g[1]/g/g[1]/svg
If I try to acces to this route in my code I've got an error
b = self.driver
wait = WebDriverWait(b, 20)
b.set_window_size(300, 300)
b.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
periods = b.find_element_by_css_selector('g.team-A').find_element_by_css_selector('g.period-list').find_elements_by_css_selector("g.period-item")
#WebDriverWait(b, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[name()='svg']"))).click()
for quarter in periods:
shots = quarter.find_element_by_css_selector("g.shot-list").find_elements_by_css_selector("g.shot-item")
for item in shots:
print(f"x: {item.find_element_by_tag_name('svg').get_attribute('x')} - y: {item.find_element_by_tag_name('svg').get_attribute('y')}")
point = b.find_element_by_xpath("/html/body/div[3]/div[3]/div/section/div[2]/div/div/ul[2]/li[6]/div/div[1]/div/div[2]/div[2]/div[2]/svg/g[1]/g/g[1]/g/g[1]/svg")
point.click()
print("\n")
The error that I've got is this:
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"/html/body/div[3]/div[3]/div/section/div[2]/div/div/ul[2]/li[6]/div/div[1]/div/div[2]/div[2]/div[2]/svg/g[1]/g/g[1]/g/g[1]/svg"}
(Session info: chrome=72.0.3626.121)
(Driver info: chromedriver=2.44.609551 (5d576e9a44fe4c5b6a07e568f1ebc753f1214634),platform=Linux 5.4.0-88-generic x86_64)
What am I doing wrong? What happend?
Edit IV:
I have modified in my code the string who gives me access to the svg but it doesn't work.
b = self.driver
wait = WebDriverWait(b, 20)
b.set_window_size(1920, 1080)
b.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
periods = b.find_element_by_css_selector('g.team-A').find_element_by_css_selector('g.period-list').find_elements_by_css_selector("g.period-item")
#WebDriverWait(b, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[name()='svg']"))).click()
for quarter in periods:
shots = quarter.find_element_by_css_selector("g.shot-list").find_elements_by_css_selector("g.shot-item")
for item in shots:
print(f"x: {item.find_element_by_tag_name('svg').get_attribute('x')} - y: {item.find_element_by_tag_name('svg').get_attribute('y')}")
point = b.find_element_by_xpath("//div[#class='shot-chart_canvas']/*[name()='svg']/*[name()='g'][1]/*[name()='g']/*[name()='g'][1]/*[name()='g']/*[name()='g'][1]/*[name()='svg']")
point.click()
print("\n")
self.assertEqual(True, True, "Error!!!")
And the error that I get is:
selenium.common.exceptions.WebDriverException: Message: unknown error: Element <svg class="shot-miss icon icon-miss clickable" x="40.5" y="99.5" width="16" height="16" viewBox="0 0 30 30" title="">...</svg> is not clickable at point (687, 917). Other element would receive the click: <div class="cc_banner cc_container cc_container--open">...</div>
(Session info: chrome=72.0.3626.121)
(Driver info: chromedriver=2.44.609551 (5d576e9a44fe4c5b6a07e568f1ebc753f1214634),platform=Linux 5.4.0-88-generic x86_64)
Sorry, but I don't understand what happend :(((
Edit V:
I have to retrive the information of all the crosses and points. I've got two teams and inside this tag we've got an array of periods and inside of each period we've got all the shoots and of each shot-item I need to retrieve the information. So, how can I make an Xpath loop to retrieve the information of each shoot?

g tag is under svg tag. so for locating g.team-A
Please use the below xpath :
//*[name()='g' and #class='team-A']
this is an xpath expression.
so the possible fix for this :
point = item.find_element_by_tag_name('svg')
point.click()
to use this :
point = item.find_element_by_xpath("//*[name()='svg']")
point.click()
What I would suggest here is to have a explicit wait defined and then you can try to click on it.
Code-trial :
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[name()='svg']"))).click()
for this explicit wait, you'd have to import these as well :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Update :
driver = webdriver.Chrome(driver_path)
driver.maximize_window()
wait = WebDriverWait(driver, 30)
driver.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
ele = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "div.shot-chart_canvas")))
driver.execute_script("arguments[0].scrollIntoView(true);", ele)
wait.until(EC.element_to_be_clickable((By.XPATH, "//*[name()='svg' and #class='chart']//*[name()='g']//descendant::*[name()='g' and #class='shot-item']"))).click()
a = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.player-profile"))).get_attribute('innerText')
print(a)
Imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
output :
Q1 09:520 0
Alexandra Stolyar
2pt jump shot missed
View Player Profile
FG 2Pts 3Pts FT Pts
In this game
2/6
33.33%
1/4
25%
1/2
50%
3/3
100%
8
Update 2 :
driver.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
ele = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "div.shot-chart_canvas")))
driver.execute_script("arguments[0].scrollIntoView(true);", ele)
all_points = wait.until(EC.presence_of_all_elements_located((By.XPATH, "//*[name()='svg' and #class='chart']//*[name()='g']//descendant::*[name()='g' and #class='shot-item']")))
print(len(all_points))
for point in all_points:
point.click()
a = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.player-profile"))).get_attribute('innerText')
print(a)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button.icon--close"))).click()
time.sleep(1)
Imports :
119
Q1 09:520 0
Alexandra Stolyar
2pt jump shot missed
View Player Profile
FG 2Pts 3Pts FT Pts
In this game
2/6
33.33%
1/4
25%
1/2
50%
3/3
100%
8
Q1 09:350 0
Karina Nizamova
3pt jump shot missed
View Player Profile
FG 2Pts 3Pts FT Pts
In this game
1/7
14.29%
1/3
33.33%
0/4
0%
2/3
66.67%
4

The elements you are trying to click on are initially out of the visible screen. Also there is a "accept cookies" banner on the bottom.
You need to close the cookies banner and scroll the page up in order to make your code working.
Please try this:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
b = self.driver
wait = WebDriverWait(b, 20)
b.set_window_size(300, 300)
b.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a.cc_btn_accept_all"))).click()
b.execute_script("window.scrollTo(0, document.body.scrollHeight);")
periods = b.find_element_by_css_selector('g.team-A').find_element_by_css_selector('g.period-list').find_elements_by_css_selector("g.period-item")
for quarter in periods:
shots = quarter.find_element_by_css_selector("g.shot-list").find_elements_by_css_selector("g.shot-item")
for item in shots:
print(f"x: {item.find_element_by_tag_name('svg').get_attribute('x')} - y: {item.find_element_by_tag_name('svg').get_attribute('y')}")
point = item.find_element_by_tag_name('svg')
point.click()
print("\n")
I would also recommend to use a normal window size, as following:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
b = self.driver
wait = WebDriverWait(b, 20)
b.set_window_size(1920, 1080)
b.get("https://www.fiba.basketball/euroleaguewomen/21-22/game/1310/MBA-Moscow-ZVVZ-USK-Praha#tab=shot_chart")
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a.cc_btn_accept_all"))).click()
b.execute_script("window.scrollTo(0, document.body.scrollHeight);")
periods = b.find_element_by_css_selector('g.team-A').find_element_by_css_selector('g.period-list').find_elements_by_css_selector("g.period-item")
for quarter in periods:
shots = quarter.find_element_by_css_selector("g.shot-list").find_elements_by_css_selector("g.shot-item")
for item in shots:
print(f"x: {item.find_element_by_tag_name('svg').get_attribute('x')} - y: {item.find_element_by_tag_name('svg').get_attribute('y')}")
point = item.find_element_by_tag_name('svg')
point.click()
print("\n")
UPD
Svg, rect,g etc are special tag names.
/svg or /g Xpath will not work. You will need to use /*[name()='svg'] or /*[name()='g'] respectively.
Also, the absolute paths are strongly NOT recommended. You need to use better locators.
For example instead of
/html/body/div[3]/div[3]/div/section/div[2]/div/div/ul[2]/li[6]/div/div[1]/div/div[2]/div[2]/div[2]/svg/g[1]/g/g[1]/g/g[1]/svg
This will work better:
//div[#class='shot-chart_canvas']/*[name()='svg']/*[name()='g'][1]/*[name()='g']/*[name()='g'][1]/*[name()='g']/*[name()='g'][1]/*[name()='svg']
So instead of
point = b.find_element_by_xpath("/html/body/div[3]/div[3]/div/section/div[2]/div/div/ul[2]/li[6]/div/div[1]/div/div[2]/div[2]/div[2]/svg/g[1]/g/g[1]/g/g[1]/svg")
point.click()
Try this:
point = b.find_element_by_xpath("//div[#class='shot-chart_canvas']/*[name()='svg']/*[name()='g'][1]/*[name()='g']/*[name()='g'][1]/*[name()='g']/*[name()='g'][1]/*[name()='svg']")
point.click()

how to use regular expression in python to get memory details in product name

I am trying to use regular expression to parse memory of the product.
product name : 1) Lg K42 Blu Tim Smartphone 64 gb
2) Xiaomi Smartphone 0.128 gb ram 4 gb. tim quadband - Redmi Note 9 128gb Grigio Tim.
How to get the 64 gb using regex in python. gb may be small or caps and memory value may be contains 2 or 3 numbers
import xlwt
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
import re
import time
from bs4 import BeautifulSoup
from datetime import date
class cometmobiles:
def __init__(self):
self.url='https://www.comet.it/smartphone-e-telefonia/smartphone-e-cellulari/smartphone'
self.country='IT'
self.currency='euro'
self.VAT='Included'
def comet(self):
#try:
wb = xlwt.Workbook()
ws = wb.add_sheet('Sheet1',cell_overwrite_ok=True)
ws.write(0,0,"Product_Url")
ws.write(0,1,"Product_Manufacturer")
ws.write(0,2,"Product_Model")
ws.write(0,3,"Product_memory")
ws.write(0,4,"Product_Price")
ws.write(0,5,"Currency")
ws.write(0,6,"VAT")
ws.write(0,7,"Shipping")
ws.write(0,8,"Country")
ws.write(0,9,"Date")
wb.save(r"C:\Users\Karthick R\Desktop\VS code\comet.xls")
driver=webdriver.Chrome()
today = date.today()
driver.maximize_window()
driver.implicitly_wait(30)
driver.get(self.url)
wait = WebDriverWait(driver, 20)
cookies = driver.find_element_by_xpath('//i[#class="btn-close-popup fas fa-times"]')
cookies.click()
time.sleep(5)
print("clicked")
x = 0
titles = []
models = []
memorys = []
prices = []
links =[]
product_links = []
while True:
containers = driver.find_elements_by_css_selector('div[class="col-12 col-sm-6 col-md-4"]')
i = 1
for container in containers:
url = container.find_element_by_css_selector('div[class="sotto-cat__products__item"]')
urls = url.find_element_by_tag_name('a').get_attribute('href')
i = i + 1
product_links.append(urls)
print(product_links)
x+=1
time.sleep(5)
driver.get(self.url+"?p="+str(x))
print("next page")
if driver.current_url == self.url:
break
for links in product_links:
driver.get(links)
time.sleep(10)
#product links
print(driver.current_url)
source = driver.page_source
soup = BeautifulSoup(source,'html.parser')
#title
title = soup.find('h1',{'class':'scheda-prodotto__info__col-sx__title'}).text
y = re.search('([^\s]+)',title)
print(title)
titles.append(y.group(1))
#models
model = re.sub(y.group(1),"",title).strip()
print(model)
models.append(model)
#memory
memory = re.search('^[0-9]{2,3}+[A-Za-z]',model).strip()
print(memory)
memorys.append(memory)
#price
price = soup.find('span',{'class':'caption__price'}).text
print(price)
i=0
while i<len(titles):
ws.write(i+1,0,str(links[i]))
ws.write(i+1,1,str(titles[i]))
ws.write(i+1,2,str(models[i]))
ws.write(i+1,3,str(memorys[i]))
ws.write(i+1,4,str(prices[i]))
ws.write(i+1,5,str(self.currency))
ws.write(i+1,6,str(self.VAT))
ws.write(i+1,7,str(self.shipping))
ws.write(i+1,8,str(self.country))
ws.write(i+1,9,str(date.today()))
i=i+1
wb.save(r"C:\Users\Karthick R\Desktop\VS code\comet.xls")
#except:
#pass
comets=cometmobiles()
comets.comet()

You can try this out.
import re
s = 'Lg K42 Blu Tim Smartphone 64 gb Xiaomi Smartphone 0.128 gb ram 4 gb. tim quadband - Redmi Note 9 128Gb Grigio Tim.'
f = re.findall(r'\b\d{2,3}\s*gb\b',s,re.I)
print(f)
['64 gb', '128 gb', '128Gb']

input_str = "Lg K42 Blu Tim Smartphone 64 gb "
import re
output_value = re.search('[0-9]*.(?=gb)',input_str)
print (output_value[0])
Output : 64
input_str = "Redmi Note 9 128gb Grigio Tim"
Output : 128

Is there any way to get the cookies and cache of a visited website from chrome to beautifulsoup in Python?

I want to scrape a certain website weather data but the default page layout gives max of 40 results but when layout changed to simple list gives 100 results and the layout is set to default which is difficult to achieve with selenium. Is there any way to get the cookies saved in chrome to be used with beautiful soup.
import requests
from bs4 import BeautifulSoup
import browser_cookie3
cj = browser_cookie3.load()
s = requests.Session()
url = "https:/something.org/titles/2"
i=1
print(cj)
for c in cj:
if 'mangadex' in str(c):
s.cookies.set_cookie(c)
r = s.get(url)
soup = BeautifulSoup(r.content, 'lxml')
for anime in soup.find_all('div', {'class': 'manga-entry col-lg-6 border-bottom pl-0 my-1'}):
det = anime.find('a', {"class": "ml-1 manga_title text-truncate"})
anime_name = det.text
anime_link = det['href']
stars = anime.select("span")[3].text
print(anime_name, anime_link, stars,i)
i=i+1

Try:
import browser_cookie3
import requests
cj = browser_cookie3.load()
s = requests.Session()
for c in cj:
if 'sitename' in str(c):
s.cookies.set_cookie(c)
r = s.get(the_site)
This code use the browsers cookies in the requests module in as Session. Simply change sitename to the site you want cookies from.
Your new code:
import requests
from bs4 import BeautifulSoup
import browser_cookie3
cj = browser_cookie3.load()
s = requests.Session()
url = "https://something.org/titles/2"
i = 1
print(cj)
for c in cj:
if 'mangadex' in str(c):
s.cookies.set_cookie(c)
r = s.get(url)
soup = BeautifulSoup(r.content, 'lxml')
for anime in soup.find_all('div', {'class': 'manga-entry row m-0 border-bottom'}):
det = anime.find('a', {"class": "ml-1 manga_title text-truncate"})
anime_name = det.text
anime_link = det['href']
stars = anime.select("span")[3].text
print(anime_name, anime_link, stars, i)
i = i + 1
prints:
-Hitogatana- /title/540/hitogatana 4 1
-PIQUANT- /title/44134/piquant 5 2
-Rain- /title/37103/rain 4 3
-SINS- /title/1098/sins 4
:radical /title/46819/radical 1 5
:REverSAL /title/3877/reversal 3 6
... /title/52206/ 7
...Curtain. ~Sensei to Kiyoraka ni Dousei~ /title/7829/curtain-sensei-to-kiyoraka-ni-dousei 8
...Junai no Seinen /title/28947/junai-no-seinen 9
...no Onna /title/10162/no-onna 2 10
...Seishunchuu! /title/19186/seishunchuu 11
...Virgin Love /title/28945/virgin-love 12
.flow - Untitled (Doujinshi) /title/27292/flow-untitled-doujinshi 2 13
.gohan /title/50410/gohan 14
.hack//4koma + Gag Senshuken /title/7750/hack-4koma-gag-senshuken 24 15
.hack//Alcor - Hagun no Jokyoku /title/24375/hack-alcor-hagun-no-jokyoku 16
.hack//G.U.+ /title/7757/hack-g-u 1 17
.hack//GnU /title/7758/hack-gnu 18
.hack//Link - Tasogare no Kishidan /title/24374/hack-link-tasogare-no-kishidan 1 19
.hack//Tasogare no Udewa Densetsu /title/5817/hack-tasogare-no-udewa-densetsu 20
.hack//XXXX /title/7759/hack-xxxx 21
.traeH /title/9789/traeh 22
(G) Edition /title/886/g-edition 1 23
(Not) a Househusband /title/22832/not-a-househusband 6 24
(R)estauraNTR /title/37551/r-estaurantr 14 25
[ rain ] 1st Story /title/25587/rain-1st-story 3 26
[another] Xak /title/24881/another-xak 27
[es] ~Eternal Sisters~ /title/4879/es-eternal-sisters 1 28
and so on to 100...

Python - Selenium has old data

I am trying to scrape data from www.blocket.se with selenium. All is going good but when i try to get a phone number, its not working. At first there is a button with "Visa telefonnummer", so i press it with selenium. Then phone number appears but when i try to get it with selenium it only gets "Visa telefonnummer" Can you please help me with getting it?
Thanks for answers
driver.get(property_link) # link is ok
driver.find_element_by_class_name("iXyZie").click() # i click the button, phone number appears on screen
phone_number = driver.find_element_by_class_name("fXYiwE").text # it gives me only "Visa telefonnummer"
This is the button before click
<div class="ShowPhoneNumberButton__StyledButton-sc-1fetcgp-0 cMGXxG">
<button type="button" class="Buttonstyles__BaseButton-hz08m4-0 iXyZie"><span class="Buttonstyles__ChildrenWrapper-hz08m4-3 fXYiwE"><svg viewBox="0 0 32 32" color="#363433" width="24" height="24" class="ShowPhoneNumberButton__StyledIconCall-sc-1fetcgp-1 ljfHXc"><defs><path id="iconCall_svg__a" d="M15.466 21.406l5.32-1.401 5.64 8.054-.402.573a8.281 8.281 0 01-2.04 2.02c-1.422.995-2.976 1.494-4.61 1.494-1.428 0-2.916-.38-4.433-1.142-3.098-1.554-6.28-4.645-9.46-9.187C2.302 17.275.487 13.227.085 9.786-.34 6.17.845 3.273 3.506 1.409A8.287 8.287 0 016.103.183L6.78 0l5.64 8.055-3.136 4.52 6.184 8.83zm7.37 7.607a6.501 6.501 0 001.123-.991l-4.011-5.728-5.32 1.4L6.845 12.58l3.136-4.52L5.97 2.332a6.475 6.475 0 00-1.317.716c-2.05 1.436-2.92 3.625-2.584 6.506.363 3.108 2.062 6.849 5.05 11.116 2.987 4.267 5.92 7.143 8.718 8.547 2.594 1.302 4.947 1.232 6.999-.204zm-7.325-12.865a1.5 1.5 0 110-3 1.5 1.5 0 010 3zm6 0a1.5 1.5 0 110-3 1.5 1.5 0 010 3zm6-3a1.5 1.5 0 110 3 1.5 1.5 0 010-3z"></path></defs><use fill="currentColor" xlink:href="#iconCall_svg__a" fill-rule="evenodd" transform="translate(1)"></use></svg><div class="TextCallout1__TextCallout1Wrapper-qzrnab-0 lkIkoj">Visa telefonnummer</div></span></button>
</div>
This is the button after click
<div class="ShowPhoneNumberButton__StyledButton-sc-1fetcgp-0 cMGXxG"><span class="Buttonstyles__ChildrenWrapper-hz08m4-3 fXYiwE">Ring 08-551 157 97</span></div>

wait = WebDriverWait(self.driver,30)
wait.until(EC.presence_of_element_located((By.XPATH, "//button[#class='Buttonstyles__BaseButton-hz08m4-0 iXyZie']//span[#class='Buttonstyles__ChildrenWrapper-hz08m4-3 fXYiwE']"))).click()
phone_number = wait.until(EC.presence_of_element_located((By.XPATH, "//a[#class='Buttonstyles__BaseButton-hz08m4-0-a Buttonstyles__BaseAnchor-hz08m4-1 hmzYTq']")))
print phone_number.text
Note please add below imports :
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait as Wait
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

driver.find_element_by_class_name("fXYiwE").__getattribute__(Visa telefonnummer)
Please try this.

Categories

Resources