Develop Reference

How to upload all images from a folder onto a website with python

I am trying to get all images from a folder which has an ".jpg" and upload it to an element on a website. I am very new to coding in general and would appreciate it someone could point me into the right direction. In the function upload_image I am trying to upload all the images to the file variable. Could any share with me how I can perform this?
import textwrap
import os
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import csv
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
import pyautogui
import time
import codecs
from PIL import Image
from selenium.common.exceptions import NoSuchElementException
opts = Options()
opts.add_experimental_option('debuggerAddress', 'localhost:3333')
driver = webdriver.Chrome(executable_path="C:\chromedriver_win32\chromedriver.exe", chrome_options=opts)
act = ActionChains(driver)
product_data_upload = '/Users/singh/PycharmProjects/ebayscraper/product_url_tables_shpock.csv'
images_folder = '/Users/singh/PycharmProjects/ebayscraper/images_folder'
types_of_encoding = ["utf8", "cp1252"]
for encoding_type in types_of_encoding:
with codecs.open(product_data_upload, 'r', encoding=encoding_type, errors='replace') as csv_file:
csv_reader = csv.reader(csv_file)
next(csv_reader)
for line in csv_reader:
driver.implicitly_wait(5)
driver.get('https://www.shpock.com/en-gb/sell')
def image_link_extraction():
os.chdir(images_folder)
driver.execute_script('window.open("https://www.google.com/");')
driver.switch_to.window(driver.window_handles[1])
driver.get(line[0])
pyautogui.moveTo(373, 896, 0.1)
pyautogui.click()
for i in range(1,6):
image_url = driver.find_element_by_xpath('//img[contains(#id, "viEnlargeImgLayer_img_ctr")]')
name = image_url.get_attribute('alt')
url = image_url.get_attribute('src')
pyautogui.moveTo(1866, 499, 0.1)
pyautogui.click()
with open(name.replace(' ', '-').replace('/', '') + '.jpg', 'wb') as f:
im = requests.get(url)
f.write(im.content)
print('writing: ', name)
print(name, url)
driver.close()
driver.switch_to.window(driver.window_handles[0])
time.sleep(10)
image_link_extraction()
def upload_image():
file = driver.find_elements_by_xpath('//input[contains(#type, "file")]')
os.startfile('/Users/singh/PycharmProjects/ebayscraper/images_folder/*.jpg', )
upload_image()

Python Selenium - Extract words using Safari and csvWriter - Empty variable and container -

Now I am having an issue where the next page is not loading completely for me to take the information down fully and the script is coming up with a NoMatchingElement error halfway through. I tried adding a WebDriverWait as shown below in the code, but it is not working. Any help is greatly appreciated. Thank you!
import sys
import csv
from selenium import webdriver
import time
import pandas as pd
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# default path to file to store data
path_to_file = "/Users/Dan/Desktop/reviews.csv"
# default number of scraped pages
num_page = 1939
# default tripadvisor website of hotel or things to do (attraction/monument)
url = "https://www.tripadvisor.com/Attraction_Review-g187791-d192285-Reviews-Colosseum-Rome_Lazio.html"
# if you pass the inputs in the command line
if (len(sys.argv) == 4):
path_to_file = sys.argv[1]
num_page = int(sys.argv[2])
url = sys.argv[3]
# import the webdriver
driver = webdriver.Safari()
driver.get(url)
# open the file to save the review
csvFile = open(path_to_file, 'a', encoding="utf-8")
csvWriter = csv.writer(csvFile)
# change the value inside the range to save more or less reviews
data=[]
for i in range(0, num_page):
# expand the review
wait = WebDriverWait(driver, 10)
element = wait.until(EC.element_to_be_clickable((By.XPATH, "//*[#id='tabs-1']/div[2]/a[#accesskey='n']")))
container = driver.find_elements_by_xpath("//*[#id='tabs-1']/div[3]/table/tbody")
for con in container:
name = con.find_element_by_xpath(".//tr[2]/td").text.replace("Sponsor Name:","")
start = con.find_element_by_xpath(".//tr[1]/td[3]").text.replace("Start Date*: ","")
data.append((name,start))
df = pd.DataFrame(data,columns=['Name','Start'])
df.to_csv('/Users/Dan/Desktop/reviews.csv', index = False)
driver.find_element_by_xpath("//*[#id='tabs-1']/div[2]/a[#accesskey='n']").click()
driver.quit()

I am not sure how to do it with csv writer but I know that you can do it like so. Just append to 2d list and then write to csv file.
driver.get("https://www.clinicaltrialsregister.eu/ctr-search/search?query=")
container = driver.find_elements_by_xpath("//*[#id='tabs-1']/div[3]/table/tbody")
data=[]
for con in container:
name = con.find_element_by_xpath(".//tr[2]/td").text
start = con.find_element_by_xpath(".//tr[1]/td[3]").text
data.append((name,start))
df = pd.DataFrame(data,columns=['Name','Start'])
#print(df)
df.to_csv('/Users/Dan/Desktop/reviews.csv', index = False)
Import
import pandas as pd

Scraped website data is not being written to a CSV

I am trying to scrape a website to get the info and output it to a CSV file. For the data I am trying to extract, there is an output to the terminal but I need that to be in a CSV file.
I have tried several different methods but cannot find a solution.
The CSV file is created but it's just empty. There is probably something really simple.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import csv
import time
from bs4 import BeautifulSoup
DRIVER_PATH = '/Users/jasonbeedle/Desktop/snaviescraper/chromedriver'
options = Options()
options.page_load_strategy = 'normal'
# Navigate to url
driver = webdriver.Chrome(options=options, executable_path=DRIVER_PATH)
driver.get("http://best4sport.tv/2hd/2020-12-10/")
options.add_argument("--window-size=1920x1080")
results = driver.find_element_by_class_name('program1_content_container')
soup = BeautifulSoup(results.text, 'html.parser')
# results = driver.find_element_by_class_name('program1_content_container')
p_data1 = soup.find_all("div", {"class_name": "program1_content_container"})
p_data2 = soup.find_all("div", {"class_name": "program_time"})
p_data3 = soup.find_all("div", {"class_name": "sport"})
p_data4 = soup.find_all("div", {"class": "program_text"})
print("Here is your data, I am off ot sleep now see ya ")
print(results.text)
# Create csv
programme_list = []
# Programme List
for item in p_data1:
try:
name = item.contents[1].find_all(
"div", {"class": "program1_content_container"})[0].text
except:
name = ''
p_data1 = [time]
programme_list.append(p_data1)
# Programme Time
for item in p_data2:
try:
time = item.contents[1].find_all(
"div", {"class": "program_time"})[0].text
except:
time = ''
p_data2 = [time]
programme_list.append(p_data2)
# Which sport
for item in p_data3:
try:
time = item.contents[1].find_all(
"div", {"class": "sport"})[0].text
except:
time = ''
p_data3 = [time]
programme_list.append(p_data3)
with open('sport.csv', 'w') as file:
writer = csv.writer(file)
for row in programme_list:
writer.writerow(row)
I have just tried to add an object called data_output Then I tried to print the data_output
data_output = [p_data1, p_data2, p_data3, p_data4]
...
print(data_output)
The output in the terminal is

Load data into pandas dataframe and export into csv.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
from bs4 import BeautifulSoup
DRIVER_PATH = '/Users/jasonbeedle/Desktop/snaviescraper/chromedriver'
driver = webdriver.Chrome(executable_path=DRIVER_PATH)
driver.get("http://best4sport.tv/2hd/2020-12-10/")
results =WebDriverWait(driver,10).until(EC.visibility_of_element_located((By.CSS_SELECTOR,".program1_content_container")))
soup = BeautifulSoup(results.get_attribute("outerHTML"), 'html.parser')
program_time=[]
sport=[]
program_text=[]
program_info=[]
for item in soup.select(".program_details "):
if item.find_next(class_='program_time'):
program_time.append(item.find_next(class_='program_time').text.strip())
else:
program_time.append("Nan")
if item.find_next(class_='sport'):
sport.append(item.find_next(class_='sport').text.strip())
else:
sport.append("Nan")
if item.find_next(class_='program_text'):
program_text.append(item.find_next(class_='program_text').text.strip())
else:
program_text.append("Nan")
if item.find_next(class_='program_info'):
program_info.append(item.find_next(class_='program_info').text.strip())
else:
program_info.append("Nan")
df=pd.DataFrame({"program_time":program_time,"sport":sport,"program_text":program_text,"program_info":program_info})
print(df)
df.to_csv("sport.csv")
csv snapshot after creation
If you don't have pandas then you need to install it.
pip install pandas

As Blue Fishy said you can try to change to w mode only, but you may run in an encoding error.
Solution that works on your data
import csv
programme_list = ['19:55','MOTORU SPORTS','Motoru sporta "5 minūte"','Iknedēļas Alda Putniņa veidots apskats par motoru sportu','20:00','BASKETBOLS','...']
with open('sport.csv', 'w', encoding='utf-8') as file:
writer = csv.writer(file, delimiter=',', lineterminator='\n')
for row in programme_list:
print(row)
writer.writerow([row])
Output
19:55
MOTORU SPORTS
"Motoru sporta ""5 minūte"""
Iknedēļas Alda Putniņa veidots apskats par motoru sportu
20:00
BASKETBOLS
...

Instead of writing binary, can you try changing wb to w?
Change
with open('sport.csv', 'wb') as file:
to
with open('sport.csv', 'w') as file:
EDITED:
Sorry for being a bit late. Here is the code modified based on your original code FYI.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import csv
import time
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.options import Options
DRIVER_PATH = '/Users/jasonbeedle/Desktop/snaviescraper/chromedriver'
options = Options()
options.page_load_strategy = 'normal'
# Navigate to url
driver = webdriver.Chrome(options=options, executable_path=DRIVER_PATH)
driver.get("http://best4sport.tv/2hd/2020-12-10/")
options.add_argument("--window-size=1920x1080")
results = driver.find_element_by_class_name('program1_content_container')
page = driver.page_source
soup = BeautifulSoup(page, 'html.parser')
# results = driver.find_element_by_class_name('program1_content_container')
p_data1 = soup.find_all("p", {"class": "program_info"})
p_data2 = soup.find_all("p", {"class": "program_time"})
p_data3 = soup.find_all("p", {"class": "sport"})
p_data4 = soup.find_all("p", {"class": "program_text"})
# Create csv
programme_list = []
# Programme List
for i in range(len(p_data1)):
programme_list.append([p_data1[i].text.strip(), p_data2[i].text.strip(), p_data3[i].text.strip(), p_data4[i].text.strip()])
with open('sport.csv', 'w', encoding='utf-8') as file:
writer = csv.writer(file)
writer.writerow(["program_info", "program_time", "sport", "program_text"])
for row in programme_list:
writer.writerow(row)
Excel Screenshot here

Why wont this code return and create the information to a .csv file?

this is my code below to web scrape the tesco website, which I cant get to work, no error codes are showing but its also not creating a .csv file. any help would be appreciated.
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
import time
import csv
def save_csv(dct):
name = "tescowhiskey.csv"
print("[INFO] saving...")
with open(name, 'a', encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow((dct['cap'],
dct['Whiskey'],
dct['TESCO'],
))
def scroll(driver):
for _ in range(1,6):
# driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.execute_script("window.scrollTo(0, 1000)")
time.sleep(7)
driver = webdriver.Firefox()
driver.get("https://www.tesco.com/groceries/en-GB/shop/drinks/spirits/whisky")
for i in range(2):
element = WebDriverWait(driver, 30)
scroll(driver)
data = driver.find_elements_by_css_selector("product-list-container")
for d in data:
items = {}
body = d.text.split("\n")
items["cap"] = body[0]
items["Whiskey"] = body[1]
items["TESCO"] = body[-3]
save_csv(items)
driver.find_element_by_css_selector(".prev-next").click()
driver.quit()

How can I open URLs from a CSV file with selenium?

I'm trying to save the data from a profile on Google Scholar into a CSV. The profile has a ‘Show More’ button, and I can get all the data from it (here I only saved the data from the table but I need all the data from the profile) but the problem is that I saved the data twice or even more times sometimes, and I think it's because I saved it while I was clicking and not after I had clicked all the ‘Show More’, so how can I do that? Also, here I used only one URL, but there are more, and I have them saved in another CSV, so how do I open each URL from there to do what I do here? (I only need the Link row) the CSV with the URLs look like this
https://drive.google.com/file/d/1zkTlzYaOQ7FVoSdd5OMnE8QgwS8NOik7/view?usp=sharing
from selenium.webdriver.support.ui import WebDriverWait as W
from selenium.webdriver.support import expected_conditions as EC
from selenium.common import exceptions as SE
from selenium import webdriver
import time
from csv import writer
chrome_path=r"C:\Users\gvste\Desktop\proyecto\chromedriver.exe"
driver = webdriver.Chrome(chrome_path)
urls = ["https://scholar.google.com/citations?hl=en&user=gQb_tFMAAAAJ"]
button_locators = "//button[#class='gs_btnPD gs_in_ib gs_btn_flat gs_btn_lrge gs_btn_lsu']"
wait_time = 2
wait = W(driver, wait_time)
for url in urls:
data = {}
driver.get(url)
button_link = wait.until(EC.element_to_be_clickable((By.XPATH, button_locators)))
while button_link:
try:
wait.until(EC.visibility_of_element_located((By.ID,'gsc_a_tw')))
data = driver.find_elements_by_class_name("gsc_a_tr")
button_link = wait.until(EC.element_to_be_clickable((By.XPATH, button_locators)))
button_link.click()
time.sleep(2)
with open('perfil.csv','a', encoding="utf-8", newline='') as s:
csv_writer =writer(s)
for i in range(len(data)):
paper = driver.find_elements_by_class_name("gsc_a_t")
citas = driver.find_elements_by_class_name("gsc_a_c")
año = driver.find_elements_by_class_name("gsc_a_y")
p = paper[i].text.replace(',', '')
c = citas[i].text.replace(',', '')
a = año[i].text.replace(',', '')
csv_writer.writerow([ p, c, a])
except SE.TimeoutException:
print(f'Página parseada {url}')
break
driver.quit()

For the first part I didn't really get what's happening. But for the second part you can change URLs from hard code to a function (put the loop in function) and you can use pandas library for CSV (it's much better). This is for getting the URLS
import pandas as pd
df = pd.read_csv(csv_file)
urls = df['column_name']

Here is the most basic way read data from CSV file:
import csv
with open('filename.csv', 'r') as file:
reader = csv.reader(filename)
for row in reader:
print(row)