Related
I am trying to scrape PFF.com for football grades with selenium, I am trying to get a specific grade for all Quarterbacks. Problem is, it doesn't seem like it's capturing the text as .text isn't working but I am not getting any NoSuchElementException.
Here's my code:
service = Service(executable_path="C:\\chromedriver.exe")
op = webdriver.ChromeOptions()
driver = webdriver.Chrome(service=service, options=op)
driver.get("https://premium.pff.com/nfl/positions/2022/REG/passing?position=QB")
sleep(2)
sign_in = driver.find_element(By.XPATH, '/html/body/div/div/header/div[3]/button')
sign_in.click()
sleep(2)
email = driver.find_element(By.XPATH, '/html/body/div/div/div/div/div/div/form/div[1]/input')
email.send_keys(my_email)
password = driver.find_element(By.XPATH,
'/html/body/div/div/div/div/div/div/form/div[2]/input')
password.send_keys(my_password)
sleep(2)
sign_in_2 = driver.find_element(By.XPATH,
'/html/body/div/div/div/div/div/div/form/button')
sign_in_2.click()
sleep(2)
all_off_grades = driver.find_elements(By.CSS_SELECTOR, '.kyber-table
.kyber-grade-badge__info-text div')
all_qb_names = driver.find_elements(By.CSS_SELECTOR, '.kyber-table .p-1 a')
qb_grades = []
qb_names = []
for grade in all_off_grades:
qb_grades.append(grade.text)
for qb_name in all_qb_names:
qb_names.append(qb_name.text)
print(qb_grades)
print(qb_names)
The lists keep showing as empty.
Here are the elements I am trying to pull, but for every QB, I already confirmed the other QB's have the same class names for their grade and name.
<div class="kyber-grade-badge__info-text">91.5</div>
need to pull the 91.5
<a class="p-1" href="/nfl/players/2022/REG/josh-allen/46601/passing">Josh Allen</a>
need to pull Josh Allen
#Jbuck3 I tried modifying the locator and it works for me. I am also giving the output I am getting. Let me know that is what you were expecting.
all_off_grades = driver.find_elements(By.CSS_SELECTOR, '.kyber-table-body__scrolling-rows-container .kyber-grade-badge__info-text')
all_qb_names = driver.find_elements(By.CSS_SELECTOR, "a[data-gtm-id = 'player_name']")
And the output I got is:
['91.5', '90.3', '74.6', '-', '-', '60.0', '84.3', '78.3', '78.1', '-', '-', '60.0', '82.8', '83.4', '-', '-', '-', '60.0']
['Josh Allen ', 'Geno Smith ', 'Kirk Cousins ', 'Marcus Mariota ', 'Jameis Winston ', 'Trey Lance ', 'Derek Carr ', 'Justin Fields ', 'Trevor Lawrence ', 'Russell Wilson ', 'Ryan Tannehill ', 'Tom Brady ', 'Tua Tagovailoa ', 'Mac Jones ', 'Davis Mills ', 'Matthew Stafford ', 'Baker Mayfield ', 'Lamar Jackson ', 'Joe Flacco ', 'Matt Ryan ', 'Jalen Hurts ', 'Daniel Jones ', 'Kyler Murray ', 'Justin Herbert ', 'Joe Burrow ', 'Aaron Rodgers ', 'Patrick Mahomes ', 'Mitchell Trubisky ', 'Dak Prescott ', 'Jacoby Brissett ', 'Carson Wentz ', 'Jared Goff ']
I made a dictionary using .groupdict() function, however, I am having a problem regarding elimination of certain output dictionaries.
For example my code looks like this (tweet is a string that contains 5 elements separated by || :
def somefuntion(pattern,tweet):
pattern = "^(?P<username>.*?)(?:\|{2}[^|]+){2}\|{2}(?P<botprob>.*?)(?:\|{2}|$)"
for paper in tweet:
for item in re.finditer(pattern,paper):
item.groupdict()
This produces an output in the form:
{'username': 'yashrgupta ', 'botprob': ' 0.30794588629999997 '}
{'username': 'sterector ', 'botprob': ' 0.39391528649999996 '}
{'username': 'MalcolmXon ', 'botprob': ' 0.05630123819 '}
{'username': 'ryechuuuuu ', 'botprob': ' 0.08492567222000001 '}
{'username': 'dpsisi ', 'botprob': ' 0.8300337045 '}
But I would like it to only return dictionaries whose botprob is above 0.7. How do I do this?
Specifically, as #WiktorStribizew notes, just skip iterations you don't want:
pattern = "^(?P<username>.*?)(?:\|{2}[^|]+){2}\|{2}(?P<botprob>.*?)(?:\|{2}|$)"
for paper in tweet:
for item in re.finditer(pattern,paper):
item = item.groupdict()
if item["botprob"] < 0.7:
continue
print(item)
This could be wrapped in a generator expression to save the explicit continue, but there's enough going on as it is without making it harder to read (in this case).
UPDATE since you are apparently in a function:
pattern = "^(?P<username>.*?)(?:\|{2}[^|]+){2}\|{2}(?P<botprob>.*?)(?:\|{2}|$)"
items = []
for paper in tweet:
for item in re.finditer(pattern,paper):
item = item.groupdict()
if float(item["botprob"]) > 0.7:
items.append(item)
return items
Or using comprehensions:
groupdicts = (item.groupdict() for paper in tweet for item in re.finditer(pattern, paper))
return [item for item in groupdicts if float(item["botprob"]) > 0.7]
I would like it to only return dictionaries whose botprob is above 0.7.
entries = [{'username': 'yashrgupta ', 'botprob': ' 0.30794588629999997 '},
{'username': 'sterector ', 'botprob': ' 0.39391528649999996 '},
{'username': 'MalcolmXon ', 'botprob': ' 0.05630123819 '},
{'username': 'ryechuuuuu ', 'botprob': ' 0.08492567222000001 '},
{'username': 'dpsisi ', 'botprob': ' 0.8300337045 '}]
filtered_entries = [e for e in entries if float(e['botprob'].strip()) > 0.7]
print(filtered_entries)
output
[{'username': 'dpsisi ', 'botprob': ' 0.8300337045 '}]
I extract information from a website and able to get the data. however, I unable to expend the data for 'K" but successfully for 'J' and ''L' .
print "hello from python 2"
from lxml import html
import requests
import csv
import pandas as pd
import MySQLdb as mdb
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
bursa = ['J','K','L']
bursalist = []
print bursalist
for x in range (len(bursa)):
try:
page = requests.get('https://www.malaysiastock.biz/Listed-Companies.aspx?type=A&value='+bursa[x])
tree = html.fromstring(page.content)
tree1 = tree.xpath('//td/h3/a[contains(text(),"(")][not(contains(text(),"(F"))]/text()')
tree2 = tree.xpath('.//td/h3/text()')
len(tree1)
alist = []
for y in range (len(tree1)):
a = tree1[y].split('(')
b = a[1].split(')')
alist.append(a[0])
alist.append(b[0])
alist.append(tree2[2*y])
alist.append(tree2[2*y+1])
a = alist
bursalist.extend(alist)
print bursalist
except Exception:
print "no data"
Notice that print bursalist only show data from 'J' and 'L' while 'K' is missing. But if I get the data from alist, data'K' is shown but unable to extend.
please advise if there is a robust way to do it
print alist will show
['K1 ', '0111', 'K-ONE TECHNOLOGY BERHAD', 'Technology Equipment ', 'KAB ', '0193', 'KEJURUTERAAN ASASTERA BERHAD', 'Industrial Engineering ', 'KAMDAR ', '8672', 'KA
MDAR GROUP (M) BERHAD', 'Retailers ', 'KANGER ', '0170', 'KANGER INTERNATIONAL BERHAD', 'Household Goods ', 'KAREX ', '5247', 'KAREX BERHAD', 'Personal Goods ', 'KAR
YON ', '0054', 'KARYON INDUSTRIES BERHAD', 'Chemicals ', 'KAWAN ', '7216', 'KAWAN FOOD BERHAD', 'Food & Beverages ', 'KEINHIN ', '7199', 'KEIN HING INTERNATIONAL BER
HAD', 'Metals ', 'KEN ', '7323', 'KEN HOLDINGS BERHAD', 'Property ', 'KENANGA ', '6483', 'KENANGA INVESTMENT BANK BERHAD', 'Other Financials ', 'KERJAYA ', '7161', '
KERJAYA PROSPEK GROUP BERHAD', 'Construction ', 'KESM ', '9334', 'KESM INDUSTRIES BERHAD', 'Semiconductors ', 'KEYASIC ', '0143', 'KEY ASIC BERHAD', 'Semiconductors
', 'KFIMA ', '6491', 'KUMPULAN FIMA BERHAD', 'Diversified Industrials ', 'KGB ', '0151', 'KELINGTON GROUP BERHAD', 'Industrial Engineering ', 'KGROUP ', '0036', 'KEY
ALLIANCE GROUP BERHAD', 'Technology Equipment ', 'KHEESAN ', '6203', 'KHEE SAN BERHAD', 'Food & Beverages ', 'KHIND ', '7062', 'KHIND HOLDINGS BERHAD', 'Household G
oods ', 'KHJB ', '0210', 'KIM HIN JOO (MALAYSIA) BERHAD', 'Retailers ', 'KIALIM ', '6211', 'KIA LIM BERHAD', 'Building Materials ', 'KIMHIN ', '5371', 'KIM HIN INDUS
TRY BERHAD', 'Building Materials ', 'KIMLUN ', '5171', 'KIMLUN CORPORATION BERHAD', 'Construction ', 'KINSTEL ', '5060', 'KINSTEEL BHD', 'Metals ', 'KIPREIT ', '5280
', 'KIP REAL ESTATE INVESTMENT TRUST', 'Real Estate Investment Trusts ', 'KKB ', '9466', 'KKB ENGINEERING BERHAD', 'Industrial Engineering ', 'KLCC ', '5235SS', 'KLC
C PROPERTY HOLDINGS BERHAD', 'Real Estate Investment Trusts ', 'KLCI1XI ', '0835EA', 'KENANGA KLCI DAILY (-1X) INVERSE ETF', 'KENANGA KLCI DAILY 2X LEVERAGED ETF', '
NUSFOR ', '5035', 'KOBAY TECHNOLOGY BERHAD', 'Industrial Materials, Components & Equipment ', 'KOBAY ', '6971', 'KOMARKCORP BERHAD', 'Packaging Materials ', 'KOMARK
KLCI2XL ', '0834EA', 'KUALA LUMPUR KEPONG BERHAD', 'Plantation ', 'KLK ', '2445', 'KLUANG RUBBER COMPANY (MALAYA) BERHAD', 'Plantation ', 'KLUANG ', '2453', 'KIM LOONG RESOURCES BERHAD', 'Plantation ', 'KMLOONG ', '5027', 'KNM GROUP BERHAD', 'Other Energy Resources ', 'KNM ', '7164', 'KNUSFORD BERHAD', 'Industrial Services ', 'KNUSFOR ', '5035', 'KOBAY TECHNOLOGY BERHAD', 'Industrial Materials, Components & Equipment ', 'KOBAY ', '6971', 'KOMARKCORP BERHAD', 'Packaging Materials ', 'KOMARK ', '7017', 'KOSSAN RUBBER INDUSTRIES BERHAD', 'Health Care Equipment & Services ', 'KOSSAN ', '7153', 'KOTRA INDUSTRIES BERHAD', 'Pharmaceuticals ', 'KOTRA ', '0002', 'KPJ HEALTHCARE BERHAD', 'Health Care Providers ', 'KPJ ', '5878', 'KUMPULAN POWERNET BERHAD', 'Personal Goods ', 'KPOWER ', '7130', 'KERJAYA PROSPEK PROPERTY BERHAD', 'Property ', 'KPPROP ', '7077', 'KUMPULAN PERANGSANG SELANGOR BERHAD', 'Diversified Industrials ', 'KPS ', '5843', 'KPS CONSORTIUM BERHAD', 'Wood & Wood Products ', 'KPSCB ', '9121', 'KRETAM HOLDINGS BERHAD', 'Plantation ', 'KRETAM ', '1996', 'KRONOLOGI ASIA BERHAD', 'Digital Services ', 'KRONO ', '0176', 'KECK SENG (MALAYSIA) BERHAD', 'Diversified Industrials ', 'KSENG ', '3476', 'KSL HOLDINGS BERHAD', 'Property ', 'KSL ', '5038', 'K.SENG SENG CORPORATION BERHAD', 'Industrial Materials, Components & Equipment ', 'KSSC ', '5192', 'K-STAR SPORTS LIMITED', 'Personal Goods ', 'KSTAR ', '5172', 'KONSORTIUM TRANSNASIONAL BERHAD', 'Travel, Leisure & Hospitality ', 'KTB ', '4847', 'KIM TECK CHEONG CONSOLIDATED BERHAD', 'Consumer Services ', 'KTC ', '0180', 'KUB MALAYSIA BERHAD', 'Industrial Services ', 'KUB ', '6874', 'KUCHAI DEVELOPMENT BERHAD', 'Other Financials ', 'KUCHAI ', '2186', 'KWANTAS CORPORATION BERHAD', 'Plantation ', 'KWANTAS ', '6572', 'KYM HOLDINGS BERHAD', 'Pack
aging Materials ', 'KYM ', '8362']
I tried lot of suggestions but I am unable to remove carriage returns. I am new python and trying it with csv file cleaning.
import csv
filepath_i = 'C:\Source Files\Data Source\Flat File Source\PatientRecords.csv'
filepath_o = 'C:\Source Files\Data Source\Flat File Source\PatientRecords2.csv'
rows = []
with open(filepath_i, 'rU', newline='') as csv_file:
#filtered = (line.replace('\r\n', '') for line in csv_file)
filtered = (line.replace('\r', '') for line in csv_file)
csv_reader = csv.reader(csv_file, delimiter=',')
i = 0
for row in csv_reader:
print(row)
i = i + 1
if(i == 10):
break
#with open(filepath_o, 'w',newline='' ) as writeFile:
# writer = csv.writer(writeFile,lineterminator='\r')
# for row in csv_reader:
# #rows.append(row.strip())
# rows.append(row.strip())
# writer.writerows(rows)
Input
DRG Definition,Provider Id,Provider Name,Provider Street Address,Provider City,Provider State,Provider Zip Code,Hospital Referral Region Description,Hospital Category,Hospital Type, Total Discharges ,Covered Charges , Total Payments ,Medicare Payments
039 - EXTRACRANIAL PROCEDURES W/O CC/MCC,10001,SOUTHEAST ALABAMA MEDICAL CENTER,1108 ROSS CLARK CIRCLE,DOTHAN,AL,36301,AL - Dothan,Specialty Centers,Government Funded,91,"$32,963.07 ","$5,777.24 ","$4,763.73 "
039 - EXTRACRANIAL PROCEDURES W/O CC/MCC,10005,MARSHALL MEDICAL CENTER SOUTH,"2505 U S HIGHWAY
431 NORTH",BOAZ,AL,35957,AL - Birmingham,Specialty Centers,Private Institution,14,"$15,131.85 ","$5,787.57 ","$4,976.71 "
039 - EXTRACRANIAL PROCEDURES W/O CC/MCC,10006,ELIZA COFFEE MEMORIAL HOSPITAL,205 MARENGO STREET,FLORENCE,AL,35631,AL - Birmingham,Rehabilitation Centers,Private Institution,24,"$37,560.37 ","$5,434.95 ","$4,453.79 "
Output (4th column 'Provider Street Address')
['DRG Definition', 'Provider Id', 'Provider Name', 'Provider Street Address', 'Provider City', 'Provider State', 'Provider Zip Code', 'Hospital Referral Region Description', 'Hospital Category', 'Hospital Type', ' Total Discharges ', 'Covered Charges ', ' Total Payments ', 'Medicare Payments']
['039 - EXTRACRANIAL PROCEDURES W/O CC/MCC', '10001', 'SOUTHEAST ALABAMA MEDICAL CENTER', '1108 ROSS CLARK CIRCLE', 'DOTHAN', 'AL', '36301', 'AL - Dothan', 'Specialty Centers', 'Government Funded', '91', '$32,963.07 ', '$5,777.24 ', '$4,763.73 ']
['039 - EXTRACRANIAL PROCEDURES W/O CC/MCC', '10005', 'MARSHALL MEDICAL CENTER SOUTH', '2505 U S HIGHWAY \n431 NORTH', 'BOAZ', 'AL', '35957', 'AL - Birmingham', 'Specialty Centers', 'Private Institution', '14', '$15,131.85 ', '$5,787.57 ', '$4,976.71 ']
I ran this on my side and it works:
with open(filepath_i, 'rU', newline='') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
row[3] = row[3].replace("\n","").replace("\r","")
print(row)
Output:
['DRG Definition', 'Provider Id', 'Provider Name', 'Provider Street Address', 'Provider City', 'Provider State', 'Provider Zip Code', 'Hospital Referral Region Description', 'Hospital Category', 'Hospital Type', ' Total Discharges ', 'Covered Charges ', ' Total Payments ', 'Medicare Payments']
['039 - EXTRACRANIAL PROCEDURES W/O CC/MCC', '10001', 'SOUTHEAST ALABAMA MEDICAL CENTER', '1108 ROSS CLARK CIRCLE', 'DOTHAN', 'AL', '36301', 'AL - Dothan', 'Specialty Centers', 'Government Funded', '91', '$32,963.07 ', '$5,777.24 ', '$4,763.73 ']
['039 - EXTRACRANIAL PROCEDURES W/O CC/MCC', '10005', 'MARSHALL MEDICAL CENTER SOUTH', '2505 U S HIGHWAY 431 NORTH', 'BOAZ', 'AL', '35957', 'AL - Birmingham', 'Specialty Centers', 'Private Institution', '14', '$15,131.85 ', '$5,787.57 ', '$4,976.71 ']
['039 - EXTRACRANIAL PROCEDURES W/O CC/MCC', '10006', 'ELIZA COFFEE MEMORIAL HOSPITAL', '205 MARENGO STREET', 'FLORENCE', 'AL', '35631', 'AL - Birmingham', 'Rehabilitation Centers', 'Private Institution', '24', '$37,560.37 ', '$5,434.95 ', '$4,453.79 ']
def exportOrders(self):
file = open("orders.txt", 'w')
file.write("\"Date\" \"Pair\" \"Amount bought/sold\" \"Pair Price\" \"Profit/Loss\" \"Order Type\"" + '\n')
for x in self.tradeHistory:
date = x['date']
pair = self.currentPair
amount = x[self.currentPair]
price = x['price']
order = x['Order Type']
if order == "buy":
spent = x['spent']
file.write(date + ' ' + pair + ' ' + amount + ' '
+ price + ' ' + float(-spent) + ' ' + order + ' \n')
if order == "sell":
obtained = x['obtained']
file.write(date + ' ' + pair + ' ' + amount + ' '
+ price + ' ' + obtained + ' ' + order + ' \n')
file.close()
self.tradeHistory is a list of dictionaries that store a date, a pair, the amount bought, the price of the pair, the money spent or obtained, and the order type.
My problem is that when the program runs for the first time into:
if order == "buy":
spent = x['spent']
file.write(date + ' ' + pair + ' ' + amount + ' '
+ price + ' ' + str(float(-spent)) + ' ' + order + ' \n')
The for loop breaks out and the orders.txt only shows the first line which is:
file.write("\"Date\" \"Pair\" \"Amount bought/sold\" \"Pair Price\" \"Profit/Loss\" \"Order Type\"" + '\n')
Thank you in advance!
edit:
Basically, my self.tradeHistory has the following content
{'date': 1505161800, 'BTC_ETH': 0.7091196761422075, 'price': 0.07050996, 'spent': 0.05, 'Order Type': 'buy'}
{'date': 1505167200, 'BTC_ETH': 0.7091196761422075, 'price': 0.07079909, 'obtained': 0.050205027771963, 'Order Type': 'sell'}
{'date': 1505236500, 'BTC_ETH': 0.7032346826344071, 'price': 0.07110002, 'spent': 0.05, 'Order Type': 'buy'}
{'date': 1505251800, 'BTC_ETH': 0.7032346826344071, 'price': 0.0707705, 'obtained': 0.04976827010737831, 'Order Type': 'sell'}
{'date': 1505680200, 'BTC_ETH': 0.715374411944349, 'price': 0.06989347, 'spent': 0.05, 'Order Type': 'buy'}
{'date': 1505699100, 'BTC_ETH': 0.715374411944349, 'price': 0.071989, 'obtained': 0.05149908854146174, 'Order Type': 'sell'}
{'date': 1505733300, 'BTC_ETH': 0.6879187705515734, 'price': 0.072683, 'spent': 0.05, 'Order Type': 'buy'}
{'date': 1505745000, 'BTC_ETH': 0.6889021311187427, 'price': 0.07257925, 'spent': 0.05, 'Order Type': 'buy'}
{'date': 1505756700, 'BTC_ETH': 1.3768209016703161, 'price': 0.0732, 'obtained': 0.10078329000226714, 'Order Type': 'sell'}
...
There are 63 items inside the list of dictionaries. My aim is to create a .txt file that looks like
"Date" "Pair" "Amount bought/sold" "Pair Price" "Profit/Loss" "Order Type"
1505161800 BTC_ETH 0.7091196761422075 0.07050996 0.05 buy
1505167200 BTC_ETH 0.7091196761422075 0.07079909 0.05 sell
...
You should not concatenate numbers with strings in Python. Use str.format instead:
file.write(
'{} {} {} {} {} {}\n'
.format(date, pair, amount, price, float(-spent), order)
)
You can also use csv module for a better implementation.
import csv
def exportOrders(self):
with open("orders.txt", 'w') as file:
writer = csv.writer(file, delimiter=' ', quotechar='"')
writer.writerow([
'Date', 'Pair', 'Amount bought/sold', 'Pair Price',
'Profit/Loss', 'Order Type'])
for x in self.tradeHistory:
date = x['date']
pair = self.currentPair
amount = x[self.currentPair]
price = x['price']
order = x['Order Type']
if order == "buy":
spent = x['spent']
writer.writerow([
date, pair, amount, price,
float(-spent), order])
if order == "sell":
obtained = x['obtained']
writer.writerow([
date, pair, amount, price,
obtained, order])