from bs4 import BeautifulSoup
import pandas as pd
import requests
import time
from datetime import datetime
def extract_source(url):
agent = {"User-Agent":"Mozilla/5.0"}
source=requests.get(url, headers=agent).text
return source
html_text = extract_source('https://www.mpbio.com/us/life-sciences/biochemicals/amino-acids')
soup = BeautifulSoup(html_text, 'lxml')
for a in soup.find_all('a', class_ = 'button button--link button--fluid catalog-list-item__actions-primary-button', href=True):
# print ("Found the URL:", a['href'])
urlof = a['href']
html_text = extract_source(urlof)
soup = BeautifulSoup(html_text, 'lxml')
table_rows = soup.find_all('tr')
first_columns = []
third_columns = []
for row in table_rows:
# for row in table_rows[1:]:
first_columns.append(row.findAll('td')[0])
third_columns.append(row.findAll('td')[1])
for first, third in zip(first_columns, third_columns):
print(first.text, third.text)
Basically I am trying to scrape data from tables from multiple links of Website. And I want to insert that data in one excel csv file in following table format
SKU 07DE9922
Analyte / Target Corticosterone
Base Catalog Number DE9922
Diagnostic Platforms EIA/ELISA
Diagnostic Solutions Endocrinology
Disease Screened Corticosterone
Evaluation Quantitative
Pack Size 96 Wells
Sample Type Plasma, Serum
Sample Volume 10 uL
Species Reactivity Mouse, Rat
Usage Statement For Research Use Only, not for use in diagnostic procedures.
To below format in excel file
SKU Analyte/Target Base Catalog Number Pack Size Sample Type
data data data data data
I am facing difficulties while converting data in proper format
I made small modification to your code. Instead of printing the data, I created a dictionary and added it to list. Then I used this list to create a DataFrame:
import pandas as pd
import requests
import time
from datetime import datetime
def extract_source(url):
agent = {"User-Agent": "Mozilla/5.0"}
source = requests.get(url, headers=agent).text
return source
html_text = extract_source(
"https://www.mpbio.com/us/life-sciences/biochemicals/amino-acids"
)
soup = BeautifulSoup(html_text, "lxml")
data = []
for a in soup.find_all(
"a",
class_="button button--link button--fluid catalog-list-item__actions-primary-button",
href=True,
):
urlof = a["href"]
html_text = extract_source(urlof)
soup = BeautifulSoup(html_text, "lxml")
table_rows = soup.find_all("tr")
first_columns = []
third_columns = []
for row in table_rows:
first_columns.append(row.findAll("td")[0])
third_columns.append(row.findAll("td")[1])
# create dictionary with values and add to the list
d = {}
for first, third in zip(first_columns, third_columns):
d[first.get_text(strip=True)] = third.get_text(strip=True)
data.append(d)
df = pd.DataFrame(data)
print(df)
df.to_csv("data.csv", index=False)
Prints:
SKU Alternate Names Base Catalog Number CAS # EC Number Format Molecular Formula Molecular Weight Personal Protective Equipment Usage Statement Application Notes Beilstein Registry Number Optical Rotation Purity UV Visible Absorbance Hazard Statements RTECS Number Safety Symbol Auto Ignition Biochemical Physiological Actions Density Melting Point pH pKa Solubility Vapor Pressure Grade Boiling Point Isoelectric Point
0 02100078-CF 2-Acetamido-5-Guanidinovaleric acid 100078 210545-23-6 205-846-6 Powder C8H16N4O3· 2H2O 216.241 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
1 02100142-CF Acetyltryptophan; DL-α-Acetylamino-3-indolepro... 100142 87-32-1 201-739-3 Powder C13H14N2O3 246.266 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... N-acetyl-DL-tryptophan, is used as stabilizer ... 89478 0° ± 2° (c=1, 1N NaOH, 24 hrs.) ~99% λ max (water)=280 ± 2 nm NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
2 02100421-CF L-2,5-Diaminopentanoic acid; 2,5-Diaminopentan... 100421 3184-13-2 221-678-6 Powder C5H12N2O2 • HCl 168.621 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... NaN 3625847 NaN ~99% NaN H319 RM2985000 GHS07 NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
3 02100520-CF Phosphocreatine Disodium Salt Tetrahydrate; So... 100520 922-32-7 213-074-6 Powder C4H8N3Na2O5P·4H2O 255.077 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... NaN NaN NaN ≥98% NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN NaN
4 02100769-CF Vitamin C; Ascorbate; Sodium ascorbate; L-Xylo... 100769 50-81-7 200-066-2 NaN C6H8O6 176.124 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... L-Ascorbic Acid is used as an Antimicrobial an... 84272 +18° to +32° (c=1, water) ≥98% NaN NaN CI7650000 NaN 1220° F (NTP, 1992) Ascorbic Acid, also known as Vitamin C, is a s... 1.65 (NTP, 1992) 374 to 378° F (decomposes) (NTP, 1992) Between 2,4 and 2,8 (2 % aqueous solution) pK1: 4.17; pK2: 11.57 greater than or equal to 100 mg/mL at 73° F (... 9.28X10-11 mm Hg at 25 deg C (est) NaN NaN NaN
5 02101003-CF Lycine; Oxyneurine; (Carboxymethyl)trimethylam... 101003 107-43-7 203-490-6 Powder C5H11NO2 117.148 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... Betaine is a reagent that is used in soldering... 3537113 NaN NaN NaN NaN DS5900000 NaN NaN End-product of oxidative metabolism of choline... NaN Decomposes around 293 deg C NaN 1.83 (Lit.) Solubility (g/100 g solvent): <a class="pubche... 1.36X10-8 mm Hg at 25 deg C (est) Anhydrous NaN NaN
6 02101806-CF (S)-2,5-Diamino-5-oxopentanoic acid; L-Glutami... 101806 56-85-9 200-292-1 NaN C5H10N2O3 146.146 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... L-glutamine is an essential amino acid, which ... 1723797 +30 ± 5° (c = 3.5, 1N HCl) ≥99% NaN NaN MA2275100 NaN NaN L-Glutamine is an essential amino acid that is... 1.364 g/cu cm 185.5 dec °C pH = 5-6 at 14.6 g/L at 25 deg C NaN Water Solubility41300 mg/L (at 25 °C) 1.9X10-8 mm Hg at 25 deg C (est) NaN NaN NaN
7 02102158-CF L-2-Amino-4-methylpentanoic acid; Leu; L; α-am... 102158 61-90-5 200-522-0 Powder C6H13NO2 131.175 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... Leucine has been used as a molecular marker in... 1721722 +14.5 ° to +16.5 ° (Lit.) NaN NaN NaN OH2850000 NaN NaN NaN 1.293 g/cu cm at 18 deg C 293 °C NaN 2.33 (-COOH), 9.74 (-NH2)(Lit.) Water Solubility21500 mg/L (at 25 °C) 5.52X10-9 mm Hg at 25 deg C (est) NaN Sublimes at 145-148 deg C. Decomposes at 293-2... 6.04(Lit.)
8 02102576-CF 4-Hydroxycinnamic acid; 3-(4-Hydroxphenyl)-2-p... 102576 7400-08-0 231-000-0 Powder C9H8O3 164.16 g/mol Dust mask, Eyeshields, Gloves Unless specified otherwise, MP Biomedical's pr... p-Coumaric acid was used as a substrate to stu... NaN NaN ≥98% NaN NaN GD9094000 NaN NaN NaN NaN 211.5 °C NaN NaN NaN NaN NaN NaN NaN
9 02102868-CF DL-2-Amino-3-hydroxypropionic acid; (±)-2-Amin... 102868 302-84-1 206-130-6 Powder C3H7NO3 105.093 g/mol Eyeshields, Gloves, respirator filter Unless specified otherwise, MP Biomedical's pr... NaN 1721405 -1° to + 1° (c = 5, 1N HCl) ≥98% NaN NaN NaN NaN NaN NMDA agonist acting at the glycine site; precu... 1.6 g/cu cm # 22 deg C 228 deg C (decomposes) NaN NaN SOL IN <a class="pubchem-internal-link CID-962... NaN NaN NaN NaN
...and so on.
And saves data.csv (screenshot from LibreOffice):
Try this -
import pandas as pd
import requests
import time
from datetime import datetime
from bs4 import BeautifulSoup
def extract_source(url):
agent = {"User-Agent": "Mozilla/5.0"}
return requests.get(url, headers=agent).text
html_text = extract_source(
'https://www.mpbio.com/us/life-sciences/biochemicals/amino-acids')
soup = BeautifulSoup(html_text, 'lxml')
result = []
for index, a in enumerate(soup.find_all('a', class_='button button--link button--fluid catalog-list-item__actions-primary-button', href=True)):
# if index >= 10:
# break
# print ("Found the URL:", a['href'])
urlof = a['href']
html_text = extract_source(urlof)
soup = BeautifulSoup(html_text, 'lxml')
table_rows = soup.find_all('tr')
first_columns = []
third_columns = []
for row in table_rows:
# for row in table_rows[1:]:
first_columns.append(row.findAll('td')[0])
third_columns.append(row.findAll('td')[1])
temp = {}
for first, third in zip(first_columns, third_columns):
third = str(third.text).strip('\n')
first = str(first.text).strip('\n')
temp[first] = third
result.append(temp)
df = pd.DataFrame(result)
# please drop useless column before saving the output to csv.
df.to_csv('out.csv', index=False)
This will give output -
SKU
Alternate Names
Base Catalog Number
CAS #
EC Number
Format
Molecular Formula
Molecular Weight
Personal Protective Equipment
Usage Statement
Application Notes
Beilstein Registry Number
Optical Rotation
Purity
UV Visible Absorbance
Hazard Statements
RTECS Number
Safety Symbol
02100078-CF
2-Acetamido-5-Guanidinovaleric acid
100078
210545-23-6
205-846-6
Powder
C8H16N4O3· 2H2O
216.241 g/mol
Eyeshields, Gloves, respirator filter
Unless specified otherwise, MP Biomedical's products are for research or further manufacturing use only, not for direct human use. For more information, please contact our customer service department.
02100142-CF
Acetyltryptophan; DL-α-Acetylamino-3-indolepropionic acid
100142
87-32-1
201-739-3
Powder
C13H14N2O3
246.266 g/mol
Eyeshields, Gloves, respirator filter
Unless specified otherwise, MP Biomedical's products are for research or further manufacturing use only, not for direct human use. For more information, please contact our customer service department.
N-acetyl-DL-tryptophan, is used as stabilizer in the human blood-derived therapeutic products normal serum albumin and plasma protein fraction.
89478
0° ± 2° (c=1, 1N NaOH, 24 hrs.)
~99%
λ max (water)=280 ± 2 nm
02100421-CF
L-2,5-Diaminopentanoic acid; 2,5-Diaminopentanoic acid monohydrochloride
100421
3184-13-2
221-678-6
Powder
C5H12N2O2 • HCl
168.621 g/mol
Eyeshields, Gloves, respirator filter
Unless specified otherwise, MP Biomedical's products are for research or further manufacturing use only, not for direct human use. For more information, please contact our customer service department.
3625847
~99%
H319
RM2985000
GHS07
Related
Can anybody help, please?
I am finding it difficult to grab the value of currency units per SDR using my python code below.
The code is working with other URLs, but for this URL, I always get a null result. I don’t understand what is wrong.
I’m using python scrappy spider.
URL: https://www.imf.org/external/np/fin/data/rms_five.aspx
I reviewed the content on the website and found that it contains some spaces. There are some spaces in the element value:
Using RSS response and XPath, I get the same result i.e. null
def start_requests(self):
urls = [
'https://www.imf.org/external/np/fin/data/rms_five.aspx'
]
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
print(response.url)
#for i in range (1, 24):
yield {
'Kurs_imf2': response.xpath('//*[#id="content"]/center/table/tbody/tr[3]/td/div/table/tbody/tr[3]/td[3]/text()').getall()
}
Well, looking only at your ultimate goal (and disregarding your eventual unfortunate choice of tools), here is one way of getting the data from that table, as a dataframe. Once you have the data, you can manipulate it as needed (like stripping the leading/trailing spaces, etc):
import requests
import pandas as pd
from bs4 import BeautifulSoup as bs
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36'
}
url = 'https://www.imf.org/external/np/fin/data/rms_five.aspx'
r = requests.get(url, headers=headers)
soup = bs(r.text, 'html.parser')
table_w_data = soup.select_one('div.fancy').select_one('table')
df = pd.read_html(str(table_w_data))[0]
print(df)
This will display in terminal:
('SDRs per Currency unit 2', 'Unnamed: 0_level_1')
('SDRs per Currency unit 2', 'September 05, 2022')
('SDRs per Currency unit 2', 'September 02, 2022')
('SDRs per Currency unit 2', 'September 01, 2022')
('SDRs per Currency unit 2', 'August 31, 2022')
('SDRs per Currency unit 2', 'August 30, 2022')
0
Chinese yuan
nan
0.111437
0.111266
0.111474
0.110906
1
Euro
nan
0.768586
0.768411
0.768436
0.769353
2
Japanese yen
nan
0.00549178
0.00550612
0.00554387
0.00553607
3
U.K. pound
nan
0.889646
0.887851
0.892615
0.898473
4
U.S. dollar
nan
0.769124
0.768104
0.768436
0.766746
5
Algerian dinar
nan
0.0054812
0.00547939
0.00547416
0.00547027
6
Australian dollar
nan
0.522235
0.524922
0.530375
0.529438
7
Botswana pula
nan
0.0593764
0.0595281
0.0600917
0.0600362
8
Brazilian real
nan
0.148273
0.147709
0.148393
0.151498
9
Brunei dollar
nan
0.548669
0.548215
0.55085
0.54893
10
Canadian dollar
nan
0.586178
0.5834
0.5861
0.586377
11
Chilean peso
nan
0.000856257
0.000855312
0.000871134
0.000860642
12
Czech koruna
nan
0.0313774
0.0313768
0.0313289
0.0312983
13
Danish krone
nan
0.103346
0.10332
0.103325
0.103441
14
Indian rupee
nan
0.0096395
0.00967422
nan
0.00961806
15
Israeli New Shekel
nan
0.227889
0.228331
0.230002
0.231996
16
Korean won
nan
0.000569131
0.000571039
0.000570268
0.000568929
17
Kuwaiti dinar
nan
nan
2.49425
2.49654
2.49105
18
Malaysian ringgit
nan
0.171526
0.171356
nan
0.170977
19
Mauritian rupee
nan
0.0172087
nan
0.0171443
0.0171741
20
Mexican peso
nan
0.0385038
0.0379361
0.0382379
0.0380585
21
New Zealand dollar
nan
0.466781
0.468236
0.471167
0.47105
22
Norwegian krone
nan
0.0768317
0.0767413
0.0773168
0.0788655
23
Omani rial
nan
nan
1.99767
1.99853
1.99414
24
Peruvian sol
nan
0.198946
0.199042
0.200166
nan
25
Philippine peso
nan
nan
0.0136744
0.0136628
0.0136826
26
Polish zloty
nan
0.162688
0.163569
0.162254
0.162412
27
Qatari riyal
nan
nan
0.211018
0.211109
0.210645
28
Russian ruble
nan
0.0127399
0.0127514
0.0127565
0.0127013
29
Saudi Arabian riyal
nan
nan
0.204828
0.204916
0.204466
30
Singapore dollar
nan
0.548669
0.548215
0.55085
0.54893
31
South African rand
nan
0.0444661
0.0448438
0.0450817
0.0455618
32
Swedish krona
nan
0.0715711
0.0718351
0.0720278
0.0718782
33
Swiss franc
nan
0.782903
0.78458
0.784038
0.789442
34
Thai baht
nan
0.0208978
0.020927
0.0210548
0.0210465
35
Trinidadian dollar
nan
0.114475
0.114048
nan
0.114091
36
U.A.E. dirham
nan
nan
0.20915
0.209241
0.20878
37
Uruguayan peso
nan
0.0188128
0.0188288
0.0187606
0.0188043
Relevant documentation:
Requests: https://requests.readthedocs.io/en/latest/
Pandas: https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_html.html
BeautifulSoup: https://beautiful-soup-4.readthedocs.io/en/latest/index.html
I have 2 dataframes one dataframe(df1) contains columns like- ISIN, Name, Currency, Value, % Weight, Asset type., comments and assumptions
So this dataframe looks like this:- df1
ISIN Name Currency Value % Weight Asset Type Comments/ Assumptions
0 NaN Transcanada Trust 5.875 08/15/76 USD 7616765.00 0.0176 NaN https://assets.cohenandsteers.com/assets/conte...
1 NaN Bp Capital Markets Plc Flt Perp USD 7348570.50 0.0169 NaN Holding value for each constituent is derived ...
2 NaN Transcanada Trust Flt 09/15/79 USD 7341250.00 0.0169 NaN NaN
3 NaN Bp Capital Markets Plc Flt Perp USD 6734022.32 0.0155 NaN NaN
4 NaN Prudential Financial 5.375% 5/15/45 USD 6508290.68 0.0150 NaN NaN
(241, 7)
whereas I have another dataframe df2 having columns like- Short Name, ISIN.
This dataframe looks like this.
Short Name ISIN
0 ABU DHABI COMMER AEA000201011
1 ABU DHABI NATION AEA002401015
2 ABU DHABI NATION AEA006101017
3 ADNOC DRILLING C AEA007301012
4 ALPHA DHABI HOLD AEA007601015
(66987, 2)
I developed a logic that compares Name(from df1) and Short Name(from df2) based on a match it extracts relevant ISIN(from df2) into df1(ISIN column which is empty at present).
Here's the logic for the same
def strMergeData(strColumnDf1):
strColumnDf1 = strColumnDf1.split()[0]
for strColumnDf2 in df2['Short Name']:
if strColumnDf1 in strColumnDf2:
return df2[df2['Short Name'] == strColumnDf2]['ISIN'].values[0]
break
else:
pass
df1['ISIN'] = df1.apply(lambda x: strMergeData(x['Name']),axis=1)
print(df1)
which gives the output as :
ISIN Name Currency Value % Weight Asset Type Comments/ Assumptions
0 NA Transcanada Trust 5.875 08/15/76 USD 7616765.00 0.0176 NaN https://assets.cohenandsteers.com/assets/conte...
1 NA Bp Capital Markets Plc Flt Perp USD 7348570.50 0.0169 NaN Holding value for each constituent is derived ...
2 NA Transcanada Trust Flt 09/15/79 USD 7341250.00 0.0169 NaN NaN
3 NA Bp Capital Markets Plc Flt Perp USD 6734022.32 0.0155 NaN NaN
4 NA Prudential Financial 5.375% 5/15/45 USD 6508290.68 0.0150 NaN NaN
The end result should look like this however because of the logic(which actually compares Name and Short Name word by word) it takes the first occurrence in the dataframe and straightaway gives ISIN which is incorrect. For eg: for Name- Bank of Scotland ISIN is 1324fdd is written as 1345o
as a result, I developed a new logic using fuzzywuzzy module which shows the exact match, if a match is not relevant wrt Name then it shows null. Here's the logic.
mat1 = []
mat2 = []
p = []
# converting dataframe column
# to list of elements
# to do fuzzy matching
list1 = df1['Name'].tolist()
list2 = df2['Short Name'].tolist()
# taking the threshold as 80
threshold = 93
# iterating through list1 to extract
# it's closest match from list2
for i in list1:
mat1.append(process.extractOne(i, list2, scorer=fuzz.token_set_ratio))
df1['matches'] = mat1
# iterating through the closest matches
# to filter out the maximum closest match
for j in df1['matches']:
if j[1] >= threshold:
p.append(j[0])
mat2.append(",".join(p))
p = []
# storing the resultant matches back
# to df1
df1['matches'] = mat2
print("\nDataFrame after Fuzzy matching using token_set_ratio():")
print(df1.tail())
and the output that I get is this:
ISIN Name Currency Value % Weight Asset Type Comments/ Assumptions matches
236 NaN Partnerre Ltd 4.875% Perp Sr:J USD 1.684069e+05 0.0004 NaN NaN
237 NaN Berkley (Wr) Corporation 5.700% 03/30/58 USD 6.955837e+04 0.0002 NaN NaN
238 NaN Tc Energy Corp Flt Perp Sr:11 USD 6.380262e+04 0.0001 NaN NaN TC ENERGY CORP
239 NaN Cash and Equivalents USD 2.166579e+07 0.0499 NaN NaN
240 NaN AUM NaN 4.338766e+08 0.9999 NaN NaN AUM IND BARC US
This output basically adds a match column on df1 and constitutes which ShortName(from df1) matches Name(from df1) however doesn't add any ISIN.
How do I add ISIN from df2 to df1 based on the above logic(fuzzywuzzy) so that in the new dataframe(df3) I get the output as:
ISIN Name Currency Value % Weight Asset Type Comments/ Assumptions
0 NA Transcanada Trust 5.875 08/15/76 USD 7616765.00 0.0176 NaN https://assets.cohenandsteers.com/assets/conte...
1 NA Bp Capital Markets Plc Flt Perp USD 7348570.50 0.0169 NaN Holding value for each constituent is derived ...
2 NA Transcanada Trust Flt 09/15/79 USD 7341250.00 0.0169 NaN NaN
3 NA Bp Capital Markets Plc Flt Perp USD 6734022.32 0.0155 NaN NaN
4 NA Prudential Financial 5.375% 5/15/45 USD 6508290.68 0.0150 NaN NaN
Please help.
One option is to use recordlinkage: https://recordlinkage.readthedocs.io/en/latest/
The code below is a quick hack, so will probably need fixing:
import recordlinkage
# Indexation step
indexer = recordlinkage.Index()
indexer.add(recordlinkage.index.Full())
candidate_links = indexer.index(df1, df2)
# Comparison step
compare_cl = recordlinkage.Compare()
compare_cl.string('Name', 'Short Name', label='name_similarity', method='jarowinkler', threshold=0.85)
matches = compare_cl.compute(candidate_links, df1, df2)
I am trying to scrape data from the following website. For the year 1993, for eg, this is the link.
https://www.ugc.ac.in/jobportal/search.aspx?tid=MTk5Mw==
Firstly, I am not sure how to navigate between the pages as the url for every page is the same.
Secondly, I wrote the following code to scrape information on any given page.
url = "https://www.ugc.ac.in/jobportal/search.aspx?tid=MTk5Mw=="
File = []
response = requests.get(url)
soup = bs(response.text,"html.parser")
entries = soup.find_all('tr',{'class': 'odd'})
for entry in entries:
columns = {}
Cells = entry.find_all("td")
columns['Gender'] = Cells[3].get_text()
columns['Category'] = Cells[4].get_text()
columns['Subject'] = Cells[5].get_text()
columns['NET Qualified'] = Cells[6].get_text()
columns['Month/Year'] = Cells[7].get_text()
File.append(columns)
df = pd.DataFrame(File)
df
I am not getting any error while running the code but I am not getting any output. I cant figure out what mistake I am doing here. Would appreciate any inputs. Thanks!
All data is stored inside <script> on that HTML page. To read it into panda's dataframe you can use next example:
import re
import requests
import pandas as pd
url = "https://www.ugc.ac.in/jobportal/search.aspx?tid=MTk5Mw=="
html_doc = requests.get(url).text
data = re.search(r'"aaData":(\[\{.*?\}]),', html_doc, flags=re.S).group(1)
df = pd.read_json(data)
print(df)
df.to_csv("data.csv", index=False)
Prints:
ugc_net_ref_no candidate_name roll_no subject gender jrf_lec cat vh_ph dob fname mname address exam_date result_date emailid mobile_no subject_code year
0 1/JRF (DEC.93) SHRI VEERENDRA KUMAR SHARMA N035010 ECONOMICS Male JRF GEN NaN NaN SHRI SATYA NARAIN SHARMA None 27 E KARANPUR (PRAYAG) ALLAHABAD U.P. 19th DEC.93 NULL NaN NaN NaN 1993
1 1/JRF (JUNE,93) SH MD TARIQUE R020005 ECONOMICS Male JRF GEN NaN NaN MD. ZAFIR ALAM None D-32, R.M. HALL, A.M.U. ALIGARH 20th June,93 NULL NaN NaN NaN 1993
2 10/JRF (DEC.93) SHRI ARGHYA GHOSH A245015 ECONOMICS Male JRF GEN NaN NaN SHRI BHOLANATH GHOSH None C/O SH.B. GHOSH 10,BAMACHARAN GHOSH LANE P.O.-BHADRESWAR,DIST.HOOGHLY CALCUTTA-24. 19th DEC.93 NULL NaN NaN NaN 1993
3 10/JRF (JUNE,93) SH SANTADAS GHOSH T210024 ECONOMICS Male JRF GEN NaN NaN SHRI HARIDAS GHOSH None P-112, MOTILAL COLONY, NO.-1 P.O. DUM DUM, CALCUTTA - 700 028 20th June,93 NULL NaN NaN NaN 1993
...
and saves data.csv (screenshot from LibreOffice):
I am trying to figure out an elegant way to scrape tables from a website. However, when running below script I am getting a ValueError: No tables found error.
from selenium import webdriver
import pandas as pd
driver = webdriver.Chrome(executable_path=r'C:...\chromedriver.exe')
driver.implicitly_wait(30)
driver.get("https://www.gallop.co.za/#meeting#20201125#3")
df_list=pd.read_html(driver.find_element_by_id("eventTab_4").get_attribute('outerHTML'))
When I look at the site elements, I notice that the code below works if the < table > tag lies neatly within the < div id="...">. However, in this case, I think the code is not working because of the following reasons:
There is a < div > within a < div > and then there is the < table > tag.
The site uses Javascript with the tables.
Grateful for advice on how to pull the tables for all races. That is, there are several tables which are made visible as the user clicks on each tab (race). I need to extract all of them into separate dataframes.
from selenium import webdriver
import time
import pandas as pd
pd.set_option('display.max_column',None)
driver = webdriver.Chrome(executable_path='C:/bin/chromedriver.exe')
driver.get("https://www.gallop.co.za/#meeting#20201125#3")
time.sleep(5)
tab = driver.find_element_by_id('tabs') #All tabs are here
li_list = tab.find_elements_by_tag_name('li') #They are in a "li"
a_list = []
for li in li_list[1:]: #First tab has nothing..We skip it
a = li.find_element_by_tag_name('a') #extract the "a" element from the "li"
a_list.append(a)
df = []
for a in a_list:
a.click() #Next Tab
time.sleep(8) #Tables take some time to load fully
page = driver.page_source #Get the HTML of the new Tab page
source = pd.read_html(page)
table = source[1] #Get 2nd table
df.append(table)
print(df)
Output
[ Silk No Horse Unnamed: 3 ACS SH CFR Trainer \
0 NaN 1 JEM ROCK NaN 4bC A NaN Eric Sands
1 NaN 2 MAISON MERCI NaN 3chC A NaN Brett Crawford
2 NaN 3 WORDSWORTH NaN 3bC AB NaN Greg Ennion
3 NaN 4 FOUND THE DREAM NaN 3bG A NaN Adam Marcus
4 NaN 5 IZAPHA NaN 3bG A NaN Andre Nel
5 NaN 6 JACKBEQUICK NaN 3grG A NaN Glen Kotzen
6 NaN 7 MHLABENI NaN 3chG A NaN Eric Sands
7 NaN 8 ORLOV NaN 3bG A NaN Adam Marcus
8 NaN 9 T'CHALLA NaN 3bC A NaN Justin Snaith
9 NaN 10 WEST COAST WONDER NaN 3bG A NaN Piet Steyn
Jockey Wgt MR Dr Odds Last 3 Runs
0 D Dillon 60 0 7 NaN NaN
continued
I have 4 Excel files that I have to merge into one Excel file.
Demography file containing ID, Initials, Age, and Sex.
Laboratory file containing ID, Initials Test name, Test date, and Test Value.
Medical History containing ID, Initials, Medical condition, Start and Stop Dates.
Medication given containing ID, Initials, Drug name, dose, frequency, start and stop dates.
There are 50 patients. The demography file contains all 50 rows of 50 patients. The rest of the files have 50 patients but between 100 to 400 rows because each patient has multiple lab tests or multiple drugs.
When I merge in pandas, I have duplicates or assignment of entities to wrong patients. The challenge is to do this a way such that where you have a patient with more medications given than lab tests, the lab test should replace the duplicates with whitespaces.
This is a shortened representation:
import pandas as pd
lab = pd.read_excel('data/data.xlsx', sheetname='lab')
drugs = pd.read_excel('data/data.xlsx', sheetname='drugs')
merged_data = pd.merge(drugs, lab, on='ID', how='left')
merged_data.to_excel('merged_data.xls')
You get this result: Pandas merge result
I would prefer this result: Prefered output
Consider using cumcount() on a groupby() and then join on both that field with ID:
drugs['GrpCount'] = (drugs.groupby(['ID'])).cumcount()
lab['GrpCount'] = (lab.groupby(['ID'])).cumcount()
merged_data = pd.merge(drugs, lab, on=['ID', 'GrpCount'], how='left').drop(['GrpCount'], axis=1)
# ID Initials_x Drug Name Frequency Route Start Date End Date Initials_y Name Result Date Result
# 0 1 AB AMPICLOX NaN Oral 21-Jun-2016 21-Jun-2016 AB Rapid Diagnostic Test 30-May-16 Abnormal
# 1 1 AB CIPROFLOXACIN Daily Oral 30-May-2016 03-Jun-2016 AB Microscopy 30-May-16 Normal
# 2 1 AB Ibuprofen Tablet 400 mg Two Times a Day Oral 06-Oct-2016 10-Oct-2016 NaN NaN NaN NaN
# 3 1 AB COARTEM NaN Oral 17-Jun-2016 17-Jun-2016 NaN NaN NaN NaN
# 4 1 AB INJECTABLE ARTESUNATE 12 Hourly Intravenous 01-Jun-2016 02-Jun-2016 NaN NaN NaN NaN
# 5 1 AB COTRIMOXAZOLE Daily Oral 30-May-2016 12-Jun-2016 NaN NaN NaN NaN
# 6 1 AB METRONIDAZOLE Two Times a Day Oral 30-May-2016 03-Jun-2016 NaN NaN NaN NaN
# 7 2 SS GENTAMICIN Daily Intravenous 04-Jun-2016 04-Jun-2016 SS Microscopy 6-Jun-16 Abnormal
# 8 2 SS METRONIDAZOLE 8 Hourly Intravenous 04-Jun-2016 06-Jun-2016 SS Complete Blood Count 6-Oct-16 Recorded
# 9 2 SS Oral Rehydration Salts Powder PRN Oral 06-Jun-2016 06-Jun-2016 NaN NaN NaN NaN
# 10 2 SS ZINC 8 Hourly Oral 06-Jun-2016 06-Jun-2016 NaN NaN NaN NaN