Develop Reference

Python Regex Capturing Multiple Matches in separate observations

I am trying to create variables location; contract items; contract code; federal aid using regex on the following text:
PAGE 1
BID OPENING DATE 07/25/18 FROM 0.2 MILES WEST OF ICE HOUSE 07/26/18 CONTRACT NUMBER 03-2F1304 ROAD TO 0.015 MILES WEST OF CONTRACT CODE 'A '
LOCATION 03-ED-50-39.5/48.7 DIVISION HIGHWAY ROAD 44 CONTRACT ITEMS
INSTALL SANDTRAPS AND PULLOUTS FEDERAL AID ACNH-P050-(146)E
PAGE 1
BID OPENING DATE 07/25/18 IN EL DORADO COUNTY AT VARIOUS 07/26/18 CONTRACT NUMBER 03-2H6804 LOCATIONS ALONG ROUTES 49 AND 193 CONTRACT CODE 'C ' LOCATION 03-ED-0999-VAR 13 CONTRACT ITEMS
TREE REMOVAL FEDERAL AID NONE
PAGE 1
BID OPENING DATE 07/25/18 IN LOS ANGELES, INGLEWOOD AND 07/26/18 CONTRACT NUMBER 07-296304 CULVER CITY, FROM I-105 TO PORT CONTRACT CODE 'B '
LOCATION 07-LA-405-R21.5/26.3 ROAD UNDERCROSSING 55 CONTRACT ITEMS
ROADWAY SAFETY IMPROVEMENT FEDERAL AID ACIM-405-3(056)E
This text is from one word file; I'll be looping my code on multiple doc files. In the text above are three location; contract items; contract code; federal aid pairs. But when I use regex to create variables, only the first instance of each pair is included.
The code I have right now is:
# imports
import os
import pandas as pd
import re
import docx2txt
import textract
import antiword
all_bod = []
all_cn = []
all_location = []
all_fedaid = []
all_contractcode = []
all_contractitems = []
all_file = []
text = ' PAGE 1
BID OPENING DATE 07/25/18 FROM 0.2 MILES WEST OF ICE HOUSE 07/26/18 CONTRACT NUMBER 03-2F1304 ROAD TO 0.015 MILES WEST OF CONTRACT CODE 'A '
LOCATION 03-ED-50-39.5/48.7 DIVISION HIGHWAY ROAD 44 CONTRACT ITEMS
INSTALL SANDTRAPS AND PULLOUTS FEDERAL AID ACNH-P050-(146)E
PAGE 1
BID OPENING DATE 07/25/18 IN EL DORADO COUNTY AT VARIOUS 07/26/18 CONTRACT NUMBER 03-2H6804 LOCATIONS ALONG ROUTES 49 AND 193 CONTRACT CODE 'C ' LOCATION 03-ED-0999-VAR 13 CONTRACT ITEMS
TREE REMOVAL FEDERAL AID NONE
PAGE 1
BID OPENING DATE 07/25/18 IN LOS ANGELES, INGLEWOOD AND 07/26/18 CONTRACT NUMBER 07-296304 CULVER CITY, FROM I-105 TO PORT CONTRACT CODE 'B '
LOCATION 07-LA-405-R21.5/26.3 ROAD UNDERCROSSING 55 CONTRACT ITEMS
ROADWAY SAFETY IMPROVEMENT FEDERAL AID ACIM-405-3(056)E'
bod1 = re.search('BID OPENING DATE \s+ (\d+\/\d+\/\d+)', text)
bod2 = re.search('BID OPENING DATE\n\n(\d+\/\d+\/\d+)', text)
if not(bod1 is None):
bod = bod1.group(1)
elif not(bod2 is None):
bod = bod2.group(1)
else:
bod = 'NA'
all_bod.append(bod)
# creating contract number
cn1 = re.search('CONTRACT NUMBER\n+(.*)', text)
cn2 = re.search('CONTRACT NUMBER\s+(.........)', text)
if not(cn1 is None):
cn = cn1.group(1)
elif not(cn2 is None):
cn = cn2.group(1)
else:
cn = 'NA'
all_cn.append(cn)
# location
location1 = re.search('LOCATION \s+\S+', text)
location2 = re.search('LOCATION \n+\S+', text)
if not(location1 is None):
location = location1.group(0)
elif not(location2 is None):
location = location2.group(0)
else:
location = 'NA'
all_location.append(location)
# federal aid
fedaid = re.search('FEDERAL AID\s+\S+', text)
fedaid = fedaid.group(0)
all_fedaid.append(fedaid)
# contract code
contractcode = re.search('CONTRACT CODE\s+\S+', text)
contractcode = contractcode.group(0)
all_contractcode.append(contractcode)
# contract items
contractitems = re.search('\d+ CONTRACT ITEMS', text)
contractitems = contractitems.group(0)
all_contractitems.append(contractitems)
This code parses the only first instance of these variables in the text.
contract-number
location
contract-items
contract-code
federal-aid
03-2F1304
03-ED-50-39.5/48.7
44
A
ACNH-P050-(146)E
But, I am trying to figure out a way to get all possible instances in different observations.
contract-number
location
contract-items
contract-code
federal-aid
03-2F1304
03-ED-50-39.5/48.7
44
A
ACNH-P050-(146)E
03-2H6804
03-ED-0999-VAR
13
C
NONE
07-296304
07-LA-405-R21.5/26.3
55
B
ACIM-405-3(056)E
The all_variables in the code are for looping over multiple word files - we can ignore that if we want :).
Any leads would be super helpful. Thanks so much!

import re
data = []
df = pd.DataFrame()
regex_contract_number =r"(?:CONTRACT NUMBER\s+(?P<contract_number>\S+?)\s)"
regex_location = r"(?:LOCATION\s+(?P<location>\S+))"
regex_contract_items = r"(?:(?P<contract_items>\d+)\sCONTRACT ITEMS)"
regex_federal_aid =r"(?:FEDERAL AID\s+(?P<federal_aid>\S+?)\s)"
regex_contract_code =r"(?:CONTRACT CODE\s+\'(?P<contract_code>\S+?)\s)"
regexes = [regex_contract_number,regex_location,regex_contract_items,regex_federal_aid,regex_contract_code]
for regex in regexes:
for match in re.finditer(regex, text):
data.append(match.groupdict())
df = pd.concat([df, pd.DataFrame(data)], axis=1)
data = []
df

Loop scrapes the same page 20 times instead of iterating through range

I'm trying to scrape IMDB for a list of the top 1000 movies and get some details about them. However, when I run it, instead of getting the first 50 movies and going to the next page for the next 50, it repeats the loop and makes the same 50 entries 20 times in my database.
# Dataframe template
data = pd.DataFrame(columns=['ID','Title','Genre','Summary'])
#Get page data function
def getPageContent(start=1):
start = 1
url = 'https://www.imdb.com/search/title/?title_type=feature&year=1950-01-01,2019-12-31&sort=num_votes,desc&start='+str(start)
r = requests.get(url)
bs = bsp(r.text, "lxml")
return bs
#Run for top 1000
for start in range(1,1001,50):
getPageContent(start)
movies = bs.findAll("div", "lister-item-content")
for movie in movies:
id = movie.find("span", "lister-item-index").contents[0]
title = movie.find('a').contents[0]
genres = movie.find('span', 'genre').contents[0]
genres = [g.strip() for g in genres.split(',')]
summary = movie.find("p", "text-muted").find_next_sibling("p").contents
i = data.shape[0]
data.loc[i] = [id,title,genres,summary]
#Clean data
# data.ID = [float(re.sub('.','',str(i))) for i in data.ID] #remove . from ID
data.head(51)
0 1. The Shawshank Redemption [Drama] [\nTwo imprisoned men bond over a number of ye...
1 2. The Dark Knight [Action, Crime, Drama] [\nWhen the menace known as the Joker wreaks h...
2 3. Inception [Action, Adventure, Sci-Fi] [\nA thief who steals corporate secrets throug...
3 4. Fight Club [Drama] [\nAn insomniac office worker and a devil-may-...
...
46 47. The Usual Suspects [Crime, Drama, Mystery] [\nA sole survivor tells of the twisty events ...
47 48. The Truman Show [Comedy, Drama] [\nAn insurance salesman discovers his whole l...
48 49. Avengers: Infinity War [Action, Adventure, Sci-Fi] [\nThe Avengers and their allies must be willi...
49 50. Iron Man [Action, Adventure, Sci-Fi] [\nAfter being held captive in an Afghan cave,...
50 1. The Shawshank Redemption [Drama] [\nTwo imprisoned men bond over a number of ye...

Delete 'start' variable inside 'getPageContent' function. It assigns 'start=1' every time.
#Get page data function
def getPageContent(start=1):
url = 'https://www.imdb.com/search/title/?title_type=feature&year=1950-01-01,2019-12-31&sort=num_votes,desc&start='+str(start)
r = requests.get(url)
bs = bsp(r.text, "lxml")
return bs

I was not able to test this code. See inline comments for what I see as the main issue.
# Dataframe template
data = pd.DataFrame(columns=['ID', 'Title', 'Genre', 'Summary'])
# Get page data function
def getPageContent(start=1):
start = 1
url = 'https://www.imdb.com/search/title/?title_type=feature&year=1950-01-01,2019-12-31&sort=num_votes,desc&start=' + str(
start)
r = requests.get(url)
bs = bsp(r.text, "lxml")
return bs
# Run for top 1000
# for start in range(1, 1001, 50): # 50 is a
# step value so this gets every 50th movie
# Try 2 loops
start = 0
for group in range(0, 1001, 50):
for item in range(group, group + 50):
getPageContent(item)
movies = bs.findAll("div", "lister-item-content")
for movie in movies:
id = movie.find("span", "lister-item-index").contents[0]
title = movie.find('a').contents[0]
genres = movie.find('span', 'genre').contents[0]
genres = [g.strip() for g in genres.split(',')]
summary = movie.find("p", "text-muted").find_next_sibling("p").contents
i = data.shape[0]
data.loc[i] = [id, title, genres, summary]
# Clean data
# data.ID = [float(re.sub('.','',str(i))) for i in data.ID] #remove . from ID
data.head(51)

Python crypto buy-sell bot

This is a simple trading bot working with gate.io api. It does almost what we want but 2 features are missing.
1-When I enter the token name in the terminal, it has to automatically buy it 3% more than the market price.
2-When the price rises as much as I have determined, it has to automatically sell back all the tokens it bought.
How can I add these features?
import ccxt, time
import utils.config
from utils.telegram_bot import post_message
# Create the GateIO exchange object using CCXT
exchange = ccxt.gateio({
'apiKey': utils.config.GATEIO_API_KEY,
'secret': utils.config.GATEIO_SECRET_KEY,
})
# Get the market and balance data from GateIO
markets = exchange.fetch_markets()
balance = exchange.fetch_total_balance()
# Get all the available symbols from GateIO
symbols = [symbol for symbol in [market['symbol'] for market in markets]]
# Placeholder for keeping the price at the time of buying
entryPrice = 0
def symbol_check(symbol):
if symbol in symbols:
print(f"Good news!, {symbol} exists in Gate.io")
return True
else:
print(f"Sorry, {symbol} does not exist in Gate.io")
return False
def buy(symbol):
global entryPrice
# Pick a price more than the last ticker data, to make sure that we can fulfill order
price = exchange.fetch_ticker(symbol)['last'] * 1.01
entryPrice = price
# Get the current USDT balance in GateIO
balance = exchange.fetch_total_balance()
coin = symbol.split('/')[0]
usdt_balance = balance['USDT']
# Calculate the amount of coin that we can buy, apply a small margin for rounding errors in balance
amount = (usdt_balance * 0.999) / (price)
# Create the limit buy order
exchange.create_limit_buy_order(symbol, amount=amount, price=price)
# Notify the user both on Telegram and CLI
message = f"You have {usdt_balance} USDT in your account. Buying {amount} {coin} for {price}"
print(message)
post_message(message)
def sell(symbol):
# Pick a price less than the last ticker data, to make sure that we can fulfill order
price = exchange.fetch_ticker(symbol)['last'] * 0.99
# Get the current coin balance in GateIO
balance = exchange.fetch_total_balance()
coin = symbol.split('/')[0]
coin_balance = balance[coin]
# Create the limit sell order
exchange.create_limit_sell_order(symbol, amount=coin_balance, price=price)
# Notify the user both on Telegram and CLI
message = f"You have {coin_balance} {coin} in your account. Selling them for {price}"
print(message)
post_message(message)
def sellBearish(symbol):
global entryPrice
bestPrice = exchange.fetch_ticker(symbol)['last']
balance = exchange.fetch_total_balance()
while True:
price = exchange.fetch_ticker(symbol)['last']
# Update the best price if we have a new best price
if price > bestPrice:
bestPrice = price
# If the price has dropped 3% percent w.r.t. the best price, sell the coins and get the profit
elif price < (0.97 * bestPrice):
coin = symbol.split('/')[0]
coin_balance = balance[coin]
exchange.create_limit_sell_order(symbol, amount=coin_balance, price=price*0.99)
# Notify the user both on Telegram and CLI
message = f"You have {coin_balance} {coin} in your account.\nSelling them for {price*0.99}\nYour profit is{price/entryPrice*100}"
print(message)
post_message(message)

BeautifulSoup trying to get text from wrapped divs but empty or "none" is being returned

Here is a picture (sorry) of the HTML that I am trying to parse:
I am using this line:
home_stats = soup.select_one('div', class_='statText:nth-child(1)').text
Thinking that I'd get the 1st child of the class statText and the outcome would be 53%.
But it's not. I get "Loading..." and none of the data that I was trying to use and display.
The full code I have so far:
soup = BeautifulSoup(source, 'lxml')
home_team = soup.find('div', class_='tname-home').a.text
away_team = soup.find('div', class_='tname-away').a.text
home_score = soup.select_one('.current-result .scoreboard:nth-child(1)').text
away_score = soup.select_one('.current-result .scoreboard:nth-child(2)').text
print("The home team is " + home_team, "and they scored " + home_score)
print()
print("The away team is " + away_team, "and they scored " + away_score)
home_stats = soup.select_one('div', class_='statText:nth-child(1)').text
print(home_stats)
Which currently does print the hone and away team and the number of goals they scored. But I can't seem to get any of the statistical content from this site.
My output plan is to have:
[home_team] had 53% ball possession and [away_team] had 47% ball possession
However, I would like to remove the "%" symbols from the parse (but that's not essential). My plan is to use these numbers for more stats later on, so the % symbol gets in the way.
Apologies for the noob question - this is the absolute beginning of my Pythonic journey. I have scoured the internet and StackOverflow and just can not find this situation - I also possibly don't know exactly what I am looking for either.
Thanks kindly for your help! May your answer be the one I pick as "correct" ;)

Assuming that this is the website that u r tryna scrape, here is the complete code to scrape all the stats:
from bs4 import BeautifulSoup
from selenium import webdriver
import pandas as pd
driver = webdriver.Chrome('chromedriver.exe')
driver.get('https://www.scoreboard.com/en/match/SO3Fg7NR/#match-statistics;0')
pg = driver.page_source #Gets the source code of the page
driver.close()
soup = BeautifulSoup(pg,'html.parser') #Creates a soup object
statrows = soup.find_all('div',class_ = "statTextGroup") #Finds all the div tags with class statTextGroup -- these div tags contain the stats
#Scrapes the team names
teams = soup.find_all('a',class_ = "participant-imglink")
teamslst = []
for x in teams:
team = x.text.strip()
if team != "":
teamslst.append(team)
stats_dict = {}
count = 0
for x in statrows:
txt = x.text
final_txt = ""
stat = ""
alphabet = False
percentage = False
#Extracts the numbers from the text
for c in txt:
if c in '0123456789':
final_txt+=c
else:
if alphabet == False:
final_txt+= "-"
alphabet = True
if c != "%":
stat += c
else:
percentage = True
values = final_txt.split('-')
#Appends the values to the dictionary
for x in values:
if stat in stats_dict.keys():
if percentage == True:
stats_dict[stat].append(x + "%")
else:
stats_dict[stat].append(int(x))
else:
if percentage == True:
stats_dict[stat] = [x + "%"]
else:
stats_dict[stat] = [int(x)]
count += 1
if count == 15:
break
index = [teamslst[0],teamslst[1]]
#Creates a pandas DataFrame out of the dictionary
df = pd.DataFrame(stats_dict,index = index).T
print(df)
Output:
Burnley Southampton
Ball Possession 53% 47%
Goal Attempts 10 5
Shots on Goal 2 1
Shots off Goal 4 2
Blocked Shots 4 2
Free Kicks 11 10
Corner Kicks 8 2
Offsides 2 1
Goalkeeper Saves 0 2
Fouls 8 10
Yellow Cards 1 0
Total Passes 522 480
Tackles 15 12
Attacks 142 105
Dangerous Attacks 44 29
Hope that this helps!
P.S: I actually wrote this code for a different question, but I didn't post it as an answer was already posted! But I didn't know that it would come in handy now! Anyways, I hope that my answer does what u need.

IF/ELSE Control flow in Python to work out price of courier service from 4 choices

I cannot seem to understand how to use if/else in the following question:
You need to design a program for a courier company to calculate the cost of sending a parcel.
Ask the user to enter the price of the package they would like to purchase.
Ask the user to enter the total distance of the delivery in kilometers.
Now, add on the delivery costs to get the final cost of the product. There are four categories to factor in when determining a parcel’s final cost, each with two options based on the customer’s delivery preferences. (Use an if else statement based on the choice they make)
Delivery via air ($0.36 per km) or via freight ($0.25 per km)
Full insurance ($50.00) or limited insurance ($25.00)
Gift option ($15.00) or not ($0.00)
Priority delivery ($100.00) or standard delivery ($20.00)
Write code to work out the total cost of the package based on the options
selected in each category.

#Mohseen Ramjan
I used some of the original code, but simplified it a bit.
I am no expert, and I'm sure even this code can be improved a lot.
===== NOTE: our currency is Rand, thus the use of the 'R' =====
But maybe you'll understand this a bit better:
main_product_price = int(float(input("""Please enter the price of the package you would like to purchase:
(Only use numbers, do not inlcude 'R')\n""")))
total_distance = int(float(input("\nPlease enter the total distance of the delivery in Kilometers:\n")))
print ("\nNow please choose your shipping preferences;")
print ("\nWould you prefere Air at R0.36 per km, or Freight at R0.25 per km?")
freight_choise = input("Enter: 'Air' or 'Freight'\n")
cost_per_distance = 0
if freight_choise in ['Freight']:
cost_per_distance = 0.25
elif freight_choise in ['Air']:
cost_per_distance = 0.36
print ("\nWould you prefere Full insurance (R50.00), or Limited insurance (R25.00)?")
insurance_choise = input("Enter: 'Full' or 'Limited'?\n")
insurance_cost = 0
if insurance_choise in ['Full']:
insurance_cost = 50
elif insurance_choise in ['Limited']:
insurance_cost = 25
print ("\nWould you like to add a Gift for R15.00?")
gift_choise = input("Enter: 'Yes please' or 'No thanks'\n")
gift_cost = 0
if gift_choise in ['Yes please']:
gift_cost = 15
elif gift_choise in ['No thanks']:
gift_cost = 0
print ("\nWould you prefere Priority delivery for R100.00, or Standard delivery for R20.00?")
delivery_choise = input("Enter: 'Priority' or 'Standard'\n")
priority_or_standard_delivery = 0
if delivery_choise in ['Priority']:
priority_or_standard_delivery = 100
elif delivery_choise in ['Standard']:
priority_or_standard_delivery = 20
total_cost = main_product_price + total_distance*cost_per_distance + insurance_cost + gift_cost + priority_or_standard_delivery
print (f"""\nThis is your Total cost:
R{total_cost}""")

# Courier cost of delivering parcel
# You can and should add your own assertions and checks if the user input is valid
# I used a list instead of '==' so that you can expand the response options
# There are many other ways to do it, this is just my quick and dirty method
# But I suppose you could iterate from here
def user_input():
price_of_package = float(input('Input price of package.\n'))
total_distance = float(input('Input total distance in km\n'))
freight_or_air = input('Input freight or air delivery?\n').lower()
full_or_limited_insurance = input('Input full or limited insurance?\n').lower()
gift_or_not = input('Is this a gift?\n').lower()
priority_or_standard = input('Is this priority or standard delivery?\n').lower()
cost_per_distance = 0
if freight_or_air in ['freight']:
cost_per_distance = 0.25
elif freight_or_air in ['air']:
cost_per_distance = 0.36
cost_of_insurance = 0
if full_or_limited_insurance in ['full']:
cost_of_insurance = 50.00
elif full_or_limited_insurance in ['limited']:
cost_of_insurance = 25.00
cost_of_gift = 0
if gift_or_not in ['yes']:
cost_of_gift = 15
cost_of_delivery = 0
if priority_or_standard in ['priority']:
cost_of_delivery = 100
elif priority_or_standard in ['standard']:
cost_of_delivery = 20
print (f'\nThe user has specified that\n\
price of package: {price_of_package}\n\
total distance: {total_distance}\n\
freight or air: {freight_or_air}\n\
cost per distance {cost_per_distance}\n\
type of insurance: {full_or_limited_insurance}\n\
cost of insurance: {cost_per_distance}\n\
if it is a gift: {gift_or_not}\n\
cost of gift: {cost_of_gift}\n\
type of delivery: {priority_or_standard}\n\
cost of delivery: {cost_of_delivery}.')
return price_of_package, total_distance, cost_per_distance,\
cost_of_insurance, cost_of_gift, cost_of_delivery
def total_cost():
price_of_package, total_distance, cost_per_distance,\
cost_of_insurance, cost_of_gift, cost_of_delivery = user_input()
total_cost = price_of_package + total_distance*cost_per_distance +\
cost_of_insurance + cost_of_gift + cost_of_delivery
print (f'\nThe total cost is {total_cost}.')
return total_cost