I'm going to start out by saying this is the 1st python program I have ever written and have no real background in the language so my code is probably pretty rough. Take it easy on me!
When I wrote this it works fine for a small number of tickers but when I have a file with 6k+ it is very slow. Now I know there are other ways I can improve performance BUT I want to try and tackle the async thing first.
I thought it was as simple as making the read_ticker_file function async and adding await in front of the yfin_options() call but obviously that didn't work.
I'm thinking possibly I need to restructure the way things are called but kind of stuck here. Hoping someone can point me in the right direction! Thanks in advance
import logging
import pyodbc
import config
import yahoo_fin as yfin
from yahoo_fin import options
from datetime import datetime, date
from selenium import webdriver
def main():
read_ticker_file()
def init_selenium():
driver = webdriver.Chrome(config.CHROME_DRIVER)
return driver
def yfin_options(symbol):
logging.basicConfig(filename='yfin.log', level=logging.INFO)
logging.basicConfig(filename='no_options.log', level=logging.ERROR)
try:
# get all options dates (in epoch) from dropdown on yahoo finance options page
dates = get_exp_dates(symbol)
# iterate each date to get all calls and insert into sql db
for date in dates:
arr = yfin.options.get_calls(symbol, date)
arr_length = len(arr.values)
i = 0
for x in range(0, arr_length):
strike = str(arr.values[i][2])
volume = str(arr.values[i][8])
open_interest = str(arr.values[i][9])
convert_epoch = datetime.fromtimestamp(int(date))
try:
sql_insert(symbol, strike, volume, open_interest, convert_epoch)
i += 1
except Exception as insert_fail:
print("I failed at sqlinsert {0}".format(insert_fail))
file_name_dir = "C:\\temp\\rh\\options{0}{1}.xlsx".format(symbol, date)
logging.info(arr.to_excel(file_name_dir))
except Exception as e:
bad_tickers_file_dir = config.BAD_TICKERS
f = open(bad_tickers_file_dir, "a")
f.write(symbol)
f.write('\n')
def sql_insert(symbol, strike, volume, open_interest, exp_date):
conn_string = ('Driver={SQL Server};'
'Server={0};'
'Database={1};'
'Trusted_Connection=yes;').format(config.SERVER, config.DATABASE)
conn = pyodbc.connect(conn_string)
cursor = conn.cursor()
insert_string = """INSERT INTO dbo.options (Ticker, Strike, Volume, OpenInterest, expDate)
VALUES
(?, ?, ?, ?, ?)"""
cursor.execute(insert_string, symbol, strike, volume, open_interest, str(exp_date))
conn.commit()
def get_exp_dates(symbol):
url = "https://finance.yahoo.com/quote/" + symbol + "/options?p=" + symbol
chromedriver = init_selenium()
chromedriver.get(url)
# Yahoo Finance options dropdown class name (find better way to do this)
select_dropdown = chromedriver.find_element_by_css_selector("div[class='Fl(start) Pend(18px)'] > select")
options_list = [x for x in select_dropdown.find_elements_by_tag_name("option")]
dates = []
for element in options_list:
dates.append(element.get_attribute("value"))
return dates
def read_ticker_file():
file1 = open(config.TICKER_FILE, 'r')
lines = file1.readlines()
count = 0
# loop to read each ticker in file
for line in lines:
count += 1
line = line.strip('\n')
line = line.strip()
yfin_options(line)
if __name__ == "__main__":
main()
Related
I have this problem, because I am measuring the temperature of my room with a serial arduino.
I also have a db of plants in azure, that could grow in x temperatures.
I want to check if the temerature on my room is okay for the plants i have on my db to grow, but i dont know how to send this temperature value to azure and check the temperature column.
Any advice?
Thanks
At the moment I thout about having two files, one in azure (i dont know how to upload it to azure) and one in my local pc
The one on the cloud:
from arduinodb_pc import *
import pyodbc
import time
server = '...'
database = '...'
username = '...'
password = '...'
driver= '{ODBC Driver 17 for SQL Server}'
conn = pyodbc.connect('DRIVER='+driver+';SERVER=tcp:'+server+';PORT=1433;DATABASE='+database+';UID='+username+';PWD='+ password)
cursor = conn.cursor()
def db_check_temperature_cold(farenh):
cursor.execute(f"SELECT * FROM my_plants WHERE minTinF >= {farenh};") # Send query and execute
# row = [id, Name, ..., maxTinF]
row = cursor.fetchone() # Move to the next row;
list_of_names = []
while row:
list_of_names.append(row)
row = cursor.fetchone() # Move to the next row
return list_of_names
def db_check_temperature_hot(farenh):
cursor.execute(f"SELECT * FROM my_plants WHERE maxTinF <= {farenh};") # Send query and execute
# row = [id, Name, ..., maxTinF]
row = cursor.fetchone() # Move to the next row;
list_of_names = []
while row:
list_of_names.append(row)
row = cursor.fetchone() # Move to the next row
return list_of_names
currentTinF =0
currentTinC=0
text_currentT=0
while True:
currentTinF, currentTinC, text_currentT = update_T()
too_cold = db_check_temperature_cold(currentTinF)
too_hot = db_check_temperature_hot(currentTinF)
if (too_cold): arduino_write_cold(arduino)
if (too_hot): arduino_write_hot(arduino)
time.sleep(15) # but every 10 minutes
The code in my local pc:
import datetime
import time
from arduinodb_cloud import *
import serial
arduino = serial.Serial(port='COM3', baudrate=115200, timeout=.1)
def update_T():
"""
Updates the label where the time and temperature are shown
"""
return arduino_read_T(arduino, -1)
def arduino_read_T(arduino, unit):
"""
In this function, the current temperature will be read from the aruino.
:param arduino: HW it is going to use for checking the T
:param unit: If the user prefers ºF or ºC
:return: f, c: current temerature in farenheit and celisus
txt_currentT: Text that will be shown on the gui
"""
global fbefore
global cbefore
x = datetime.datetime.now()
arduino.write(bytes("readT", 'utf-8'))
time.sleep(0.05)
value = arduino.readline()
try:
value = str(value).lstrip("b'")
value = str(value).rstrip("'")
f, c = str(value).split(",")
except:
try:
f = fbefore
c = cbefore
except:
f=65.3
c=18.5
fbefore=f
cbefore=c
if unit == 1: txt_currentT = (f"Today, {x.day}/{x.month}/{x.year} at {x.hour}:{x.minute}:{x.second} it's {f}ºF")
elif unit == 0: txt_currentT = (f"Today, {x.day}/{x.month}/{x.year} at {x.hour}:{x.minute}:{x.second} it's {c}ºC")
else: txt_currentT = ""
return float(f), float(c), txt_currentT
def arduino_write_cold(arduino):
arduino.write(bytes("cold", 'utf-8'))
def arduino_write_hot(arduino):
arduino.write(bytes("hot", 'utf-8'))
import time
import subprocess
from tkinter import *
from w1thermsensor import W1ThermSensor
import time
import datetime
import sqlite3
root = Tk()
id = 1
conn = sqlite3.connect('temp_sensor2.db')
c = conn.cursor()
sensor = W1ThermSensor()
temperature = sensor.get_temperature()
t = int(time.time())
date = str (datetime.datetime.fromtimestamp(t).strftime('%d-%m-%Y %H:%M:%S'))
global tempLabel1
def get_temp(period_ms):
temperature = sensor.get_temperature()
tempLabel1['text'] = temperature
tempLabel1.after(period_ms, get_temp, period_ms)
c.execute('''INSERT INTO datetemp VALUES (?, ?, ?)''',(id, date, temperature));
conn.commit()
root.title('Temperature')
tempLabel2 = Label(root, text="Temperature is ")
tempLabel2.pack()
tempLabel1 = Label(root, width=25)
tempLabel1.pack()
get_temp(1000)
root.mainloop()
I have a program here that monitors temperature and automatically updates in a tkinter label. I also wish to have it update a SQLite DB, however it enters multiple entries into the table with the exact same datestamp (although temperature reading will be different). Any ideas would be appreciated!
t = int(time.time())
date = str (datetime.datetime.fromtimestamp(t).strftime('%d-%m-%Y %H:%M:%S'))
These two lines of code run once, when your program starts up, and the date variable never changes after that.
If you move these lines inside the get_temp() function, then it will use the current timestamp.
I am using Python 3.6, pyodbc, and connect to SQL Server.
I am trying make connection to a database, then creating a query with parameters.
Here is the code:
import sys
import pyodbc
# connection parameters
nHost = 'host'
nBase = 'base'
nUser = 'user'
nPasw = 'pass'
# make connection start
def sqlconnect(nHost,nBase,nUser,nPasw):
try:
return pyodbc.connect('DRIVER={SQL Server};SERVER='+nHost+';DATABASE='+nBase+';UID='+nUser+';PWD='+nPasw)
print("connection successfull")
except:
print ("connection failed check authorization parameters")
con = sqlconnect(nHost,nBase,nUser,nPasw)
cursor = con.cursor()
# make connection stop
# if run WITHOUT parameters THEN everything is OK
ask = input ('Go WITHOUT parameters y/n ?')
if ask == 'y':
# SQL without parameters start
res = cursor.execute('''
SELECT * FROM TABLE
WHERE TABLE.TIMESTAMP BETWEEN '2017-03-01T00:00:00.000' AND '2017-03-01T01:00:00.000'
''')
# SQL without parameters stop
# print result to console start
row = res.fetchone()
while row:
print (row)
row = res.fetchone()
# print result to console stop
# if run WITH parameters THEN ERROR
ask = input ('Go WITH parameters y/n ?')
if ask == 'y':
# parameters start
STARTDATE = "'2017-03-01T00:00:00.000'"
ENDDATE = "'2017-03-01T01:00:00.000'"
# parameters end
# SQL with parameters start
res = cursor.execute('''
SELECT * FROM TABLE
WHERE TABLE.TIMESTAMP BETWEEN :STARTDATE AND :ENDDATE
''', {"STARTDATE": STARTDATE, "ENDDATE": ENDDATE})
# SQL with parameters stop
# print result to console start
row = res.fetchone()
while row:
print (row)
row = res.fetchone()
# print result to console stop
When I run the program without parameters in SQL, it works.
When I try running it with parameters, an error occurred.
Parameters in an SQL statement via ODBC are positional, and marked by a ?. Thus:
# SQL with parameters start
res = cursor.execute('''
SELECT * FROM TABLE
WHERE TABLE.TIMESTAMP BETWEEN ? AND ?
''', STARTDATE, ENDDATE)
# SQL with parameters stop
Plus, it's better to avoid passing dates as strings. Let pyodbc take care of that using Python's datetime:
from datetime import datetime
...
STARTDATE = datetime(year=2017, month=3, day=1)
ENDDATE = datetime(year=2017, month=3, day=1, hour=0, minute=0, second=1)
then just pass the parameters as above. If you prefer string parsing, see this answer.
If you're trying to use pd.to_sql() like me I fixed the problem by passing a parameter called chunksize.
df.to_sql("tableName", engine ,if_exists='append', chunksize=50)
hope this helps
i tryied and have a lot of different errors: 42000, 22007, 07002 and others
The work version is bellow:
import sys
import pyodbc
import datetime
# connection parameters
nHost = 'host'
nBase = 'DBname'
nUser = 'user'
nPasw = 'pass'
# make connection start
def sqlconnect(nHost,nBase,nUser,nPasw):
try:
return pyodbc.connect('DRIVER={SQL Server};SERVER='+nHost+';DATABASE='+nBase+';UID='+nUser+';PWD='+nPasw)
except:
print ("connection failed check authorization parameters")
con = sqlconnect(nHost,nBase,nUser,nPasw)
cursor = con.cursor()
# make connection stop
STARTDATE = '11/2/2017'
ENDDATE = '12/2/2017'
params = (STARTDATE, ENDDATE)
# SQL with parameters start
sql = ('''
SELECT * FROM TABLE
WHERE TABLE.TIMESTAMP BETWEEN CAST(? as datetime) AND CAST(? as datetime)
''')
# SQL with parameters stop
# print result to console start
query = cursor.execute(sql, params)
row = query.fetchone()
while row:
print (row)
row = query.fetchone()
# print result to console stop
say = input ('everething is ok, you can close console')
I fixed this issue with code if you are using values through csv.
for i, row in read_csv_data.iterrows():
cursor.execute('INSERT INTO ' + self.schema + '.' + self.table + '(first_name, last_name, email, ssn, mobile) VALUES (?,?,?,?,?)', tuple(row))
I had a similar issue. Saw that downgrading the version of PyODBC to 4.0.6 and SQLAlchemy to 1.2.9 fixed the error,using Python 3.6
Im using sqlite to store a series of prices for certain stocks
I wasnt able to log in to the website, so i saved the webpage locally a number of times to track different prices.
My code now works for the two saves ive made previously, but when i try and use any saves from today i get the following error
UnicodeDecodeError: 'charmap' codec can't decode byte 0x90 in position 415367:character maps to <undefined>
my code is as follows
from bs4 import BeautifulSoup
from time import gmtime, strftime
import sqlite3
from sqlclean import *
count = 0
def create_tables(stock):
sql_command = """CREATE TABLE """ + stock +"""(
Stock_number INTEGER PRIMARY KEY,
BuyPrice REAL,
SellPrice REAL,
Time VARCHAR(30));"""
cursor.execute(sql_command)
def fill():
y = 0
for i in stock:
string = sqstring(i)
stock[y] = string
y = y + 1
for i in stock:
create_tables(str(i))
def populate():
x = 0
for i in stock:
cursor.execute("""
INSERT INTO """+ i +"""
(SellPrice,BuyPrice)
VALUES
(""" + sell[x]+""","""+ buy[x] +""")
""")
x = x + 1
def get_stocks(soup):
global count
rep1 = 0
rep2 = 0
if count == 0:
count = count + 1
for price in soup.find_all('span',{"class" : "tbox-list-button-sell"}):
sell.append(price.text)
for price in soup.find_all('span',{"class" : "tbox-list-button-buy"}):
buy.append(price.text)
for price in soup.find_all('div',{"class" : "window_title list-title"}):
a = price.text.strip()
stock.append(a)
fill()
populate()
else:
for price in soup.find_all('span',{"class" : "tbox-list-button-sell"}):
sell[rep1] = (price.text)
rep1 = rep1 + 1
for price in soup.find_all('span',{"class" : "tbox-list-button-buy"}):
buy[rep2] = (price.text)
rep2 = rep2 + 1
populate()
connection = sqlite3.connect("stocks.db")
cursor = connection.cursor()
web = ["C:/Users/Luke_2/Desktop/Computing/Coursework/Practice/Stocks1/demo.trading212.com.html","C:/Users/Luke_2/Desktop/Computing/Coursework/live/Stocks1/demo.trading212.com1.html","C:/Users/Luke_2/Desktop/Computing/Coursework/live/Stocks1/demo.trading212.com10.24.html"]
stock=[]
sell = []
buy = []
def run():
for i in web:
soup = BeautifulSoup(open(i),"html.parser")
get_stocks(soup)
run()
connection.commit()
connection.close()
You didn't tell the open() function what codec to use when reading the file e.g. if you are using utf-8 in your file:
In Python 2.7:
import io
...
def run():
for i in web:
with io.open(i, encoding='utf-8') as infile:
soup = BeautifulSoup(infile,"html.parser")
For Python3
def run():
for i in web:
with open(i, encoding='utf-8') as infile:
soup = BeautifulSoup(infile,"html.parser")
I keep getting the following error when trying to access a variable from one function inside another function.
NameError: global name 'savemovieurl' is not defined
how can i access the "savemovieurl" from the function "tmdb_posters" inside "dynamic_data_entry" to save it to the database?
i've tried adding global to the variable name, and had no success.
import requests
import urllib
import sqlite3
import time
import datetime
import random
movie = raw_input('Enter your movie: ')
print('You searched for: ', movie)
def imdb_id_from_title(title):
""" return IMDb movie id for search string
Args::
title (str): the movie title search string
Returns:
str. IMDB id, e.g., 'tt0095016'
None. If no match was found
"""
pattern = 'http://www.imdb.com/xml/find?json=1&nr=1&tt=on&q={movie_title}'
url = pattern.format(movie_title=urllib.quote(title))
r = requests.get(url)
res = r.json()
# sections in descending order or preference
for section in ['popular','exact','substring']:
key = 'title_' + section
if key in res:
return res[key][0]['id']
if __name__=="__main__":
title = movie
imdb_info_returned = ("{1}".format(title, imdb_id_from_title(title)))
print imdb_info_returned
import os
import requests
CONFIG_PATTERN = 'http://api.themoviedb.org/3/configuration?api_key={key}'
IMG_PATTERN = 'http://api.themoviedb.org/3/movie/{imdbid}/images?api_key={key}'
KEY = '47db65094c31430c5a2b65112088d70e'
imdb_id_input = imdb_info_returned
print('You searched for: ', imdb_id_input)
def _get_json(url):
r = requests.get(url)
return r.json()
def _download_images(urls, path='.'):
"""download all images in list 'urls' to 'path' """
for nr, url in enumerate(urls):
r = requests.get(url)
filetype = r.headers['content-type'].split('/')[-1]
filename = 'poster_{0}.{1}'.format(nr+1,filetype)
filepath = os.path.join(path, filename)
with open(filepath,'wb') as w:
w.write(r.content)
def get_poster_urls(imdbid):
""" return image urls of posters for IMDB id
returns all poster images from 'themoviedb.org'. Uses the
maximum available size.
Args:
imdbid (str): IMDB id of the movie
Returns:
list: list of urls to the images
"""
config = _get_json(CONFIG_PATTERN.format(key=KEY))
base_url = config['images']['base_url']
sizes = config['images']['poster_sizes']
"""
'sizes' should be sorted in ascending order, so
max_size = sizes[-1]
should get the largest size as well.
"""
def size_str_to_int(x):
return float("inf") if x == 'original' else int(x[1:])
max_size = max(sizes, key=size_str_to_int)
posters = _get_json(IMG_PATTERN.format(key=KEY,imdbid=imdbid))['posters']
poster_urls = []
rel_path = posters[0]['file_path']
url = "{0}{1}{2}".format(base_url, max_size, rel_path)
poster_urls.append(url)
return poster_urls
def tmdb_posters(imdbid, count=None, outpath='.'):
urls = get_poster_urls(imdbid)
if count is not None:
urls = urls[:count]
_download_images(urls, outpath)
savemovieurl = urls
print savemovieurl
conn = sqlite3.connect('tutorial.db')
c = conn.cursor()
def create_table():
c.execute("CREATE TABLE IF NOT EXISTS movies(unix REAL, datestamp TEXT, keyword TEXT, value REAL, moviename TEXT, movieimage TEXT, movieurl TEXT)")
def data_entry():
c.execute("INSERT INTO movies VALUES(1452549219,'2016-01-11 13:53:39','Python',6,'movienamehere1', 'savemovieurl', 'movieurlhere1')")
conn.commit()
c.close()
conn.close()
def dynamic_data_entry(argument) :
unix = time.time()
date = str(datetime.datetime.fromtimestamp(unix).strftime('%Y-%m-%d %H: %M: %S'))
keyword = 'keyword_string'
movieurl = 'bing.com'
value = random.randrange(0,10)
savemovieurl2 = 'testimageurl.com'
print argument
c.execute("INSERT INTO movies (unix, datestamp, keyword, value, moviename, movieimage, movieurl) VALUES (?, ?, ?, ?, ?, ?, ?)", (unix, date, keyword, value, movie, savemovieurl2, movieurl))
conn.commit()
create_table()
#data_entry()
for i in range(10) :
dynamic_data_entry(savemovieurl)
time.sleep(1)
c.close()
conn.close()
if __name__=="__main__":
tmdb_posters(imdb_id_input)
I think this has already been answered here: How do I use a variable so that it is inside and outside of a function
I know I should comment this however for some reason I can't so I just thought I'd write it as an answer instead. I hope this helps.