I've got this netcdf of weather data (one of thousands that require postgresql ingestion). I'm currently capable of inserting each band into a postgis-enabled table at a rate of about 20-23 seconds per band. (for monthly data, there is also daily data that i have yet to test.)
I've heard of different ways of speeding this up using COPY FROM, removing the gid, using ssds, etc... but I'm new to python and have no idea how to store the netcdf data to something I could use COPY FROM or what the best route might be.
If anyone has any other ideas on how to speed this up, please share!
Here is the ingestion script
import netCDF4, psycopg2, time
# Establish connection
db1 = psycopg2.connect("host=localhost dbname=postgis_test user=********** password=********")
cur = db1.cursor()
# Create Table in postgis
print(str(time.ctime()) + " CREATING TABLE")
try:
cur.execute("DROP TABLE IF EXISTS table_name;")
db1.commit()
cur.execute(
"CREATE TABLE table_name (gid serial PRIMARY KEY not null, thedate DATE, thepoint geometry, lon decimal, lat decimal, thevalue decimal);")
db1.commit()
print("TABLE CREATED")
except:
print(psycopg2.DatabaseError)
print("TABLE CREATION FAILED")
rawvalue_nc_file = 'netcdf_file.nc'
nc = netCDF4.Dataset(rawvalue_nc_file, mode='r')
nc.variables.keys()
lat = nc.variables['lat'][:]
lon = nc.variables['lon'][:]
time_var = nc.variables['time']
dtime = netCDF4.num2date(time_var[:], time_var.units)
newtime = [fdate.strftime('%Y-%m-%d') for fdate in dtime]
rawvalue = nc.variables['tx_max'][:]
lathash = {}
lonhash = {}
entry1 = 0
entry2 = 0
lattemp = nc.variables['lat'][:].tolist()
for entry1 in range(lat.size):
lathash[entry1] = lattemp[entry1]
lontemp = nc.variables['lon'][:].tolist()
for entry2 in range(lon.size):
lonhash[entry2] = lontemp[entry2]
for timestep in range(dtime.size):
print(str(time.ctime()) + " " + str(timestep + 1) + "/180")
for _lon in range(lon.size):
for _lat in range(lat.size):
latitude = round(lathash[_lat], 6)
longitude = round(lonhash[_lon], 6)
thedate = newtime[timestep]
thevalue = round(float(rawvalue.data[timestep, _lat, _lon] - 273.15), 3)
if (thevalue > -100):
cur.execute("INSERT INTO table_name (thedate, thepoint, thevalue) VALUES (%s, ST_MakePoint(%s,%s,0), %s)",(thedate, longitude, latitude, thevalue))
db1.commit()
cur.close()
db1.close()
print(" Done!")
If you're certain most of the time is spent in PostgreSQL, and not in any other code of your own, you may want to look at the fast execution helpers, namely cur.execute_values() in your case.
Also, you may want to make sure you're in a transaction, so the database doesn't fall back to an autocommit mode. ("If you do not issue a BEGIN command, then each individual statement has an implicit BEGIN and (if successful) COMMIT wrapped around it.")
Something like this could do the trick -- not tested though.
for timestep in range(dtime.size):
print(str(time.ctime()) + " " + str(timestep + 1) + "/180")
values = []
cur.execute("BEGIN")
for _lon in range(lon.size):
for _lat in range(lat.size):
latitude = round(lathash[_lat], 6)
longitude = round(lonhash[_lon], 6)
thedate = newtime[timestep]
thevalue = round(
float(rawvalue.data[timestep, _lat, _lon] - 273.15), 3
)
if thevalue > -100:
values.append((thedate, longitude, latitude, thevalue))
psycopg2.extras.execute_values(
cur,
"INSERT INTO table_name (thedate, thepoint, thevalue) VALUES %s",
values,
template="(%s, ST_MakePoint(%s,%s,0), %s)"
)
db1.commit()
Related
Just started using ib_insync. I am trying to get the tick data into a dataframe.
Here is the relevant code:
def onPendingTickers(tickers, conn=conn):
for t in tickers:
# 'CREATE TABLE IF NOT EXISTS {} (timestamp timestamp, bid_qty INT, bid REAL, ask REAL, ' \
# 'ask_qty INT, high REAL, low REAL, close REAL, open REAL, contractID INT)'
# print(t)
c.execute('INSERT INTO {} (timestamp, bid_qty, bid, ask, ask_qty, high, low, close, open, contractID)'
' VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);'.format(t.contract.pair()),
(t.time, t.bidSize, t.bid, t.ask, t.askSize, t.high, t.low, t.close, t.open, t.contract.conId))
# print(t.time, t.bidSize, t.bid, t.ask, t.askSize, t.high, t.low, t.close, t.open, t.contract.conId)
conn.commit()
ib.pendingTickersEvent += onPendingTickers
ib.sleep(60*60)
ib.pendingTickersEvent -= onPendingTickers
When I run this code in a terminal, it prints the ticker, I am not sure what exactly needs to be changed here.
If you just want to get ticks without displaying the information, here's some sample code that you should be able to run:
from ib_insync import *
import pandas as pd
import numpy as np
# Connect to IB; args are (IP address, device number, client ID)
def ibConnect(port,clientID):
connection = ib.connect('127.0.0.1', port, clientID)
ib.sleep(0)
return ()
# Disconnect from IB
def ibDisconnect():
ib.disconnect()
ib.sleep(0)
return
# Set up a futures contract
def ibFuturesContract(symbol, expirationDate, exchange):
futuresContract = Future(symbol, expirationDate, exchange)
return futuresContract
# Realtime Ticks Subscription
def ibGetTicker (contract):
ticker = ib.ticker(contract)
return [ticker]
ib = IB()
ibConnect(7496,300)
contract = ibFuturesContract('YM',20210618,'ECBOT')
# Start the real-time tick subscription
ib.reqMktData(contract, '', False, False)
# Real Time Ticks
global ticker
ticker = ibGetTicker(contract)
# Get just the last tick each second and put it into a data table
x = 0
while x < 10:
ib.sleep(1)
if ticker is not None:
df = util.df(ticker)
if (x == 0):
dt = df
else:
dt = dt.append(df)
x = x + 1
print (dt)
ib.cancelMktData(contract)
ibDisconnect()
I have this query that I use on a weekly basis to create a report using pd.read_sql. I want to be able to update the case statement of store, update the Date Add, and store IN at the end of the statement without having to manually change the store numbers and the dates. Is there any way that I can edit the query to make the updates?
This is the query
dataframe = pd.read_sql("""
SELECT Top(10)
CAST( Store as VARCHAR) + 'þ' as Store,
CONVERT( VARCHAR, Tran_Dt2, 101 ) + 'þ' as Tran_Dt,
CONVERT(char(5), Start_Time, 108) + 'þ' as Start_Time,
[Count]
FROM
(
SELECT
CASE
WHEN [Store] = 313 THEN 3174
WHEN [Store] = 126 THEN 3191
END AS Store
, DATEADD (YEAR, +2, DATEADD(DAY, +4, Tran_Dt2)) as Tran_Dt2
,[Start_Time]
,[Count]
,Store as Sister_Store
FROM
(
SELECT
Store,
CONVERT(datetime, Tran_Dt) as Tran_Dt2,
Start_Time,
Count
FROM [VolumeDrivers].[dbo].[SALES_DRIVERS_ITC_Signup_65wks]
WHERE CONVERT(datetime, Tran_Dt) between CONVERT(datetime,'2/8/2019') and CONVERT(datetime,'3/15/2019')
AND
Store IN (313, 126)
--Single Store: Store = Store #
) AS A
) AS B
ORDER BY Tran_Dt2, Store
""", con = conn)
I would want to be able to do something like declare a variable and have it populate in the code such as something like:
oldstore1 = 313
newstore1 = 3174
oldstore2 = 126
newstore2 = 3191
daframe = pd.ready_sql("""...
...
SELECT
CASE
WHEN [Store] = oldstore1 THEN newstore1
WHEN [Store] = oldstore2 THEN newstore2
...
UPDATE----
I am currently at this point and had the query working until my kernel restarted and I lost my code. Any tips on why it isn't working anymore?
#Declare variables for queries
old_store1 = 313
new_store1 = 3157
old_store2 = 126
new_store2 = 3196
datefrom = '2/8/2019'
dateto = '3/15/2019'
yearadd = '+2'
dayadd = '+4'
ITC = pd.read_sql("""SELECT
CAST( Store as VARCHAR) + 'þ' as Store,
CONVERT( VARCHAR, Tran_Dt2, 101 ) + 'þ' as Tran_Dt,
CONVERT(char(5), Start_Time, 108) + 'þ' as Start_Time,
[Count]
FROM
(
SELECT
CASE
WHEN [Store] = {old_store1} THEN {new_store1}
WHEN [Store] = {old_store2} THEN {new_store2}
END AS Store
, DATEADD (YEAR, {yearadd}, DATEADD(DAY, {dayadd}, Tran_Dt2)) as Tran_Dt2
,[Start_Time]
,[Count]
,Store as Sister_Store
FROM
(
SELECT
Store,
CONVERT(datetime, Tran_Dt) as Tran_Dt2,
Start_Time,
Count
FROM [VolumeDrivers].[dbo].[SALES_DRIVERS_ITC_Signup_65wks]
WHERE CONVERT(datetime, Tran_Dt) between CONVERT(datetime,{datefrom}) and CONVERT(datetime,{dateto})
AND
Store IN ({old_store1}, {old_store2})
--Single Store: Store = Store #
) AS A
) AS B
ORDER BY Tran_Dt2, Store
""", con = conn)
Was able to figure out why it wasn't working. I guess python 3 and beyond has a built in function that allows you to place "f" in front of the query and will let you pass the variables you created. I know this isn't the most secure way of executing the script but I'll look into creating a for loop in the future that will allow it to be more secure. Thanks for all the help!
#Declare variables for queries
old_store1 = 313
new_store1 = 3157
old_store2 = 126
new_store2 = 3196
datefrom = '2/15/2019'
dateto = '3/22/2019'
yearadd = '+2'
dayadd = '+4'
ITC = pd.read_sql(f"""SELECT
CAST( Store as VARCHAR) + 'þ' as Store,
CONVERT( VARCHAR, Tran_Dt2, 101 ) + 'þ' as Tran_Dt,
CONVERT(char(5), Start_Time, 108) + 'þ' as Start_Time,
[Count]
FROM
(
SELECT
CASE
WHEN [Store] = {old_store1} THEN {new_store1}
WHEN [Store] = {old_store2} THEN {new_store2}
.....
#run code and verify it works
Sales_Drivers_ITCSignup = pd.read_sql(ITCQuery, con = conn, index_col='Store')
Sales_Drivers_ITCSignup.head()
Is there anyway i can compare two different databases (postgresl, sql server) and return the missing rows? I am missing one row in the postgresql table that is not in the sql server one and have no clue how to return that answer to me.
I have two connections opened for postgresql (bpo_table_results) and for sql server(rps_table_results)
postgresql table:
date count amount
1/1/21 500 1,234,654.12
sql server table:
date count amount
1/1/21 500 1,234,654.12
1/2/21 4541 3,457,787.24
expected results:
The row in the amount of 3,457,787.24 is missing from your posgresql table.
code:
def queryRPS(sql_server_conn, sql_server_cursor):
rps_item_count_l = []
rps_icl_amt_l = []
rps_table_q_2 = f"""select * from rps..sendfile where processingdate = '{cd}' and datasetname like '%ICL%' """
rps_table_results = sql_server_cursor.execute(rps_table_q_2).fetchall()
for row in rps_table_results:
rps_item_count = row[16]
rps_item_count_l.append(rps_item_count)
rps_icl_amt = row[18]
rps_icl_amt_l.append(rps_icl_amt)
def queryBPO(postgres_conn, postgres_cursor,rps_item_count_l, rps_icl_amt_l):
bpo_results_l = []
rps_results_l = []
for rps_count, rps_amount in zip(rps_item_count_l, rps_icl_amt_l):
rps_amount_f = str(rps_amount).rstrip('0')
rps_amount_f = ("{:,}".format(float(rps_amount_f)))
bpo_icl_awk_q_2 = """select * from ppc_data.icl_awk where num_items = '%s' and
file_total = '%s' """ % (str(rps_count), str(rps_amount_f))
postgres_cursor.execute(bpo_icl_awk_q_2)
bpo_table_results = postgres_cursor.fetchall()
rps_table_q_2 = f"""select * from rps..sendfile where processingdate = '{cd}' and datasetname like '%ICL%' """
rps_table_results = sql_server_cursor.execute(rps_table_q_2).fetchall()
rps_item_count_l, rps_icl_amt_l = queryRPS(sql_server_conn, sql_server_cursor)
queryBPO(postgres_conn, postgres_cursor, rps_item_count_l, rps_icl_amt_l)
I'm trying to query one table against the other using Python and MySQLdb. Here's what I've got so far:
db = MySQLdb.connect( host = 'localhost', user = 'user', passwd=
'password', db = 'vacants')
cursor = db.cursor()
numrows = cursor.rowcount
query = "SELECT address, ((20903520) * acos (cos ( radians(38.67054) )* cos(
radians( lat ) ) * cos( radians( `long` ) - radians(-90.22942) ) + sin (
radians(38.67054) ) * sin( radians( lat ) ))) AS distance FROM vacants HAVING
distance < 100;"
cursor.execute(query)
I have one table, cfs, and another, vacants. I want to see for each row in cfs is there a vacant property within 100 feet. So for the ( radians(38.67054) and radians(-90.22942), I need to loop through the cfs table so that each cfs latitude and longitude replaces those two numbers. (That's just a test latitude and longitude we used)
In the end I'd like to have (in a .csv) the vacant property address, the distance from the call for service, and the type of call (which are two separate fields in the calls for service database). Something like this, which is from the query above:
Here's example data - calls for service coordinates:
38.595767638008056,-90.2316138251402
38.57283495467307,-90.24649031378685
38.67497061776659,-90.28415976525395
38.67650431524285,-90.25623757427952
38.591971519414784,-90.27782710145746
38.61272746420862,-90.23292862245287
38.67312983860098,-90.23591869583113
38.625956494342674,-90.18853950906939
38.69044465638584,-90.24339061920696
38.67745024638241,-90.20657832034047
And vacants:
38.67054,-90.22942
38.642956,-90.21466
38.671535,-90.27293
38.666367,-90.23749
38.65339,-90.23141
38.645996,-90.20334
38.60214,-90.224815
38.67265,-90.214134
38.665504,-90.274414
38.668354,-90.269966
This is not the final solution as there is insufficient info (address field? and 20903520 ?) in the question but it might help you get on track by showing how to iterate through both tables and substitute lat, lon from the CFS table into the query applied to the vacants table:
import mysql.connector
cnx1 = mysql.connector.connect(user='root',password='xxxx',host='127.0.0.1',database=db)
cursor1 = cnx1.cursor()
cnx2 = mysql.connector.connect(user='root',password='xxxx',host='127.0.0.1',database=db)
cursor2 = cnx2.cursor()
sql_cfs = ('select lat,lon from cfs')
cursor1.execute(sql_cfs)
for cfs in cursor1:
[cfs_lat,cfs_lon] = cfs
print (cfs_lat,cfs_lon)
query = ("SELECT address, ((20903520) * " \
"acos (cos(radians(lon)) *" \
"cos(radians({})) * " \
"cos(radians({})-radians(lat)) + sin(radians(lon)) * " \
"sin( radians({})))) AS distance " \
"FROM vacants HAVING distance < 100;".format(cfs_lat,cfs_lon,cfs_lat))
print (query)
cursor2.execute(query)
for vacants in cursor2:
print (vacants)
Using psycopg2, I'm able to select data from a table in one PostgreSQL database connection and INSERT it into a table in a second PostgreSQL database connection.
However, I'm only able to do it by setting the exact feature I want to extract, and writing out separate variables for each column I'm trying to insert.
Does anyone know of a good practice for either:
moving an entire table between databases, or
iterating through features while not having to declare variables for every column you want to move
or...?
Here's the script I'm currently using where you can see the selection of a specific feature, and the creation of variables (it works, but this is not a practical method):
import psycopg2
connDev = psycopg2.connect("host=host1 dbname=dbname1 user=postgres password=*** ")
connQa = psycopg2.connect("host=host2 dbname=dbname2 user=postgres password=*** ")
curDev = connDev.cursor()
curQa = connQa.cursor()
sql = ('INSERT INTO "tempHoods" (nbhd_name, geom) values (%s, %s);')
curDev.execute('select cast(geom as varchar) from "CCD_Neighborhoods" where nbhd_id = 11;')
tempGeom = curDev.fetchone()
curDev.execute('select nbhd_name from "CCD_Neighborhoods" where nbhd_id = 11;')
tempName = curDev.fetchone()
data = (tempName, tempGeom)
curQa.execute (sql, data)
#commit transactions
connDev.commit()
connQa.commit()
#close connections
curDev.close()
curQa.close()
connDev.close()
connQa.close()
One other note is that python allows the ability to explicitly work with SQL functions / data type casting, which for us is important as we work with the GEOMETRY data type. Above you can see I'm casting it to TEXT then dumping it into an existing geometry column in the source table - this will work with MSSQL Server, which is a huge feature in the geospatial community...
In your solution (your solution and your question have a different order of statements) change the lines which start with 'sql = ' and the loop before '#commit transactions' comment to
sql_insert = 'INSERT INTO "tempHoods" (nbhd_id, nbhd_name, typology, notes, geom) values '
sql_values = ['(%s, %s, %s, %s, %s)']
data_values = []
# you can make this larger if you want
# ...try experimenting to see what works best
batch_size = 100
sql_stmt = sql_insert + ','.join(sql_values*batch_size) + ';'
for i, row in enumerate(rows, 1):
data_values += row[:5]
if i % batch_size == 0:
curQa.execute (sql_stmt , data_values )
data_values = []
if (i % batch_size != 0):
sql_stmt = sql_insert + ','.join(sql_values*(i % batch_size)) + ';'
curQa.execute (sql_stmt , data_values )
BTW, I don't think you need to commit. You don't begin any transactions. So there should not be any need to commit them. Certainly, you don't need to commit a cursor if all you did was a bunch of selects on it.
Here's my updated code based on Dmitry's brilliant solution:
import psycopg2
connDev = psycopg2.connect("host=host1 dbname=dpspgisdev user=postgres password=****")
connQa = psycopg2.connect("host=host2 dbname=dpspgisqa user=postgres password=****")
curDev = connDev.cursor()
curQa = connQa.cursor()
print "Truncating Source"
curQa.execute('delete from "tempHoods"')
connQa.commit()
#Get Data
curDev.execute('select nbhd_id, nbhd_name, typology, notes, cast(geom as varchar) from "CCD_Neighborhoods";') #cast geom to varchar and insert into geometry column!
rows = curDev.fetchall()
sql_insert = 'INSERT INTO "tempHoods" (nbhd_id, nbhd_name, typology, notes, geom) values '
sql_values = ['(%s, %s, %s, %s, %s)'] #number of columns selecting / inserting
data_values = []
batch_size = 1000 #customize for size of tables...
sql_stmt = sql_insert + ','.join(sql_values*batch_size) + ';'
for i, row in enumerate(rows, 1):
data_values += row[:5] #relates to number of columns (%s)
if i % batch_size == 0:
curQa.execute (sql_stmt , data_values )
connQa.commit()
print "Inserting..."
data_values = []
if (i % batch_size != 0):
sql_stmt = sql_insert + ','.join(sql_values*(i % batch_size)) + ';'
curQa.execute (sql_stmt, data_values)
print "Last Values..."
connQa.commit()
# close connections
curDev.close()
curQa.close()
connDev.close()
connQa.close()