I would like to produce the following set up in Python 3.4, SQLite3 v.3.8.11:
(1) Create an in-memory shared-cache SQLite3 database:
(2) Create one connection that only writes to this DB from one thread
(3) Create multiple connections that concurrently read from this DB from various other threads
This is what I have created to test this:
import time
import zmq
import random
from threading import Thread
import sqlite3
def producer(context):
zmq_socket = context.socket(zmq.PUB)
zmq_socket.bind("inproc://test_pub")
while True:
msg = random.random()
zmq_socket.send(str(msg).encode())
wait_time = random.uniform(0, 0.05)
time.sleep(wait_time)
def subscriber_writer(context):
# Create database connection for writing to memory
write_con = sqlite3.connect('file::memory:?cache=shared', uri=True)
cursor = write_con.cursor()
zmq_socket = context.socket(zmq.SUB)
zmq_socket.connect("inproc://test_pub")
zmq_socket.setsockopt(zmq.SUBSCRIBE, b'')
while True:
msg = float(zmq_socket.recv().decode())
cursor.execute('UPDATE TEST SET Value=? WHERE Key="Val"', [msg])
write_con.commit()
def consumer(context):
# Create database connection for reading from memory in read-only mode
read_con = sqlite3.connect('file::memory:?cache=shared&mode=ro', uri=True)
cursor = read_con.cursor()
while True:
cursor.execute('SELECT Value FROM TEST WHERE Key="Val"')
row = cursor.fetchone()
result = row[0]
print(str(result))
wait_time = random.uniform(0, 0.05)
time.sleep(wait_time)
def main():
# Create context
context = zmq.Context()
# Create database
con = sqlite3.connect('file::memory:?cache=shared', uri=True)
# Create db table
cursor = con.cursor()
cursor.execute('CREATE TABLE TEST(Key TEXT, Value NUMERIC)')
cursor.execute('INSERT INTO TEST VALUES (?,?)', ["Val", 0.00])
con.commit()
Thread(target=subscriber_writer, args=(context,)).start()
Thread(target=producer, args=(context,)).start()
Thread(target=consumer, args=(context,)).start()
if __name__ == '__main__':
main()
This works for a while .....but then I get the following error:
...
0.2504188310554989
0.2504188310554989
0.8038719720740617
0.42408896748682956
0.21361498908206744
0.3404497358396832
0.010459475861968603
0.5070540941748318
0.5070540941748318
0.23151535812095037
0.636881359928549
0.4184038089576615
0.9920311052908629
Exception in thread Thread-3:
Traceback (most recent call last):
File "E:\Python34-64\lib\threading.py", line 911, in _bootstrap_inner
self.run()
File "E:\Python34-64\lib\threading.py", line 859, in run
self._target(*self._args, **self._kwargs)
File "test.py", line 43, in consumer
cursor.execute('SELECT Value FROM TEST WHERE Key="Val"')
sqlite3.OperationalError: database table is locked: TEST
How can I make this work?
As a side note, CREATING ONLY CONNECTION WITH check_same_thread=False and sharing this across the whole process works even when the wait times are eliminated....is this advisable to do instead? See below:
import time
import zmq
import random
from threading import Thread
import sqlite3
def producer(context):
zmq_socket = context.socket(zmq.PUB)
zmq_socket.bind("inproc://test_pub")
while True:
msg = random.random()
zmq_socket.send(str(msg).encode())
# wait_time = random.uniform(0, 0.05)
# time.sleep(wait_time)
def subscriber_writer(context, con):
zmq_socket = context.socket(zmq.SUB)
zmq_socket.connect("inproc://test_pub")
zmq_socket.setsockopt(zmq.SUBSCRIBE, b'')
cursor = con.cursor()
while True:
msg = float(zmq_socket.recv().decode())
cursor.execute('UPDATE TEST SET Value=? WHERE Key="Val"', [msg])
def consumer(context, con):
cursor = con.cursor()
while True:
cursor.execute('SELECT Value FROM TEST WHERE Key="Val"')
row = cursor.fetchone()
result = row[0]
print(str(result))
# wait_time = random.uniform(0, 0.05)
# time.sleep(wait_time)
def main():
# Create context
context = zmq.Context()
# Create database
con = sqlite3.connect('file::memory:?cache=shared', uri=True, isolation_level=None, check_same_thread=False)
# Create db table
cursor = con.cursor()
cursor.execute('CREATE TABLE TEST(Key TEXT, Value NUMERIC)')
cursor.execute('INSERT INTO TEST VALUES (?,?)', ["Val", 0.00])
Thread(target=subscriber_writer, args=(context, con)).start()
Thread(target=producer, args=(context,)).start()
Thread(target=consumer, args=(context, con)).start()
if __name__ == '__main__':
main()
Related
I'm trying to read the data from MySql database to OPC UA server. I tested it with the following code and sample database it is working. However, I'm not sure if it runs in a real time environment as the database has 40+ tables and 30+ columns in each table recording 1 minute data. Can someone please suggest the optimal way to do this.
from opcua import ua, uamethod, Server
from time import sleep
import logging
import mysql.connector
mydb = mysql.connector.connect(
host="127.0.0.1",
port=3306,
user="root",
password="root",
database="classicmodels")
mycursor = mydb.cursor(buffered=True , dictionary=True)
sql = "SELECT * FROM classicmodels.customers"
mycursor.execute(sql)
myresult = mycursor.fetchone()
sql1 = "SELECT * FROM classicmodels.employees"
mycursor.execute(sql1)
myresult1 = mycursor.fetchone()
if __name__ == "__main__":
"""
OPC-UA-Server Setup
"""
server = Server()
endpoint = "opc.tcp://127.0.0.1:4848"
server.set_endpoint(endpoint)
servername = "Python-OPC-UA-Server"
server.set_server_name(servername)
"""
OPC-UA-Modeling
"""
root_node = server.get_root_node()
object_node = server.get_objects_node()
idx = server.register_namespace("OPCUA_SERVER")
myobj = object_node.add_object(idx, "DA_UA")
myobj1 = object_node.add_object(idx, "D_U")
"""
OPC-UA-Server Add Variable
"""
for key, value in myresult.items():
myobj.add_variable(idx, key, str(value))
for key, value in myresult1.items():
myobj1.add_variable(idx, key, str(value))
"""
OPC-UA-Server Start
"""
server.start()
'''
import psycopg2
import time
import threading
def initiate():
conn = psycopg2.connect(host='localhost', user='postgres', password='password', port='5432', database='test')
conn.set_isolation_level(1)
conn.autocommit = False
cursor1 = conn.cursor()
cursor2 = conn.cursor()
cursor3 = conn.cursor()
t1 = threading.Thread(target=test1, args=(cursor1, conn))
t2 = threading.Thread(target=test2, args=(cursor2, conn))
t3 = threading.Thread(target=test3, args=(cursor3, conn))
t1.start()
t2.start()
t3.start()
def test1(cursor, conn):
cursor.execute("INSERT INTO test_sch.tb_for_test1(col_for_t12, col_for_t13, col_for_t14) VALUES ('test_col1', 'test_col1', 1)")
time.sleep(10)
conn.commit()
cursor.close()
print("completed test1")
def test2(cursor, conn):
cursor.execute("INSERT INTO test_sch.tb_for_test1(col_for_t12, col_for_t13, col_for_t14) VALUES ('test_col2', 'test_col2', 2)")
time.sleep(5)
conn.commit()
cursor.close()
print("completed test2")
def test3(cursor, conn):
cursor.execute("INSERT INTO test_sch.tb_for_test1(col_for_t12, col_for_t13, col_for_t14) VALUES ('test_col3', 'test_col3', 3)")
time.sleep(15)
conn.commit()
cursor.close()
print("completed test3")
initiate()
In the above code I have tried to insert three records to database using three different threads running three different methods. When method test2 gets completed it is committing all three records instead waiting for transaction at method test1 and test3. I can understand that psycopg2 transaction is per connection and not per cursor. So any suggestions on how it can be moved to per cursor basis ?. One of my limitation is not to open too many connections
#!/usr/bin/env python
import pika
def doQuery( conn, i ) :
cur = conn.cursor()
cur.execute("SELECT * FROM table OFFSET %s LIMIT 100000", (i,))
return cur.fetchall()
print "Using psycopg2"
import psycopg2
myConnection = psycopg2.connect( host=hostname, user=username,
password=password, dbname=database )
connection =
pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='task_queue2')
endloop = False
i = 1
while True:
results = doQuery( myConnection, i )
j = 0
while j < 10000:
try:
results[j][-1]
except:
endloop = True
break
message = str(results[j][-1]).encode("hex")
channel.basic_publish(exchange='',
routing_key='task_queue2',
body=message
#properties=pika.BasicProperties(
#delivery_mode = 2, # make message persistent
)#)
j = j + 1
# if i % 10000 == 0:
# print i
if endloop == False:
break
i = i + 10000
The SQL query is taking too long to execute when i gets to 100,000,000, but I have about two billion entries I need to put into the queue. Anyone know of a more efficient SQL query that I can run so that I can get all those two billion into the queue faster?
psycopg2 supports server-side cursors, that is, a cursor that is managed on the database server rather than in the client. The full result set is not transferred all at once to the client, rather it is fed to it as required via the cursor interface.
This will allow you to perform the query without using paging (as LIMIT/OFFSET implements), and will simplify your code. To use a server side cursor use the name parameter when creating the cursor.
import pika
import psycopg2
with psycopg2.connect(host=hostname, user=username, password=password, dbname=database) as conn:
with conn.cursor(name='my_cursor') as cur: # create a named server-side cursor
cur.execute('select * from table')
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()
channel.queue_declare(queue='task_queue2')
for row in cur:
message = str(row[-1]).encode('hex')
channel.basic_publish(exchange='', routing_key='task_queue2', body=message)
You might want to tweak cur.itersize to improve performance if necessary.
What is wrong with my code? Thank you
import os
import os.path
import time
global item_count
#-*- coding: cp936 -*-
import MySQLdb
import MySQLdb.cursors
import threading
import multiprocessing
from time import sleep,ctime
def qucun():
#connect to mysql
conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong")
cursor=conn.cursor()
try:
cursor.execute("""create database if not exists quad""")
except:
print 'Quad is exist'
conn.select_db('quad')
conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong",db="quad")
#get cursor
cursor=conn.cursor()
try:
cursor.execute("""create table if not exists record(fn1 varchar(100),
fn2 varchar(100),fn3 varchar(100),fn4 varchar(100),
fn5 varchar(100),fn6 varchar(100),fn7 varchar(100),fn8 varchar(100))""")
except:
print 'Table record is exist'
loops=['2013071818_1.txt','2013071818_2.txt','2013071818_3.txt','2013071818_4.txt','2013071818_5.txt']
def loop(nloop,filename):
print 'This loop%s start at:'%nloop,ctime()
#connect to quad
conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong",db="quad")
conn.select_db('quad')
#get cursor
cursor=conn.cursor()
newitem=open('C:\\Python27\\caihong\\%s'%filename,'r')
data=[line.strip() for line in newitem.readlines()]
print data
##put data into value
values=['%s'%data[0],'%s'%data[1],'%s'%data[2],'%s'%data[3],'%s'%data[4],
'%s'%data[5],'%s'%data[6],'%s'%data[7]]
cursor.execute("""insert into record values(%s,%s,%s,%s,%s,%s,%s,%s)""",values);
conn.commit()
cursor.close()
sleep(2)
print 'This loop done at',ctime()
if __name__=='__main__':
print 'starting at:',ctime()
threads=[]
nloops=range(len(loops))
pool=multiprocessing.Pool(processes=2)
for i in nloops:
t=pool.apply_async(loop,(i,loops[i]))
pool.close()
pool.join()
if t.successful():
print 'successful'
print 'all Done at:',ctime()
os.system("pause")
qucun()
You are attempting to call locally defined function in async.
You are trying to share an open connection between processes.
First is tricky to implement in 2.7 and second is impossible in any multiprocessing
You have to use separate connection for each process in process pool.
import os
import os.path
import time
global item_count
#-*- coding: cp936 -*-
import MySQLdb
import MySQLdb.cursors
import threading
import multiprocessing
from time import sleep,ctime
CONNECTION = None
def close_connection():
CONNECTION.close()
def get_connection():
global CONNECTION
#If this process pool member launched for a first time - create connection
if CONNECTION is None:
conn = MySQLdb.connect( host="localhost",
user="root",
passwd="caihong")
cursor = conn.cursor()
try:
cursor.execute("""create database if not exists quad""")
except:
print 'Quad is exist'
conn.select_db('quad')
CONNECTION = MySQLdb.connect(host="localhost",
user="root",
passwd="caihong",
db="quad")
cursor = CONNECTION.cursor()
try:
cursor.execute("""create table if not exists record(fn1 varchar(100),
fn2 varchar(100),fn3 varchar(100),fn4 varchar(100),
fn5 varchar(100),fn6 varchar(100),fn7 varchar(100),fn8 varchar(100))""")
except:
print 'Table record is exist'
# we dont need to close connection after each insert.
# insted - register a finalizer once
# so it will be called right before Pool.close()
multiprocessing.util.Finalize(CONNECTION, close_connection, exitpriority=1)
#use existing connection
return CONNECTION
def loop(nloop, filename):
conn = get_connection()
cursor = conn.cursor()
print 'This loop %s start at: %s'%(nloop, ctime())
with open('C:\\Python27\\caihong\\%s'%filename, 'r') as newitem:
data = [line.strip() for line in newitem.readlines()]
# values=['%s'%data[0],'%s'%data[1],'%s'%data[2],'%s'%data[3],'%s'%data[4],
# '%s'%data[5],'%s'%data[6],'%s'%data[7]]
# ^^^ Thats a bad way to stringify list
cursor.execute('insert into record values(%s)', ','.join(data));
conn.commit()
# we dont need to close connection after each insert.
# cursor.close()
print 'This loop done at', ctime()
LOOPS = ['2013071818_1.txt', '2013071818_2.txt', '2013071818_3.txt', '2013071818_4.txt', '2013071818_5.txt']
if __name__=='__main__':
pool = multiprocessing.Pool(processes=2)
results = []
for i, loopfile in enumerate(LOOPS):
results.apply(pool.apply_async(loop, (i, loopfile)))
pool.close()
pool.join()
if all((res.successful() for res in results)):
print 'successful'
print 'all Done at:', ctime()
os.system('pause')
I'm trying use Python and pyodbc to access SQL server 2008. The first connection works. Then, after the program finishes its job, it closes the connection. When the program tries to access the database and connect to it again, it fails in the statement:
self.conn = pyodbc.connect(DRIVER=self.DRIVER, SERVER=self.SERVER, DATABASE=self.DATABASE, UID=self.UID, PWD=self.PWD, charset="UTF-8")
but the first time is OK. So does anyone know why? Below is the Python code:
class ODBC_MS:
def __init__(self, DRIVER,SERVER, DATABASE, UID, PWD):
''' initialization '''
self.DRIVER = DRIVER
self.SERVER = SERVER
self.DATABASE = DATABASE
self.UID = UID
self.PWD = PWD
def _GetConnect(self):
''' Connect to the DB '''
if not self.DATABASE:
raise(NameError,"no getting db name")
try:
self.conn = pyodbc.connect(DRIVER=self.DRIVER, SERVER=self.SERVER,
DATABASE=self.DATABASE, UID=self.UID,
PWD=self.PWD, charset="UTF-8")
except Exception,e:
print e.message
else:
self.cur = self.conn.cursor()
if not self.cur:
raise(NameError,"connected failed!")
else:
return self.cur, self.conn
def ExecNoQuery(self,conn, cursor, sql):
cursor.execute(sql)
ret = conn.commit()
return ret
def _UnConnect(self,conn, cursor):
conn.close()
if __name__ == '__main__':
ms = ODBC_MS('{SQL SERVER}', r'<server>', '<db>', '<user>', '<password>')
cursor, conn = ms._GetConnect() #connection
sql = "create table XX for example"
ret = ms.ExecNoQuery(conn, cursor,sql) #sql operation
ms._UnConnect(conn, cursor) #close db
#access the database the second time.
ms = ODBC_MS('{SQL SERVER}', r'<server>', '<db>', '<user>', '<password>')
cursor, conn = ms._GetConnect() # not success, I don't know why
sql = "create table XX for example"
ret = ms.ExecNoQuery(conn, cursor,sql) #sql operation
ms._UnConnect(conn, cursor) #close db
The second time when the program calls ms.GetConnect(), the statement self.conn = pyodbc.connect(DRIVER=self.DRIVER, SERVER=self.SERVER, DATABASE=self.DATABASE, UID=self.UID, PWD=self.PWD, charset="UTF-8") fails.