I'm new to python and trying to work out how to insert some JSON into MySQL database in different python code. It works fine when run separately, but not works when i try to connect 2nd to 1st script. How to links 2nd python in 1st python script so it can works together?
I have 1st code like bellow, this code serves to send the image to API and generate a json file data.json
import glob
import argparse
import requests
import json
import time
import os
import cv2
import numpy as np
from pprint import pprint
import json_mysql
def main():
result = []
regions = ['id']
time_to_wait = np.inf
time_counter = 0
while True:
files = glob.glob(os.path.join("./image_dir*/*.jpg"))
files.sort(key=os.path.getmtime)
for file in files:
if os.path.isfile(file):
with open(file, 'rb') as fp:
response = requests.post(
'https://API/',
data=dict(regions=regions),
files=dict(upload=fp),
headers={'Authorization': 'Token ' + 'XXXX'})
result.append(response.json())
with open('data.json', 'w') as outfile:
json.dump(result, outfile)
time.sleep(1)
pprint(response.json())
os.remove(file)
time.sleep(1)
time_counter += 1
if time_counter > time_to_wait: break
print("waiting for file... ")
if __name__ == '__main__':
main()
json_mysql.create_db()
it generate json file like this:
And 2nd code is for create and store 'data.json' to MySQL database:
from urllib.request import urlopen
import urllib
import json
import sys
import pymysql
def dbconnect():
try:
db = pymysql.connect(
host="localhost",
user="root",
passwd="YYYY",
)
except Exception as e:
sys.exit("Can't connect to Database")
return db
def create_db():
db_name="plate_recognizer"
table_name="vehicles"
db = dbconnect()
cursor = db.cursor()
cursor.execute("SET sql_notes = 0;")
cursor.execute("CREATE DATABASE IF NOT EXISTS {}".format(db_name))
cursor.execute("SET sql_notes = 0;")
cursor.execute(
"""CREATE TABLE IF NOT EXISTS {}.{}(time varchar(150),plate varchar(20),region varchar(150), score varchar(20), filename varchar(50), tipe varchar(10));""".format(db_name, table_name))
cursor.execute("SET sql_notes = 1;")
with open('data.json') as f:
data = json.load(f)
for i in data:
cursor.execute(
"""INSERT INTO {}.{}(time, plate, region, score, filename, tipe) VALUES(%s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE plate =%s """.format
(db_name, table_name),
(i['timestamp'], i['results'][0]['plate'].upper(), i['results'][0]['region']['code'], i['results'][0]['score'], i['filename'], i['results'][0]['vehicle']['type'], i['results'][0]['plate'].upper()))
db.commit()
db.close()
if __name__ == '__main__':
create_db()
Thanks in advance.
You should divide your code in several methods. Each method should be responsible to perform a particular task, independent of other methods. For example you can use an insert_data() method to just insert your data.
# creates your db
def create_db():
db_name="plate_recognizer"
table_name="vehicles"
db = dbconnect()
cursor = db.cursor()
cursor.execute("SET sql_notes = 0;")
cursor.execute("CREATE DATABASE IF NOT EXISTS {}".format(db_name))
cursor.execute("SET sql_notes = 0;")
cursor.execute(
"""CREATE TABLE IF NOT EXISTS {}.{}(time varchar(150),plate varchar(20),region varchar(150), score varchar(20), filename varchar(50), tipe varchar(10));""".format(db_name, table_name))
cursor.execute("SET sql_notes = 1;")
# pass data to be inserted
def insert_data(cursor, data):
for i in data:
cursor.execute(
"""INSERT INTO {}.{}(time, plate, region, score, filename, tipe) VALUES(%s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE plate =%s """.format
(db_name, table_name),
(i['timestamp'], i['results'][0]['plate'].upper(), i['results'][0]['region']['code'], i['results'][0]['score'], i['filename'], i['results'][0]['vehicle']['type'], i['results'][0]['plate'].upper()))
db.commit()
db.close()
def main():
db = dbconnect()
cursor = db.cursor()
# your logic
while True:
files = glob.glob(os.path.join("./image_dir*/*.jpg"))
files.sort(key=os.path.getmtime)
for file in files:
if os.path.isfile(file):
with open(file, 'rb') as fp:
response = requests.post(
'https://API/',
data=dict(regions=regions),
files=dict(upload=fp),
headers={'Authorization': 'Token ' + 'XXXX'})
result.append(response.json())
with open('data.json', 'w') as outfile:
json.dump(result, outfile)
time.sleep(1)
pprint(response.json())
# insert data in db
insert_data(cursor, response.json())
os.remove(file)
time.sleep(1)
time_counter += 1
if time_counter > time_to_wait: break
print("waiting for file... ")
Related
I have a piece of code which is taking Windows logs and inserting various pieces of information into an mySQL database. The code is running perfectly with no errors, but does not actually input the data into the table. The table remains blank. I pulled my mySQL syntax from an example with some modification, so I'm not entirely sure what is going wrong. I have a feeling it has either to do with the data types, or some changes I made to the syntax.
import sys
import pymysql
import pymysql.cursors
import win32evtlog # requires pywin32 pre-installed
import win32evtlogutil
import time
server = 'localhost' # name of the target computer to get event logs
logtype = 'System' # 'Application' # 'Security'
hand = win32evtlog.OpenEventLog(server,logtype)
flags =
win32evtlog.EVENTLOG_BACKWARDS_READ|win32evtlog.EVENTLOG_SEQUENTIAL_READ
events = win32evtlog.ReadEventLog(hand, flags,0)
while True:
for event in events:
evt_tp = event.EventType
if evt_tp != (1 or 2 or 8):
eve_cat = str(('Event Category:', event.EventCategory))
eve_timegen = str(('Time Generated:', event.TimeGenerated))
eve_srcnm = str(('Source Name:', event.SourceName))
eve_id = str(('Event ID:', event.EventID))
eve_typ = str(('Event Type:', event.EventType))
data = event.StringInserts
if data:
print ('Event Data:')
for msg in data:
print(msg)
print(type(eve_cat))
print(type(eve_timegen))
print(type(eve_srcnm))
print(type(eve_id))
print(type(eve_typ))
print(type(data))
time.sleep(10)
else:
eve_cat = ('Event Category:', event.EventCategory)
eve_timegen = ('Time Generated:', event.TimeGenerated)
eve_srcnm = ('Source Name:', event.SourceName)
eve_id = ('Event ID:', event.EventID)
eve_typ = ('Event Type:', event.EventType)
data = event.StringInserts
print('There were no errors found')
print(eve_cat)
print(eve_timegen)
print(eve_srcnm)
print(eve_id)
print(eve_typ)
print(data)
time.sleep(10)
# Connect to the database
connection = pymysql.connect(host='localhost',
user='root',
password='',
db='ptest',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
try:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `win_logs` (`Category`, `TimeGenerated`, 'SourceName',
'EventID', 'Type') VALUES (%s, %s, %s, %s, %s)"
cursor.execute(sql, (eve_cat, eve_timegen, eve_srcnm, eve_id, eve_typ))
# connection is not autocommit by default. So you must commit to save
# your changes.
connection.commit()
with connection.cursor() as cursor:
# Read a single record
sql = "SELECT `id`, `Type` FROM `win_logs` WHERE `Category`=%s"
cursor.execute(sql, ('webmaster#python.org',))
result = cursor.fetchone()
print(result)
finally:
connection.close()
I can be very wrong.
But this is python.
Indentation matter.
Try just:
try:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `win_logs` (`Category`, `TimeGenerated`, 'SourceName`, 'EventID', 'Type') VALUES (%s, %s, %s, %s, %s)"
cursor.execute(sql, (eve_cat, eve_timegen, eve_srcnm, eve_id, eve_typ))
I guess your cursor is out of with scope
I have a python script below that attempts to make a connection to a sql db. The database was having some issues and I want the script to try to reconnect to the db so I added try / except. Now however I get the error message of "Unable to connect to database". If I take out the try / except statement the script works perfectly. If anyone can help me with getting the try / except statement working I would greatly appreciate it!
Updated code with connection inside of Try
from __future__ import print_function
try:
import psycopg2
except ImportError:
raise ImportError('\n\033[33mpsycopg2 library missing. pip install psycopg2\033[1;m\n')
sys.exit(1)
import re
import sys
import json
import pprint
outfilepath = "crtsh_output/crtsh_flat_file"
DB_HOST = 'crt.sh'
DB_NAME = 'certwatch'
DB_USER = 'guest'
#conn = psycopg2.connect("dbname={0} user={1} host={2}".format(DB_NAME, DB_USER, DB_HOST))
#cursor = conn.cursor()
def connect_to_db():
filepath = 'forager.txt'
conn = psycopg2.connect("dbname={0} user={1} host={2}".format(DB_NAME, DB_USER, DB_HOST))
cursor = conn.cursor()
with open(filepath) as fp:
unique_domains = ''
while True:
try:
for cnt, domain_name in enumerate(fp):
print("Line {}: {}".format(cnt, domain_name))
print(domain_name)
domain_name = domain_name.rstrip()
cursor.execute('''SELECT c.id, x509_commonName(c.certificate), x509_issuerName(c.certificate), x509_notBefore(c.certificate), x509_notAfter(c.certificate), x509_issuerName(c.certificate)$
FROM certificate c, certificate_identity ci WHERE
c.id= ci.certificate_id AND ci.name_type = 'dNSName' AND lower(ci.name_value) =
lower(%s) AND x509_notAfter(c.certificate) > statement_timestamp()''', (domain_name,))
unique_domains = cursor.fetchall()
pprint.pprint(unique_domains)
outfilepath = "crtsh1" + ".json"
with open(outfilepath, 'a') as outfile:
outfile.write(json.dumps(unique_domains, sort_keys=True, indent=4, default=str, ensure_ascii = False))
break
except:
pass
#print("\n\033[1;31m[!] Unable to connect to the database\n\033[1;m")
if __name__ == "__main__":
connect_to_db()
You'll want to do a few things:
move this code:
conn = psycopg2.connect("dbname={0} user={1} host={2}".format(DB_NAME, DB_USER, DB_HOST))
cursor = conn.cursor()
Into your try, that way when it loops if it dies it will remake the connection.
Like this:
try:
conn = psycopg2.connect("dbname={0} user={1} host={2}".format(DB_NAME, DB_USER, DB_HOST))
cursor = conn.cursor()
for cnt, domain_name in enumerate(fp):
print("Line {}: {}".format(cnt, domain_name))
print(domain_name)
domain_name = domain_name.rstrip()
I would toss a pause between connections. Your DBA might have code to detect too many connections in a period of time, like this:
#at the top of your code
import time
#in your loop add:
time.sleep(1)
I'm trying to get a script working that takes each line from a file and use the line as input to run the SQL query. Specifically I'm trying to use a file that has a list of domains and use those domains names to query a postgresql database. Any help would be greatly appreciated!
from __future__ import print_function
try:
import psycopg2
except ImportError:
raise ImportError('\n\033[33mpsycopg2 library missing. pip install psycopg2\033[1;m\n')
sys.exit(1)
import re
import sys
import json
DB_HOST = 'crt.sh'
DB_NAME = 'certwatch'
DB_USER = 'guest'
def connect_to_db(domain_name):
try:
conn = psycopg2.connect("dbname={0} user={1} host={2}".format(DB_NAME, DB_USER, DB_HOST))
cursor = conn.cursor()
cursor.execute("SELECT ci.NAME_VALUE NAME_VALUE FROM certificate_identity ci WHERE ci.NAME_TYPE = 'emailAddress' AND reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower('%{}'));".format(domain_name))
except:
print("\n\033[1;31m[!] Unable to connect to the database\n\033[1;m")
return cursor
def get_unique_emails(cursor, domain_name):
unique_emails = []
for result in cursor.fetchall():
matches=re.findall(r"\'(.+?)\'",str(result))
for email in matches:
#print(email)
if email not in unique_emails:
if "{}".format(domain_name) in email:
unique_emails.append(email)
return unique_emails
def print_unique_emails(unique_emails):
print("\033[1;32m[+] Total unique emails found: {}\033[1;m".format(len(unique_emails)))
for unique_email in sorted(unique_emails):
print(unique_email)
def write_unique_emails(unique_emails):
with open('unique_emails.json', 'w') as outfile:
json.dump(unique_emails, outfile, sort_keys=True, indent=4)
def get_domain_name():
filepath = 'file.txt'
with open(filepath) as fp:
for cnt, line in enumerate(fp):
print("Line {}: {}".format(cnt, line))
return line
if __name__ == '__main__':
domain_name = get_domain_name()
cursor = connect_to_db(domain_name)
unique_emails = get_unique_emails(cursor, domain_name)
print_unique_emails(unique_emails)
write_unique_emails(unique_emails)
Code below using sys.argv
from __future__ import print_function
try:
import psycopg2
except ImportError:
raise ImportError('\n\033[33mpsycopg2 library missing. pip install psycopg2\033[1;m\n')
sys.exit(1)
import re
import sys
import json
DB_HOST = 'crt.sh'
DB_NAME = 'certwatch'
DB_USER = 'guest'
def connect_to_db(domain_name):
try:
conn = psycopg2.connect("dbname={0} user={1} host={2}".format(DB_NAME, DB_USER, DB_HOST))
cursor = conn.cursor()
cursor.execute("SELECT ci.NAME_VALUE NAME_VALUE FROM certificate_identity ci WHERE ci.NAME_TYPE = 'emailAddress' AND reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower('%{}'));".format(domain_name))
cursor.execute("SELECT ci.NAME_VALUE NAME_VALUE FROM certificate_identity ci WHERE ci.NAME_TYPE = 'serialNumber' AND reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower('%{}'));".format(domain_name))
except:
print("\n\033[1;31m[!] Unable to connect to the database\n\033[1;m")
return cursor
def get_unique_emails(cursor, domain_name):
unique_emails = []
for result in cursor.fetchall():
matches=re.findall(r"\'(.+?)\'",str(result))
for email in matches:
#print(email)
if email not in unique_emails:
if "{}".format(domain_name) in email:
unique_emails.append(email)
return unique_emails
def get_unique_serialNumber(cursor, domains):
unique_domains = []
for result in cursor.fetchall():
matches=re.findall(r"\'(.+?)\'",str(result))
for serialNumber in matches:
if serialNumber not in unique_serialNumber:
if ".{}".format(domain_name) in serialNumber:
unique_serialNumber.append(serialNumber)
return unique_serialNumber
def print_unique_serialNumber(unique_serialNumber):
for unique_serialNumber in sorted(unique_serialNumber):
print(unique_serialNumber)
def print_unique_emails(unique_emails):
print("\033[1;32m[+] Total unique emails found: {}\033[1;m".format(len(unique_emails)))
for unique_email in sorted(unique_emails):
print(unique_email)
def write_unique_emails(unique_emails):
with open('read.json', 'w') as outfile:
json.dump(unique_emails, outfile, sort_keys=True, indent=4)
def get_domain_name():
if len(sys.argv) <= 1:
print("\n\033[33mUsage: python emails_from_ct_logs.py <target_domain>\033[1;m\n")
sys.exit(1)
else:
return sys.argv[1]
if __name__ == '__main__':
domain_name = get_domain_name()
cursor = connect_to_db(domain_name)
unique_emails = get_unique_emails(cursor, domain_name)
print_unique_emails(unique_emails)
write_unique_emails(unique_emails)
unique_serialNumber = get_unique_serialNumber(cursor, domain_name)
print_unique_serialNumber(unique_serialNumber)
Check out Psycopg2. Without knowing all the details of your db it's going to be impossible to do a "cut & paste" code dump. The basics are covered here, which hopefully is enough to get you going. When or if you have more specific questions create a new thread.
Iteration of each record then print by key display nothing
#!/usr/bin/env python
# Print necessary headers.
print("Content-Type: text/html")
print()
import pymysql.cursors
# Connect to the database.
import pymysql
conn = pymysql.connect(db='candidate', user='root',passwd='123',host='localhost')
print ("connect successful!!")
c = conn.cursor()
c.execute("SELECT can_id,can_name,state,total_votes FROM voting")
result_set = c.fetchall()
for row in result_set:
print(row["can_id"])
print(row) works fine
c = conn.cursor(pymysql.cursors.DictCursor)
or
conn = pymysql.connect(db='candidate', user='root', passwd='123', host='localhost', cursorclass=pymysql.cursors.DictCursor)
I propose some changes using the PyMySQL documentation as reference:
#!/usr/bin/env python
# -*- conding: utf-8 -*-
# Print necessary headers.
print("Content-Type: text/html")
import pymysql.cursors
# Connect to the database.
# That is important: the order of connection parameters
# 1st host, 2nd user, ...
conn = pymysql.connect(
host='localhost'
user='root',
password='123',
db='candidate',
cursorclass=pymysql.cursors.DictCursor # <-- declare cursor class
)
print ("connect successful!!")
try:
with conn.cursor() as c:
query = "SELECT can_id, can_name, state, total_votes FROM voting;"
c.execute(query)
with c as pointer:
for row in pointer.fetchall():
print("Name: %s" % row['can_name'])
finally:
conn.close()
What is wrong with my code? Thank you
import os
import os.path
import time
global item_count
#-*- coding: cp936 -*-
import MySQLdb
import MySQLdb.cursors
import threading
import multiprocessing
from time import sleep,ctime
def qucun():
#connect to mysql
conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong")
cursor=conn.cursor()
try:
cursor.execute("""create database if not exists quad""")
except:
print 'Quad is exist'
conn.select_db('quad')
conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong",db="quad")
#get cursor
cursor=conn.cursor()
try:
cursor.execute("""create table if not exists record(fn1 varchar(100),
fn2 varchar(100),fn3 varchar(100),fn4 varchar(100),
fn5 varchar(100),fn6 varchar(100),fn7 varchar(100),fn8 varchar(100))""")
except:
print 'Table record is exist'
loops=['2013071818_1.txt','2013071818_2.txt','2013071818_3.txt','2013071818_4.txt','2013071818_5.txt']
def loop(nloop,filename):
print 'This loop%s start at:'%nloop,ctime()
#connect to quad
conn=MySQLdb.connect(host="localhost",user="root",passwd="caihong",db="quad")
conn.select_db('quad')
#get cursor
cursor=conn.cursor()
newitem=open('C:\\Python27\\caihong\\%s'%filename,'r')
data=[line.strip() for line in newitem.readlines()]
print data
##put data into value
values=['%s'%data[0],'%s'%data[1],'%s'%data[2],'%s'%data[3],'%s'%data[4],
'%s'%data[5],'%s'%data[6],'%s'%data[7]]
cursor.execute("""insert into record values(%s,%s,%s,%s,%s,%s,%s,%s)""",values);
conn.commit()
cursor.close()
sleep(2)
print 'This loop done at',ctime()
if __name__=='__main__':
print 'starting at:',ctime()
threads=[]
nloops=range(len(loops))
pool=multiprocessing.Pool(processes=2)
for i in nloops:
t=pool.apply_async(loop,(i,loops[i]))
pool.close()
pool.join()
if t.successful():
print 'successful'
print 'all Done at:',ctime()
os.system("pause")
qucun()
You are attempting to call locally defined function in async.
You are trying to share an open connection between processes.
First is tricky to implement in 2.7 and second is impossible in any multiprocessing
You have to use separate connection for each process in process pool.
import os
import os.path
import time
global item_count
#-*- coding: cp936 -*-
import MySQLdb
import MySQLdb.cursors
import threading
import multiprocessing
from time import sleep,ctime
CONNECTION = None
def close_connection():
CONNECTION.close()
def get_connection():
global CONNECTION
#If this process pool member launched for a first time - create connection
if CONNECTION is None:
conn = MySQLdb.connect( host="localhost",
user="root",
passwd="caihong")
cursor = conn.cursor()
try:
cursor.execute("""create database if not exists quad""")
except:
print 'Quad is exist'
conn.select_db('quad')
CONNECTION = MySQLdb.connect(host="localhost",
user="root",
passwd="caihong",
db="quad")
cursor = CONNECTION.cursor()
try:
cursor.execute("""create table if not exists record(fn1 varchar(100),
fn2 varchar(100),fn3 varchar(100),fn4 varchar(100),
fn5 varchar(100),fn6 varchar(100),fn7 varchar(100),fn8 varchar(100))""")
except:
print 'Table record is exist'
# we dont need to close connection after each insert.
# insted - register a finalizer once
# so it will be called right before Pool.close()
multiprocessing.util.Finalize(CONNECTION, close_connection, exitpriority=1)
#use existing connection
return CONNECTION
def loop(nloop, filename):
conn = get_connection()
cursor = conn.cursor()
print 'This loop %s start at: %s'%(nloop, ctime())
with open('C:\\Python27\\caihong\\%s'%filename, 'r') as newitem:
data = [line.strip() for line in newitem.readlines()]
# values=['%s'%data[0],'%s'%data[1],'%s'%data[2],'%s'%data[3],'%s'%data[4],
# '%s'%data[5],'%s'%data[6],'%s'%data[7]]
# ^^^ Thats a bad way to stringify list
cursor.execute('insert into record values(%s)', ','.join(data));
conn.commit()
# we dont need to close connection after each insert.
# cursor.close()
print 'This loop done at', ctime()
LOOPS = ['2013071818_1.txt', '2013071818_2.txt', '2013071818_3.txt', '2013071818_4.txt', '2013071818_5.txt']
if __name__=='__main__':
pool = multiprocessing.Pool(processes=2)
results = []
for i, loopfile in enumerate(LOOPS):
results.apply(pool.apply_async(loop, (i, loopfile)))
pool.close()
pool.join()
if all((res.successful() for res in results)):
print 'successful'
print 'all Done at:', ctime()
os.system('pause')