I have an exercise to do:
Through library psycopg2 connect with my database and get top 10 customers along with their credit limit:
import psycopg2
connection = psycopg2.connect(
host='host',
user='user',
password='password',
dbname='db_name',
)
cursor = connection.cursor()
cursor.execute('SELECT customername, creditlimit FROM customers order by 2 desc limit 10')
for row in cursor:
row[0]
row[1]
print(f"{row[0]}: {row[1]} USD")
This is the result of the query above:
As per next step I need to connect with API and convert this creditlimit to different currencies (EUR, JPY, GBP), so the final output will be like that:
Euro+ Shopping Channel: 227600.00 USD / 187183.93EUR / 24017262.40 JPY /160578.63 GBP
...
This is the code for API (the key is valid, feel free to test):
headers = {
"apikey": "b6iGXz0JZcr2iyV0SeLcd0dEpqOr8ngK"
}
currency = ['EUR', 'JPY', 'GBP']
params = {
"from": "USD",
"to": 'EUR',
"amount": row[1]
}
url = "https://api.apilayer.com/exchangerates_data/convert"
response = requests.get(url, headers=headers, params=params)
if response.status_code == 200:
data = response.json()
I connected with API, but I have difficulties to append it to my sql data, please advise.
Related
This is what I've done for testing database
#patch.object(query_functions, 'connection')
def test_service_reg_return_correct_data(self, mock_sqlalchmey):
data = {
"user_name": "testuser53",
"password": "123456",
"email_address": "testuser53#example.com",
"dob": "2022-06-07 00:00:00",
"address": "Ahmedabad",
}
# json_data = json.dumps(data)
# print("jsob", json_data)
m1 = mock.MagicMock()
m1.get_json.return_value = data
print("m", m1)
with mock.patch("backened.routes.register.request", m1):
response =user_register()
self.assertTrue(mock_sqlalchmey.called)
self.assertEqual(response.json, {'Message': 'New user Created'},201)
I've tested this function
def user_register():
request_data = request.get_json()
insert = service_register(request_data)
print(type(insert))
return insert
inside service_register(), I've run_insert() for execution of queries i.e query_functions.py file.
query_functions.py
def connection():
engine = create_engine('mysql://root:admin#172.17.0.2:3306/flask', poolclass = NullPool)
conn = engine.connect()
return conn
def run_insert(data):
conn = connection()
sql = text(CONST_INSERT)
print(sql)
conn.execute(sql, data)
conn.close()
return ("executed")
I want to know whether it is right or wrong as I was trying to mock my database and don't want to insert data into DB. I'm not sure how database mock happened in this.
im a beginner in python language , and im trying to insert my json file data into my database table using python but the problem is i have no errors i just get :
tweet number 49634 is uploading to the server
i don't get where the problem is, please i would appreciate any help
import mysql.connector
import json
mydb = mysql.connector.connect(host='localhost', port='3306', user='root', password='nihad147', database='tweets')
mycursor = mydb.cursor()
sql_request='insert ignore into tweet_location (latitude, longitude, tweet_id) values (%s,%s,%s)'""
myJsonFile = open('tweet.json', encoding="utf-8")
c = 0
for line in myJsonFile:
c = c + 1
print("tweet number ", c, " is uploading to the server")
data = json.loads(line)
#line = line.replace('','')
tweet = "SELECT * FROM tweet WHERE tweet_id = '" + str(data['tweet_id']) + "'"
mycursor.execute(tweet)
myresult = mycursor.fetchall()
row_count = mycursor.rowcount
if row_count == 0:
if 'location' in data.keys() and data['location'] != None and 'address' in data['location']:
latitude = data ['location']['lat']
longitude=data ['location']['lon']
mycursor.execute(sql_request, (latitude,longitude,data['tweet_id']))
print ('------------')
mydb.commit ()
here's an example of my json file data:
{"tweet_id":"1261276320878788609",
"date":"Fri May 15 12:44:42 +0000 2020",
"raw_text":"برنامج وطني لدعم المبدعين في مواجهة #كورو"
"geo_source":"user_location",
"location":{"address":
{"country":"Tunisia","country_code":"tn","state_district":"غزالة","county":"العرب","state":"Bizerte"},
"response":
"{'place_id': 235309103, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'relation', 'osm_id': 7124228, 'boundingbox': ['37.105957', '37.2033466', '9.4739053', '9.6124953'], 'lat': '37.1551868', 'lon': '9.54834183807249', 'display_name': 'العرب, غزالة, Bizerte, Tunisia', 'class': 'boundary', 'type': 'administrative', 'importance': 0.45, 'icon': '/data/nominatimimages/mapicons/poi_boundary_administrative.p.20.png','address':{'county': 'العرب', 'state_district': 'غزالة', 'state': 'Bizerte', 'country': 'Tunisia', 'country_code': 'tn'}}",
"geohash":"snwg37buskzd","query_term":"arab","lon":9.54834183807249,"lat":37.1551868},
"user_friends_count":61,"user_description":"I love UAE and his great leadership",
"user_created_at":"Wed Oct 09 11:41:41 +0000 2013",
"user_screen_name":"SikandarMirani",
"user_id_str":"706377881",
"user_verified":false,
"user_statuses_count":50804,
"user_followers_count":946,
"user_location":"Dubai United Arab Emirates"}
and this my table's attributes tweet:
tweet_id,
id_user,
text,
tweet_location,
created_at,
name_screen,
categorie_id,
Don't read the JSON file one line at a time. Use json.load() to load the entire file into a dictionary.
Use a parameter in the query that selects the tweet, rather that concatenating data['tweet_id'] into the SQL.
The code that inserts the new row should be inside all the if statements that set latitude and longitude from the data. In fact, you might as well put all the database code inside the if statement that checks whether location is set in the JSON.
import mysql.connector
import json
mydb = mysql.connector.connect(host='localhost', port='3306', user='root', password='nihad147', database='tweets')
mycursor = mydb.cursor()
sql_request='insert ignore into tweet_location (latitude, longitude, tweet_id) values (%s,%s,%s)'""
with open('tweet.json', encoding="utf-8") as myJsonFile:
data = json.load(myJsonFile)
if data.get('location') and 'address' in data['location']:
tweet = "SELECT 1 FROM tweet WHERE tweet_id = %s"
mycursor.execute(tweet, (data['tweet_id'],))
myresult = mycursor.fetchall()
row_count = len(myresult)
if row_count == 0:
print(f"Inserting {data['tweet_id']} to the database");
latitude = data['location']['lat']
longitude = data['location']['lon']
mycursor.execute(sql_request, (latitude,longitude,data['tweet_id']))
mydb.commit ()
print ('------------')
else:
print(f"Tweet {data['tweet_id']} is already in the database")
im a beginner in python , Trying to connect access database to python with a json file loaded in my program so I can read it and eventually analyze it for certain things. But I can't connect to it and tried different approaches still getting the same error.
import mysql.connector
import json
# create the key
from mysql.connector import cursor
mydb = mysql.connector.connect(host='localhost', port='3306', user='root', password='nihad147', database='tweets')
mycursor = mydb.cursor()
sql_tweet = """INSERT INTO tweet ( tweet_id,
id_user,
text,
tweet_location,
created_at,
name_screen,
categorie_id,
)
VALUES (%s,%s,%s,%s,%s,%s,%s)"""
sql_user = """INSERT INTO tweetuser (
id_user,
name_screen,
location_user,
count_followers,
friends_count,
statuse_count)
VALUES (%s,%s,%s,%s,%s,%s)"""
sql_location = """"insert into tweet_location (
location_id,
latitude,
longitude
tweet_id
VALUES(%s,%s,%s,%s)"""
myJsonFile = open('tweets.json', encoding="utf-8")
mycursor.execute("DELETE FROM tweet")
mycursor.execute("DELETE FROM tweetuser")
mycursor.execute("DELETE FROM tweet_location")
c = 0
for line in myJsonFile:
c = c + 1
print("tweet number ", c, " is uploading to the server")
data = json.loads(line)
# insert into tweet
val_tweet = (
data['tweet_id'], data['user_id_str'], data['raw_text'],data['location']['address']['city'],data['date'], data['user_screen_name'])
mycursor.execute(sql_tweet,sql_location, val_tweet)
mydb.commit()
# testing ifthe user already exist
user = "SELECT * FROM tweetuser WHERE id_user = '" + str(data['user_id_str']) + "'"
mycursor.execute(user)
myresult = mycursor.fetchall()
row_count = mycursor.rowcount
if row_count == 0:
val_user = (data['user_id_str'], data['user_screen_name'], data['location']['address']['city'],data['user_followers_count'],
data['user_friends_count'], data['user_statuses_count'])
mycursor.execute(sql_user, val_user)
mydb.commit()
print('done')
here's an example of json file data :
{
"tweet_id":"1261276320878788609",
"date":"Fri May 15 12:44:42 +0000 2020",
"raw_text":"برنامج وطني لدعم المبدعين في مواجهة #كورون",
"geo_source":"user_location",
"location":{
"address":{
"country":"Tunisia",
"country_code":"tn",
"state_district":"غزالة",
"county":"العرب",
"state":"Bizerte"
},
"response":"{'place_id': 235309103, 'licence': 'Data © OpenStreetMap contributors, ODbL 1.0. https://osm.org/copyright', 'osm_type': 'relation', 'osm_id': 7124228, 'boundingbox': ['37.105957', '37.2033466', '9.4739053', '9.6124953'], 'lat': '37.1551868', 'lon': '9.54834183807249', 'display_name': 'العرب, غزالة, Bizerte, Tunisia', 'class': 'boundary', 'type': 'administrative', 'importance': 0.45, 'icon': '/data/nominatimimages/mapicons/poi_boundary_administrative.p.20.png','address':{'county': 'العرب', 'state_district': 'غزالة', 'state': 'Bizerte', 'country': 'Tunisia', 'country_code': 'tn'}}",
"geohash":"snwg37buskzd",
"query_term":"arab",
"lon":9.54834183807249,
"lat":37.1551868
},
"user_friends_count":61,
"user_description":"I love UAE and his great leadership",
"user_created_at":"Wed Oct 09 11:41:41 +0000 2013",
"user_screen_name":"SikandarMirani",
"user_id_str":"706377881",
"user_verified":false,
"user_statuses_count":50804,
"user_followers_count":946,
"user_location":"Dubai United Arab Emirates"
}
thanks to you guys , i was able to solve the previous error since i didn't check tha data type of the id user it has to be bigint not int since it's a large data .
i had no problem connecting my jsonfile to my database but it got inserted only in tweetuser table but not in tweet table .
the tweet table is empty.
i would appreciate any kind of help thank you
The error
mysql.connector.errors.DataError: 1264 (22003): Out of range value for column 'id_user' at row 1
suggests that the value you are trying to use as the id_user is numerically too large.
Since you haven't posted the table definitions, my guess is you are using MEDIUMINT or SMALLINT or TINYINT for id_user and the actual user ID that you are trying to write into the database is too large for that data type.
In your example user_id_str is 706377881, however, the maximum value for MEDIUMINT is 8388607 and 16777215 (unsigned), respectively.
Check the data types in the table definitions.
You are connecting to your DB, that is not the problem.
The problem is that the user id that you are trying to insert has a length that surpasses the maximum allowed by MySQL for the datatype of that field. See here and here for more info related to your error.
I have a Postgres table in pgadmin4 and it has 2 columns. ID (big serial, not null) and json (json). I keep getting the value null in my json column when trying to insert my json data from it which I got by doing a get request. Under the column settings for json I have not selected 'not null'.
I am still learning; my code might be a bit messy.
import os
import requests
import psycopg2
import json
def get_request():
token = os.environ['TOKEN']
url = os.environ['URL']
payload = {}
headers = {
'Accept': 'application/json',
'Authorization': 'Bearer ' + token
}
response = requests.request('GET', url, headers=headers, data=payload)
result = (response.text.encode('utf8'))
parsed = json.loads(result)
print(parsed)
print(type(parsed))
def db_connection(parsed):
con = psycopg2.connect(
host=os.environ['HOST'],
dbname=os.environ['DBNAME'],
user=os.environ['USER'],
password=os.environ['PASSWORD'],
port=os.environ['PORT'])
print("CONNECTED")
cur = con.cursor()
cur.execute("INSERT INTO ingest.occupations_group (json) VALUES (%s)", (parsed,))
con.commit()
cur.close()
con.close()
get_request()
db_connection(get_request())
And when I run the code for some reason it display the results twice. I did not include the all of the api results as it is super long.
{'#odata.context': 'https://apis.burning-glass.com/v3.5/ontology/$metadata#occupationgroups', 'value': [{'id': 2, 'ontologyIRI':
<class 'dict'>
{'#odata.context': 'https://apis.burning-glass.com/v3.5/ontology/$metadata#occupationgroups', 'value': [{'id': 2, 'ontologyIRI':
<class 'dict'>
CONNECTED
Process finished with exit code 0
I have a case study where i need to take data from a REST API do some analysis on the data using aggregate function,joins etc and use the response data in JSON format to plot some retail grahs.
Approaches being followed till now:
Read the data from JSON store these in python variable and use insert to hit the SQL query. Obviously it is a costly operation because for every JSON line read it is inserting into database.For 33k rows it is taking more than 20 mins which is inefficient.
This can be handled in elastic search for faster processing but complex operation like joins are not present in elastic search.
If anybody can suggest what would be the best approach (like preprocessing or post processing in python) to follow for handling such scenerios it would be helpful.
Thanks in advance
Sql Sript
def store_data(AccountNo)
db=MySQLdb.connect(host=HOST, user=USER, passwd=PASSWD, db=DATABASE, charset="utf8")
cursor = db.cursor()
insert_query = "INSERT INTO cstore (AccountNo) VALUES (%s)"
cursor.execute(insert_query, (AccountNo))
db.commit()
cursor.close()
db.close()
return
def on_data(file_path):
#This is the meat of the script...it connects to your mongoDB and stores the tweet
try:
# Decode the JSON from Twitter
testFile = open(file_path)
datajson = json.load(testFile)
#print (len(datajson))
#grab the wanted data from the Tweet
for i in range(len(datajson)):
for cosponsor in datajson[i]:
AccountNo=cosponsor['AccountNo']
store_data( AccountNo)
Edit1: Json Added
{
"StartDate": "1/1/18",
"EndDate": "3/30/18",
"Transactions": [
{
"CSPAccountNo": "41469300",
"ZIP": "60098",
"ReportDate": "2018-03-08T00:00:00",
"POSCode": "00980030003",
"POSCodeModifier": "0",
"Description": "TIC TAC GUM WATERMEL",
"ActualSalesPrice": 1.59,
"TotalCount": 1,
"Totalsales": 1.59,
"DiscountAmount": 0,
"DiscountCount": 0,
"PromotionAmount": 0,
"PromotionCount": 0,
"RefundAmount": 0,
"RefundCount": 0
},
{
"CSPAccountNo": "41469378",
"ZIP": "60098",
"ReportDate": "2018-03-08T00:00:00",
"POSCode": "01070080727",
"POSCodeModifier": "0",
"Description": "PAYDAY KS",
"ActualSalesPrice": 2.09,
"TotalCount": 1,
"Totalsales": 2.09,
"DiscountAmount": 0,
"DiscountCount": 0,
"PromotionAmount": 0,
"PromotionCount": 0,
"RefundAmount": 0,
"RefundCount": 0
}
]
}
I do not have your json file so not know if it is runnable, but I would have tried something like below: I read just your account infos to a list and than try to write to the db at once with executemany I expect it to have a better(less) execution time than 20 mins.
def store_data(AccountNo):
db = MySQLdb.connect(host=HOST, user=USER, passwd=PASSWD, db=DATABASE, charset="utf8")
cursor = db.cursor()
insert_query = "INSERT INTO cstore (AccountNo,ZIP,ReportDate) VALUES (:AccountNo,:ZIP,:ReportDate)"
cursor.executemany(insert_query, AccountNo)
db.commit()
cursor.close()
db.close()
return
def on_data(file_path):
# This is the meat of the script...it connects to your mongoDB and stores the tweet
try:
#declare an empty list for the all accountno's
accountno_list = list()
# Decode the JSON from Twitter
testFile = open(file_path)
datajson = json.load(testFile)
# print (len(datajson))
# grab the wanted data from the Tweet
for row in datajson[0]['Transactions']:
values = dict()
values['AccountNo'] = row['CSPAccountNo']
values['ZIP'] = row['ZIP']
values['ReportDate'] = row['ReportDate']
#from here on you can populate the attributes you need in a similar way..
accountno_list.append(values)
except:
pass
store_data(accountno_list)