I would like to save a JSON file from the variable 'data' through an API.
I've created a function writetoJsonfile to do it but it does not work.
Can you guys help me out? Thank you.
import requests,json,io
import xmltodict
import logging as log
import pandas as pd
from io import BytesIO
from zipfile import ZipFile
from datetime import datetime
def writeToJSONFile(path, filename, cs ):
path = 'C:/Users/Skelaton/Desktop'
filename = 'compromised_systems'
ext = 'json'
filePathNameWExt = path + filename + ext
with open(filePathNameWExt, 'w') as fp :
json.dump(cs, fp)
def getdata(person_id):
log.info("Downloading all data from people {}".format(person_id))
payload = {'format' : 'csv'}
r = requests.get
("https://example.com/people/{}/reports/person".format(person_id),
auth=(api_key,''), proxies=proxy,params=payload)
if r.status_code == 200:
data = pd.read_csv(BytesIO(r.content),error_bad_lines=False)
data = data.to_json(orient="records")
data = json.loads(data)
return data
else:
log.error("Unable to download all data due to status code :
{}".format(r.status_code))
return False
Your file path doesn't seem right.
This works:
path = 'C:/Users/Skelaton/Desktop/'
filename = 'compromised_systems'
ext = '.json'
filePathNameWExt = path + filename + ext
def write_to_file(filename, data):
with open(filename, 'w+') as fp:
fp.write(json.dumps(data))
Related
I want to store numpyarrays(images), data and a timestamp to a file in Python.
Later i want to scroll through the images and show the data.
I wrote my data to a json file, but it takes long and with one image(640x480) and one set of data it gets 10 Mb in size.
Here is my aproach:
import json as json
import base64
import io
import sys
import cv2
import numpy as np
import os
from numpy import asarray
from json import JSONEncoder
path = r'C:/Videotool/Data'
name = 'testfile'
cam = cv2.VideoCapture(0)
result, image = cam.read()
if result:
print(f'image:{image}')
else:
print('no image')
def json_write_to_file(path, name, data):
file_exists = os.path.exists(name+'.json')
file = path + '/' + name + '.json'
readed_data = []
data_list = []
sum_dict = []
if file_exists:
print(f'file {name}.json exist\n')
with open(file) as fp:
readed_data = json.load(fp)
print(f'readed_data:{readed_data}\n')
print(f'data:{data}\n')
readed_data.append(data)
sum_dict = readed_data
print(f'sum_dict:{sum_dict}\n')
with open (file, 'w') as fp:
json.dump(sum_dict, fp, indent=2)
print(f'length of readed_data: {len(readed_data)}\n')
#fp.write('\n')
else:
sum_dict.append(data)
print(f'file {name}.json not exists\n')
with open (file, 'w') as fp:
json.dump(sum_dict, fp, indent=2)
print(f'Number of Dimensions of image:{image.ndim}')
print(f'Shape of image:{image.shape}')
print(f'Size of image:{image.size}')
low_image_array = image.reshape(-1)
ori_array = np.zeros((2,3,4))
print(f'Number of Dimensions of ori_array:{ori_array.ndim}')
print(f'Shape of ori_array:{ori_array.shape}')
print(f'Size of ori_array:{ori_array.size}')
print(f'ori_array:{ori_array}')
low_1_array = ori_array.reshape(-1)
print(f'Number of Dimensions of low_1_array:{low_1_array.ndim}')
print(f'Shape of low_1_array:{low_1_array.shape}')
print(f'Size of low_1_array:{low_1_array.size}')
print(f'low_1_array:{low_1_array}')
data_set = {}
data_set['Stream_1'] = low_1_array.tolist()
data_set['Stream_2'] = low_image_array.tolist()
print(f'data_set: {data_set}')
json_write_to_file(path, name, data_set)
Is there a better way to store images and data in a file?
I am currently trying to create a function within a Django app to download a pandas dataframe as an image. I wanted to create the image as a temporary file, download it, then delete it. Does anyone know how to integrate tempfile into this code?
Views.py
def download_png(request, study):
Names(study) #Funciton to get (name) variable
Retrieve(study) #Function to get (data) variable
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
pngfilename = str(name) + "_" + str(current_time) + ".png"
temp = tempfile.NamedTemporaryFile(suffix=".png")
fig = temp.write(df2img.plot_dataframe(data))
filepath = temp.name
response = HttpResponse(df2img.save_dataframe(fig=fig, filename=filepath), content_type=mimetypes.guess_type(filepath))
response['Content-Disposition'] = "attachment; filename=%s" % pngfilename
return response
Update 06/30/22:
I had a lot of difficulty integrating the tempfile module because it requires data to be converted to bytes-like objects. Instead I simply resolved on deleting the file after creating it.
def download_png(request, study):
Names(study)
Retrieve(study)
pngfilename = str(name) + "_" + str(current_time) + ".png"
mime_type, _ = mimetypes.guess_type(pngfilename)
fig = df2img.plot_dataframe(data)
df2img.save_dataframe(fig=fig, filename=pngfilename)
response = FileResponse(open(pngfilename, 'rb'), content_type=mime_type)
response['Content-Disposition'] = "attachment; filename=%s" % pngfilename
os.remove(pngfilename)
return response
This is how you can use the tempfile library:
tempfile = tempfile.mktemp(suffix='.png')
with open(temp, 'w') as f:
f.write(df2img.plot_dataframe(data))
do_something_with_your_file(temp)
os.unlink(temp)
Edit: was just reading the tempfile documentation and apparently tempfile.mktemp is prone to race conditions, so you should either use tempfile.mkstemp like:
fd, tempf = tempfile.mkstemp()
Or there is a context manager:
with tempfile.NamedTemporaryFile() as tmp:
with open(tmp.name, 'w') as f:
f.write('some data')
# do something with your tempfile file
And then it will automatically delete the tempfile afterwards.
I am trying to add a prefix from one column to a filename. This code downloads a picture from a given URL in a CSV. That file name should have the prefix + filename.
The CSV has two columns, the first with the URLs and the second with a uniqueID named "idnumber"
Its been a while since i coded and maybe mixing SQL with Python...
Here is what I have so far:
import pandas as pd
import urllib.request
def url_to_jpg(i, url,file_path ):
filename = 'idname-image-{}.jpg'.format(i)
full_path = '{}{}'.format(file_path, filename)
urllib.request.urlretrieve(url, full_path)
print('{} saved.'.format(filename))
return None
FILENAME = 'Image_Download.csv'
FILE_PATH = 'images/'
urls = pd.read_csv(FILENAME)
idnumber = urls.idnumber
for i, URL in enumerate(urls.values):
url_to_jpg(i, URL[0],FILE_PATH)
import urllib.request
import time
def url_to_jpg(i, name,url,file_path ):
time.sleep(5)
filename = name
full_path = '{}{}'.format(file_path, filename)
urllib.request.urlretrieve(url, full_path)
print('{} saved.'.format(filename))
return None
FILENAME = 'Image_Download.csv'
FILE_PATH = 'images/'
df = pd.read_csv(FILENAME)
name = df.name
for i, URL, in enumerate(df.values):
url_to_jpg(i, name[i] ,URL[0],FILE_PATH)
I'm looking for a solution about path making for glob and for pandas to_csv anyone have a solution ?
My code :
from glob import glob
import json
import pandas as pd
PathIn = 'c:\\Users\\***\\PycharmProjects\\Project\\In'
PathOut = 'c:\\Users\\***\\PycharmProjects\\Project\\Out'
for fileName in glob(PathIn + '*.json', recursive=True):
with open(fileName, 'rb') as f:
json_dict = json.load(f)
print(json_dict)
.
.
.
.
.
.
df.to_csv(PathOut + fileName + '.csv', sep=";")
He doesn't print me my JSON file so don't take any file in my In. And I don't have any CSV in my Output.
the key here is you want to create the output file in the relevant user dir based on the input file, so you could instead just get a list of the users dirs and iterate over each of them settting the in and output file then search the json files and create the csv in the coresponding dir. something like.
import json
from glob import glob
import os.path as op
basepath = r'C:\Users\***\PycharmProjects'
_in = 'In'
_out = 'Out'
suffix = '\*.json'
output_suffix = '.csv'
for path in glob(basepath):
in_dir = op.join(path, _in)
out_dir = op.join(path, _out)
for json_file in glob(in_dir + suffix, recursive=True):
in_file_name = op.basename(json_file)
out_file_name = in_file_name.split('.')[0] + output_suffix
output_file = op.join(out_dir, out_file_name)
with open(json_file) as jf:
json_data = json.load(jf)
print(json_data)
###do some stuff with the json
with open(output_file, 'w') as of:
of.write("some data or json stuff")
Just slightly modifying your code I think you missed a \ when writing the path for searching in the input directory.
For the output directory you need to build your filename by replacing the extension .json with .csv. There are many ways to do that:
for fileName in glob(PathIn + '\*.json', recursive=True):
with open(fileName, 'rb') as f:
json_dict = json.load(f)
print(json_dict)
out_file_name = os.path.split(fileName)[0] + '.csv'
out_file_dir = os.path.join(PathOut, out_file_name)
# Here do something with your output file
I need to download several files via http in Python.
The most obvious way to do it is just using urllib2:
import urllib2
u = urllib2.urlopen('http://server.com/file.html')
localFile = open('file.html', 'w')
localFile.write(u.read())
localFile.close()
But I'll have to deal with the URLs that are nasty in some way, say like this: http://server.com/!Run.aspx/someoddtext/somemore?id=121&m=pdf. When downloaded via the browser, the file has a human-readable name, ie. accounts.pdf.
Is there any way to handle that in python, so I don't need to know the file names and hardcode them into my script?
Download scripts like that tend to push a header telling the user-agent what to name the file:
Content-Disposition: attachment; filename="the filename.ext"
If you can grab that header, you can get the proper filename.
There's another thread that has a little bit of code to offer up for Content-Disposition-grabbing.
remotefile = urllib2.urlopen('http://example.com/somefile.zip')
remotefile.info()['Content-Disposition']
Based on comments and #Oli's anwser, I made a solution like this:
from os.path import basename
from urlparse import urlsplit
def url2name(url):
return basename(urlsplit(url)[2])
def download(url, localFileName = None):
localName = url2name(url)
req = urllib2.Request(url)
r = urllib2.urlopen(req)
if r.info().has_key('Content-Disposition'):
# If the response has Content-Disposition, we take file name from it
localName = r.info()['Content-Disposition'].split('filename=')[1]
if localName[0] == '"' or localName[0] == "'":
localName = localName[1:-1]
elif r.url != url:
# if we were redirected, the real file name we take from the final URL
localName = url2name(r.url)
if localFileName:
# we can force to save the file as specified name
localName = localFileName
f = open(localName, 'wb')
f.write(r.read())
f.close()
It takes file name from Content-Disposition; if it's not present, uses filename from the URL (if redirection happened, the final URL is taken into account).
Combining much of the above, here is a more pythonic solution:
import urllib2
import shutil
import urlparse
import os
def download(url, fileName=None):
def getFileName(url,openUrl):
if 'Content-Disposition' in openUrl.info():
# If the response has Content-Disposition, try to get filename from it
cd = dict(map(
lambda x: x.strip().split('=') if '=' in x else (x.strip(),''),
openUrl.info()['Content-Disposition'].split(';')))
if 'filename' in cd:
filename = cd['filename'].strip("\"'")
if filename: return filename
# if no filename was found above, parse it out of the final URL.
return os.path.basename(urlparse.urlsplit(openUrl.url)[2])
r = urllib2.urlopen(urllib2.Request(url))
try:
fileName = fileName or getFileName(url,r)
with open(fileName, 'wb') as f:
shutil.copyfileobj(r,f)
finally:
r.close()
2 Kender:
if localName[0] == '"' or localName[0] == "'":
localName = localName[1:-1]
it is not safe -- web server can pass wrong formatted name as ["file.ext] or [file.ext'] or even be empty and localName[0] will raise exception.
Correct code can looks like this:
localName = localName.replace('"', '').replace("'", "")
if localName == '':
localName = SOME_DEFAULT_FILE_NAME
Using wget:
custom_file_name = "/custom/path/custom_name.ext"
wget.download(url, custom_file_name)
Using urlretrieve:
urllib.urlretrieve(url, custom_file_name)
urlretrieve also creates the directory structure if not exists.
You need to look into 'Content-Disposition' header, see the solution by kender.
How to download a file using python in a 'smarter' way?
Posting his solution modified with a capability to specify an output folder:
from os.path import basename
import os
from urllib.parse import urlsplit
import urllib.request
def url2name(url):
return basename(urlsplit(url)[2])
def download(url, out_path):
localName = url2name(url)
req = urllib.request.Request(url)
r = urllib.request.urlopen(req)
if r.info().has_key('Content-Disposition'):
# If the response has Content-Disposition, we take file name from it
localName = r.info()['Content-Disposition'].split('filename=')[1]
if localName[0] == '"' or localName[0] == "'":
localName = localName[1:-1]
elif r.url != url:
# if we were redirected, the real file name we take from the final URL
localName = url2name(r.url)
localName = os.path.join(out_path, localName)
f = open(localName, 'wb')
f.write(r.read())
f.close()
download("https://example.com/demofile", '/home/username/tmp')
I have just updated the answer of kender for python3