I am trying to make a copy of smart sheet data on my local disk. I am able to copy all the smart sheet data except for the cell images. Below is the code am using. This code works perfectly fine to copy the data but not the cell images
NOTE: I am not trying to copy the attachments from smart sheets; only the cell the images and data.
Could someone help me to enhance this code to copy the cell images as well?
import json
import os
import requests
import time
token = "Bearer <TOken>"
backed_up_sheets = {"Attach": 86960044478894,"test2":6659760455684}
dir = r'C:\Users\\me\SmartSheetsBackup\WorkSheet' + time.strftime("-%m_%d_%Y_%H_%M")
API_URL = "https://api.smartsheet.com/2.0/sheets/"
payload = {"Authorization": token,
"Accept": "application/vnd.ms-excel,image/*"}
amount = len(backed_up_sheets)
i = 1
for el in backed_up_sheets:
r = requests.get(API_URL + str(backed_up_sheets[el]) , headers=payload)
if r.status_code != 200:
print ('Some problem with connections please retry later0')
pass
if not os.path.exists(dir):
os.makedirs(dir)
with open(dir + el + time.strftime("-%m_%d_%Y_%H_%M") + ".xlsx", 'wb') as output:
output.write(r.content)
print ('Progress in sheets: ' + str(i) + '/' + str(amount))
i += 1
Here's a complete code sample:
# Download an image in a cell
def download_cell_image(client, sheet_id, row_id, column_id, default_filename):
# Get desired row
row = client.Sheets.get_row(sheet_id, row_id)
cell = row.get_column(column_id)
image = cell.image
filename = getattr(image, 'alt_text', default_filename)
# Obtain a temporary image URL
imageUrl = ss_client.models.ImageUrl( { "imageId": image.id } )
response = ss_client.Images.get_image_urls([imageUrl])
url = response.image_urls[0].url
# Download the image
import requests
response = requests.get(url)
if response.status_code == 200:
with open(filename, 'wb') as f:
f.write(response.content)
Note that this requires SDK version 1.3.0 or later
The same steps illustrated in the cURL example should work in Python. (Apologies that we don't have an complete published sample)
Get the image id from the cell object, as returned from get_sheet
Convert the image id to a download url, using images.get_image_urls (docs)
Download the image from the url, probably using the Requests library.
Related
I have an excel list of DOIs of papers I'm interested in. Based on this list, I would like to download all the papers.
I tried to do it with request, as recommended in their documentation. But the pdf files I get are damaged. They are just some KB big. I changed the chunk_size several times from None till 1024*1024 and I have read many posts already. Nothing helps.
Please, what are your ideas?
import pandas as pd
import os
import requests
def get_pdf(doi, file_to_save_to):
url = 'http://api.elsevier.com/content/article/doi:'+doi+'?view=FULL'
headers = {
'X-ELS-APIKEY': "keykeykeykeykeykey",
'Accept': 'application/pdf'
}
r = requests.get(url, stream=True, headers=headers)
if r.status_code == 200:
for chunk in r.iter_content(chunk_size=1024*1024):
file_to_save_to.write(chunk)
return True
doi_list = pd.read_excel('list.xls')
doi_list.columns = ['DOIs']
count = 0
for doi in doi_list['DOIs']:
doi = doi.replace('DOI:','')
pdf = doi.replace('/','%')
if not os.path.exists(f'path/{pdf}.pdf'):
file = open(f'path/{pdf}.pdf', 'wb')
get_pdf(doi, file)
count += 1
print(f"Dowloaded: {count} of {len(doi_list['DOIs'])} articles")
I think your problem is the return True in for chunk in r.iter_content. With that line, you'll only ever write one chunk of the PDF of size chunk_size.
You should also open files using with; as is, you'll never close the file handles.
import pandas as pd
import os
import requests
HEADERS = {
'X-ELS-APIKEY': "keykeykeykeykeykey",
'Accept': 'application/pdf'
}
def get_pdf(doi, file_to_save_to):
url = f'http://api.elsevier.com/content/article/doi:{doi}?view=FULL'
with requests.get(url, stream=True, headers=HEADERS) as r:
if r.status_code == 200:
for chunk in r.iter_content(chunk_size=1024*1024):
file_to_save_to.write(chunk)
doi_list = pd.read_excel('list.xls')
doi_list.columns = ['DOIs']
count = 0
for doi in doi_list['DOIs']:
doi = doi.replace('DOI:','')
pdf = doi.replace('/','%')
if not os.path.exists(f'path/{pdf}.pdf'):
with open(f'path/{pdf}.pdf', 'wb') as file:
get_pdf(doi, file)
count += 1
print(f"Dowloaded: {count} of {len(doi_list['DOIs'])} articles")
I am trying to download and save in a folder all the PDFs contained in some webs with dynamic elements i.e: https://www.bankinter.com/banca/nav/documentos-datos-fundamentales
Every PDF in this url have similar href. Here they are two of them:
"https://bancaonline.bankinter.com/publico/DocumentacionPrixGet?doc=workspace://SpacesStore/fb029023-dd29-47d5-8927-31021d834757;1.0&nameDoc=ISIN_ES0213679FW7_41-Bonos_EstructuradosGarantizad_19.16_es.pdf"
"https://bancaonline.bankinter.com/publico/DocumentacionPrixGet?doc=workspace://SpacesStore/852a7524-f21c-45e8-a8d9-1a75ce0f8286;1.1&nameDoc=20-Dep.Estruc.Cont.Financieros_18.1_es.pdf"
Here it is what I did for another web, this code is working as desired:
link = 'https://www.bankia.es/estaticos/documentosPRIIPS/json/jsonSimple.txt'
base = 'https://www.bankia.es/estaticos/documentosPRIIPS/{}'
dirf = os.environ['USERPROFILE'] + "\Documents\TFM\PdfFolder"
if not os.path.exists(dirf2):os.makedirs(dirf2)
os.chdir(dirf2)
res = requests.get(link,headers={"User-Agent":"Mozilla/5.0"})
for item in res.json():
if not 'nombre_de_fichero' in item: continue
link = base.format(item['nombre_de_fichero'])
filename_bankia = item['nombre_de_fichero'].split('.')[-2] + ".PDF"
with open(filename_bankia, 'wb') as f:
f.write(requests.get(link).content)
You have to make a post http requests with appropriate json parameter. Once you get the response, you have to parse two fields objectId and nombreFichero to use them to build right links to the pdf's. The following should work:
import os
import json
import requests
url = 'https://bancaonline.bankinter.com/publico/rs/documentacionPrix/list'
base = 'https://bancaonline.bankinter.com/publico/DocumentacionPrixGet?doc={}&nameDoc={}'
payload = {"cod_categoria": 2,"cod_familia": 3,"divisaDestino": None,"vencimiento": None,"edadActuarial": None}
dirf = os.environ['USERPROFILE'] + "\Desktop\PdfFolder"
if not os.path.exists(dirf):os.makedirs(dirf)
os.chdir(dirf)
r = requests.post(url,json=payload)
for item in r.json():
objectId = item['objectId']
nombreFichero = item['nombreFichero'].replace(" ","_")
filename = nombreFichero.split('.')[-2] + ".PDF"
link = base.format(objectId,nombreFichero)
with open(filename, 'wb') as f:
f.write(requests.get(link).content)
After executing the above script, wait a little for it to work as the site is real slow.
I'm trying to download an image from a website but I get a 404 error. I tried to add a user agent with no sucess.
Here is the code:
import requests
import shutil
with open(r'C:\Users\home\Desktop\urls.csv') as file:
csv = []
for row in file:
csv.append(row.split(";"))
row = 0
while row < len(csv):
r = requests.get(csv[row][0], stream=True, headers={'User-agent': 'Mozilla/5.0'})
if r.status_code == 200:
with open(r"C:\Users\home\Desktop\images\house" + str(row) + ".jpg", 'wb') as f:
r.raw.decode_content = True
shutil.copyfileobj(r.raw, f)
row +=1
The url is:
https://example.com/wp-content/uploads/2018/10/toronto-curbed-8.jpg
replace example by cdn.icepop
I want to automate a report I created in Namely using python, how can I do this with the Namely API?
Here's a python script I made that should cover it:
#Imports
import http.client
import json
import os
import time
import smtplib
#Constants
namelyDomain = "company.namely.com" #change this to your company's namely
csvName = "C:\\Path\\To_Write\\Your_CSV\\Report.csv" #absolute path to write csv
reportID = "0a12bac7-eac4-4bae-b18f-63ea3173gbb4" #report ID (find in URL)
APIkey = "yuIo4fH7f4z4dgabsSqXzxm9IMbW1ixLhjP0eh8jPuIo9vUI1nij9qZmG822al54" #get this from Namely>API>Personal Access Tokens
server = smtplib.SMTP()
#Variables
line = ""
columnCount = 0
#run report with get request
conn = http.client.HTTPSConnection(namelyDomain)
payload = "{}"
headers = { 'authorization': "Bearer " + APIkey }
conn.request("GET", "/api/v1/reports/" + reportID + ".json", payload, headers)
res = conn.getresponse()
if(res.status != 200):
print("failed to connect")
exit()
data = res.read() #returns json object
#Delete if it exists (overwrite)
if os.path.exists(csvName):
os.remove(csvName)
#make the csv
f = open(csvName,"w")
#get objects to loop from
dataHeader = dataRow = json.loads(data)
#Print headers to CSV
for data in dataHeader['reports'][0]['columns']:
columnCount = columnCount + 1
line = line + str(data['label']) + ","
line = line.rstrip(",")
f.write(line + chr(10))
#Print rows to CSV
for data in dataRow['reports'][0]['content']:
line = '"'
for ndx in range(0,columnCount):
line = line + str(data[ndx]) + '","'
line = line.replace("None","").replace('\u202d','').replace('\u202c','').rstrip('"').rstrip(",")
f.write(line + chr(10))
Just replace:
namelyDomain with your company's namely domain
csvName with the absolute path of where you want to write the csv report
reportID with the id of the report you want to generate
APIkey with the personal access token from namely
Useful Link: https://developers.namely.com/1.0/reports/show-report
I am trying to retrieve files from the internet, save them as dataframe (without saving the files on computer), do some
calculation, save resulting dataframes in the required format as csv files, zip them and send them back to the orginal source(see below my code).
This works perfectly fine but as I have to send hundred of files and was wondering if there is a way of
saving the dataframe to the csv file and then zipping it on the go (i.e. without saving it on computer)? Below is my code;
file_names=['A','B','C']
hdll_url = 'http://XYZ'
saving_folder_url = 'C:\Users\Desktop\Testing_Folder'
general = []
for i in file_names:
url = ('http://')
data1 = requests.get(url)
Z1 = zipfile.ZipFile(StringIO.StringIO(data1.content))
x=pd.read_csv(Z1.open('%s.tsv'%i), sep = '\t', names=["Datetime","Power"])
x['Datetime'] = pd.to_datetime(x['Datetime'])
x = x.set_index("Datetime")
x = x.resample('H', how= 'sum')
general.append(x)
ABC = pd.DataFrame(general[0]['Power'] + general[1]['Power'] + general[3]['Power'] * 11.363)
ABC.to_csv('%s\ABC.tsv'%saving_folder_url,
sep='\t',header=False,date_format='%Y-%m-%dT%H:%M:%S.%f')
ABC_z1 = zipfile.ZipFile('%s\ABC.zip'%saving_folder_url,mode='w')
ABC_z1.write("%s\ABC.tsv"%saving_folder_url)
ABC_z1.close()
ABC_files1 = {'file': open("%s\ABC.zip"%saving_folder_url,'rb')}
ABC_r1 = requests.put(hdll_url, files=ABC_files1, headers = {'content-type':'application/zip',
'Content-Disposition': 'attachment;filename=ABC.zip'
} )
XYZ = pd.DataFrame(general[0]['Power'] + general[1]['Power']*100 )
XYZ.to_csv('%s\XYZ.tsv'%saving_folder_url,
sep='\t',header=False,date_format='%Y-%m-%dT%H:%M:%S.%f')
XYZ_z1 = zipfile.ZipFile('%s\XYZ.zip'%saving_folder_url,mode='w')
XYZ_z1.write("%s\XYZ.tsv"%saving_folder_url)
XYZ_z1.close()
XYZ_files1 = {'file': open("%s\XYZ.zip"%saving_folder_url,'rb')}
XYZ_r1 = requests.put(hdll_url, files=XYZ_files1, headers = {'content-type':'application/zip',
'Content-Disposition': 'attachment;filename=XYZ.zip'
} )
P.S: I am only allowed to send files in .zip format.
Have a look at BytesIO():
memory_file = BytesIO()
with zipfile.ZipFile(memory_file, 'w') as zf:
zi = zipfile.ZipInfo('your_filename.txt')
zi.date_time = time.localtime(time.time())[:6]
zi.compress_type = zipfile.ZIP_DEFLATED
zf.writestr(zi, 'your file content goes here')
memory_file.seek(0)
Then hand it to the put method:
requests.put(<your_url>, data=memory_file, header=<your_headers>)
Or something along the lines of that. I didn't test the code, but the first part taken from a flask app, which uses send_file() to hand the file over to the client.