Develop Reference

ConnectionResetError: [Errno 54] Connection reset by peer & Other Errno messages

I have a script that generates a CSV by calling an API lots of times that has successfully run in the past, however now I am getting Errno messages. Most commonly [Errno 54] Connection reset by peer, [Errno 60] Operation timed out, and sometimes [Errno 8] nodename nor servname provided, or not known.
This specific script calls an API around 16-24k times (I am 100% sure this is below the ratelimit) and when it ran successfully took around 2-3 hours to complete. For some reason, I can't seem to get it to run successfully without getting an error of some sort a while into the script running. So it will call an API route hundreds/thousands of time and then out of nowhere spit out one of the errors for calling the same exact one it's already been calling successfully. It seems strange. I've provided the error messages below as well.
I've removed some sensitive information such as the token and API routes as this is private data! I've left the errors in tact though.
import http.client
import json
import csv
class mgrData:
def __init__(self):
self.token = '[REMOVED]'
self.userDict = {}
def getPeakData(self, route):
conn = http.client.HTTPSConnection("[REMOVED]")
bod = ''
headers = {
'Authorization': 'Bearer '+ str(self.token),
'Content-Type': 'application/json'
}
conn.request("GET", route, bod, headers)
res = conn.getresponse()
data = res.read()
return json.loads(data)
def PeakExport(self):
print("Please wait, data generating...")
raw = self.getPeakData("[REMOVED]")
totalrecords = raw['meta']['page']['total']
resultspp = len(raw['data'])
if totalrecords % resultspp == 0:
numpages = int(totalrecords/resultspp)
elif totalrecords % resultspp != 0:
numpages = int(totalrecords//resultspp + 1)
mainDct = {}
for i in range(1,numpages+1):
route = "[REMOVED]"
dct = self.getPeakData(route)
data = dct['data']
for i in data:
pkEmpID = i['attributes']['identifier'].split('_')[0]
name = i['attributes']['name']
# segID = i['id']
hrisIDget = self.getPeakData("[REMOVED]")
hrisID = hrisIDget['urn:ietf:params:scim:schemas:extension:enterprise:2.0:User']['employeeNumber']
if pkEmpID in mainDct:
if i['attributes']['direct'] == True:
dSegID = i['id']
mainDct[pkEmpID].append({'dSegID' : dSegID})
if i['attributes']['direct'] == False:
aSegID = i['id']
mainDct[pkEmpID].append({'aSegID' : aSegID})
else:
mainDct.update({pkEmpID : []})
mainDct[pkEmpID].append({'name': name})
mainDct[pkEmpID].append({'EID' : hrisID})
if i['attributes']['direct'] == True:
dSegID = i['id']
mainDct[pkEmpID].append({'dSegID' : dSegID})
if i['attributes']['direct'] == False:
aSegID = i['id']
mainDct[pkEmpID].append({'aSegID' : aSegID})
urllist = []
for key in mainDct:
switch = False
for i in mainDct[key]:
if 'aSegID' in i:
switch = True
data = self.getPeakData("[REMOVED]" % (i['aSegID']))
if 'mean' in data['data']['attributes']['scores']:
score = data['data']['attributes']['scores']['mean']
participation = data['data']['attributes']['participation']['mean']
nps = data['data']['attributes']['scores']['nps']['score']
mainDct[key].append({'engagement all': score})
mainDct[key].append({'nps all': nps})
mainDct[key].append({'participation all' : participation})
elif 'anonymity' in data['data']['attributes']['scores']:
mainDct[key].append({'engagement all':'Anonymity'})
mainDct[key].append({'nps all': 'Anonymity'})
mainDct[key].append({'participation all': 'Anonymity'})
else:
mainDct[key].append({'engagement all': 'N/A'})
mainDct[key].append({'nps all':'N/A'})
mainDct[key].append({'participation all': 'N/A'})
if switch == False:
mainDct[key].append({'aSegID' : 'None'})
mainDct[key].append({'engagement all' :'N/A'})
mainDct[key].append({'nps all': 'N/A'})
mainDct[key].append({'participation all: N/A'})
for key in mainDct:
switch = False
for i in mainDct[key]:
if 'dSegID' in i:
switch = True
data = self.getPeakData("[REMOVED]" % (i['dSegID']))
if 'mean' in data['data']['attributes']['scores']:
score = data['data']['attributes']['scores']['mean']
participation = data['data']['attributes']['participation']['mean']
nps = data['data']['attributes']['scores']['nps']['score']
mainDct[key].append({'engagement direct': score})
mainDct[key].append({'nps direct': nps})
mainDct[key].append({'participation direct': participation})
elif 'anonymity' in data['data']['attributes']['scores']:
mainDct[key].append({'engagement direct': 'Anonymity'})
mainDct[key].append({'nps direct': 'Anonymity'})
mainDct[key].append({'participation direct': 'Anonymity'})
else:
mainDct[key].append({'engagement direct': 'N/A'})
mainDct[key].append({'nps direct': 'N/A'})
mainDct[key].append({'participation direct': 'N/A'})
if switch == False:
mainDct[key].append({'dSegID' : 'None'})
mainDct[key].append({'engagement direct': 'N/A'})
mainDct[key].append({'nps direct': 'N/A'})
mainDct[key].append({'participation direct': 'N/A'})
with open('Output.csv', 'w') as csv_file:
csvwriter = csv.writer(csv_file, delimiter=',')
csvwriter.writerow(['Manager ID', 'Manager Name','Direct Engagement', 'All Engagement', 'Direct NPS', 'All NPS', 'Direct Participation', 'All Participation'])
for key in mainDct:
for i in mainDct[key]:
if 'name' in i:
name = i['name']
if 'EID' in i:
EID = i['EID']
if 'nps direct' in i:
ND = i['nps direct']
if 'engagement direct' in i:
ED = i['engagement direct']
if 'participation direct' in i:
PD = i['participation direct']
if 'engagement all' in i:
EA = i['engagement all']
if 'nps all' in i:
NA = i['nps all']
if 'participation all' in i:
PA = i['participation all']
csvwriter.writerow([EID,name,ED,EA,ND,NA,PD,PA])
def main():
thisSync = mgrData()
thisSync.PeakExport()
main()
Some errors I receive:
Please wait, data generating...
Traceback (most recent call last):
File "/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py", line 209, in <module>
main()
File "/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py", line 204, in main
thisSync.PeakExport()
File "/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py", line 72, in PeakExport
hrisIDget = self.getPeakData("/scim/v2/users/employee_"+str(pkEmpID))
File "/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py", line 31, in getPeakData
res = conn.getresponse()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py", line 1332, in getresponse
response.begin()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py", line 303, in begin
version, status, reason = self._read_status()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py", line 264, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
ConnectionResetError: [Errno 54] Connection reset by peer
[Finished in 1263.8s with exit code 1]
[cmd: ['/Library/Frameworks/Python.framework/Versions/3.8/bin/python3', '-u', '/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py']]
[dir: /Volumes/GoogleDrive/My Drive/BI Scripts]
[path: /Library/Frameworks/Python.framework/Versions/3.8/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin]
Please wait, data generating...
Traceback (most recent call last):
File "/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py", line 209, in <module>
main()
File "/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py", line 204, in main
thisSync.PeakExport()
File "/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py", line 72, in PeakExport
hrisIDget = self.getPeakData("/scim/v2/users/employee_"+str(pkEmpID))
File "/Volumes/GoogleDrive/My Drive/BI Scripts/Random.py", line 31, in getPeakData
res = conn.getresponse()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py", line 1332, in getresponse
response.begin()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py", line 303, in begin
version, status, reason = self._read_status()
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/http/client.py", line 264, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
TimeoutError: [Errno 60] Operation timed out

Python giving an HTTP error 500 from Thingspeak intermiittantly

Here is my code
import os
import glob
import time
import sys
import datetime
import urllib2
baseURL = "https://api.thingspeak.com/update?api_key=DCL2IZ1REQT5GUSM"
os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')
base_dir = 'sys/bus/w1/devices/'
#Temp1 temperature device location
temp1_file = '/sys/bus/w1/devices/28-0000087787c4/w1_slave'
#Temp2 temperature Device Location
temp2_file = '/sys/bus/w1/devices/28-000008762fa3/w1_slave'
#Determine Temp1 Temperature
def read_rawtemp_temp1():
y = open(temp1_file,'r')
lines = y.readlines()
y.close
return lines
def read_temp_temp1():
lines = read_rawtemp_temp1()
while lines[0].strip()[-3:] !='YES':
time.sleep(0.2)
lines = read_rawtemp_temp1()
equals_pos = lines[1].find('t=')
if equals_pos !=-1:
temp_string = lines[1][equals_pos+2:]
temp_temp1 = ((float(temp_string)/1000.0)*1.8)-32
return temp_temp1
#Determine Temp2 Temperature
def read_rawtemp_temp2():
x = open(temp2_file,'r')
lines = x.readlines()
x.close
return lines
def read_temp_temp2():
lines = read_rawtemp_temp2()
while lines[0].strip()[-3:] !='YES':
time.sleep(0.2)
lines = read_rawtemp_temp1()
equals_pos = lines[1].find('t=')
if equals_pos !=-1:
temp_string = lines[1][equals_pos+2:]
temp_temp2 = float(temp_string)/1000.0
return temp_temp2
while True: #Loop
#Send Temp1 Temperature to Thingspeak
temp1_tempin = read_temp_temp1()
#Send temp2 temperature to Thingspeak
temp2_tempin = read_temp_temp2()
#Pull results together
values = [datetime.datetime.now(), temp1_tempin, temp2_tempin]
#Open Thingspeak channel and assign fields to temperatures
j = urllib2.urlopen(baseURL + "&field1=%s" % (temp1_tempin) + "&field2=%s" % (temp2_tempin))
try:
search_response = urllib2.urlopen(search_request)
except urllib2.HTTPError:
pass
#Time to next loop
time.sleep(600)
Here is the error that will stop the script from running after a few hours.
Traceback (most recent call last):
File "tempdatalog.py", line 137, in <module>
j = urllib2.urlopen(baseURL + "&field1=%s" % (temp1_tempin) + "&field2=%s" % (temp2_tempin))
File "/usr/lib/python2.7/urllib2.py", line 154, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python2.7/urllib2.py", line 437, in open
response = meth(req, response)
File "/usr/lib/python2.7/urllib2.py", line 550, in http_response
'http', request, response, code, msg, hdrs)
File "/usr/lib/python2.7/urllib2.py", line 475, in error
return self._call_chain(*args)
File "/usr/lib/python2.7/urllib2.py", line 409, in _call_chain
result = func(*args)
File "/usr/lib/python2.7/urllib2.py", line 558, in http_error_default
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
urllib2.HTTPError: HTTP Error 500: Internal Server Error
I added:
try:
search_response = urllib2.urlopen(search_request)
except urllib2.HTTPError:
pass
this to try to get it to pass over the error, but it didn't work. Any advice that would be given would be great, this is just the start to a monitor for a medical system in a mobile.

Looking at your stacktrace, it looks like you wrapped the wrong line in the try/except clause.
File "tempdatalog.py", line 137, in
j = urllib2.urlopen(baseURL + "&field1=%s" % (temp1_tempin) +
"&field2=%s" % (temp2_tempin))
This version at least catches the exception:
#Open Thingspeak channel and assign fields to temperatures
try:
j = urllib2.urlopen(baseURL + "&field1=%s" % (temp1_tempin) + "&field2=%s" % (temp2_tempin))
search_response = urllib2.urlopen(search_request)
except urllib2.HTTPError:
pass
That still does not explain why the error appears in the first place. My guess would be a form of rate limiting

Finding File size on a Website using python3

This is My Code >>
When I use this Code , I Am getting the correct output
url = "http://songs.djmazadownload.com/music/indian_movies/Creature%20%282014%29/01%20-%20Creature%20-%20Sawan%20Aaya%20Hai%20%5BDJMaza.Info%5D.mp3"
site = urllib.request.urlopen(url)
print ( ( round((site.length / 1024),2) / 1024) , "Mb")
But , When I use this code
url = "http://songs.djmazadownload.com/music/indian_movies/Creature (2014)/01 - Creature - Sawan Aaya Hai [DJMaza.Info].mp3"
site = urllib.request.urlopen(url)
print ( ( round((site.length / 1024),2) / 1024) , "Mb")
I am Getting some error .
Traceback (most recent call last):
File "C:\Python_Mass_downloader\New folder\download.py", line 35, in <module>
site = urllib.request.urlopen(url)
File "C:\Python34\lib\urllib\request.py", line 153, in urlopen
return opener.open(url, data, timeout)
File "C:\Python34\lib\urllib\request.py", line 461, in open
response = meth(req, response)
File "C:\Python34\lib\urllib\request.py", line 571, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Python34\lib\urllib\request.py", line 499, in error
return self._call_chain(*args)
File "C:\Python34\lib\urllib\request.py", line 433, in _call_chain
result = func(*args)
File "C:\Python34\lib\urllib\request.py", line 579, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 400: Bad Request
My Full Code is >>
def audios(link):
print("Enter the Minimum size of file to be downloaded\n")
size1 = input('>\t')
url = urllib.request.urlopen(link)
content = url.read()
soup = BeautifulSoup(content)
links = [a['href'] for a in soup.find_all('a',href=re.compile('http.*\.(mp3|wav|ogg|wma|flac)'))]
print (str(len(links)) + " Audios Found ")
print("\n".join(links))
#For Downloading
dest = "C:\\Downloads\\" # or '~/Downloads/' on linux
for i in range(len(links)):
a_link = links[i]
#a_link2 = urllib.request.urlopen(a_link)
site = urllib.request.urlopen(a_link)
size2 = int((site.length / 1024) / 1024)
if size 1 >= size2:
obj = SmartDL(a_link, dest)
obj.start()
path = obj.get_dest()
**So I need to Find the size of the file before downloading . But the links i got from the page is in this format >> http://songs.djmazadownload.com/music/indian_movies/Creature (2014)/01 - Creature - Sawan Aaya Hai [DJMaza.Info].mp3
How to get the links in other format ?
http://songs.djmazadownload.com/music/indian_movies/Creature%20%282014%29/01%20-%20Creature%20-%20Sawan%20Aaya%20Hai%20%5BDJMaza.Info%5D.mp3
**

use
from urllib.parse import quote_plus
....
url = quote_plus(url)
site = urllib.request.urlopen(url)
print ( ( round((site.length / 1024),2) / 1024) , "Mb")
...
before opening and tell e what you get.
you also have syntax error here:
if size 1 >= size2:
#should be:
if size1 >= size2:

add header in request SOAP message in python

I try to add header when sending SOAP request in Python.
The header in SOAP is:
> <SOAP-ENV:Header>
> <ns3:userCredentials
> xsi:type="https://4psa.com/HeaderData.xsd/2.0.0">
> <username>admin</username>
> <password>welcome</password>
> </ns3:userCredentials>
> </SOAP-ENV:Header>
I have used:
from suds.client import Client
from suds.xsd.doctor import ImportDoctor, Import
wsdl = 'https://192.168.1.15//soap2/schema/2.5.0/Report/Report.wsdl'
client = Client(wsdl)
and I don`t know how to add header to this code.
Please suggest how to add it.
And I tried:
> >>> from suds.client import Client
> >>> from suds.xsd.doctor import ImportDoctor, Import
> >>> imp = Import('http://schemas.xmlsoap.org/soap/encoding/')
> >>> url = 'https://192.168.1.15//soap2/schema/2.5.0/Report/Report.wsdl'
> >>> client = Client(url)
> >>> userid = 'admin'
> >>> passwd = '12345678#X'
> >>> client.set_options(soapheaders=(userid,passwd))
> >>> print client a get error when run:
>
> >>> client.service.CallCosts(1) Traceback (most recent call last):
> File "<stdin>", line 1, in <module>
> File
> "/usr/local/lib/python2.6/dist-packages/suds-0.4-py2.6.egg/suds/client.py",
> line 542, in __call__
> return client.invoke(args, kwargs) File
> "/usr/local/lib/python2.6/dist-packages/suds-0.4-py2.6.egg/suds/client.py",
> line 602, in invoke
> result = self.send(soapenv) File "/usr/local/lib/python2.6/dist-packages/suds-0.4-py2.6.egg/suds/client.py",
> line 637, in send
> reply = transport.send(request) File
> "/usr/local/lib/python2.6/dist-packages/suds-0.4-py2.6.egg/suds/transport/https.py",
> line 64, in send
> return HttpTransport.send(self, request) File
> "/usr/local/lib/python2.6/dist-packages/suds-0.4-py2.6.egg/suds/transport/http.py", line 77, in send
> fp = self.u2open(u2request) File "/usr/local/lib/python2.6/dist-packages/suds-0.4-py2.6.egg/suds/transport/http.py", line 118, in u2open
> return url.open(u2request, timeout=tm) File
> "/usr/lib/python2.6/urllib2.py", line
> 391, in open
> response = self._open(req, data) File "/usr/lib/python2.6/urllib2.py",
> line 409, in _open
> '_open', req) File "/usr/lib/python2.6/urllib2.py", line
> 369, in _call_chain
> result = func(*args) File "/usr/lib/python2.6/urllib2.py", line
> 1169, in https_open
> return self.do_open(httplib.HTTPSConnection,
> req) File
> "/usr/lib/python2.6/urllib2.py", line
> 1136, in do_open
> raise URLError(err) urllib2.URLError: <urlopen error
> [Errno 111] Connection refused>
Please suggest if you know what is the problem here.

I did something like this
def client_with_token(token):
header_ns = ('ns1', "http://4psa.com/HeaderData.xsd/3.5.0")
access_token = Element('accessToken', ns=header_ns).setText(token)
auth_header = Element('userCredentials', ns=header_ns)
auth_header.append(access_token)
client = Client(url)
auth_header = get_auth_header()
client.set_options(soapheaders=auth_header, *args, **kwargs)
return client

The suds client has many that may be used to control the behavior of the library. Some are general options and others are transport options. Although, the options objects are exposed, the preferred and supported way to set/unset options is through:
The Client constructor
The Client.set_options()
The Transport constructor(s).
soapheaders – Provides for soap headers.
https://fedorahosted.org/suds/wiki/Documentation#OPTIONS
http://jortel.fedorapeople.org/suds/doc/suds.options.Options-class.html
http://jortel.fedorapeople.org/suds/doc/suds.client.Client-class.html#set_options

How to debug socket error

1 upload_odl function
import os
import urllib2_files
import urllib2
user = 'patrick'
password = 'mypass'
url = 'http://localhost:8000/api/odl/'
password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_manager.add_password(
None, url, user, password
)
auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
def upload_odl(filename, sky_code=46800):
f = open(filename)
params = {
'installer_ac': str(sky_code),
'pdf': {
'fd': f,
'filename': os.path.basename(filename),
},
}
try:
the_page = urllib2.urlopen(url, params)
#response = urllib2.urlopen(req)
#the_page = response.read()
except urllib2.HTTPError, error:
print error.read()
and this second snippet that search and uploads some documents
import os
import time
from rest import upload_odl
start_time = time.time()
ODL_DIRECTORY = '/Users/patrick/Documents/ODL'
UPLOAD_DIR = 'CARICA'
directories = os.listdir(ODL_DIRECTORY)
files = 0
try:
for dir in directories:
c = os.path.join(ODL_DIRECTORY, dir)
if os.path.isdir(c):
u = os.path.join(c, UPLOAD_DIR)
if not os.path.exists(u):
continue
for x in os.listdir(u):
upload_odl(os.path.join(u, x))
#time.sleep(1)
files += 1
print 'Uploaded %d files in ' % len(to_upload),
print time.time() - start_time, " seconds"
except IOError, e:
#edit
import sys, import traceback
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
print e, os.path.join(u, x)
when I run the code, I get this error, after 5 uploads:
patrick:odl_uploader patrick$ python __init__.py
[Errno 32] Broken pipe /Users/patrick/Documents/ODL/name.pdf
EDIT:
Traceback output:
File "__init__.py", line 25, in <module>
upload_odl(os.path.join(u, x))
File "/Users/patrick/Documents/django/installer/installer_crm/tools/odl_uploader/rest/__init__.py", line 32, in upload_odl
the_page = urllib2.urlopen(url, params)
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 124, in urlopen
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 383, in open
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 401, in _open
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/urllib2.py", line 361, in _call_chain
File "/Users/patrick/Documents/django/installer/installer_crm/tools/rest/urllib2_files.py", line 207, in http_open
File "/Users/patrick/Documents/django/installer/installer_crm/tools/rest/urllib2_files.py", line 281, in do_open
File "/Users/patrick/Documents/django/installer/installer_crm/tools/rest/urllib2_files.py", line 194, in send_data
File "/System/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/httplib.py", line 719, in send
File "<string>", line 1, in sendall
[Errno 32] Brok
en pipe /Users/patrick/Documents/ODL/name.pdf