I am trying to work on a website and the code works fine, but sometimes the response text has a specific string error happened. If that string appears, I need to send request to that item again
Here's my try but I still got some results with that string error happened
for item in mylist:
while True:
response = requests.get(f'myurl/{item}', headers=headers)
res_text = response.text
if 'SUCESSFUL EXECUTION' in res_text:
scraped_item = (item, 'PAY IT')
else:
json_data=json.loads(res_text)
scraped_item = (item, json_data['errorMsg'])
print(scraped_item)
results.append(scraped_item)
if not 'error happened' in res_text:break
I could solve it using while True trick
for item in mylist:
while True:
response = requests.get(f'myurl/{item}', headers=headers)
res_text = response.text
if not 'error happened' in res_text:
break
if 'SUCESSFUL EXECUTION' in res_text:
scraped_item = (item, 'PAY IT')
else:
json_data=json.loads(res_text)
scraped_item = (item, json_data['errorMsg'])
print(scraped_item)
results.append(scraped_item)
time.sleep(1)
Related
I have a python program, which will make some api calls and print some output depending upon the api's response. I am using flush = true argument in the print function call. Sometimes the python program looks hung (It won't print anything to the terminal). But when i press CTRL-C, some output is printed to the output and python (looks to)resumes again. I have not written any signal handler for SIGINT. Why is this behavior?
UPDATE (Added python code)
import requests
import config
import json
from ipaddress import IPv4Address, IPv4Network
ip_list = []
filepath = "ip_address_test.txt"
with open(filepath) as fp:
line = fp.readline()
ip_list.append(line.rstrip())
while line:
line = fp.readline()
ip_list.append(line.rstrip())
print(ip_list)
del ip_list[-1]
sysparm_offset = 0
sysparm_limit = 2
flag = 1
for ip in ip_list:
hosts = []
while flag == 1:
print("ip is "+str(ip), flush=True)
# Set the request parameters
url = 'https://xxx.service-now.com/api/now/table/u_cmdb_ci_subnet?sysparm_query=u_lifecycle_status!=end of life^GOTOname>='+ip+'&sysparm_limit='+str(sysparm_limit)+'&sysparm_offset='+str(sysparm_offset)
# Eg. User name="username", Password="password" for this code sample.
user = config.asknow_username
pwd = config.asknow_password
headers = {"Accept":"application/json"}
# Do the HTTP request
response = requests.get(url, auth=(user, pwd), headers=headers,verify=False)
# Check for HTTP codes other than 200
if response.status_code != 200:
print('Status:', response.status_code, 'Headers:', response.headers, 'Error Response:', response.content)
exit()
# Decode the json response into a dictionary and use the data
result = json.loads(response.content)['result']
iter = 0
while iter < sysparm_limit:
print("iter = "+str(iter),flush=True)
print("checking "+result[iter]["name"], flush=True)
id = result[iter]["u_guid"]
url = "https://xxx.service-now.com/api/now/cmdb/instance/u_cmdb_ci_subnet/" + str(id)
response = requests.get(url, auth=(user, pwd), headers=headers,verify=False)
# Check for HTTP codes other than 200
if response.status_code != 200:
print('Status:', response.status_code, 'Headers:', response.headers, 'Error Response:', response.content)
exit()
result1 = json.loads(response.content)['result']
for relation in result1["outbound_relations"]:
if relation["type"]["display_value"] == "IP Connection::IP Connection":
if IPv4Address(relation["target"]["display_value"]) in IPv4Network(ip):
print("appending host", flush=True)
hosts.append(relation["target"]["display_value"])
else:
print("ip not in subnet", flush=True)
flag = 0
break
if flag == 0:
break
iter = iter+1
sysparm_offset = sysparm_offset + 2
with open('output.txt',"a+") as output_file:
for value in hosts:
output_file.write(str(value)+"\n")
print("completed "+ip,flush=True)
flag = 1
I am not sure if my issue is the same since you didn't mention clicking inside the command prompt and you didn't mention if it also resumes when pressing something besides ctrl-c, but it sounds a lot like what I had.
A little more googling led me to an insight I wish I had found as an answer to your question a while ago:
https://www.davici.nl/blog/the-mystery-of-hanging-batch-script-until-keypress
I had no idea a QuickEdit mode existed but besides following the steps in the article, you can also simply open the commandprompt, right-click the title bar, choose "defaults", and then under "Edit Options" disable "QuickEdit Mode".
I hope this helps you (or someone else) as much as it just helped me
I do a loop of requests and unfortunately sometimes there happens a server timeout. Thats why I want to check the status code first and if it is not 200 I want to go back and repeat the last request until the status code is 200. An example of the code looks like this:
for i in range(0, len(table)):
var = table.iloc[i]
url = 'http://example.request.com/var/'
response = requests.get(url)
if response.status_code == 200:
data = response.json()
else:
"go back to response"
I am appending the response data of every i, so I would like to go back as long as the response code is 200 and then go on with the next i in the loop.
Is there any easy solution?
I believe you want to do something like this:
for i in range(0, len(table)):
var = table.iloc[i]
url = 'http://example.request.com/var/'
response = requests.get(url)
while response.status_code != 200:
response = requests.get(url)
data = response.json()
I made a small example, used an infinite loop and used break to demonstrate when the status code is = 200
while True:
url = 'https://stackoverflow.com/'
response = requests.get(url)
if response.status_code == 200:
# found code
print('found exemple')
break
The code below produces these errors:
'Response' object has no attribute 'read'
Please help me understand what I did wrong. I made sure un and pwd are correct
user = "un"
password = 'pwd'
datanodes = ["https://server040:25000/"]
for i, datanode in enumerate(datanodes):
print("Checking {0}: {1}".format(i, datanode))
try:
print "trying"
response = requests.get(datanode + "queries?json",auth=(user,
password), verify='certs.pem')
print response
data = json.loads(response.read())
print data
if data["num_waiting_queries"] > 0:
print(data["num_waiting_queries"])
for in_flight_query in data["in_flight_queries"]:
if in_flight_query["waiting"] is True and
in_flight_query['state'] == "FINISHED":
cancel_url = datanode + "cancel_query?query_id=
{}".format(in_flight_query['query_id'])
print(cancel_url)
except IOError as ioe:
print ioe
except Exception as e:
print(e)
I have tried both json.load(reponse) and json.loads(response.read())
I was able to get around the issues by adding HTTPDigestAuth and changing data = json.loads(response.read()) to data = response.json()
A lot of requests now return page-tokens and I was curious what the most appropriate way of dealing with them was?
I have usually just gone with recursion:
def get_followers(user_id,
access_token,
cursor=''):
url = 'https://api.instagram.com/v1/users/%s/followed-by?access_token=%s&cursor=%s' % (my_user,
access_token,
cursor)
print(url)
response = requests.get(url)
json_out = json.loads(response.text)
for user in json_out['data']:
usrname = user['username']
my_followers.add_follower(usrname)
print("Added: %s" % usrname)
try:
cursor = json_out['pagination']['next_url']
get_followers(user_id, access_token, cursor)
except KeyError:
print("Finished with %d followers" % len(my_followers.followers))
return
However, perhaps a:
while True:
..
if condition:
break
Or some other implementation is seen as more efficient/pythonic?
You could change this to a generator function that yields the next follower when the structure is accessed, fetching more data only when necessary.
def get_followers(...)
...
while token is not None:
# fetch data
...
for user in json_out['data']:
yield user
...
# fetch new token
Then, iterate through this generator to apply your data handling. This also has the advantage of separating data acquisition and handling logic.
followers = get_followers(...)
for user in followers:
username = user['username']
my_followers.add_follower(username)
print("Added: %s" % username)
Thanks for the help, I have never used generators apart from the implicit (range vs xrange) distinction. I was curious if in this case it provides any speed/memory advantage?
My final code-snippet:
def get_followers(user_id,
access_token,
cursor=''):
""" Create a generator which will be accessed later to get the usernames of the followers"""
while cursor is not None:
# fetch data
url = 'https://api.instagram.com/v1/users/%s/followed-by?access_token=%s&cursor=%s' % (my_user,
access_token,
cursor)
print(url)
response = requests.get(url)
json_out = json.loads(response.text)
for user in json_out['data']:
yield user
# fetch new token
try:
cursor = json_out['pagination']['next_url']
except KeyError:
print("Finished with %d followers" % len(my_followers.followers))
cursor = None
followers_gen = get_followers(my_user, access_token) # Get followers
for usr in followers_gen:
usrname = usr['username']
my_followers.add_follower(usrname)
print("Added: %s" % usrname)
Compared to not using a generator:
def get_followers(user_id,
access_token,
cursor=''):
""" Create a generator which will be accessed later to get the usernames of the followers"""
while cursor is not None:
# fetch data
url = 'https://api.instagram.com/v1/users/%s/followed-by?access_token=%s&cursor=%s' % (my_user,
access_token,
cursor)
print(url)
response = requests.get(url)
json_out = json.loads(response.text)
for usr in json_out['data']:
usrname = usr['username']
my_followers.add_follower(usrname)
print("Added: %s" % usrname)
# fetch new token
try:
cursor = json_out['pagination']['next_url']
get_followers(user_id, access_token, cursor)
except KeyError:
print("Finished with %d followers" % len(my_followers.followers))
cursor = None
get_followers(my_user, access_token) # Get followers
i'm vey new to python and i wrote this code
def geturl(url):
url = urllib.quote(url,'/:')
log( __name__ ,"Getting url:[%s]" % url)
try:
req = urllib2.Request(url)
response = urllib2.urlopen(req)
content = response.read()
return_url = response.geturl()
response.close()
except Exception, e:
log( __name__ ,"Failed to get url because %s" % str(e))
content = None
return_url = None
return(content, return_url)
def myFunc():
searchurl="http://www.google.com"
socket.setdefaulttimeout(3)
content, response_url = geturl(searchurl)
if content is None:
log( __name__ , "Content is None!!")
content = content.replace("string1", "string2")
when i run it, i get this error: NoneType object is not callable.
i can't understand why... i learned that NoneType is the type of None, but i check if "content" variable is None and it's not.
thx for your help
You're not giving the erroneous line, however I suspect the following to be wrong:
if content is None:
log( __name__ , "Content is None!!")
content = content.replace("string1", "string2")
When content is indeed None, a log is emitted but the function keeps executing, until the call to replace raises an exception.