So, I'm still at the noob level when it comes to python. I know... I know... there's probably a more efficient way to do what I'm trying but still learning and hopefully, I'll get better with practice.
For a training project, I'm writing a script to do various DNS operations against a domain. I found DNSPython and it seemed to be exactly what I needed to use and I thought I was done with it but when I tried it against a different domain it keeps failing at the zone transfer.
I've got two domains hardcoded right now for testing. The megacorpone domain iw was working as I expected however, now it's failing (with no code change) in order to get it to work I had to filter the first record '#' that was returned otherwise it failed as well.
However, the zonetransfer.me domain sometimes completes the script with error but fails errors sporadically as well, but it never displays the host records for some reason and I've not been able to figure out how to fix it yet, been banging my head against it for a while now.
The megacoprone run was working every time earlier now it's not working at all. The only thing I can think of so far is that it may be a timing issue.
Run with megacoprpone
Attempting zone transfers for megacorpone.com
Traceback (most recent call last):
File "/home/kali/Exercises/Module_7/dns-axfer.py", line 56, in zoneXFR
zone = dns.zone.from_xfr(dns.query.xfr(str(server).rstrip('.'), domain))
File "/usr/lib/python3/dist-packages/dns/zone.py", line 1106, in from_xfr
for r in xfr:
File "/usr/lib/python3/dist-packages/dns/query.py", line 627, in xfr
raise TransferError(rcode)
dns.query.TransferError: Zone transfer error: REFUSED
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/kali/Exercises/Module_7/dns-axfer.py", line 73, in <module>
zoneXFR()
File "/home/kali/Exercises/Module_7/dns-axfer.py", line 66, in zoneXFR
print ("\nResults for",server, "\nZone origin:", str(zone.origin).rstrip('.'))
UnboundLocalError: local variable 'zone' referenced before assignment
Run 1 with zonetransfer.me
Attempting zone transfers for zonetransfer.me
Results for nsztm1.digi.ninja.
Zone origin: zonetransfer.me
---------------------------------------------------------------------------
Results for nsztm1.digi.ninja.
Zone origin: zonetransfer.me
---------------------------------------------------------------------------
[*] Error: <class 'dns.resolver.NoAnswer'> The DNS response does not contain an answer to the question: _acme-challenge.zonetransfer.me. IN A
Results for nsztm2.digi.ninja.
Zone origin: zonetransfer.me
---------------------------------------------------------------------------
Results for nsztm2.digi.ninja.
Zone origin: zonetransfer.me
---------------------------------------------------------------------------
[*] Error: <class 'dns.resolver.NoAnswer'> The DNS response does not contain an answer to the question: _acme-challenge.zonetransfer.me. IN A
Run 2 with no code change (zonetransfer.me)
Attempting zone transfers for zonetransfer.me
Traceback (most recent call last):
File "/home/kali/Exercises/Module_7/dns-axfer.py", line 56, in zoneXFR
zone = dns.zone.from_xfr(dns.query.xfr(str(server).rstrip('.'), domain))
File "/usr/lib/python3/dist-packages/dns/zone.py", line 1106, in from_xfr
for r in xfr:
File "/usr/lib/python3/dist-packages/dns/query.py", line 596, in xfr
_net_write(s, tcpmsg, expiration)
File "/usr/lib/python3/dist-packages/dns/query.py", line 364, in _net_write
current += sock.send(data[current:])
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/kali/Exercises/Module_7/dns-axfer.py", line 73, in <module>
zoneXFR()
File "/home/kali/Exercises/Module_7/dns-axfer.py", line 66, in zoneXFR
print ("\nResults for",server, "\nZone origin:", str(zone.origin).rstrip('.'))
UnboundLocalError: local variable 'zone' referenced before assignment
My script: bash away... I can always take constructive criticism.
#!/usr/bin/python3
import sys, argparse
import dns.query
import dns.zone
import dns.resolver
from colorama import Fore, Style
bracket = f"{Fore.BLUE}[{Fore.GREEN}*{Fore.BLUE}]{Style.RESET_ALL} "
bracket_err = f"{Fore.BLUE}[{Fore.RED}*{Fore.BLUE}]{Style.RESET_ALL} "
'''
parser = argparse.ArgumentParser()
parser.add_argument('domain')
args = parser.parse_args()
'''
# domain = (sys.argv[1])
domain = 'megacorpone.com'
#domain = 'zonetransfer.me'
def line():
print ('-' * 75)
return None
def resolveDNS(system):
resolver = dns.resolver.Resolver()
results = resolver.query(system , "A")
return results
def getNS ():
name_servers = dns.resolver.query(domain, 'NS')
print ("\nThe name servers for " + domain + " are:")
line()
for system in name_servers:
A_records = resolveDNS(str(system))
for item in A_records:
answer = ','.join([str(item)])
print (bracket, "{:30}".format(str(system).rstrip('.')), "{:15}".format(answer))
return name_servers
def getMX():
mail_server = dns.resolver.query(domain, 'MX')
print("\nMail servers for", domain)
line()
for system in mail_server:
A_records = resolveDNS(str(system.exchange))
for item in A_records:
answer = ','.join([str(item)])
print(bracket, "{:30}".format(str(system.exchange).rstrip('.')), "{:15}".format(str(answer)), '\t', "{:5}".format("Preference:"), str(system.preference))
return None
def zoneXFR():
print ("\nAttempting zone transfers for", domain,)
for server in name_servers:
try:
zone = dns.zone.from_xfr(dns.query.xfr(str(server).rstrip('.'), domain))
print ("\nResults for",server, "\nZone origin:", str(zone.origin).rstrip('.'))
line()
for host in zone:
if str(host) != '#':
A_records = resolveDNS(str(host) + "." + domain)
for item in A_records:
answer = ','.join([str(item)])
print(bracket, "{:30}".format(str(host) + "." + domain), answer)
except Exception as e:
print ("\nResults for",server, "\nZone origin:", str(zone.origin).rstrip('.'))
line()
print (bracket_err, f"{Fore.RED}Error:{Style.RESET_ALL}", e.__class__, e)
name_servers = getNS()
getMX()
zoneXFR()
print("\n")
I see that you are trying well-known name servers that are specifically set up for testing. However, for the benefit of other readers, I will add a couple explanations.
As you are probably aware, most name servers will not allow zone transfers nowadays. That being said, it is possible that each of the name servers for a given domain name will behave differently (they could have different configurations and even be running different software).
In the case of megacorpone.com there are 3 name servers listed:
ns2.megacorpone.com.
ns3.megacorpone.com.
ns1.megacorpone.com.
ns2.megacorpone.com is the only one that did allow a zone transfer.
This message
dns.query.TransferError: Zone transfer error: REFUSED
means what it means: your query was refused. Probably you talked to the wrong name server.
Then you have another error which suggest a variable scoping issue:
UnboundLocalError: local variable 'zone' referenced before assignment
You are calling functions in this order:
name_servers = getNS()
getMX()
zoneXFR()
If name_servers fails, then the subsequent call to zoneXFR will fail too. Because this code:
for server in name_servers:
will try to iterate over an empty list.
Intermittent DNS resolution failures are common so a few checks are required here. At the very least, make sure that the list of NS is not empty.
Another issue: you start a for loop outside of the try block so your control structure is broken right in the middle:
for server in name_servers:
try:
zone = dns.zone.from_xfr(dns.query.xfr(str(server).rstrip('.'), domain))
print ("\nResults for",server, "\nZone origin:", str(zone.origin).rstrip('.'))
line()
Do this instead:
try:
for server in name_servers:
zone = dns.zone.from_xfr(dns.query.xfr(str(server).rstrip('.'), domain))
print ("\nResults for",server, "\nZone origin:", str(zone.origin).rstrip('.'))
...
I suspect that your script fails intermittently because the list of name servers is not always returned in the same order. If the first NS returned is ns1.megacorpone.com. or ns3.megacorpone.com. then the code crashes. If the scripts starts with ns2.megacorpone.com (the sole NS allowing zone transfers) then it seems to work OK.
When this code fails (AXFR denied):
zone = dns.zone.from_xfr(dns.query.xfr(str(server).rstrip('.'), domain))
then zone is not defined, and that's why you cannot print it in your exception block. Instead, show the domain name or some other variables that you know are defined and valid.
So if the AXFR is denied, your script should handle this exception dns.query.TransferError, and quietly move on to the next NS if any, until the list has been exhausted.
Another bit of advice: you try to resolve resources names that are different than '#'. Instead, look at the record type. You should only resolve CNAME, MX or NS. The other common types are TXT, A, AAAA, SOA. The rest are more exotic such as NAPTR, LOC or SRV. Nothing that should be resolved I think.
Fixed up your code, doesn't look great yet, but it works
#!/usr/bin/python3
# you might want to run python3 -m pip install dnspython before running this script
import sys
import dns.query
import dns.zone
import dns.resolver
# formatting setup
from colorama import Fore, Style
bracket = f"{Fore.BLUE}[{Fore.GREEN}*{Fore.BLUE}]{Style.RESET_ALL} "
bracket_err = f"{Fore.BLUE}[{Fore.RED}*{Fore.BLUE}]{Style.RESET_ALL} "
def drawLine():
print ('-' * 75)
# read arguments
try:
domain = (sys.argv[1])
except:
print("[!] USAGE: python3 zt.py DOMAIN_NAME")
sys.exit(0)
# DNS functions
def resolveDNS(name):
resolver = dns.resolver.Resolver()
results = resolver.query(name , "A")
return results
def getNS (domain):
mapping = {}
name_servers = dns.resolver.query(domain, 'NS')
print ("\nThe name servers for " + domain + " are:")
drawLine()
for name_server in name_servers:
A_records = resolveDNS(str(name_server))
for item in A_records:
answer = ','.join([str(item)])
mapping[str(name_server)] = answer
print (bracket, "{:30}".format(str(name_server).rstrip('.')), "{:15}".format(answer))
return mapping
def zoneXFR(server):
try:
zone = dns.zone.from_xfr(dns.query.xfr(str(server).rstrip('.'), domain))
except Exception as e:
print (bracket_err, f"{Fore.RED}Error:{Style.RESET_ALL}", e.__class__, e)
else:
print ("\nResults for",server, "\nZone origin:", str(zone.origin).rstrip('.'))
drawLine()
for host in zone:
if str(host) != '#':
A_records = resolveDNS(str(host) + "." + domain)
for item in A_records:
answer = ','.join([str(item)])
print(bracket, "{:30}".format(str(host) + "." + domain), answer)
drawLine()
name_servers = getNS(domain)
for server in name_servers:
print ("\nAttempting zone transfers for", server,name_servers[server])
zoneXFR(name_servers[server])
I am pulling images from the Internet Archive as a test of some python code and I am incorporating the requests module. My code is as follows: (note, not the entire code, just the relevant section)
image_results = []
image_hashes = []
session = requests.Session()
for image in image_list:
if txtUrl not in image:
continue
try:
self.rslBox.AppendText("[v] Downloading %s" % image + "\n")
self.rslBox.Refresh()
response = session.get(image)
except:
self.rslBox.AppendText("[!] Failed to download: %s" % image + "\n")
self.rslBox.Refresh()
# continue
if "image" in response.headers['content-type']:
sha1 = hashlib.sha1(response.content).hexadigest()
if sha1 not in image_hashes:
image_hashes.append(sha1)
image_path = "WayBackImages/%s-%s" % (sha1.image.split("/")[-1])
with open(image_path, "wb") as fd:
fd.write(response.content)
self.rslBox.AppendText("[*] Saved %s" % images + "\n")
self.rslBox.Refresh()
info = pyexifinfo.get_json(image_path)
info[0]['ImageHash'] = sha1
image_results.append(info[0])
image_results_json = json.dumps(image_results)
data_frame = pandas.read_json(image_results_json)
csv = data_frame.to_csv('results.csv')
self.rslBox.AppendText("[*] Finished writing CSV to results.csv" + '\n')
self.rslBox.Refresh()
return
When I run my code, I get the following message:
Traceback (most recent call last):
File "C:\eclipse-workspace\test\tabbedPage.py", line 136, in OnSearch
if "image" in response.headers['content-type']:
NameError: name 'response' is not defined
But response is defined in the try statement - or so I would think. It only complains on the if "image" section - why??
I am new to python and I am using python3.6 and pydev with Eclipse.
Thanks!
Something inside your try failed. Your except caught it handle the error but since there is no raise in it, it continues execution, but response is not set.
It's because you are declaring response in the try block. If the exception gets thrown then response is not declared.
A work around for this would be putting the code that relies on response being declared into that try block.
I have a script that calls a subprocess (speedtest-cli).
The script seems to randomly fail with the following error message:-
ERROR: timed out
ERROR: 'speedtest-cli --share' failed (exit code 1).
Retrieving speedtest.net configuration... Cannot retrieve speedtest configuration
Traceback (most recent call last):
File "/home/steve/speedtest_dev.py", line 80, in <module>
data[1] = data[1].strip("'") ##Finish date and time
IndexError: list index out of range
As far as I can tell it looks like there are two errors in here:-
a) Speedtest-cli fails by timing out
ERROR: timed out
ERROR: 'speedtest-cli --share' failed (exit code 1).
and
b) The data strip then fails as one would expect because there is no data.
Traceback (most recent call last):
File "/home/steve/speedtest_dev.py", line 80, in <module>
data[1] = data[1].strip("'") ##Finish date and time
IndexError: list index out of range
I would like to catch the 1st error if possible and re-run the subprocess after an interval (60 seconds?).
I have tried creating a function:-
def run_speedtest():
outfile = open(dataFile, "w+")
subprocess.call(["/home/steve/speedtest-cli-extras/speedtest-csv", "--share"], stdout=outfile)
outfile.close()
and then using a try statement like:-
try:
run_speedtest()
except:
print("1st attempt failed") #for testing only
time.sleep(60)
run_speedtest()
For some reason I only manage to run the first part of this and when that errors out the except statement doesn't seem to run. The script then does this:-
#Separate Values from csv string
with open(dataFile, "r+") as f:
data = f.read()
data = data.strip()
data = data.replace("\t","|")
f.seek(0)
f.write(data)
f.truncate
f.close()
#Open file and process
with open(dataFile, "r") as g:
data = g.read()
data = data.split("|")
writes to a database and sends an email when the one of the parameters is less than a defined value.
It all works fine unless the initial run_speedtest() fails.
Any help would be appreciated.
I had the same issue after upgraded the recent python version and the solution can be:
export PYTHONHTTPSVERIFY=0
python your_script
It works for me though.
I am trying to learn how to use Python-click. I was not able to use a help parameter with one of my options so I finally gave up and changed the code to not include help for that option. However, despite closing and restarting Python and now rebooting my computer the error message associated with trying to use the help parameter is still appearing.
Code:
import click
def something():
pass
#click.command()
#click.argument('dest_dir',type=click.Path(exists=True, readable=True,
resolve_path=True, dir_okay=True),
help='Location of directory where results will be saved')
#click.option('--use_terms', is_flag=True,
help='Process strings based on terms or phrases')
#click.option('--use_query', is_flag=True, help='Process string based on
search query')
#click.option('--search_phrase', '-s', multiple=True)
def do_process(dest_dir,use_terms,use_query,*search_phrase):
""" testing setting parameters for snip tables"""
outref = open('e:\\myTemp\\testq.txt')
ms = dest_dir + '\n'
if use_terms:
ms += use_term + '\n'
else:
ms += use_query + '\n'
for each in search_phrase:
x = something()
ms += each + '\n'
outref.writelines(ms)
outref.close()
if __name__ == "__main__":
do_process()
Originally for the last #click.option I had
#click.option('--search_phrase', '-s', multiple=True, help='The search phrase to use')
I kept getting an error message that I could not solve relating to having an unknown parameter help. I ditched it, changed to what is above and now I am getting a similar error,
I then shut down Python, I closed my module and then restarted Python opened and ran my code again and still getting this error message
Traceback:
Traceback (most recent call last):
File "C:\Program Files\PYTHON\snipTables\test_snip_click.py", line 14, in <module>
#click.option('--search_phrase', '-s', multiple=True)
File "C:\Program Files\PYTHON\lib\site-packages\click\decorators.py", line 148, in decorator
_param_memo(f, ArgumentClass(param_decls, **attrs))
File "C:\Program Files\PYTHON\lib\site-packages\click\core.py", line 1618, in __init__
Parameter.__init__(self, param_decls, required=required, **attrs)
TypeError: __init__() got an unexpected keyword argument 'help'
So then I shut down Python Idle, I saved and closed my code and then restarted Python, reopened my code, but I am still getting the same traceback except notice that the traceback has the line of code I switched to after beating my head hard against the monitor and giving up
I am getting ready to reboot but am really curious as to the cause.
I rebooted and still am getting the same error
Renaming the file and running again did not change outcome - same traceback
The problem is that click does not accept a help string with an argument parameter. It is interesting behavior. The help string associated with the argument will be the string in the function that processes the argument and options.
The error message will always show up associated with the last option. So the correct code for this example would be
import click
def something():
pass
#click.command()
#click.argument('dest_dir',type=click.Path(exists=True, readable=True,
resolve_path=True, dir_okay=True)
##Notice no help string here
#click.option('--use_terms', is_flag=True,
help='Process strings based on terms or phrases')
#click.option('--use_query', is_flag=True, help='Process string based on
search query')
#click.option('--search_phrase', '-s', multiple=True)
def do_process(dest_dir,use_terms,use_query,*search_phrase):
""" testing setting parameters for snip tables"""
outref = open('e:\\myTemp\\testq.txt')
ms = dest_dir + '\n'
if use_terms:
ms += use_term + '\n'
else:
ms += use_query + '\n'
for each in search_phrase:
x = something()
ms += each + '\n'
outref.writelines(ms)
outref.close()
if __name__ == "__main__":
do_process()
This runs fine the problem I was originally having is that click was not doing a good job of explaining the source of the error. Above, even though I got rid of the help string in the option the click parser associates the help string from the argument with the last option it parses.
Maybe you renamed the source file and you are running an old version that was compiled with the previous name?
try deleting *.pyc files
The following code works almost perfect, thanks to the help received here:
import urllib.request
import zipfile
import subprocess
urls = ["http://url.com/archive1.zip", "http://url.com/archive2.zip", "http://url.com/archive3.zip"]
filename = "C:/test/test.zip"
destinationPath = "C:/test"
for url in urls:
try:
urllib.request.urlretrieve(url,filename)
sourceZip = zipfile.ZipFile(filename, 'r')
break
except ValueError:
pass
for name in sourceZip.namelist():
sourceZip.extract(name, destinationPath)
sourceZip.close()
subprocess.call(r'C:\WINDOWS\system32\cmd.exe /C "C:\test\test.exe"')
Except that when none of the url's successfully download, the final subprocess.call command never gets executed and I get the following error:
Traceback (most recent call last):
File "test.py", line 29, in <module>
for name in sourceZip.namelist():
NameError: name 'sourceZip' is not defined
I have been playing around a bit with the try: and except, but can not come to a working solution. I believe it is because the try: except command has to be inside the sourceZip loop, but since the sourceZip variable never gets declared, it fails. How would I alter this code to have the subprocess.call always get executed at the end regardless whether the download is successfull or not? (very new to Python)
Set sourceZip = None prior to the for url in urls line. Then test if sourceZip is None after the for loop to determine if you were able to successfully fetch the file. For example:
sourceZip = None
for url in urls:
try:
urllib.request.urlretrieve(url,filename)
sourceZip = zipfile.ZipFile(filename, 'r')
break
except ValueError:
pass
if sourceZip is not None:
for name in sourceZip.namelist():
sourceZip.extract(name, destinationPath)
sourceZip.close()
subprocess.call(r'C:\WINDOWS\system32\cmd.exe /C "C:\test\test.exe"')
Initialize sourceZip to None. Then check if it is not none do the extraction and closing.