This script works perfectly on Mac OS and Linux but when I try it on Windows it does not work.
When I try to try it on Windows, this error appears...
I imagine the problem from switch_user(dev).
I tried it with her (dev) but I couldn't solve it.
Whereas when Check GPT is verified, it stops and the error occurs.
PS C:\Users\motc-pc\Desktop\amonet-karnak-\amonet\modules> python main.py
[2020-06-03 22:03:49.653199] Waiting for bootrom
[2020-06-03 22:03:59.339064] Found port = COM22
[2020-06-03 22:03:59.348800] Handshake
* * * If you have a short attached, remove it now * * *
* * * Press Enter to continue * * *
[2020-06-03 22:04:02.368897] Init crypto engine
[2020-06-03 22:04:02.422396] Disable caches
[2020-06-03 22:04:02.424535] Disable bootrom range checks
[2020-06-03 22:04:02.459386] Load payload from ../brom-payload/build/payload.bin = 0x4888 bytes
[2020-06-03 22:04:02.469524] Send payload
[2020-06-03 22:04:03.440416] Let's rock
[2020-06-03 22:04:03.442368] Wait for the payload to come online...
[2020-06-03 22:04:04.163004] all good
[2020-06-03 22:04:04.165239] Check GPT
Traceback (most recent call last):
File "main.py", line 450, in <module>
main()
File "main.py", line 361, in main
switch_user(dev)
File "main.py", line 321, in switch_user
block = dev.emmc_read(0)
File "main.py", line 196, in emmc_read
raise RuntimeError("read fail")
RuntimeError: read fail
This script is full.
As I said earlier when I trying it on Linux or MacOS it works normally without problems
import struct
import os
import sys
import time
from handshake import handshake
from load_payload import load_payload, UserInputThread
from logger import log
import struct
import glob
import serial
from logger import log
BAUD = 115200
TIMEOUT = 5
CRYPTO_BASE = 0x10210000 # for karnak
def serial_ports ():
""" Lists available serial ports
:raises EnvironmentError:
On unsupported or unknown platforms
:returns:
A set containing the serial ports available on the system
"""
if sys.platform.startswith("win"):
ports = [ "COM{0:d}".format(i + 1) for i in range(256) ]
elif sys.platform.startswith("linux"):
ports = glob.glob("/dev/ttyACM*")
elif sys.platform.startswith("darwin"):
ports = glob.glob("/dev/cu.usbmodem*")
else:
raise EnvironmentError("Unsupported platform")
result = set()
for port in ports:
try:
s = serial.Serial(port, timeout=TIMEOUT)
s.close()
result.add(port)
except (OSError, serial.SerialException):
pass
return result
def p32_be(x):
return struct.pack(">I", x)
class Device:
def __init__(self, port=None):
self.dev = None
if port:
self.dev = serial.Serial(port, BAUD, timeout=TIMEOUT)
def find_device(self,preloader=False):
if self.dev:
raise RuntimeError("Device already found")
if preloader:
log("Waiting for preloader")
else:
log("Waiting for bootrom")
old = serial_ports()
while True:
new = serial_ports()
# port added
if new > old:
port = (new - old).pop()
break
# port removed
elif old > new:
old = new
time.sleep(0.25)
log("Found port = {}".format(port))
self.dev = serial.Serial(port, BAUD, timeout=TIMEOUT)
def check(self, test, gold):
if test != gold:
raise RuntimeError("ERROR: Serial protocol mismatch")
def check_int(self, test, gold):
test = struct.unpack('>I', test)[0]
self.check(test, gold)
def _writeb(self, out_str):
self.dev.write(out_str)
return self.dev.read()
def handshake(self):
# look for start byte
while True:
c = self._writeb(b'\xa0')
if c == b'\x5f':
break
self.dev.flushInput()
# complete sequence
self.check(self._writeb(b'\x0a'), b'\xf5')
self.check(self._writeb(b'\x50'), b'\xaf')
self.check(self._writeb(b'\x05'), b'\xfa')
def handshake2(self, cmd='FACTFACT'):
# look for start byte
c = 0
while c != b'Y':
c = self.dev.read()
log("Preloader ready, sending " + cmd)
command = str.encode(cmd)
self.dev.write(command)
self.dev.flushInput()
def read32(self, addr, size=1):
result = []
self.dev.write(b'\xd1')
self.check(self.dev.read(1), b'\xd1') # echo cmd
self.dev.write(struct.pack('>I', addr))
self.check_int(self.dev.read(4), addr) # echo addr
self.dev.write(struct.pack('>I', size))
self.check_int(self.dev.read(4), size) # echo size
self.check(self.dev.read(2), b'\x00\x00') # arg check
for _ in range(size):
data = struct.unpack('>I', self.dev.read(4))[0]
result.append(data)
self.check(self.dev.read(2), b'\x00\x00') # status
# support scalar
if len(result) == 1:
return result[0]
else:
return result
def write32(self, addr, words, status_check=True):
# support scalar
if not isinstance(words, list):
words = [ words ]
self.dev.write(b'\xd4')
self.check(self.dev.read(1), b'\xd4') # echo cmd
self.dev.write(struct.pack('>I', addr))
self.check_int(self.dev.read(4), addr) # echo addr
self.dev.write(struct.pack('>I', len(words)))
self.check_int(self.dev.read(4), len(words)) # echo size
self.check(self.dev.read(2), b'\x00\x01') # arg check
for word in words:
self.dev.write(struct.pack('>I', word))
self.check_int(self.dev.read(4), word) # echo word
if status_check:
self.check(self.dev.read(2), b'\x00\x01') # status
def run_ext_cmd(self, cmd):
self.dev.write(b'\xC8')
self.check(self.dev.read(1), b'\xC8') # echo cmd
cmd = bytes([cmd])
self.dev.write(cmd)
self.check(self.dev.read(1), cmd)
self.dev.read(1)
self.dev.read(2)
def wait_payload(self):
data = self.dev.read(4)
if data != b"\xB1\xB2\xB3\xB4":
raise RuntimeError("received {} instead of expected pattern".format(data))
def emmc_read(self, idx):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x1000))
# block to read
self.dev.write(p32_be(idx))
data = self.dev.read(0x200)
if len(data) != 0x200:
raise RuntimeError("read fail")
return data
def emmc_write(self, idx, data):
if len(data) != 0x200:
raise RuntimeError("data must be 0x200 bytes")
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x1001))
# block to write
self.dev.write(p32_be(idx))
# data
self.dev.write(data)
code = self.dev.read(4)
if code != b"\xd0\xd0\xd0\xd0":
raise RuntimeError("device failure")
def emmc_switch(self, part):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x1002))
# partition
self.dev.write(p32_be(part))
def reboot(self):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x3000))
def kick_watchdog(self):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x3001))
def rpmb_read(self):
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x2000))
data = self.dev.read(0x100)
if len(data) != 0x100:
raise RuntimeError("read fail")
return data
def rpmb_write(self, data):
if len(data) != 0x100:
raise RuntimeError("data must be 0x100 bytes")
# magic
self.dev.write(p32_be(0xf00dd00d))
# cmd
self.dev.write(p32_be(0x2001))
# data
self.dev.write(data)
def switch_boot0(dev):
dev.emmc_switch(1)
block = dev.emmc_read(0)
if block[0:9] != b"EMMC_BOOT" and block != b"\x00" * 0x200:
dev.reboot()
raise RuntimeError("what's wrong with your BOOT0?")
dev.kick_watchdog()
def flash_data(dev, data, start_block, max_size=0):
while len(data) % 0x200 != 0:
data += b"\x00"
if max_size and len(data) > max_size:
raise RuntimeError("data too big to flash")
blocks = len(data) // 0x200
for x in range(blocks):
print("[{} / {}]".format(x + 1, blocks), end='\r')
dev.emmc_write(start_block + x, data[x * 0x200:(x + 1) * 0x200])
if x % 10 == 0:
dev.kick_watchdog()
print("")
def flash_binary(dev, path, start_block, max_size=0):
with open(path, "rb") as fin:
data = fin.read()
while len(data) % 0x200 != 0:
data += b"\x00"
flash_data(dev, data, start_block, max_size=0)
def dump_binary(dev, path, start_block, max_size=0):
with open(path, "w+b") as fout:
blocks = max_size // 0x200
for x in range(blocks):
print("[{} / {}]".format(x + 1, blocks), end='\r')
fout.write(dev.emmc_read(start_block + x))
if x % 10 == 0:
dev.kick_watchdog()
print("")
def force_fastboot(dev, gpt):
switch_user(dev)
block = list(dev.emmc_read(gpt["MISC"][0]))
block[0:16] = "FASTBOOT_PLEASE\x00".encode("utf-8")
dev.emmc_write(gpt["MISC"][0], bytes(block))
block = dev.emmc_read(gpt["MISC"][0])
def force_recovery(dev, gpt):
switch_user(dev)
block = list(dev.emmc_read(gpt["MISC"][0]))
block[0:16] = "boot-recovery\x00\x00\x00".encode("utf-8")
dev.emmc_write(gpt["MISC"][0], bytes(block))
block = dev.emmc_read(gpt["MISC"][0])
def switch_user(dev):
dev.emmc_switch(0)
block = dev.emmc_read(0)
dev.kick_watchdog()
def parse_gpt(dev):
data = dev.emmc_read(0x400 // 0x200) + dev.emmc_read(0x600 // 0x200) + dev.emmc_read(0x800 // 0x200) + dev.emmc_read(0xA00 // 0x200)
num = len(data) // 0x80
parts = dict()
for x in range(num):
part = data[x * 0x80:(x + 1) * 0x80]
part_name = part[0x38:].decode("utf-16le").rstrip("\x00")
part_start = struct.unpack("<Q", part[0x20:0x28])[0]
part_end = struct.unpack("<Q", part[0x28:0x30])[0]
parts[part_name] = (part_start, part_end - part_start + 1)
return parts
def main():
minimal = False
dev = Device()
dev.find_device()
# 0.1) Handshake
handshake(dev)
# 0.2) Load brom payload
load_payload(dev, "../brom-payload/build/payload.bin")
dev.kick_watchdog()
if len(sys.argv) == 2 and sys.argv[1] == "minimal":
thread = UserInputThread(msg = "Running in minimal mode, assuming LK, TZ, LK-payload and TWRP to have already been flashed.\nIf this is correct (i.e. you used \"brick\" option in step 1) press enter, otherwise terminate with Ctrl+C")
thread.start()
while not thread.done:
dev.kick_watchdog()
time.sleep(1)
minimal = True
# 1) Sanity check GPT
log("Check GPT")
switch_user(dev)
# 1.1) Parse gpt
gpt = parse_gpt(dev)
log("gpt_parsed = {}".format(gpt))
if "lk" not in gpt or "tee1" not in gpt or "boot" not in gpt or "recovery" not in gpt:
raise RuntimeError("bad gpt")
# 2) Sanity check boot0
log("Check boot0")
switch_boot0(dev)
# 3) Sanity check rpmb
log("Check rpmb")
rpmb = dev.rpmb_read()
if rpmb[0:4] != b"AMZN":
thread = UserInputThread(msg = "rpmb looks broken; if this is expected (i.e. you're retrying the exploit) press enter, otherwise terminate with Ctrl+C")
thread.start()
while not thread.done:
dev.kick_watchdog()
time.sleep(1)
# Clear preloader so, we get into bootrom without shorting, should the script stall (we flash preloader as last step)
# 4) Downgrade preloader
log("Clear preloader header")
switch_boot0(dev)
flash_data(dev, b"EMMC_BOOT" + b"\x00" * ((0x200 * 8) - 9), 0)
# 5) Zero out rpmb to enable downgrade
log("Downgrade rpmb")
dev.rpmb_write(b"\x00" * 0x100)
log("Recheck rpmb")
rpmb = dev.rpmb_read()
if rpmb != b"\x00" * 0x100:
dev.reboot()
raise RuntimeError("downgrade failure, giving up")
log("rpmb downgrade ok")
dev.kick_watchdog()
if not minimal:
# 6) Install preloader
log("Flash preloader")
switch_boot0(dev)
flash_binary(dev, "../bin/preloader.bin", 8)
flash_binary(dev, "../bin/preloader.bin", 520)
# 6) Install lk-payload
log("Flash lk-payload")
switch_boot0(dev)
flash_binary(dev, "../lk-payload/build/payload.bin", 1024)
# 7) Downgrade tz
log("Flash tz")
switch_user(dev)
flash_binary(dev, "../bin/tz.img", gpt["tee1"][0], gpt["tee1"][1] * 0x200)
# 8) Downgrade lk
log("Flash lk")
switch_user(dev)
flash_binary(dev, "../bin/lk.bin", gpt["lk"][0], gpt["lk"][1] * 0x200)
# 9) Flash microloader
log("Inject microloader")
switch_user(dev)
boot_hdr1 = dev.emmc_read(gpt["boot"][0]) + dev.emmc_read(gpt["boot"][0] + 1)
boot_hdr2 = dev.emmc_read(gpt["boot"][0] + 2) + dev.emmc_read(gpt["boot"][0] + 3)
flash_binary(dev, "../bin/microloader.bin", gpt["boot"][0], 2 * 0x200)
if boot_hdr2[0:8] != b"ANDROID!":
flash_data(dev, boot_hdr1, gpt["boot"][0] + 2, 2 * 0x200)
if not minimal:
log("Force fastboot")
force_fastboot(dev, gpt)
else:
log("Force recovery")
force_recovery(dev, gpt)
# 10) Downgrade preloader
log("Flash preloader header")
switch_boot0(dev)
flash_binary(dev, "../bin/preloader.hdr0", 0, 4)
flash_binary(dev, "../bin/preloader.hdr1", 4, 4)
# Reboot (to fastboot or recovery)
log("Reboot")
dev.reboot()
if __name__ == "__main__":
main()
Related
i keep having a problem in my code coming back as Test Failed: unsupported operand type(s) for +: 'int' and 'tuple'.
i am a super beginner who is not very good at coding, so i cannot figure out what the issue is.
here is the full code.
i am making a simple icmp pinger program.
thanks everyone for your help!!!
(this is my first question here so please let me know if i need to edit anything)
from socket import *
import os
import sys
import struct
import time
import select
import statistics
import binascii
# Should use stdev
ICMP_ECHO_REQUEST = 8
def checksum(string):
csum = 0
countTo = (len(string) // 2) * 2
count = 0
while count < countTo:
thisVal = (string[count + 1]) * 256 + (string[count])
csum += thisVal
csum &= 0xffffffff
count += 2
if countTo < len(string):
csum += (string[len(string) - 1])
csum &= 0xffffffff
csum = (csum >> 16) + (csum & 0xffff)
csum = csum + (csum >> 16)
answer = ~csum
answer = answer & 0xffff
answer = answer >> 8 | (answer << 8 & 0xff00)
return answer
def receiveOnePing(mySocket, ID, timeout, destAddr):
timeLeft = timeout
while 1:
startedSelect = time.time()
whatReady = select.select([mySocket], [], [], timeLeft)
howLongInSelect = (time.time() - startedSelect)
if whatReady[0] == []: # Timeout
return "Request timed out."
timeReceived = time.time()
recPacket, addr = mySocket.recvfrom(1024)
# Fetch the ICMP header from the IP packet
header = recPacket[20:28]
type, code, checksum, packID, seqNo = struct.unpack("bbHHh", header)
if type == 0 and packID == ID:
bytesInDouble = struct.calcsize("d")
timeSent = struct.unpack("d", recPacket[28:28 + bytesInDouble])[0]
ttls = struct.unpack("c", recPacket[8:9])[0]
rtt = timeReceived - timeSent
return (rtt, ttls)
timeLeft = timeLeft - howLongInSelect
if timeLeft <= 0:
return "Request timed out."
def sendOnePing(mySocket, destAddr, ID):
# Header is type (8), code (8), checksum (16), id (16), sequence (16)
myChecksum = 0
# Make a dummy header with a 0 checksum
# struct -- Interpret strings as packed binary data
header = struct.pack("bbHHh", ICMP_ECHO_REQUEST, 0, myChecksum, ID, 1)
data = struct.pack("d", time.time())
# Calculate the checksum on the data and the dummy header.
myChecksum = checksum(header + data)
# Get the right checksum, and put in the header
if sys.platform == 'darwin':
# Convert 16-bit integers from host to network byte order
myChecksum = htons(myChecksum) & 0xffff
else:
myChecksum = htons(myChecksum)
header = struct.pack("bbHHh", ICMP_ECHO_REQUEST, 0, myChecksum, ID, 1)
packet = header + data
mySocket.sendto(packet, (destAddr, 1)) # AF_INET address must be tuple, not str
# Both LISTS and TUPLES consist of a number of objects
# which can be referenced by their position number within the object.
def doOnePing(destAddr, timeout):
icmp = getprotobyname("icmp")
# SOCK_RAW is a powerful socket type. For more details: http://sockraw.org/papers/sock_raw
mySocket = socket(AF_INET, SOCK_RAW, icmp)
myID = os.getpid() & 0xFFFF # Return the current process i
sendOnePing(mySocket, destAddr, myID)
delay = receiveOnePing(mySocket, myID, timeout, destAddr)
mySocket.close()
return delay
def ping(host, timeout=1):
# timeout=1 means: If one second goes by without a reply from the server, # the client assumes that either the client's ping or the server's pong is lost
dest = gethostbyname(host)
# print("Pinging " + dest + " using Python:")
# print("")
# Calculate vars values and return them
count = 0
val = []
# Send ping requests to a server separated by approximately one second
for i in range(0,4):
delay = doOnePing(dest, timeout)
val.append(delay)
# print(delay)
time.sleep(1) # one second
if len(val) > 0:
packet_min = min(val) * 1000
packet_avg = sum(val) / len(val) * 1000
packet_max = max(val) * 1000
stdev_var = list(val) * 1000
vars = [str(round(packet_min, 2)), str(round(packet_avg, 2)), str(round(packet_max, 2)),str(round(stdev(stdev_var), 2))]
else:
vars = ['0', '0.0', '0', '0.0']
return vars
if __name__ == '__main__':
ping("google.co.il")
You have return (rtt, ttls) in function receiveOnePing and then you return the same tuple from function doOnePing. After that, you append this tuple to list and are trying to sum this list of tuples. This leads to the error you mentioned.
You need val.append(delay[0]) in ping function (line 122).
You also use undefined function stdev. Should be statistics.stdev.
Please note that your script will crash in case of timeout because you return a string in this.
Also the code is runnable only by root.
UPD
Below is fixed code.
rom socket import *
import os
import sys
import struct
import time
import select
import statistics
import binascii
# Should use stdev
ICMP_ECHO_REQUEST = 8
def checksum(string):
csum = 0
countTo = (len(string) // 2) * 2
count = 0
while count < countTo:
thisVal = (string[count + 1]) * 256 + (string[count])
csum += thisVal
csum &= 0xffffffff
count += 2
if countTo < len(string):
csum += (string[len(string) - 1])
csum &= 0xffffffff
csum = (csum >> 16) + (csum & 0xffff)
csum = csum + (csum >> 16)
answer = ~csum
answer = answer & 0xffff
answer = answer >> 8 | (answer << 8 & 0xff00)
return answer
def receiveOnePing(mySocket, ID, timeout, destAddr):
timeLeft = timeout
while 1:
startedSelect = time.time()
whatReady = select.select([mySocket], [], [], timeLeft)
howLongInSelect = (time.time() - startedSelect)
if whatReady[0] == []: # Timeout
return "Request timed out."
timeReceived = time.time()
recPacket, addr = mySocket.recvfrom(1024)
# Fetch the ICMP header from the IP packet
header = recPacket[20:28]
type, code, checksum, packID, seqNo = struct.unpack("bbHHh", header)
if type == 0 and packID == ID:
bytesInDouble = struct.calcsize("d")
timeSent = struct.unpack("d", recPacket[28:28 + bytesInDouble])[0]
ttls = struct.unpack("c", recPacket[8:9])[0]
rtt = timeReceived - timeSent
return (rtt, ttls)
timeLeft = timeLeft - howLongInSelect
if timeLeft <= 0:
return "Request timed out."
def sendOnePing(mySocket, destAddr, ID):
# Header is type (8), code (8), checksum (16), id (16), sequence (16)
myChecksum = 0
# Make a dummy header with a 0 checksum
# struct -- Interpret strings as packed binary data
header = struct.pack("bbHHh", ICMP_ECHO_REQUEST, 0, myChecksum, ID, 1)
data = struct.pack("d", time.time())
# Calculate the checksum on the data and the dummy header.
myChecksum = checksum(header + data)
# Get the right checksum, and put in the header
if sys.platform == 'darwin':
# Convert 16-bit integers from host to network byte order
myChecksum = htons(myChecksum) & 0xffff
else:
myChecksum = htons(myChecksum)
header = struct.pack("bbHHh", ICMP_ECHO_REQUEST, 0, myChecksum, ID, 1)
packet = header + data
mySocket.sendto(packet, (destAddr, 1)) # AF_INET address must be tuple, not str
# Both LISTS and TUPLES consist of a number of objects
# which can be referenced by their position number within the object.
def doOnePing(destAddr, timeout):
icmp = getprotobyname("icmp")
# SOCK_RAW is a powerful socket type. For more details: http://sockraw.org/papers/sock_raw
mySocket = socket(AF_INET, SOCK_RAW, icmp)
myID = os.getpid() & 0xFFFF # Return the current process i
sendOnePing(mySocket, destAddr, myID)
delay = receiveOnePing(mySocket, myID, timeout, destAddr)
mySocket.close()
return delay
def ping(host, timeout=1):
# timeout=1 means: If one second goes by without a reply from the server, # the client assumes that either the client's ping or the server's pong is lost
dest = gethostbyname(host)
# print("Pinging " + dest + " using Python:")
# print("")
# Calculate vars values and return them
count = 0
val = []
# Send ping requests to a server separated by approximately one second
for i in range(0,4):
delay = doOnePing(dest, timeout)
val.append(delay[0])
# print(delay)
time.sleep(1) # one second
print(val)
if len(val) > 0:
packet_min = min(val) * 1000
packet_avg = sum(val) / len(val) * 1000
packet_max = max(val) * 1000
stdev_var = list(val) * 1000
vars = [str(round(packet_min, 2)), str(round(packet_avg, 2)), str(round(packet_max, 2)),str(round(statistics.stdev(stdev_var), 2))]
else:
vars = ['0', '0.0', '0', '0.0']
return vars
if __name__ == '__main__':
ping("google.co.il")
>sudo python3 ping.py
[0.0778355598449707, 0.07866811752319336, 0.07798004150390625, 0.07628297805786133]
The error occurs in this line:
packet_avg = sum(val) / len(val) * 1000
Therefore val is suspect. Follow the logic back through doOnePing to receiveOnePing and you will find that function does not return suitable types.
You have a few issues. What I can see:
you are trying to checksum but in python, using an index returns a string, not a character, so this generates a string, not a number:
thisVal = (string[count + 1]) * 256 + (string[count])
What you probably want in this case is:
thisVal = ord(string[count + 1]) * 256 + ord(string[count])
and also on this line:
csum += (string[len(string) - 1])
to
csum += ord(string[len(string) - 1])
Then, you are putting tuples and possibly strings into your val array.
You need to decide how you want to handle the errors/time out of the ping.
You could just ignore them for now:
for i in range(0,4):
delay = doOnePing(dest, timeout)
if isinstance(delay, tuple):
val.append(delay[0])
time.sleep(1) # one second
That will only add if you received a tuple, and only add the first member of the tuple, which appears to be the delay you want.
I have written a Python script that downloads a single file using 32 connections if available.
I have written a multiconnection downloader that works fine without pausing, but won't stop downloading after resuming, the progress would go beyond 100%...
Like this:
Download mode: Multi-thread (press Space to pause/resume, press Escape to stop)
[████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 120% completed, paused: False
Download mode: Multi-thread (press Space to pause/resume, press Escape to stop)
1798.08 MiB downloaded, 1489.83 MiB total, -308.25 MiB remaining, download speed: 22.73 MiB/s
Minimum speed: 0.00 MiB/s, average speed: 4.54 MiB/s, maximum speed: 75.00 MiB/s
Task started on 2021-08-09 16:57:03, 00:06:35 elapsed, ETA: -1:59:47
After progress exceeds 100%, there will be error messages like this:
Exception in thread Thread-78:
Traceback (most recent call last):
File "C:\Program Files\Python39\lib\threading.py", line 973, in _bootstrap_inner
self.run()
File "C:\Program Files\Python39\lib\threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "D:\MyScript\downloader.py", line 70, in multidown
mm[position: position+len(chunk)] = chunk
IndexError: mmap slice assignment is wrong size
(The above doesn't include all of the error message)
I have encountered all sorts of errors after resuming, but most importantly, the server will often send extra bytes from previous request, whose connection is dead and needless to say this breaks the whole code.
How should I implement pause and resume correctly?
I am thinking about multiprocessing, I assume the sessions and connections are all PID and port number related, and so far I haven't encountered a new run of the script that received extra bytes from previous runs of the script, so I guess using another process with a new PID and new port number plus requests.session() plus {'connection': 'close'} for each download should guarantee that no extra bytes from previous connections will be received, I just don't know how to share variables between processes...
The code:
downloader.py
import json
import keyboard
import os
import re
import requests
import sys
import time
import validators
from collections import deque
from datetime import datetime, timedelta
from math import inf
from mmap import mmap
from pathlib import Path
from ping3 import ping
from reprint import output
from threading import Thread
def timestring(sec):
sec = int(sec)
m, s = divmod(sec, 60)
h, m = divmod(m, 60)
return f'{h:02d}:{m:02d}:{s:02d}'
class Downloader:
def __init__(self):
self.recent = deque([0] * 12, maxlen=12)
self.recentspeeds = deque([0] * 200, maxlen=200)
self.paused = False
self.progress = dict()
class Multidown:
def __init__(self, obj, id):
self.count = 0
self.position = 0
self.completed = False
self.id = id
self.parent = obj
def multidown(self, url, start, end):
interrupted = False
s = requests.session()
s.headers.update({'connection': 'close', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0'})
r = s.get(
url, headers={'range': 'bytes={0}-{1}'.format(start, end)}, stream=True)
length = int(r.headers['content-length'])
while end - length + (self.id != self.parent.progress['connections'] - 1) != start or r.status_code != 206:
r.close()
s.close()
del r
del s
time.sleep(0.02)
s = requests.session()
r = s.get(
url, headers={'range': 'bytes={0}-{1}'.format(start, end)}, stream=True)
length = int(r.headers['content-length'])
self.position = start
for chunk in r.iter_content(1048576):
if self.parent.paused:
self.parent.mm.flush()
r.connection.close()
r.close()
s.close()
del r
del s
interrupted = True
break
if chunk:
self.parent.mm[self.position: self.position+len(chunk)] = chunk
self.count += len(chunk)
self.position += len(chunk)
self.parent.progress[self.id]['count'] = self.count
self.parent.progress[self.id]['position'] = self.position
if not interrupted:
r.close()
s.close()
if self.count == self.parent.progress[self.id]['length']:
self.completed = True
self.parent.progress[self.id]['completed'] = True
self.parent.mm.flush()
class Singledown:
def __init__(self):
self.count = 0
def singledown(self, url, path):
with requests.get(url, stream=True) as r:
with path.open('wb') as file:
for chunk in r.iter_content(1048576):
if chunk:
self.count += len(chunk)
file.write(chunk)
def download(self, url, filepath, num_connections=32, overwrite=False):
singlethread = False
threads = []
bcontinue = False
filepath = filepath.replace('\\', '/')
if (not re.match('^[a-zA-Z]:/(((?![<>:"/|?*]).)+((?<![ .])/)?)*$', filepath) or
not Path(filepath[:3]).exists()):
print('Invalid windows file path has been inputted, process will now stop.')
return
if not validators.url(url):
print('Invalid url been inputted, process will now stop.')
return
if url.lower().startswith('ftp://'):
print(
"`requests` module doesn't suport File Transfer Protocol, process will now stop")
return
path = Path(filepath)
if not path.exists():
bcontinue = True
else:
if path.is_file():
if overwrite:
bcontinue = True
else:
while True:
answer = input(
f'`{filepath}` already exists, do you want to overwrite it? \n(Yes, No):').lower()
if answer in ['y', 'yes', 'n', 'no']:
if answer.startswith('y'):
os.remove(filepath)
bcontinue = True
break
else:
print('Invalid input detected, retaking input.')
if not bcontinue:
print(
f'Overwritting {filepath} has been aborted, process will now stop.')
return
bcontinue = False
server = url.split('/')[2]
ok = ping(server, timeout=2)
if ok == False:
print(
'The server of the inputted url is non-existent, process will now stop.')
return
if ok:
bcontinue = True
if not ok:
print('Connection has timed out, will reattempt to ping server 5 times.')
for i in range(5):
print(
f'Reattempting to ping server, retrying {i + 1} out of 5')
ok = ping(server, timeout=2)
if ok:
print(
f'Connection successful on retry {i + 1}, process will now continue.')
bcontinue = True
break
else:
print(f'Retry {i + 1} out of 5 timed out' + (i != 4)
* ', reattempting in 1 second.' + (i == 4) * '.')
time.sleep(1)
if not bcontinue:
print('Failed to connect server, connection timed out, process will now stop')
return
bcontinue = False
head = requests.head(url)
if head.status_code == 200:
bcontinue = True
else:
for i in range(5):
print(f'Server responce is invalid, retrying {i + 1} out of 5')
head = requests.head(url)
if head.status_code == 200:
print(
f'Connection successful on retry {i + 1}, process will now continue.')
bcontinue = True
break
else:
print(f'Retry {i + 1} out of 5 failed to access data' +
(i != 4) * ', reattempting in 1 second.' + (i == 4) * '.')
time.sleep(1)
if not bcontinue:
print("Can't establish a connection with access to data, can't download target file, process will now stop.")
return
folder = '/'.join(filepath.split('/')[:-1])
Path(folder).mkdir(parents=True, exist_ok=True)
headers = head.headers
total = headers.get('content-length')
if not total:
print(
f'Cannot find the total length of the content of {url}, the file will be downloaded using a single thread.')
started = datetime.now()
print('Task started on %s.' %
started.strftime('%Y-%m-%d %H:%M:%S'))
sd = self.Singledown()
th = Thread(target=sd.singledown, args=(url, path))
threads.append(sd)
th.start()
total = inf
singlethread = True
else:
total = int(total)
if not headers.get('accept-ranges'):
print(
'Server does not support the `range` parameter, the file will be downloaded using a single thread.')
started = datetime.now()
print('Task started on %s.' %
started.strftime('%Y-%m-%d %H:%M:%S'))
sd = self.Singledown()
th = Thread(target=sd.singledown, args=(url, path))
threads.append(sd)
th.start()
singlethread = True
else:
segment = total / num_connections
started = datetime.now()
lastpressed = started
path.touch()
file = path.open('wb')
file.seek(total - 1)
file.write(b'\0')
file.close()
file = path.open(mode='r+b')
self.mm = mmap(file.fileno(), 0)
print('Task started on %s.' %
started.strftime('%Y-%m-%d %H:%M:%S'))
self.progress['total'] = total
self.progress['connections'] = num_connections
for i in range(num_connections):
md = self.Multidown(self, i)
start = int(segment * i)
end = int(segment * (i + 1)) - (i != num_connections - 1)
length = end - start + (i != num_connections - 1)
th = Thread(target=md.multidown, args=(
url, start, end))
threads.append(md)
self.progress[i] = dict()
self.progress[i]['start'] = start
self.progress[i]['position'] = start
self.progress[i]['end'] = end
self.progress[i]['count'] = 0
self.progress[i]['length'] = length
self.progress[i]['completed'] = False
th.start()
Path(filepath + '.progress.json').write_text(json.dumps(self.progress, indent=4))
downloaded = 0
totalMiB = total / 1048576
speeds = []
interval = 0.04
with output(initial_len=5, interval=0) as dynamic_print:
while True:
Path(filepath + '.progress.json').write_text(json.dumps(self.progress, indent=4))
status = sum([i.completed for i in threads])
downloaded = sum(i.count for i in threads)
self.recent.append(downloaded)
done = int(100 * downloaded / total)
doneMiB = downloaded / 1048576
gt0 = len([i for i in self.recent if i])
if not gt0:
speed = 0
else:
recent = list(self.recent)[12 - gt0:]
if len(recent) == 1:
speed = recent[0] / 1048576 / interval
else:
diff = [b - a for a, b in zip(recent, recent[1:])]
speed = sum(diff) / len(diff) / 1048576 / interval
speeds.append(speed)
self.recentspeeds.append(speed)
nzspeeds = [i for i in speeds if i]
if nzspeeds:
minspeed = min(nzspeeds)
else:
minspeed = 0
maxspeed = max(speeds)
now = datetime.now()
elapsed = (now - started).total_seconds()
meanspeed = downloaded / elapsed / 1048576
remaining = totalMiB - doneMiB
dynamic_print[0] = '[{0}{1}] {2}'.format(
'\u2588' * done, '\u00b7' * (100-done), str(done)) + '% completed' + (not singlethread) * ', paused: {0}'.format(self.paused)
dynamic_print[1] = 'Download mode: ' + singlethread * \
'Single-thread' + (not singlethread) * 'Multi-thread (press Space to pause/resume, press Escape to stop)'
dynamic_print[2] = '{0:.2f} MiB downloaded, {1:.2f} MiB total, {2:.2f} MiB remaining, download speed: {3:.2f} MiB/s'.format(
doneMiB, totalMiB, remaining, speed)
if speed and total != inf:
eta = timestring(remaining / speed)
else:
eta = '99:59:59'
dynamic_print[3] = 'Minimum speed: {0:.2f} MiB/s, average speed: {1:.2f} MiB/s, maximum speed: {2:.2f} MiB/s'.format(
minspeed, meanspeed, maxspeed)
dynamic_print[4] = 'Task started on {0}, {1} elapsed, ETA: {2}'.format(
started.strftime('%Y-%m-%d %H:%M:%S'), timestring(elapsed), eta)
if keyboard.is_pressed('space'):
if not singlethread:
pressed = datetime.now()
if (pressed - lastpressed).total_seconds() > 0.5:
lastpressed = pressed
if self.paused:
for i, md in enumerate(threads):
if not md.completed:
th = Thread(target=md.multidown, args=(
url, self.progress[i]['position'], self.progress[i]['end']))
th.start()
self.paused = not self.paused
if keyboard.is_pressed('esc'):
if not singlethread:
ended = datetime.now()
self.paused = True
break
if status == len(threads):
if not singlethread:
self.mm.close()
ended = datetime.now()
break
time.sleep(interval)
time_spent = (ended - started).total_seconds()
meanspeed = total / time_spent / 1048576
status = sum([i.completed for i in threads])
if status == len(threads):
print('Task completed on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))
else:
print('Task interrupted on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))
if __name__ == '__main__':
d = Downloader()
d.download(*sys.argv[1:])
For testing purposes this is a dumbed-down version of the script, with all checks removed while retaining the same functionality (sorry it really takes all these lines to show the download information):
import json
import os
import requests
import sys
import time
from collections import deque
from datetime import datetime, timedelta
from math import inf
from mmap import mmap
from pathlib import Path
from reprint import output
from threading import Thread
def timestring(sec):
sec = int(sec)
m, s = divmod(sec, 60)
h, m = divmod(m, 60)
return f'{h:02d}:{m:02d}:{s:02d}'
class Downloader:
def __init__(self):
self.recent = deque([0] * 12, maxlen=12)
self.recentspeeds = deque([0] * 200, maxlen=200)
self.paused = False
self.progress = dict()
self.UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0'
class Multidown:
def __init__(self, obj, id):
self.count = 0
self.position = 0
self.completed = False
self.id = id
self.parent = obj
self.UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0'
def multidown(self, url, start, end):
interrupted = False
s = requests.session()
s.headers.update({'connection': 'close', 'user-agent': self.UA})
r = s.get(
url, headers={'range': 'bytes={0}-{1}'.format(start, end)}, stream=True)
length = int(r.headers['content-length'])
while end - length + (self.id != self.parent.progress['connections'] - 1) != start or r.status_code != 206:
r.close()
s.close()
del r
del s
time.sleep(0.02)
s = requests.session()
r = s.get(
url, headers={'range': 'bytes={0}-{1}'.format(start, end)}, stream=True)
length = int(r.headers['content-length'])
self.position = start
for chunk in r.iter_content(1048576):
if self.parent.paused:
self.parent.mm.flush()
r.connection.close()
r.close()
s.close()
del r
del s
interrupted = True
break
if chunk:
self.parent.mm[self.position: self.position+len(chunk)] = chunk
self.count += len(chunk)
self.position += len(chunk)
self.parent.progress[self.id]['count'] = self.count
self.parent.progress[self.id]['position'] = self.position
if not interrupted:
r.close()
s.close()
if self.count == self.parent.progress[self.id]['length']:
self.completed = True
self.parent.progress[self.id]['completed'] = True
self.parent.mm.flush()
def download(self, url, filepath, num_connections=32, overwrite=False):
singlethread = False
threads = []
bcontinue = False
filepath = filepath.replace('\\', '/')
if Path(filepath).exists():
os.remove(filepath)
folder = '/'.join(filepath.split('/')[:-1])
Path(folder).mkdir(parents=True, exist_ok=True)
head = requests.head(url, headers={'user-agent': self.UA})
path = Path(filepath)
headers = head.headers
total = headers.get('content-length')
if total:
total = int(total)
if headers.get('accept-ranges'):
segment = total / num_connections
started = datetime.now()
lastpressed = started
path.touch()
file = path.open('wb')
file.seek(total - 1)
file.write(b'\0')
file.close()
file = path.open(mode='r+b')
self.mm = mmap(file.fileno(), 0)
print('Task started on %s.' %
started.strftime('%Y-%m-%d %H:%M:%S'))
self.progress['total'] = total
self.progress['connections'] = num_connections
for i in range(num_connections):
md = self.Multidown(self, i)
start = int(segment * i)
end = int(segment * (i + 1)) - (i != num_connections - 1)
length = end - start + (i != num_connections - 1)
th = Thread(target=md.multidown, args=(
url, start, end))
threads.append(md)
self.progress[i] = dict()
self.progress[i]['start'] = start
self.progress[i]['position'] = start
self.progress[i]['end'] = end
self.progress[i]['count'] = 0
self.progress[i]['length'] = length
self.progress[i]['completed'] = False
th.start()
Path(filepath + '.progress.json').write_text(json.dumps(self.progress, indent=4))
downloaded = 0
totalMiB = total / 1048576
speeds = []
interval = 0.04
with output(initial_len=5, interval=0) as dynamic_print:
while True:
Path(filepath + '.progress.json').write_text(json.dumps(self.progress, indent=4))
status = sum([i.completed for i in threads])
downloaded = sum(i.count for i in threads)
self.recent.append(downloaded)
done = int(100 * downloaded / total)
doneMiB = downloaded / 1048576
gt0 = len([i for i in self.recent if i])
if not gt0:
speed = 0
else:
recent = list(self.recent)[12 - gt0:]
if len(recent) == 1:
speed = recent[0] / 1048576 / interval
else:
diff = [b - a for a, b in zip(recent, recent[1:])]
speed = sum(diff) / len(diff) / 1048576 / interval
speeds.append(speed)
self.recentspeeds.append(speed)
nzspeeds = [i for i in speeds if i]
if nzspeeds:
minspeed = min(nzspeeds)
else:
minspeed = 0
maxspeed = max(speeds)
now = datetime.now()
elapsed = (now - started).total_seconds()
meanspeed = downloaded / elapsed / 1048576
remaining = totalMiB - doneMiB
dynamic_print[0] = '[{0}{1}] {2}'.format(
'\u2588' * done, '\u00b7' * (100-done), str(done)) + '% completed' + (not singlethread) * ', paused: {0}'.format(self.paused)
dynamic_print[1] = 'Download mode: ' + singlethread * \
'Single-thread' + (not singlethread) * 'Multi-thread (press Space to pause/resume, press Escape to stop)'
dynamic_print[2] = '{0:.2f} MiB downloaded, {1:.2f} MiB total, {2:.2f} MiB remaining, download speed: {3:.2f} MiB/s'.format(
doneMiB, totalMiB, remaining, speed)
if speed and total != inf:
eta = timestring(remaining / speed)
else:
eta = '99:59:59'
dynamic_print[3] = 'Minimum speed: {0:.2f} MiB/s, average speed: {1:.2f} MiB/s, maximum speed: {2:.2f} MiB/s'.format(
minspeed, meanspeed, maxspeed)
dynamic_print[4] = 'Task started on {0}, {1} elapsed, ETA: {2}'.format(
started.strftime('%Y-%m-%d %H:%M:%S'), timestring(elapsed), eta)
if PAUSE:
if not singlethread:
pressed = datetime.now()
if (pressed - lastpressed).total_seconds() > 0.5:
lastpressed = pressed
if self.paused:
for i, md in enumerate(threads):
if not md.completed:
th = Thread(target=md.multidown, args=(
url, self.progress[i]['position'], self.progress[i]['end']))
th.start()
self.paused = not self.paused
if status == len(threads):
if not singlethread:
self.mm.close()
ended = datetime.now()
break
time.sleep(interval)
time_spent = (ended - started).total_seconds()
meanspeed = total / time_spent / 1048576
status = sum([i.completed for i in threads])
if status == len(threads):
print('Task completed on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))
else:
print('Task interrupted on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))
if __name__ == '__main__':
import hashlib
global PAUSE
PAUSE = False
chash = '5674E59283D95EFE8C88770515A9BBC80CBB77CB67602389FD91DEF26D26AED2'
d = Downloader()
if sys.argv[1] == '0':
d.download('http://ipv4.download.thinkbroadband.com/1GB.zip', 'C:/test/1GB.zip')
elif sys.argv[1] == '1':
th1 = Thread(target=d.download, args=('http://ipv4.download.thinkbroadband.com/1GB.zip', 'C:/test/1GB.zip'))
th1.start()
def test():
while th1.is_alive():
global PAUSE
PAUSE = not PAUSE
time.sleep(10)
th2 = Thread(target=test)
th2.start()
while th1.is_alive():
pass
sha256_hash = hashlib.sha256()
with open('C:/test/1GB.zip',"rb") as f:
for byte_block in iter(lambda: f.read(1048576),b""):
sha256_hash.update(byte_block)
print(sha256_hash.hexdigest().lower() == chash.lower())
The url isn't accessible without a VPN in my locale, and test 0 always results True, that is, if the connection hasn't gone dead during the download, and test 1 sometimes results True, sometimes results False, sometimes it doesn't finish(progress bar goes beyond 100%)...
How can my code be salvaged?
This might not be your only problem but you have a race condition that could show up if you pause and resume quickly (where the definition of quickly varies greatly depending on your circumstances). Consider that you've got 32 threads each requesting a MB chunk, let's call them threads 0-31. They are sitting their downloading and you pause. The threads do not know that you paused until they get a chunk of data as they are sitting in blocking io. Not sure what speed your connection is or how many cores your machine has (threads will sometimes act in parallel when they don't need the GIL,) but this process could take a lot longer than you expect. Then you unpause and your code creates new threads 32-63 but some or all of threads 0-31 are still waiting for the next chunk. You set threads 32-63 in motion and then you turn off your pause flag. Those threads that didn't end from 0-31 then wake up and see that things aren't paused. Now you have multiple threads accessing the same state variables
self.parent.mm[self.position: self.position + len(chunk)] = chunk
self.count += len(chunk)
self.position += len(chunk)
self.parent.progress[self.id]['count'] = self.count
self.parent.progress[self.id]['position'] = self.position
so if thread 0 is downloading the same chunk as thread 31 they both keep updating all the same state and they add to position and count even though they are downloading overlapping parts of the file. You even reuse the objects that the threads live inside of so that state can get really really messed up.
for i, md in enumerate(threads):
if not md.completed:
th = Thread(target=md.multidown, args=(url, self.progress[i]['position'], self.progress[i]['end']))
th.start()
There might be some other problems in your code and it is a lot to sort through so I suggest taking the time to do some refactoring to eliminate duplicate code and organise things into more functions. I don't believe in crazy tiny functions, but you could use a few sub functions like download_multi(download_state) and download_single maybe. I am relatively confident however that your current problem will be solved if you ensure the threads you have running actually end after you pause. To do so you need to actually hold references to your threads
somewhere:
actual_threads = []
When you create your threads (the first time and after you unpause, or preferably this would be in a function and you'd do it there and return the list):
th = Thread(target=md.multidown, args=(
url, start, end))
threads.append(md)
actual_threads.append(th)
Then when you unpause:
self.paused = not self.paused
for th in actual_threads:
th.join()
This way you have the threads working, they quit when you pause and you rebuild them. So join should return as soon as they break out of the blocking io call to iter_content. This way those threads are always dead before you make the new ones.
What I would do myself however would be to create sockets from each thread to the main process. When pause is detected the threads shut down the request and save any data that's already waiting in the OS buffer then go into a blocking receive on the socket (there might be a way to use select with a socket and requests to allow you to even break out of the blocking io involved in r.iter_content immediately but I leave that for your research). When the program is unpaused the main process would send some value to indicate the program should restart (you'd want at least two signals the threads would recognise, one for quitting gracefully and one to resume. The codes can be single characters.) When the value is sent to each thread that thread will unblock and can then restart the download using requests and its previous state like nothing happened.
import random
import socket
import time
import ipaddress
import struct
from threading import Thread
def checksum(source_string):
sum = 0
count_to = (len(source_string) / 2) * 2
count = 0
while count < count_to:
this_val = ord(source_string[count + 1]) * 256 + ord(source_string[count])
sum = sum + this_val
sum = sum & 0xffffffff
count = count + 2
if count_to < len(source_string):
sum = sum + ord(source_string[len(source_string) - 1])
sum = sum & 0xffffffff
sum = (sum >> 16) + (sum & 0xffff)
sum = sum + (sum >> 16)
answer = ~sum
answer = answer & 0xffff
answer = answer >> 8 | (answer << 8 & 0xff00)
return answer
def create_packet(id):
header = struct.pack('bbHHh', 8, 0, 0, id, 1)
data = 192 * 'Q'
my_checksum = checksum(header + data)
header = struct.pack('bbHHh', 8, 0, socket.htons(my_checksum), id, 1)
return header + data
def ping(addr, timeout=1):
try:
my_socket = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
except Exception as e:
print (e)
packet_id = int((id(timeout) * random.random()) % 65535)
packet = create_packet(packet_id)
my_socket.connect((addr, 80))
my_socket.sendall(packet)
my_socket.close()
def rotate(addr, file_name, wait, responses):
print ("Sending Packets", time.strftime("%X %x %Z"))
for ip in addr:
ping(str(ip))
time.sleep(wait)
print ("All packets sent", time.strftime("%X %x %Z"))
print ("Waiting for all responses")
time.sleep(2)
# Stoping listen
global SIGNAL
SIGNAL = False
ping('127.0.0.1') # Final ping to trigger the false signal in listen
print (len(responses), "hosts found!")
print ("Writing File")
hosts = []
for response in sorted(responses):
ip = struct.unpack('BBBB', response)
ip = str(ip[0]) + "." + str(ip[1]) + "." + str(ip[2]) + "." + str(ip[3])
hosts.append(ip)
file = open(file_name, 'w')
file.write(str(hosts))
print ("Done", time.strftime("%X %x %Z"))
def listen(responses):
global SIGNAL
s = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
s.bind(('', 1))
print ("Listening")
while SIGNAL:
packet = s.recv(1024)[:20][-8:-4]
responses.append(packet)
print ("Stop Listening")
s.close()
SIGNAL = True
responses = []
ips = '200.131.0.0/20' # Internet network
wait = 0.002 # Adjust this based in your bandwidth (Faster link is Lower wait)
file_name = 'log1.txt'
ip_network = ipaddress.ip_network(unicode(ips), strict=False)
t_server = Thread(target=listen, args=[responses])
t_server.start()
t_ping = Thread(target=rotate, args=[ip_network, file_name, wait, responses])
t_ping.start()
I tried:
ip_network = ipaddress.ip_network( ips, strict=False) instead of ip_network = ipaddress.ip_network(unicode(ips), strict=False)
because of the error: ""NameError: name 'unicode' is not defined"
after:
I got: my_checksum = checksum(header + data) -> TypeError: can't concat bytes to str
so I tried:
data = bytes(192 * 'Q').encode('utf8') instead of data = 192 * 'Q'
Now, the error is : ""data = bytes (192 * 'Q').encode('utf8') TypeError: string argument without an encoding"
Could anyone help me to port the code to Python 3 ?
import random
import socket
import time
import ipaddress
import struct
from threading import Thread
def checksum(source_string):
sum = 0
count_to = (len(source_string) / 2) * 2
count = 0
while count < count_to:
this_val = source_string[count + 1] * 256 + source_string[count]
sum = sum + this_val
sum = sum & 0xffffffff
count = count + 2
if count_to < len(source_string):
sum = sum + source_string[len(source_string) - 1]
sum = sum & 0xffffffff
sum = (sum >> 16) + (sum & 0xffff)
sum = sum + (sum >> 16)
answer = ~sum
answer = answer & 0xffff
answer = answer >> 8 | (answer << 8 & 0xff00)
return answer
def create_packet(id):
header = struct.pack('bbHHh', 8, 0, 0, id, 1)
data = 192 * b'Q'
my_checksum = checksum(header + data)
header = struct.pack('bbHHh', 8, 0, socket.htons(my_checksum), id, 1)
return header + data
def ping(addr, timeout=1):
try:
my_socket = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
except Exception as e:
print (e)
packet_id = int((id(timeout) * random.random()) % 65535)
packet = create_packet(packet_id)
my_socket.connect((addr, 80))
my_socket.sendall(packet)
my_socket.close()
def rotate(addr, file_name, wait, responses):
print ("Sending Packets", time.strftime("%X %x %Z"))
for ip in addr:
ping(str(ip))
time.sleep(wait)
print ("All packets sent", time.strftime("%X %x %Z"))
print ("Waiting for all responses")
time.sleep(2)
# Stoping listen
global SIGNAL
SIGNAL = False
ping('127.0.0.1') # Final ping to trigger the false signal in listen
print (len(responses), "hosts found!")
print ("Writing File")
hosts = set()
for response in sorted(responses):
ip = struct.unpack('BBBB', response)
ip = str(ip[0]) + "." + str(ip[1]) + "." + str(ip[2]) + "." + str(ip[3])
hosts.add(ip)
with open(file_name, 'w') as file:
file.write('\n'.join(sorted(hosts, key=lambda item: socket.inet_aton(item))))
print ("Done", time.strftime("%X %x %Z"))
def listen(responses):
global SIGNAL
s = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)
s.bind(('', 1))
print ("Listening")
while SIGNAL:
packet = s.recv(1024)[:20][-8:-4]
responses.append(packet)
print ("Stop Listening")
s.close()
SIGNAL = True
responses = []
ips = '192.168.1.0/28' # Internet network
wait = 0.002 # Adjust this based in your bandwidth (Faster link is Lower wait)
file_name = 'log1.txt'
ip_network = ipaddress.ip_network(ips, strict=False)
t_server = Thread(target=listen, args=[responses])
t_server.start()
t_ping = Thread(target=rotate, args=[ip_network, file_name, wait, responses])
t_ping.start()
I am using for loop to get the mac address from database but the loop printed the ble mac address 17 times. I tried using if else to compare the mac address with the mac address stored in database but the output scan nothing. Is there any other way to filter out other ble mac address?
import os
import sys
import struct
import bluetooth._bluetooth as bluez
import MySQLdb
# Open database connection
db = MySQLdb.connect("localhost","DB_USER","DB_PASSWORD","DB_NAME" )
# prepare a cursor object using cursor() method
cursor = db.cursor()
sql = "SELECT * FROM Mac_address"
cursor.execute(sql)
# Fetch all the rows in a list of lists.
results = cursor.fetchall()
for row in results:
NAME = row[0]
mac_address = row[1]
time = row[2]
LE_META_EVENT = 0x3e
LE_PUBLIC_ADDRESS=0x00
LE_RANDOM_ADDRESS=0x01
LE_SET_SCAN_PARAMETERS_CP_SIZE=7
OGF_LE_CTL=0x08
OCF_LE_SET_SCAN_PARAMETERS=0x000B
OCF_LE_SET_SCAN_ENABLE=0x000C
OCF_LE_CREATE_CONN=0x000D
LE_ROLE_MASTER = 0x00
LE_ROLE_SLAVE = 0x01
# these are actually subevents of LE_META_EVENT
EVT_LE_CONN_COMPLETE=0x01
EVT_LE_ADVERTISING_REPORT=0x02
EVT_LE_CONN_UPDATE_COMPLETE=0x03
EVT_LE_READ_REMOTE_USED_FEATURES_COMPLETE=0x04
# Advertisment event types
ADV_IND=0x00
ADV_DIRECT_IND=0x01
ADV_SCAN_IND=0x02
ADV_NONCONN_IND=0x03
ADV_SCAN_RSP=0x04
def returnnumberpacket(pkt):
myInteger = 0
multiple = 256
for c in pkt:
myInteger += struct.unpack("B",c)[0] * multiple
multiple = 1
return myInteger
def returnstringpacket(pkt):
myString = "";
for c in pkt:
myString += "%02x" %struct.unpack("B",c)[0]
return myString
def printpacket(pkt):
for c in pkt:
sys.stdout.write("%02x " % struct.unpack("B",c)[0])
def get_packed_bdaddr(bdaddr_string):
packable_addr = []
addr = bdaddr_string.split(':')
addr.reverse()
for b in addr:
packable_addr.append(int(b, 16))
return struct.pack("<BBBBBB", *packable_addr)
def packed_bdaddr_to_string(bdaddr_packed):
return ':'.join('%02x'%i for i in struct.unpack("<BBBBBB", bdaddr_packed[::-1]))
def hci_enable_le_scan(sock):
hci_toggle_le_scan(sock, 0x01)
def hci_disable_le_scan(sock):
hci_toggle_le_scan(sock, 0x00)
def hci_toggle_le_scan(sock, enable)
cmd_pkt = struct.pack("<BB", enable, 0x00)
bluez.hci_send_cmd(sock, OGF_LE_CTL, OCF_LE_SET_SCAN_ENABLE, cmd_pkt)
def hci_le_set_scan_parameters(sock):
old_filter = sock.getsockopt( bluez.SOL_HCI, bluez.HCI_FILTER, 14)
SCAN_RANDOM = 0x01
OWN_TYPE = SCAN_RANDOM
SCAN_TYPE = 0x01
def parse_events(sock, loop_count=100):
old_filter = sock.getsockopt( bluez.SOL_HCI, bluez.HCI_FILTER, 14)
# perform a device inquiry on bluetooth device #0
# The inquiry should last 8 * 1.28 = 10.24 seconds
# before the inquiry is performed, bluez should flush its cache of
# previously discovered devices
flt = bluez.hci_filter_new()
bluez.hci_filter_all_events(flt)
bluez.hci_filter_set_ptype(flt, bluez.HCI_EVENT_PKT)
sock.setsockopt( bluez.SOL_HCI, bluez.HCI_FILTER, flt )
done = False
results = []
myFullList = []
for i in range(0, loop_count):
pkt = sock.recv(255)
ptype, event, plen = struct.unpack("BBB", pkt[:3])
#print "--------------"
if event == bluez.EVT_INQUIRY_RESULT_WITH_RSSI:
i =0
elif event == bluez.EVT_NUM_COMP_PKTS:
i =0
elif event == bluez.EVT_DISCONN_COMPLETE:
i =0
elif event == LE_META_EVENT:
subevent, = struct.unpack("B", pkt[3])
pkt = pkt[4:]
if subevent == EVT_LE_CONN_COMPLETE:
le_handle_connection_complete(pkt)
elif subevent == EVT_LE_ADVERTISING_REPORT:
#print "advertising report"
num_reports = struct.unpack("B", pkt[0])[0]
report_pkt_offset = 0
for i in range(0, num_reports):
Mac = packed_bdaddr_to_string(pkt[report_pkt_offset + 3:report_pkt_offset + 9])
for Mac in row[1]:
print "-------------"
#print "\tfullpacket: ", printpacket(pkt)
print "\tMAC address: ", row[1] # commented out - don't know what this byte is. It's NOT TXPower
txpower, = struct.unpack("b", pkt[report_pkt_offset -2])
print "\t(Unknown):", txpower
rssi, = struct.unpack("b", pkt[report_pkt_offset -1])
print "\tRSSI:", rssi
break
# build the return string
Adstring = packed_bdaddr_to_string(pkt[report_pkt_offset + 3:report_pkt_offset + 9])
Adstring += ","
Adstring += returnstringpacket(pkt[report_pkt_offset -22: report_pkt_offset - 6])
Adstring += ","
Adstring += "%i" % returnnumberpacket(pkt[report_pkt_offset -6: report_pkt_offset - 4])
Adstring += ","
Adstring += "%i" % returnnumberpacket(pkt[report_pkt_offset -4: report_pkt_offset - 2])
Adstring += ","
Adstring += "%i" % struct.unpack("b", pkt[report_pkt_offset -2])
Adstring += ","
Adstring += "%i" % struct.unpack("b", pkt[report_pkt_offset -1])
#print "\tAdstring=", Adstring
myFullList.append(Adstring)
done = True
sock.setsockopt( bluez.SOL_HCI, bluez.HCI_FILTER, old_filter )
return myFullList
db.close()
The above is save as blescan.py. I run this file below to do a ble scan.
import blescan
import sys
import bluetooth._bluetooth as bluez
dev_id = 0
try:
sock = bluez.hci_open_dev(dev_id)
print "ble thread started"
except:
print "error accessing bluetooth device..."
sys.exit(1)
blescan.hci_le_set_scan_parameters(sock)
blescan.hci_enable_le_scan(sock)
while True:
returnedList = blescan.parse_events(sock, 10)
print "----------"
I am working on a ICMP pinger program and have run into a problem. When i run the program i first do sudo su in the terminal so i have root access since we're required to use raw sockets and then when i run the program i get this traceback
sh-3.2# python3 icmp.py
Pinging 31.13.66.112 using Python:
Traceback (most recent call last):
File "icmp.py", line 149, in <module>
ping("www.facebook.com")
File "icmp.py", line 141, in ping
delay = doOnePing(dest,timeout)
File "icmp.py", line 123, in doOnePing
sendOnePing(mySocket,destAddr,myID)
File "icmp.py", line 111, in sendOnePing
mySocket.sendto(packet,(destAddr,1))
OSError: [Errno 56] Socket is already connected
here is the code I am running
import os
from socket import *
from sys import *
from struct import *
from time import *
from select import *
from binascii import *
ICMP_ECHO_REQUEST = 8
def checksum (val):
csum = 0
countTo = (len(val) // 2) * 2
count = 0
while count < countTo:
thisVal = val[count+1] * 256 + val[count]
csum = csum + thisVal
csum = csum & 0xffffffff
count = count + 2
if countTo < len(val):
csum = csum + val[len(val) - 1]
csum = csum & 0xffffffff
csum = (csum >> 16) + (csum & 0xffff)
csum = csum + (csum >> 16)
answer = ~csum
answer = answer & 0xffff
answer = answer >> 8 | (answer << 8 & 0xff00)
return answer
def receiveOnePing(mySocket,ID,timeout,destAddr):
timeLeft = timeout
while 1:
startedSelect = time()
whatReady = select([mySocket],[],[],timeLeft)
howLongInSelect = (time() - startedSelect)
if whatReady[0] == []: #Timeout
return "Request timed out."
timeReceived = time()
recPacket, addr = mySocket.recvfrom(1024)
icmpHeader = recPacket[20:28]
kind,code,checksum,idNum,sequence = unpack("bbHHh",icmpHeader)
if idNum == ID:
sizeofdouble = calcsize("d")
timeSent = unpack("d",recPacket[28:28+sizeofdouble])[0]
print(("TYPE: %d CODE: %d CHECKSUM: 0x%08x ID: %d SEQ: %d TIME: %d ms\n" % (kind,code,checksum,idNum,sequence,timeReceived - timeSent)*1000))
timeLeft = timeLeft - howLongInSelect
if timeLeft <= 0:
return "Request timed out."
else:
return "Reply from %s successfully." % destAddr
def sendOnePing(mySocket, destAddr, ID):
myChecksum = 0
header = pack("bbHHh", ICMP_ECHO_REQUEST, 0, myChecksum, ID, 1)
data = pack("d",time())
myChecksum = checksum(header + data)
if platform == 'darwin':
myChecksum = htons(myChecksum) & 0xffff
else:
myChecksum = htons(myChecksum)
header = pack("bbHHh", ICMP_ECHO_REQUEST,0,myChecksum,ID,1)
packet = header + data
mySocket.sendto(packet,(destAddr,1))
def doOnePing(destAddr,timeout):
icmp = getprotobyname("icmp")
mySocket = socket(AF_INET,SOCK_RAW,icmp)
mySocket.connect((destAddr,80))
myID = os.getpid() & 0xFFFF
sendOnePing(mySocket,destAddr,myID)
delay = receiveOnePing(mySocket,myID,timeout,destAddr)
mySocket.close()
return delay
def ping(host,timeout = 1):
dest = gethostbyname(host)
print("Pinging " + dest + " using Python:")
print()
while 1:
delay = doOnePing(dest,timeout)
print(delay)
sleep(1)
return delay
ping("www.facebook.com")
we were given a lab sheet with the "skeleton" program and we had to fill in the missing parts so Here is the lab sheet where the skeleton code was given
note: the skeleton and my program wont match perfectly due to the fact that i had to change things to fix other errors that occurred before this one.
I thank you for your help in advance,
Tyler
You should use "bind", instead of "connect". Then this program you should run as root like "sudo python3 icmp.py" for Linux or Mac OS.
Hope this help.
Thanks
So, you connect the socket (that is, tell it the peer address to send data to), and then call sendto providing a peer address again. The socket gets confused even if those two addresses are identical.
Either do not connect, or call send.