I'm new to Python and posting on SO.
I'm doing a class project on trying to create an automated climate control system with a Raspberry pi(Pi4 4GB) and a DHT.
Here is my current error and code and I'm not sure what's causing the fail in the loop. It works for a while then errors out. It seems to error when it gets a bad read from the sensor, but until then it works fine.
Traceback (most recent call last):
File "/home/pi/raphael-kit/python/Climatecontrol.py", line 156, in <module>
main()
File "/home/pi/raphael-kit/python/Climatecontrol.py", line 118, in main
humidity, temperature = readDht11()
TypeError: cannot unpack non-iterable bool object
import RPi.GPIO as GPIO
import time
import LCD1602
dhtPin = 17
relayPin = 23 # Set GPIO23 as control pin
set_temp = 25 # this is the required temperature
GPIO.setmode(GPIO.BCM)
# Set relayPin's mode to output,
# and initial level to High(3.3v)
GPIO.setup(relayPin, GPIO.OUT, initial=GPIO.HIGH)
MAX_UNCHANGE_COUNT = 100
STATE_INIT_PULL_DOWN = 1
STATE_INIT_PULL_UP = 2
STATE_DATA_FIRST_PULL_DOWN = 3
STATE_DATA_PULL_UP = 4
STATE_DATA_PULL_DOWN = 5
def readDht11():
GPIO.setup(dhtPin, GPIO.OUT)
GPIO.output(dhtPin, GPIO.HIGH)
time.sleep(0.05)
GPIO.output(dhtPin, GPIO.LOW)
time.sleep(0.02)
GPIO.setup(dhtPin, GPIO.IN, GPIO.PUD_UP)
unchanged_count = 0
last = -1
data = []
while True:
current = GPIO.input(dhtPin)
data.append(current)
if last != current:
unchanged_count = 0
last = current
else:
unchanged_count += 1
if unchanged_count > MAX_UNCHANGE_COUNT:
break
state = STATE_INIT_PULL_DOWN
lengths = []
current_length = 0
for current in data:
current_length += 1
if state == STATE_INIT_PULL_DOWN:
if current == GPIO.LOW:
state = STATE_INIT_PULL_UP
else:
continue
if state == STATE_INIT_PULL_UP:
if current == GPIO.HIGH:
state = STATE_DATA_FIRST_PULL_DOWN
else:
continue
if state == STATE_DATA_FIRST_PULL_DOWN:
if current == GPIO.LOW:
state = STATE_DATA_PULL_UP
else:
continue
if state == STATE_DATA_PULL_UP:
if current == GPIO.HIGH:
current_length = 0
state = STATE_DATA_PULL_DOWN
else:
continue
if state == STATE_DATA_PULL_DOWN:
if current == GPIO.LOW:
lengths.append(current_length)
state = STATE_DATA_PULL_UP
else:
continue
if len(lengths) != 40:
#print ("Data not good, skip")
return False
shortest_pull_up = min(lengths)
longest_pull_up = max(lengths)
halfway = (longest_pull_up + shortest_pull_up) / 2
bits = []
the_bytes = []
byte = 0
for length in lengths:
bit = 0
if length > halfway:
bit = 1
bits.append(bit)
#print ("bits: %s, length: %d" % (bits, len(bits)))
for i in range(0, len(bits)):
byte = byte << 1
if (bits[i]):
byte = byte | 1
else:
byte = byte | 0
if ((i + 1) % 8 == 0):
the_bytes.append(byte)
byte = 0
#print (the_bytes)
checksum = (the_bytes[0] + the_bytes[1] + the_bytes[2] + the_bytes[3]) & 0xFF
#if the_bytes[4] != checksum:
#print ("Data not good, skip")
#return False
return the_bytes[0], the_bytes[2]
def main():
while True:
humidity, temperature = readDht11()
if humidity is not None and temperature is not None:
print("Temp={0:0.1f}*C Humidity={1:0.1f}%".format(temperature, humidity))
# test for low temperature
if temperature < set_temp:
print(GPIO.output(relayPin, GPIO.LOW))
# test for high temperature
if temperature > (set_temp + 1):
print(GPIO.output(relayPin, GPIO.HIGH))
else:
print("Failed to retrieve data from humidity sensor")
time.sleep(5) #this is the time between taking readings and acting on them you can reduce it but not below 5 seconds
# Define a destroy function for clean up everything after
# the script finished
def setup():
LCD1602.init(0x27, 1) # init(slave address, background light)
LCD1602.write(0, 0, 'Temperature: %s C')
LCD1602.write(1, 1, 'humidity: %s %%')
time.sleep(2)
def destroy():
# set relay to off
GPIO.output(relayPin, GPIO.LOW)
# Release resource
GPIO.cleanup()
if __name__ == '__main__':
try:
setup()
except KeyboardInterrupt:
destroy()
as a note I haven't managed to get the LCD working in tandem but im more worried about the main functionality
I have written a Python script that downloads a single file using 32 connections if available.
I have written a multiconnection downloader that works fine without pausing, but won't stop downloading after resuming, the progress would go beyond 100%...
Like this:
Download mode: Multi-thread (press Space to pause/resume, press Escape to stop)
[████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████] 120% completed, paused: False
Download mode: Multi-thread (press Space to pause/resume, press Escape to stop)
1798.08 MiB downloaded, 1489.83 MiB total, -308.25 MiB remaining, download speed: 22.73 MiB/s
Minimum speed: 0.00 MiB/s, average speed: 4.54 MiB/s, maximum speed: 75.00 MiB/s
Task started on 2021-08-09 16:57:03, 00:06:35 elapsed, ETA: -1:59:47
After progress exceeds 100%, there will be error messages like this:
Exception in thread Thread-78:
Traceback (most recent call last):
File "C:\Program Files\Python39\lib\threading.py", line 973, in _bootstrap_inner
self.run()
File "C:\Program Files\Python39\lib\threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "D:\MyScript\downloader.py", line 70, in multidown
mm[position: position+len(chunk)] = chunk
IndexError: mmap slice assignment is wrong size
(The above doesn't include all of the error message)
I have encountered all sorts of errors after resuming, but most importantly, the server will often send extra bytes from previous request, whose connection is dead and needless to say this breaks the whole code.
How should I implement pause and resume correctly?
I am thinking about multiprocessing, I assume the sessions and connections are all PID and port number related, and so far I haven't encountered a new run of the script that received extra bytes from previous runs of the script, so I guess using another process with a new PID and new port number plus requests.session() plus {'connection': 'close'} for each download should guarantee that no extra bytes from previous connections will be received, I just don't know how to share variables between processes...
The code:
downloader.py
import json
import keyboard
import os
import re
import requests
import sys
import time
import validators
from collections import deque
from datetime import datetime, timedelta
from math import inf
from mmap import mmap
from pathlib import Path
from ping3 import ping
from reprint import output
from threading import Thread
def timestring(sec):
sec = int(sec)
m, s = divmod(sec, 60)
h, m = divmod(m, 60)
return f'{h:02d}:{m:02d}:{s:02d}'
class Downloader:
def __init__(self):
self.recent = deque([0] * 12, maxlen=12)
self.recentspeeds = deque([0] * 200, maxlen=200)
self.paused = False
self.progress = dict()
class Multidown:
def __init__(self, obj, id):
self.count = 0
self.position = 0
self.completed = False
self.id = id
self.parent = obj
def multidown(self, url, start, end):
interrupted = False
s = requests.session()
s.headers.update({'connection': 'close', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0'})
r = s.get(
url, headers={'range': 'bytes={0}-{1}'.format(start, end)}, stream=True)
length = int(r.headers['content-length'])
while end - length + (self.id != self.parent.progress['connections'] - 1) != start or r.status_code != 206:
r.close()
s.close()
del r
del s
time.sleep(0.02)
s = requests.session()
r = s.get(
url, headers={'range': 'bytes={0}-{1}'.format(start, end)}, stream=True)
length = int(r.headers['content-length'])
self.position = start
for chunk in r.iter_content(1048576):
if self.parent.paused:
self.parent.mm.flush()
r.connection.close()
r.close()
s.close()
del r
del s
interrupted = True
break
if chunk:
self.parent.mm[self.position: self.position+len(chunk)] = chunk
self.count += len(chunk)
self.position += len(chunk)
self.parent.progress[self.id]['count'] = self.count
self.parent.progress[self.id]['position'] = self.position
if not interrupted:
r.close()
s.close()
if self.count == self.parent.progress[self.id]['length']:
self.completed = True
self.parent.progress[self.id]['completed'] = True
self.parent.mm.flush()
class Singledown:
def __init__(self):
self.count = 0
def singledown(self, url, path):
with requests.get(url, stream=True) as r:
with path.open('wb') as file:
for chunk in r.iter_content(1048576):
if chunk:
self.count += len(chunk)
file.write(chunk)
def download(self, url, filepath, num_connections=32, overwrite=False):
singlethread = False
threads = []
bcontinue = False
filepath = filepath.replace('\\', '/')
if (not re.match('^[a-zA-Z]:/(((?![<>:"/|?*]).)+((?<![ .])/)?)*$', filepath) or
not Path(filepath[:3]).exists()):
print('Invalid windows file path has been inputted, process will now stop.')
return
if not validators.url(url):
print('Invalid url been inputted, process will now stop.')
return
if url.lower().startswith('ftp://'):
print(
"`requests` module doesn't suport File Transfer Protocol, process will now stop")
return
path = Path(filepath)
if not path.exists():
bcontinue = True
else:
if path.is_file():
if overwrite:
bcontinue = True
else:
while True:
answer = input(
f'`{filepath}` already exists, do you want to overwrite it? \n(Yes, No):').lower()
if answer in ['y', 'yes', 'n', 'no']:
if answer.startswith('y'):
os.remove(filepath)
bcontinue = True
break
else:
print('Invalid input detected, retaking input.')
if not bcontinue:
print(
f'Overwritting {filepath} has been aborted, process will now stop.')
return
bcontinue = False
server = url.split('/')[2]
ok = ping(server, timeout=2)
if ok == False:
print(
'The server of the inputted url is non-existent, process will now stop.')
return
if ok:
bcontinue = True
if not ok:
print('Connection has timed out, will reattempt to ping server 5 times.')
for i in range(5):
print(
f'Reattempting to ping server, retrying {i + 1} out of 5')
ok = ping(server, timeout=2)
if ok:
print(
f'Connection successful on retry {i + 1}, process will now continue.')
bcontinue = True
break
else:
print(f'Retry {i + 1} out of 5 timed out' + (i != 4)
* ', reattempting in 1 second.' + (i == 4) * '.')
time.sleep(1)
if not bcontinue:
print('Failed to connect server, connection timed out, process will now stop')
return
bcontinue = False
head = requests.head(url)
if head.status_code == 200:
bcontinue = True
else:
for i in range(5):
print(f'Server responce is invalid, retrying {i + 1} out of 5')
head = requests.head(url)
if head.status_code == 200:
print(
f'Connection successful on retry {i + 1}, process will now continue.')
bcontinue = True
break
else:
print(f'Retry {i + 1} out of 5 failed to access data' +
(i != 4) * ', reattempting in 1 second.' + (i == 4) * '.')
time.sleep(1)
if not bcontinue:
print("Can't establish a connection with access to data, can't download target file, process will now stop.")
return
folder = '/'.join(filepath.split('/')[:-1])
Path(folder).mkdir(parents=True, exist_ok=True)
headers = head.headers
total = headers.get('content-length')
if not total:
print(
f'Cannot find the total length of the content of {url}, the file will be downloaded using a single thread.')
started = datetime.now()
print('Task started on %s.' %
started.strftime('%Y-%m-%d %H:%M:%S'))
sd = self.Singledown()
th = Thread(target=sd.singledown, args=(url, path))
threads.append(sd)
th.start()
total = inf
singlethread = True
else:
total = int(total)
if not headers.get('accept-ranges'):
print(
'Server does not support the `range` parameter, the file will be downloaded using a single thread.')
started = datetime.now()
print('Task started on %s.' %
started.strftime('%Y-%m-%d %H:%M:%S'))
sd = self.Singledown()
th = Thread(target=sd.singledown, args=(url, path))
threads.append(sd)
th.start()
singlethread = True
else:
segment = total / num_connections
started = datetime.now()
lastpressed = started
path.touch()
file = path.open('wb')
file.seek(total - 1)
file.write(b'\0')
file.close()
file = path.open(mode='r+b')
self.mm = mmap(file.fileno(), 0)
print('Task started on %s.' %
started.strftime('%Y-%m-%d %H:%M:%S'))
self.progress['total'] = total
self.progress['connections'] = num_connections
for i in range(num_connections):
md = self.Multidown(self, i)
start = int(segment * i)
end = int(segment * (i + 1)) - (i != num_connections - 1)
length = end - start + (i != num_connections - 1)
th = Thread(target=md.multidown, args=(
url, start, end))
threads.append(md)
self.progress[i] = dict()
self.progress[i]['start'] = start
self.progress[i]['position'] = start
self.progress[i]['end'] = end
self.progress[i]['count'] = 0
self.progress[i]['length'] = length
self.progress[i]['completed'] = False
th.start()
Path(filepath + '.progress.json').write_text(json.dumps(self.progress, indent=4))
downloaded = 0
totalMiB = total / 1048576
speeds = []
interval = 0.04
with output(initial_len=5, interval=0) as dynamic_print:
while True:
Path(filepath + '.progress.json').write_text(json.dumps(self.progress, indent=4))
status = sum([i.completed for i in threads])
downloaded = sum(i.count for i in threads)
self.recent.append(downloaded)
done = int(100 * downloaded / total)
doneMiB = downloaded / 1048576
gt0 = len([i for i in self.recent if i])
if not gt0:
speed = 0
else:
recent = list(self.recent)[12 - gt0:]
if len(recent) == 1:
speed = recent[0] / 1048576 / interval
else:
diff = [b - a for a, b in zip(recent, recent[1:])]
speed = sum(diff) / len(diff) / 1048576 / interval
speeds.append(speed)
self.recentspeeds.append(speed)
nzspeeds = [i for i in speeds if i]
if nzspeeds:
minspeed = min(nzspeeds)
else:
minspeed = 0
maxspeed = max(speeds)
now = datetime.now()
elapsed = (now - started).total_seconds()
meanspeed = downloaded / elapsed / 1048576
remaining = totalMiB - doneMiB
dynamic_print[0] = '[{0}{1}] {2}'.format(
'\u2588' * done, '\u00b7' * (100-done), str(done)) + '% completed' + (not singlethread) * ', paused: {0}'.format(self.paused)
dynamic_print[1] = 'Download mode: ' + singlethread * \
'Single-thread' + (not singlethread) * 'Multi-thread (press Space to pause/resume, press Escape to stop)'
dynamic_print[2] = '{0:.2f} MiB downloaded, {1:.2f} MiB total, {2:.2f} MiB remaining, download speed: {3:.2f} MiB/s'.format(
doneMiB, totalMiB, remaining, speed)
if speed and total != inf:
eta = timestring(remaining / speed)
else:
eta = '99:59:59'
dynamic_print[3] = 'Minimum speed: {0:.2f} MiB/s, average speed: {1:.2f} MiB/s, maximum speed: {2:.2f} MiB/s'.format(
minspeed, meanspeed, maxspeed)
dynamic_print[4] = 'Task started on {0}, {1} elapsed, ETA: {2}'.format(
started.strftime('%Y-%m-%d %H:%M:%S'), timestring(elapsed), eta)
if keyboard.is_pressed('space'):
if not singlethread:
pressed = datetime.now()
if (pressed - lastpressed).total_seconds() > 0.5:
lastpressed = pressed
if self.paused:
for i, md in enumerate(threads):
if not md.completed:
th = Thread(target=md.multidown, args=(
url, self.progress[i]['position'], self.progress[i]['end']))
th.start()
self.paused = not self.paused
if keyboard.is_pressed('esc'):
if not singlethread:
ended = datetime.now()
self.paused = True
break
if status == len(threads):
if not singlethread:
self.mm.close()
ended = datetime.now()
break
time.sleep(interval)
time_spent = (ended - started).total_seconds()
meanspeed = total / time_spent / 1048576
status = sum([i.completed for i in threads])
if status == len(threads):
print('Task completed on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))
else:
print('Task interrupted on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))
if __name__ == '__main__':
d = Downloader()
d.download(*sys.argv[1:])
For testing purposes this is a dumbed-down version of the script, with all checks removed while retaining the same functionality (sorry it really takes all these lines to show the download information):
import json
import os
import requests
import sys
import time
from collections import deque
from datetime import datetime, timedelta
from math import inf
from mmap import mmap
from pathlib import Path
from reprint import output
from threading import Thread
def timestring(sec):
sec = int(sec)
m, s = divmod(sec, 60)
h, m = divmod(m, 60)
return f'{h:02d}:{m:02d}:{s:02d}'
class Downloader:
def __init__(self):
self.recent = deque([0] * 12, maxlen=12)
self.recentspeeds = deque([0] * 200, maxlen=200)
self.paused = False
self.progress = dict()
self.UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0'
class Multidown:
def __init__(self, obj, id):
self.count = 0
self.position = 0
self.completed = False
self.id = id
self.parent = obj
self.UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0'
def multidown(self, url, start, end):
interrupted = False
s = requests.session()
s.headers.update({'connection': 'close', 'user-agent': self.UA})
r = s.get(
url, headers={'range': 'bytes={0}-{1}'.format(start, end)}, stream=True)
length = int(r.headers['content-length'])
while end - length + (self.id != self.parent.progress['connections'] - 1) != start or r.status_code != 206:
r.close()
s.close()
del r
del s
time.sleep(0.02)
s = requests.session()
r = s.get(
url, headers={'range': 'bytes={0}-{1}'.format(start, end)}, stream=True)
length = int(r.headers['content-length'])
self.position = start
for chunk in r.iter_content(1048576):
if self.parent.paused:
self.parent.mm.flush()
r.connection.close()
r.close()
s.close()
del r
del s
interrupted = True
break
if chunk:
self.parent.mm[self.position: self.position+len(chunk)] = chunk
self.count += len(chunk)
self.position += len(chunk)
self.parent.progress[self.id]['count'] = self.count
self.parent.progress[self.id]['position'] = self.position
if not interrupted:
r.close()
s.close()
if self.count == self.parent.progress[self.id]['length']:
self.completed = True
self.parent.progress[self.id]['completed'] = True
self.parent.mm.flush()
def download(self, url, filepath, num_connections=32, overwrite=False):
singlethread = False
threads = []
bcontinue = False
filepath = filepath.replace('\\', '/')
if Path(filepath).exists():
os.remove(filepath)
folder = '/'.join(filepath.split('/')[:-1])
Path(folder).mkdir(parents=True, exist_ok=True)
head = requests.head(url, headers={'user-agent': self.UA})
path = Path(filepath)
headers = head.headers
total = headers.get('content-length')
if total:
total = int(total)
if headers.get('accept-ranges'):
segment = total / num_connections
started = datetime.now()
lastpressed = started
path.touch()
file = path.open('wb')
file.seek(total - 1)
file.write(b'\0')
file.close()
file = path.open(mode='r+b')
self.mm = mmap(file.fileno(), 0)
print('Task started on %s.' %
started.strftime('%Y-%m-%d %H:%M:%S'))
self.progress['total'] = total
self.progress['connections'] = num_connections
for i in range(num_connections):
md = self.Multidown(self, i)
start = int(segment * i)
end = int(segment * (i + 1)) - (i != num_connections - 1)
length = end - start + (i != num_connections - 1)
th = Thread(target=md.multidown, args=(
url, start, end))
threads.append(md)
self.progress[i] = dict()
self.progress[i]['start'] = start
self.progress[i]['position'] = start
self.progress[i]['end'] = end
self.progress[i]['count'] = 0
self.progress[i]['length'] = length
self.progress[i]['completed'] = False
th.start()
Path(filepath + '.progress.json').write_text(json.dumps(self.progress, indent=4))
downloaded = 0
totalMiB = total / 1048576
speeds = []
interval = 0.04
with output(initial_len=5, interval=0) as dynamic_print:
while True:
Path(filepath + '.progress.json').write_text(json.dumps(self.progress, indent=4))
status = sum([i.completed for i in threads])
downloaded = sum(i.count for i in threads)
self.recent.append(downloaded)
done = int(100 * downloaded / total)
doneMiB = downloaded / 1048576
gt0 = len([i for i in self.recent if i])
if not gt0:
speed = 0
else:
recent = list(self.recent)[12 - gt0:]
if len(recent) == 1:
speed = recent[0] / 1048576 / interval
else:
diff = [b - a for a, b in zip(recent, recent[1:])]
speed = sum(diff) / len(diff) / 1048576 / interval
speeds.append(speed)
self.recentspeeds.append(speed)
nzspeeds = [i for i in speeds if i]
if nzspeeds:
minspeed = min(nzspeeds)
else:
minspeed = 0
maxspeed = max(speeds)
now = datetime.now()
elapsed = (now - started).total_seconds()
meanspeed = downloaded / elapsed / 1048576
remaining = totalMiB - doneMiB
dynamic_print[0] = '[{0}{1}] {2}'.format(
'\u2588' * done, '\u00b7' * (100-done), str(done)) + '% completed' + (not singlethread) * ', paused: {0}'.format(self.paused)
dynamic_print[1] = 'Download mode: ' + singlethread * \
'Single-thread' + (not singlethread) * 'Multi-thread (press Space to pause/resume, press Escape to stop)'
dynamic_print[2] = '{0:.2f} MiB downloaded, {1:.2f} MiB total, {2:.2f} MiB remaining, download speed: {3:.2f} MiB/s'.format(
doneMiB, totalMiB, remaining, speed)
if speed and total != inf:
eta = timestring(remaining / speed)
else:
eta = '99:59:59'
dynamic_print[3] = 'Minimum speed: {0:.2f} MiB/s, average speed: {1:.2f} MiB/s, maximum speed: {2:.2f} MiB/s'.format(
minspeed, meanspeed, maxspeed)
dynamic_print[4] = 'Task started on {0}, {1} elapsed, ETA: {2}'.format(
started.strftime('%Y-%m-%d %H:%M:%S'), timestring(elapsed), eta)
if PAUSE:
if not singlethread:
pressed = datetime.now()
if (pressed - lastpressed).total_seconds() > 0.5:
lastpressed = pressed
if self.paused:
for i, md in enumerate(threads):
if not md.completed:
th = Thread(target=md.multidown, args=(
url, self.progress[i]['position'], self.progress[i]['end']))
th.start()
self.paused = not self.paused
if status == len(threads):
if not singlethread:
self.mm.close()
ended = datetime.now()
break
time.sleep(interval)
time_spent = (ended - started).total_seconds()
meanspeed = total / time_spent / 1048576
status = sum([i.completed for i in threads])
if status == len(threads):
print('Task completed on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))
else:
print('Task interrupted on {0}, total time elapsed: {1}, average speed: {2:.2f} MiB/s'.format(
ended.strftime('%Y-%m-%d %H:%M:%S'), timestring(time_spent), meanspeed))
if __name__ == '__main__':
import hashlib
global PAUSE
PAUSE = False
chash = '5674E59283D95EFE8C88770515A9BBC80CBB77CB67602389FD91DEF26D26AED2'
d = Downloader()
if sys.argv[1] == '0':
d.download('http://ipv4.download.thinkbroadband.com/1GB.zip', 'C:/test/1GB.zip')
elif sys.argv[1] == '1':
th1 = Thread(target=d.download, args=('http://ipv4.download.thinkbroadband.com/1GB.zip', 'C:/test/1GB.zip'))
th1.start()
def test():
while th1.is_alive():
global PAUSE
PAUSE = not PAUSE
time.sleep(10)
th2 = Thread(target=test)
th2.start()
while th1.is_alive():
pass
sha256_hash = hashlib.sha256()
with open('C:/test/1GB.zip',"rb") as f:
for byte_block in iter(lambda: f.read(1048576),b""):
sha256_hash.update(byte_block)
print(sha256_hash.hexdigest().lower() == chash.lower())
The url isn't accessible without a VPN in my locale, and test 0 always results True, that is, if the connection hasn't gone dead during the download, and test 1 sometimes results True, sometimes results False, sometimes it doesn't finish(progress bar goes beyond 100%)...
How can my code be salvaged?
This might not be your only problem but you have a race condition that could show up if you pause and resume quickly (where the definition of quickly varies greatly depending on your circumstances). Consider that you've got 32 threads each requesting a MB chunk, let's call them threads 0-31. They are sitting their downloading and you pause. The threads do not know that you paused until they get a chunk of data as they are sitting in blocking io. Not sure what speed your connection is or how many cores your machine has (threads will sometimes act in parallel when they don't need the GIL,) but this process could take a lot longer than you expect. Then you unpause and your code creates new threads 32-63 but some or all of threads 0-31 are still waiting for the next chunk. You set threads 32-63 in motion and then you turn off your pause flag. Those threads that didn't end from 0-31 then wake up and see that things aren't paused. Now you have multiple threads accessing the same state variables
self.parent.mm[self.position: self.position + len(chunk)] = chunk
self.count += len(chunk)
self.position += len(chunk)
self.parent.progress[self.id]['count'] = self.count
self.parent.progress[self.id]['position'] = self.position
so if thread 0 is downloading the same chunk as thread 31 they both keep updating all the same state and they add to position and count even though they are downloading overlapping parts of the file. You even reuse the objects that the threads live inside of so that state can get really really messed up.
for i, md in enumerate(threads):
if not md.completed:
th = Thread(target=md.multidown, args=(url, self.progress[i]['position'], self.progress[i]['end']))
th.start()
There might be some other problems in your code and it is a lot to sort through so I suggest taking the time to do some refactoring to eliminate duplicate code and organise things into more functions. I don't believe in crazy tiny functions, but you could use a few sub functions like download_multi(download_state) and download_single maybe. I am relatively confident however that your current problem will be solved if you ensure the threads you have running actually end after you pause. To do so you need to actually hold references to your threads
somewhere:
actual_threads = []
When you create your threads (the first time and after you unpause, or preferably this would be in a function and you'd do it there and return the list):
th = Thread(target=md.multidown, args=(
url, start, end))
threads.append(md)
actual_threads.append(th)
Then when you unpause:
self.paused = not self.paused
for th in actual_threads:
th.join()
This way you have the threads working, they quit when you pause and you rebuild them. So join should return as soon as they break out of the blocking io call to iter_content. This way those threads are always dead before you make the new ones.
What I would do myself however would be to create sockets from each thread to the main process. When pause is detected the threads shut down the request and save any data that's already waiting in the OS buffer then go into a blocking receive on the socket (there might be a way to use select with a socket and requests to allow you to even break out of the blocking io involved in r.iter_content immediately but I leave that for your research). When the program is unpaused the main process would send some value to indicate the program should restart (you'd want at least two signals the threads would recognise, one for quitting gracefully and one to resume. The codes can be single characters.) When the value is sent to each thread that thread will unblock and can then restart the download using requests and its previous state like nothing happened.
I've two separate code bases for Tf-mini(Lidar) and servo motor. I want the both code to run at the same time. I tried to combine them in to a single code, But only one code runs at a time. Please help to rewrite or edit the code.
Code for servo motor here:
import RPi.GPIO as GPIO
import time
GPIO.setwarnings(False)
GPIO.setmode(GPIO.BOARD)
GPIO.setup(7, GPIO.OUT)
p = GPIO.PWM(7, 100)
t = 0.007
r = 20
p.start(r)
while True:
for i in range(5,r):
p.ChangeDutyCycle(i)
print(i)
time.sleep(t)
for i in range(r,5,-1):
p.ChangeDutyCycle(i)
print(i)
time.sleep(t)`
Code for lidar here:
import time
import serial
ser = serial.Serial("/dev/ttyAMA0", 115200)
def read_data():
while True:
counter = ser.in_waiting
if counter > 8:
bytes_serial = ser.read(9)
ser.reset_input_buffer()
if bytes_serial[0] == 0x59 and bytes_serial[1] == 0x59: # this portion is for python3
print("Printing python3 portion")
distance = bytes_serial[2] + bytes_serial[3]*256 # multiplied by 256, because the binary
data is shifted by 8 to the left (equivalent to "<< 8").
# Dist_L, could simply be added resulting in 16-bit data of Dist_Total.
strength = bytes_serial[4] + bytes_serial[5]*256
temperature = bytes_serial[6] + bytes_serial[7]*256
temperature = (temperature/8) - 256
print("Distance:"+ str(distance))
print("Strength:" + str(strength))
if temperature != 0:
print("Temperature:" + str(temperature))
ser.reset_input_buffer()
if bytes_serial[0] == "Y" and bytes_serial[1] == "Y":
distL = int(bytes_serial[2].encode("hex"), 16)
distH = int(bytes_serial[3].encode("hex"), 16)
stL = int(bytes_serial[4].encode("hex"), 16)
stH = int(bytes_serial[5].encode("hex"), 16)
distance = distL + distH*256
strength = stL + stH*256
tempL = int(bytes_serial[6].encode("hex"), 16)
tempH = int(bytes_serial[7].encode("hex"), 16)
temperature = tempL + tempH*256
temperature = (temperature/8) - 256
print("Printing python2 portion")
print("Distance:"+ str(distance) + "\n")
print("Strength:" + str(strength) + "\n")
print("Temperature:" + str(temperature) + "\n")
ser.reset_input_buffer()
if __name__ == "__main__":
try:
if ser.isOpen() == False:
ser.open()
read_data()
except KeyboardInterrupt(): # ctrl + c in terminal.
if ser != None:
ser.close()
print("program interrupted by the user")
Please help me to combine these two code to run at the same time.
I think I see your problem. Both segments of code are sequentially driven and hence it will create problems. Here is what you need to do (pseudo code):
loop forever:
t1=start_time
change duty cycle #remove the wait code
poll the serial port #the ladar code
t2=end_time
if t2 - t1 < t:
wait (t - (t2-t1))
Being an old Arduino fan myself, I see the struggles of handling the servo motor. I have a simple library to do most of the dirty work for me. You are most welcome to use my library via any of the two links:
https://github.com/vikramdayal/RaspberryMotors or https://pypi.org/project/RaspberryMotors/#description
I am wondering how to calculate the amount of time it would take to example:
Complete a brute force word list.
I know how to use the time function and measure in time,
but the problem is i need to find out how long it would take in the program itself...
Here is the code i made this yesterday
import itertools, math
import os
Alphabet = ("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") # Add or remove whatevs you think will be in the password you're cracking (example, [symbols])
counter = 1
CharLength = 1
range_num = int(raw_input("Enter range: "))
stopper = range_num + 1
filename = "bruteforce_%r.txt" % (range_num)
f = open(filename, 'a')
#n_1 = len(Alphabet)
#n_2 = n_1 - 1 # <-- total useless peice of garbage that could of been great in vurtual life
#n_3 = '0' * n_2
#n = '1' + n_3
x = range_num
y = len(Alphabet)
amount = math.pow(y, x)
total_items = math.pow(y, x)
for CharLength in range(range_num, stopper):
passwords = (itertools.product(Alphabet, repeat = CharLength))
for i in passwords:
counter += 1
percentage = (counter / total_items) * 100
amount -= 1
i = str(i)
i = i.replace("[", "")
i = i.replace("]", "")
i = i.replace("'", "")
i = i.replace(" ", "")
i = i.replace(",", "")
i = i.replace("(", "")
i = i.replace(")", "")
f.write(i)
f.write('\n')
print "Password: %r\tPercentage: %r/100\tAmount left: %r" % (i, int(percentage), amount)
if i == '0'* range_num:
print "*Done"
f.close()
exit(0)
else:
pass
This is my timer function i managed to make
#import winsound # Comment this out if your using linux
import os
import time
from sys import exit
print "This is the timer\nHit CTRL-C to stop the timer\nOtherwise just let it rip untill the time's up"
hours = int(raw_input('Enter the hours.\n>>> '))
os.system('clear') # Linux
#os.system('cls') # Windows
minutes = int(raw_input('Enter the minutes.\n>>> '))
os.system('clear') # linux
#os.system('cls') # Windows
seconds = int(raw_input('Enter the seconds.\n>>> '))
os.system('clear') # Linux
#os.system('cls') # Windows
stop_time = '%r:%r:%r' % (hours, minutes, seconds)
t_hours = 00
t_minutes = 00
t_seconds = 00
while t_seconds <= 60:
try:
os.system('clear') # Linux
#os.system('cls') # Windows
current_time = '%r:%r:%r' % (t_hours, t_minutes, t_seconds)
print current_time
time.sleep(1)
t_seconds+=1
if current_time == stop_time:
print "// Done"
#winsound.Beep(500,1000)
#winsound.Beep(400,1000)
break
elif t_seconds == 60:
t_minutes+=1
t_seconds=0
elif t_minutes == 60:
t_hours+=1
t_minutes = 00
except KeyboardInterrupt:
print "Stopped at: %r:%r:%r" % (t_hours, t_minutes, t_seconds)
raw_input("Hit enter to continue\nHit CTRL-C to end")
try:
pass
except KeyboardInterrupt:
exit(0)
Now i just cant figure out how to make this again but to calculate how long it will take rather than how long it is taking...
You cannot predict the time a script is going to take.
Firstly because two machines wouldn't run the script in the same time, and secondly, because the execution time on one machine can vary from on take to another.
What you can do, however, is compute the percentage of execution.
You need to figure out, for example, how many iterations your main loop will do, and calculate at each iteration the ratio current iteration count / total number of iterations.
Here is a minimalist example of what you can do:
n = 10000
for i in range(n):
print("Processing file {} ({}%)".format(i, 100*i//n))
process_file(i)
You can take it further and add the time as an additional info:
n = 10000
t0 = time.time()
for i in range(n):
t1 = time.time()
print("Processing file {} ({}%)".format(i, 100*i//n), end="")
process_file(i)
t2 = time.time()
print(" {}s (total: {}s)".format(t2-t1, t2-t0))
The output will look like this:
...
Processing file 2597 (25%) 0.2s (total: 519.4s)
Processing file 2598 (25%) 0.3s (total: 519.7s)
Processing file 2599 (25%) 0.1s (total: 519.8s)
Processing file 2600 (25%)
This is my implementation, which returns time elapsed, time left, and finish time in H:M:S format.
def calcProcessTime(starttime, cur_iter, max_iter):
telapsed = time.time() - starttime
testimated = (telapsed/cur_iter)*(max_iter)
finishtime = starttime + testimated
finishtime = dt.datetime.fromtimestamp(finishtime).strftime("%H:%M:%S") # in time
lefttime = testimated-telapsed # in seconds
return (int(telapsed), int(lefttime), finishtime)
Example:
import time
import datetime as dt
start = time.time()
cur_iter = 0
max_iter = 10
for i in range(max_iter):
time.sleep(5)
cur_iter += 1
prstime = calcProcessTime(start,cur_iter ,max_iter)
print("time elapsed: %s(s), time left: %s(s), estimated finish time: %s"%prstime)
Output:
time elapsed: 5(s), time left: 45(s), estimated finish time: 14:28:18
time elapsed: 10(s), time left: 40(s), estimated finish time: 14:28:18
time elapsed: 15(s), time left: 35(s), estimated finish time: 14:28:18
....
You will never ever be able to know exactly how long it is going to take to finish. The best you can do is calculate was percentage of the work you have finished and how long that has taken you and then project that out.
For example if you are doing some work on the range of numbers from 1 to 100 you could do something such as
start_time = get the current time
for i in range(1, 101):
# Do some work
current_time = get the current time
elapsed_time = current_time - start_time
time_left = 100 * elapsed_time / i - elapsed_time
print(time_left)
Please understand that the above is largely pseudo-code
The following function will calculate the remaining time:
last_times = []
def get_remaining_time(i, total, time):
last_times.append(time)
len_last_t = len(last_times)
if len_last_t > 5:
last_times.pop(0)
mean_t = sum(last_times) // len_last_t
remain_s_tot = mean_t * (total - i + 1)
remain_m = remain_s_tot // 60
remain_s = remain_s_tot % 60
return f"{remain_m}m{remain_s}s"
The parameters are:
i : The current iteration
total : the total number of iterations
time : the duration of the last iteration
It uses the average time taken by the last 5 iterations to calculate the remaining time. You can the use it in your code as follows:
last_t = 0
iterations = range(1,1000)
for i in iterations:
t = time.time()
# Do your task here
last_t = time.time() - t
get_remaining_time(i, len(iterations), last_t)
I have already made a program that prints the max amplitude or volume, then I made a program that only prints the volume when the threshold is above 2300, now I want to make a program that closes the stream when the threshold is below 2300 for 2 seconds, so not right after it but in 2 seconds only if the threshold hasn't raised again within 2 seconds. Here is what I have:
import pyaudio
import struct
import audioop
import time
INITIAL_THRESHOLD = 0.010
FORMAT = pyaudio.paInt16
SHORT_NORMALIZE = (1.0/32768.0)
CHANNELS = 2
RATE = 44100
INPUT_BLOCK_TIME = 0.05
INPUT_FRAMES_PER_BLOCK = int(RATE*INPUT_BLOCK_TIME)
OVERSENSITIVE = 15.0/INPUT_BLOCK_TIME
UNDERSENSITIVE = 120.0/INPUT_BLOCK_TIME
MAX_BLOCKS = 0.15/INPUT_BLOCK_TIME
class TEST(object):
def __init__(self):
self.pa = pyaudio.PyAudio()
self.stream = self.open_mic_stream()
self.tap_threshold = INITIAL_THRESHOLD
self.noisycount = MAX_BLOCKS+1
self.quietcount = 0
self.errorcount = 0
def stop(self):
self.stream.close()
def find_input_device(self):
device_index = None
for i in range( self.pa.get_device_count() ):
devinfo = self.pa.get_device_info_by_index(i)
print( "Device %d: %s"%(i,devinfo["name"]) )
for keyword in ["mic","input"]:
if keyword in devinfo["name"].lower():
print( "Found an input: device %d - %s"%(i,devinfo["name"]) )
device_index = i
return device_index
if device_index == None:
print( "No preferred input found; using default input device." )
return device_index
def open_mic_stream( self ):
device_index = self.find_input_device()
stream = self.pa.open( format = FORMAT,
channels = CHANNELS,
rate = RATE,
input = True,
input_device_index = device_index,
frames_per_buffer = INPUT_FRAMES_PER_BLOCK)
return stream
def listen(self):
try:
chunk = self.stream.read(INPUT_FRAMES_PER_BLOCK)
except IOError, e:
self.errorcount += 1
print( "(%d) Error recording: %s"%(self.errorcount,e) )
self.noisycount = 1
return
while True:
mx = audioop.max(chunk, 2)
if mx > 2300: #print the volume level whenever the volume is above 2300
print(mx)
elif mx < 2300: #If the volume is below 2300:
time.sleep(2) #Wait two seconds
if mx > 2300: #If after two seconds the volume is back up, repeat
continue
elif mx < 2300: #If after two seconds the volume is still down, break the loop
break
print("You're Done")
self.stream.close()
print("Stream Closed")
if __name__ == "__main__":
tt = TEST()
for i in range(1000):
tt.listen()
All that does is either print the initial volume level, over and over forever, or it just closes the stream immediately. This depends on whether there is sound when I start the program or not.
EDIT:
While the methods posted in my answer outline ways to break your loop, the real reason your program is not working, is not because of the loop, its because the portion of code where you actually read the audio input is not even in your loop.
You need to read the input stream each time around the loop otherwise you will just keep making a check against the value when when the listen method is called.
def listen(self):
while True:
try:
chunk = self.stream.read(INPUT_FRAMES_PER_BLOCK)
except IOError, e:
self.errorcount += 1
print( "(%d) Error recording: %s"%(self.errorcount,e) )
self.noisycount = 1
return
mx = audioop.max(chunk, 2)
if mx > 2300: #print the volume level whenever the volume is above 2300
print(mx)
elif mx < 2300: #If the volume is below 2300:
time.sleep(2) #Wait two seconds
if mx > 2300: #If after two seconds the volume is back up, repeat
continue
You need to listen during those two seconds, instead of passively waiting.
start = -1
while True:
try:
chunk = self.stream.read(INPUT_FRAMES_PER_BLOCK)
except IOError, e:
self.errorcount += 1
print( "(%d) Error recording: %s"%(self.errorcount,e) )
self.noisycount = 1
return
mx = audioop.max(chunk, 2)
if mx > 2300: #print the volume level whenever the volume is above 2300
print(mx)
start = -1
elif mx < 2300: #If the volume is below 2300:
if start < 0: # INITIALIZE
start = time.time()
else:
if time.time() - start >= 2: #Wait two seconds
break
The problem is you aren't updating mx while (or after) you sleep. You should set a variable for when you last got a big sound, and break/restart based on new samples
last_sound = time.time() # time of last loud sound
while True:
mx = audioop.max(chunk, 2) # new sample
if mx > 2300:
print(mx)
last_sound = time.time() # reset the "timer" because we're loud
else: # I changed this to else
now = time.time()
if now - last_sound >= 2:
break # two seconds of silence!
else:
print(now-last_sound) # this should count up to 2