Develop Reference

Python multiprocessing shared memory without copy

I'm currently working on a project for fun, involving calculating way too many numbers of the Fibonacci sequence. Thing is, I want it to go fast and I also don't want to loose too much progress if I have to do an update or have a computer issues....
So I over-engineered that, and I am currently looking to implement it with multiprocessing, but everywhere I look the only way to share memory in that case is to make a duplicate of it and that takes about an hour for (Yeah I have very big numbers)
Is there any way to share a dictionary with multiprocessing without making a copy of it?
Here is the code, currently with my last implementation using threading instead of using multiprocessing. (ignore the bad timekeeping of this, i'll fix it later)
import threading
import time
def printProgressBar(iteration, total, prefix="", suffix="", decimals=1, length=100, fill="█", printEnd="\r"):
"""
Call in a loop to create terminal progress bar
#params:
iteration - Required : current iteration (Int)
total - Required : total iterations (Int)
prefix - Optional : prefix string (Str)
suffix - Optional : suffix string (Str)
decimals - Optional : positive number of decimals in percent complete (Int)
length - Optional : character length of bar (Int)
fill - Optional : bar fill character (Str)
printEnd - Optional : end character (e.g. "\r", "\r\n") (Str)
"""
percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
filledLength = int(length * iteration // total)
bar = fill * filledLength + "-" * (length - filledLength)
print(
f"\r{prefix} |{bar}| {percent}% {suffix} {time.strftime('%dd %H:%M:%S', time.gmtime(time.time() - start_time - 86400)).replace('31d', '0d')}",
end=printEnd,
)
# Print New Line on Complete
if iteration == total:
print()
dict47 = {0: 0, 1: 1, 2: 1}
dict47[0] = 2
def fibSequence(n):
if n not in dict47:
dict47[n] = fibSequence(n - 1) + fibSequence(n - 2)
if n > dict47[0]:
dict47[0] = n
if n > 5 and not ((n - 3) % (iterations // 100) == 0) and not ((n - 2) % (iterations // 100) == 0):
dict47.pop(n - 3, None)
return dict47[n]
def makeBackup(start, num, total):
x = threading.Thread(target=writeBackup, args=(num,), daemon=False)
y = threading.Thread(target=writeBackup, args=(num - 1,), daemon=False)
x.start()
y.start()
x.join()
y.join()
time.sleep(1)
print(
f'{num/10000000}% done after {time.strftime("%dd %H:%M:%S", time.gmtime(time.time() - start - 86400)).replace("31d", "0d")}'
)
timings = open("times.txt", "a")
timings.write(str(int(time.time() - start)) + "\n")
timings.close()
def writeBackup(num):
file = open(f".temp/fib{num}.txt", "a")
file.write(str(num) + " : " + str(dict47[num]))
file.close()
dict47.pop(num, None)
def loadDict():
from pathlib import Path
maximum = 0
for n in range(1, 100):
if Path(f".temp/fib{n*10000000}.txt").is_file():
maximum = n * 10000000
print("Maximum number found:", maximum)
if maximum != 0:
file = open(f".temp/fib{maximum}.txt", "r")
temp = "".join(file.readlines())
dict47[maximum] = int(temp.lstrip(str(maximum) + " : "))
file.close()
file = open(f".temp/fib{maximum - 1}.txt", "r")
temp = "".join(file.readlines())
dict47[maximum - 1] = int(temp.lstrip(str(maximum - 1) + " : "))
file.close()
dict47[0] = maximum
print("Dictionary loaded at ", maximum)
else:
print("No dictionary found, starting from scratch")
if __name__ == "__main__":
try:
timings = open("times.txt", "r")
lastTime = int(timings.readlines()[-1])
start_time = time.time() - lastTime
timings.close()
except:
start_time = time.time() + 86400
print("Start duration:", time.strftime("%dd %H:%M:%S", time.gmtime(time.time() - start_time)).replace("31d", "0d"))
try:
iterations = int(input("Enter the number of iterations: "))
except:
iterations = 1000000000
print(iterations, "iterations will be performed")
loadDict()
num = dict47[0]
while num < iterations:
if num == 2:
num += 248
else:
num += 250
fibSequence(num)
if num % 1000 == 0:
printProgressBar(num, iterations, prefix="Progress:", suffix="Complete", length=100)
if num % (iterations // 100) == 0:
save = threading.Thread(
target=makeBackup,
args=(start_time, num, iterations),
daemon=False,
)
save.start()
file = open("fib.txt", "a")
file.write(str(iterations) + " : " + str(fibSequence(iterations)) + "\n")
file.close()
try:
save.join()
except:
print("No save thread running, exiting...")

fastest way to check a .rar file password - other than rarfile extractall

So i have written a little .rar password "cracker" based on tutorials, using the code underneath. It works fine, but is very slow when the file size is big. The best reason i could find is, that ever so often when you put in a wrong password, it extracts the whole file, before refusing the password. With small files that is not a big problem, but with big files it slows the process a lot.
Is there a way to just check a hashed version of the password against a iterated hash?
import itertools
import rarfile
import time
rarfile.UNRAR_TOOL = "path"
rar = rarfile.RarFile("path")
done = False
n = 0
inputList = ["A","B","1","2"]
class h():
startword = ""
rep = 1
start = 0
itrTot = 0
f = open("save.txt")
for x,each in enumerate(f):
if x == 0:
h.rep = int(each)
else:
h.start = int(each)-3
f.close()
if h.start < 0:
h.start = 0
h.itrTot = len(inputList)**h.rep
def pw_guess():
res = itertools.product(inputList, repeat=h.rep)
for guess in res:
yield guess
start_time = time.time()
while True:
guess_generator = pw_guess()
for guess in guess_generator:
n += 1
if h.startword == "":
h.startword = guess
else:
if guess == h.startword:
h.rep += 1
n = 0
h.itrTot = len(inputList)**h.rep
h.start = 0
print("next rotation, itr rep: "+str(h.rep))
h.startword = ""
break
if n < h.start:
continue
txt = f"({n}/{h.itrTot}, {round((100/h.itrTot)*n,2)}%) - {h.rep}: {''.join(guess)}"
print(txt)
try:
rar.extractall(path="path",members=None,pwd=''.join(guess))
print("Pass found!")
print(str(n) + " - " + str(h.rep) + ": " + str(''.join(guess)))
done = True
txt2 = f"({n}/{h.itrTot}, {round((100/h.itrTot)*n,2)}%) - {h.rep}: {''.join(guess)}\n"
f = open("pass.txt", "a")
f.write(txt2)
f.close()
break
except:
f = open("save.txt", "w")
f.write(str(h.rep) + "\n" + str(n))
f.close()
if done:
end_time = time.time()
break
print("End time: " + str(end_time-start_time))

John the ripper is the answer. +20k passwords checked in 2 minutes. But using parts of the script for wordlist generation, is still very fast and functional.
wordlist generator i used:
import itertools
inputList = ["A","B","C","D","1","2","3","4","5"]
itr = 7
WL_path = "path"
f = open(WL_path,"w")
f.write("")
f.close()
class h():
startword = ""
rep = 1
itrTot = 0
txt = ""
h.itrTot = len(inputList)**itr
print("Wordlist length: " + str(h.itrTot))
def pw_guess():
res = itertools.product(inputList, repeat=h.rep)
for guess in res:
yield guess
while True:
guess_generator = pw_guess()
for guess in guess_generator:
if h.startword == "":
h.startword = guess
else:
if guess == h.startword:
h.rep += 1
print("next rotation, itr rep: " + str(h.rep))
h.startword = ""
break
h.txt = ''.join(guess)
f = open(WL_path, "a")
f.write(h.txt+"\n")
f.close()
if h.rep > itr:
break

Python 3 ASCII code 219 issue

I compared code to other training example code out there and did a compare and everything matches. Here is my issue: I have a health bar and when I take damage the bar decreases.
The problem is when in my code I use █ the pipe line after the bar never stays put, it always move dynamically. When I use / the pipe bar always stays put and no issue. I'm guessing there is something in my terminal tool (using Pycharm) that it doesn't like ascii code 219. If it matters from reading forums the tool is set to UTF-8. example below so might not format correctly but top part you can see the | being shifted when using █ and the bottom part is fine when using /.
______________________________ __________
CARLOS: 2210/3260 |■■■■■■■■■■■■■■■■■ | 132/132 |//////////|
__________________ __________
CARLOS: 2219/3260 |///////////////// | 132/132 |//////////|
Code:
def get_stats(self):
hp_bar = ""
bar_ticks = (self.hp / self.maxhp) * 100 / 4
mp_bar = ""
mp_ticks = (self.mp / self.maxmp) * 100 / 10
while bar_ticks > 0:
hp_bar += '█'
bar_ticks -= 1
#num_spaces_needed = (100/4) - len(hp_bar)
#str_spaces_needed = ""
#while num_spaces_needed > 0:
#str_spaces_needed += " "
#num_spaces_needed -= 1
while len(hp_bar) < 25:
hp_bar += " "
while mp_ticks > 0:
mp_bar += "/"
mp_ticks -= 1
while len(mp_bar) < 10:
mp_bar += " "
hp_string = str(self.hp) + "/" + str(self.maxhp)
current_hp = ""
if len(hp_string) < 9:
decreased = 9 - len(hp_string)
while decreased > 0:
current_hp += " "
decreased -= 1
current_hp += hp_string
else:
current_hp = hp_string
mp_string = str(self.mp) + "/" + str(self.maxmp)
current_mp = ""
if len(mp_string) < 7:
decreased = 7 - len(mp_string)
while decreased > 0:
current_mp += " "
decreased -= 1
current_mp += mp_string
else:
current_mp = mp_string
print(" _______________________________ __________ ")
print(bcolors.BOLD + self.name + " " +
current_hp + " |" + bcolors.BAR + hp_bar + bcolors.ENDC + "| " +
current_mp + " |" + bcolors.OKBLUE + mp_bar + bcolors.ENDC + "| ")

You have to change your font type
Navigate to File -> Settings -> Editor -> Font
Font: Source Code Pro
Apply -> Ok
Rerun the program

Try the following at the top of your script:
import sys
reload(sys)
sys.setdefaultencoding('utf8')

Python stange behaviour when acessing list elements

This small scripts makes exactly what I need.
#!/usr/bin/python
import os
import fileinput
import sys
import shutil
import glob
import time
def replaceAll1(files,searchExp,replaceExp):
for line in fileinput.input(files, inplace=1):
if searchExp in line:
line = line.replace(searchExp,replaceExp)
sys.stdout.write(line)
param1 = [1,2,3]
param2 = [1,2,3]
param3 = [1,2,3]
for i in xrange(len(param1)):
for ii in xrange(len(param2)):
for iii in xrange(len(param3)):
os.system("cp -a cold.in input.in")
old_param1 = "param1 = 1"
old_param2 = "param2 = 1"
old_param3 = "param3 = 1"
new_param1 = "param1 = " + str(param1[i])
new_param2 = "param2 = " + str(param2[ii])
new_param3 = "param3 = " + str(param3[iii])
replaceAll1('input.in',old_param1,new_param1)
replaceAll1('input.in',old_param2,new_param2)
replaceAll1('input.in',old_param3,new_param3)
time.sleep(4)
It enters in a configuration file and replaces sequentially the input parameters according to the lists that are accessed by the loop indexes. It is simple a combination of all the three parameters between each other.
# Input file
param1 = 1 # --- Should be [1,2,3]
param2 = 1 # --- Should be [1,2,3]
param3 = 1 # --- Should be [1,2,3]
The problem is that his big brother is not behaving like it. When it loops through the lists, it gets lost in scheme = 2 and puts dissp_scheme = 2 (freezed) when it should be dissp_scheme = 1. I printed out every single variable that goes inside the function replaceAll marked with comments but when I turn on the other calls it mess up everything. Here is the script.
#!/usr/bin/python
import os
import fileinput
import sys
import shutil
import glob
import time
os.chdir(os.getcwd())
# Replaces the input file parameters
def replaceAll(files,searchExp,replaceExp):
for line in fileinput.input(files, inplace=1):
if searchExp in line:
line = line.replace(searchExp,replaceExp)
sys.stdout.write(line)
# Gets a number inside my input file.
def get_parameter(variable,file_name):
f = open(file_name,'r').readlines()
for i in xrange(len(f)):
index = f[i].find(variable)
if index != -1:
pre_found = f[i].split('=')[1]
return pre_found
# Gets the discretization scheme name.
def get_sheme(number):
if number == 1:
return "Simple Centered Scheme"
elif number == 2:
return "Lax-Wendroff Scheme"
elif number == 3:
return "MacCormack Scheme"
elif number == 4:
return "Beam-Warming Scheme"
elif number == 5:
return "Steger-Warming 1st Order Scheme"
elif number == 6:
return "Steger-Warming 2nd Order Scheme"
elif number == 7:
return "Van Leer 1st Order Scheme"
elif number == 8:
return "Van Leer 2nd Order Scheme"
elif number == 9:
return "Roe Scheme"
elif number == 10:
return "AUSM Scheme"
# Gets the dissipation scheme name.
def get_dissip(number):
if number == 1:
return "Pullian Non-Linear dissipation"
elif number == 2:
return "Second difference dissipation"
elif number == 3:
return "Fourth difference dissipation"
elif number == 4:
return "B&W dissipation"
# Generates the density gnuplot scripts.
def gnuplot(variable,pressure_ratio,scheme,dissip_scheme):
#gnuplot('Density',10,get_sheme(3),'Pullian')
# Building name of the output file.
outFileName = variable.lower() + '_ratio' + str(int(pressure_ratio)) + '_' + scheme.replace(" ","") + '_dissp' + dissip_scheme.replace(" ","") + '.tex'
gnuFileName = variable.lower() + '_ratio' + str(int(pressure_ratio)) + '_' + scheme.replace(" ","") + '_dissp' + dissip_scheme.replace(" ","") + '.gnu'
# Build title of the plot
title = 'Analytical vs Numerical | ' + scheme
f = open(gnuFileName,'w')
f.write("set term cairolatex monochrome size 15.0cm, 8cm\n")
f.write('set output "' + outFileName + '"\n')
f.write("set grid\n")
f.write('set xtics font "Times-Roman, 10\n')
f.write('set ytics font "Times-Roman, 10\n')
f.write('set xlabel "x position" center\n')
f.write('set ylabel "' + variable + '" center\n')
f.write('set title "Analytical vs Numerical Results | ' + variable + '" \n')
f.write('set pointsize 0.5\n')
f.write('set key font ",10"\n')
fortran_out_analytical = 'a' + variable.lower() + '.out'
fortran_out_numerical = variable.lower() + 'Output.out'
f.write('plot "' + fortran_out_analytical +'" u 1:2 with linespoints lt -1 lw 1 pt 4 title "Analytical",\\\n')
f.write( '"' + fortran_out_numerical + '" u 1:2 with lines lw 5 title "Numerical"\n')
f.close()
# Generate latex code.
def generate_latex(text_image_file,caption):
latex.write("\\begin{figure}[H]\n")
latex.write(" \centering\n")
latex.write(" \input{" + text_image_file + "}\n")
latex.write(" \caption{"+ caption +"}\n")
latex.write(" \label{fig:digraph}\n")
latex.write("\\end{figure}\n")
latex.write("\n\n")
# -----------------------------------------------------------------------
# Main loop.
# -----------------------------------------------------------------------
pressure_ratios = [5.0]
schemes = [1,2,3]
dissips = [1,2,3]
# Define replace lines for replace all.
scheme_line = "scheme = "
dissip_line = "dissp_scheme = "
# Open Latex export file.
latex = open("bizu.txt",'w')
i = 0
# ratios.
for i in xrange(len(pressure_ratios)):
print "----------------------------------------"
print " + Configuring File for pressure ratio: " + str(pressure_ratios[i])
print "----------------------------------------\n"
# Schemes
for jj in xrange(len(schemes)):
print " + Configuring file for scheme: " + get_sheme(schemes[jj]) + "\n"
for kkk in xrange(len(dissips)):
print " + Configuring file for dissip: " + get_dissip(dissips[kkk])
# We always work with a brand new file.
os.system("rm input.in")
os.system("cp -a cold.in input.in")
# Replace pressures.
p1_line_old = 'p1 = 5.0d0'
rho1_line_old = 'rho1 = 5.0d0'
p1_line_new = 'p1 = ' + str(pressure_ratios[i]) + 'd0'
rho1_line_new = 'rho1 = ' + str(pressure_ratios[i]) + 'd0'
replaceAll('input.in',p1_line_old,p1_line_new)
replaceAll('input.in',rho1_line_old,rho1_line_new)
# Replace discretization scheme.
old_scheme = scheme_line + "1"
new_scheme = scheme_line + str(schemes[jj])
#==========================================================
# This call is messing everything up when scheme turns to 2
#==========================================================
replaceAll('input.in',old_scheme,new_scheme)
# Replace dissipation scheme.
old_dissp_scheme = dissip_line + "1"
new_dissp_scheme = dissip_line + str(dissips[kkk])
print p1_line_old
print new_scheme
print new_dissp_scheme
replaceAll('input.in',old_dissp_scheme, new_dissp_scheme)
time.sleep(3)
# ### Calling program
# os.system("./sod")
#
latex.close()
And the input file that the it works on is:
&PAR_physical
p1 = 5.0d0
p4 = 1.0d0
rho1 = 5.0d0
rho4 = 1.0d0
fgamma = 1.4d0
R_const = 287.0d0
F_Cp = 1004.5
F_Cv = 717.5
/
&PAR_geometry
total_mesh_points = 1001
start_mesh_point = -5.0d0
final_mesh_point = 5.0d0
print_step = 100
/
&PAR_numeric
scheme = 3
iterations = 10000
time_step = 0.0001d0
/
&PAR_dissip
dissp_scheme = 3
dissip_omega = 0.5d0
/
Thank you all !

Python For loop repeating endlessly

I'm currently writing a program to check speeds of cars and their license plates, and I want to repeat the function that does this x number of times, the problem I'm having though is that the function is repeating endlessly and is not adhering to the number of times I want it to loop.
Here is what I have so far:
if correctMatch:
pass
else:
with open('Camera Output.txt', 'a') as f:
print("DATA RECORDED TO: Camera Output.txt")
exactTime2 = datetime.now()
f.write("{} has a non-standard license plate and has been recorded at {}.".format(licensePlate,
exactTime2) + "\n")
f.write("---------------------------------------------------------\n")
if speedCarMph > 60:
with open('Camera Output.txt', 'a') as f:
print("DATA RECORDED TO: Camera Output.txt")
exactTime= datetime.now()
f.write("{} was travelling at {}MPH, recorded at {} and has broken the law.".format(licensePlate,
speedCarMph, exactTime) + "\n")
f.write("----------------------------------------------------------\n")
licensePlateCheck()
for x in range(N):
repeatNum = 0
while repeatNum < 10:
repeatNum += 1
licensePlateCheck()
if repeatNum == 10:
print("Completed generation")
I also attempted to use a thread but that didn't work. If you need any more of the code, just ask.
The full code is here (excluding an unrelated function and the function choice):
import re
import threading
from queue import Queue
def licensePlateCheck():
camInput1 = datetime.now()
print(camInput1)
print("Car is travelling...")
time.sleep(0.1)
print("Car has passed cam2")
camInput2 = timedelta(seconds = random.uniform(5, 10))
distance = 200
duration = camInput2.total_seconds()
print("Time Delta is equal to: {0}".format(duration))
speedCarMs = distance/duration
print("Car is travelling in m/s at: {0}".format(speedCarMs))
speedCarMph = 2.237*speedCarMs
print("Car is travelling in MPH at: {0}".format(speedCarMph))
licenseCharNum = randint(2,9)
licensePlate = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(licenseCharNum))
licensePlateLayout = re.compile('[A-Z][A-Z]\d\d[A-Z][A-Z][A-Z]')
correctMatch = licensePlateLayout.match(licensePlate)
if correctMatch:
pass
else:
with open('Camera Output.txt', 'a') as f:
print("DATA RECORDED TO: Camera Output.txt")
exactTime2 = datetime.now()
f.write("{} has a non-standard license plate and has been recorded at {}.".format(licensePlate,
exactTime2) + "\n")
f.write("----------------------------------------------------------\n")
if speedCarMph > 60:
with open('Camera Output.txt', 'a') as f:
print("DATA RECORDED TO: Camera Output.txt")
exactTime= datetime.now()
f.write("{} was travelling at {}MPH, recorded at {} and has broken the law.".format(licensePlate,
speedCarMph, exactTime) + "\n")
f.write("----------------------------------------------------------\n")
licensePlateCheck()
for x in range(N):
repeatNum = 0
while repeatNum < 10:
repeatNum += 1
licensePlateCheck()
if repeatNum == 10:
print("Completed generation")

In that case you have unnecessary used while loop:)
for x in range(N): // will iterate x times
licensePlateCheck()
print("Completed generation")
With nested while loop, your method would execute:
x * 10 times:
x - for loop
10 - while loop
Both For and While are correct, the choice is up to you.