Develop Reference

My code is meant to find image duplicates using numpy, it worked for some but doesn't seem to work for all images even if they look exactly alike

I thought this would be a basic task and I am still kind of at a beginner level so this is what my code is at the moment. It is supposed to break the images into byte arrays and then into numpy arrays. There are so many variables because I keep getting a storage error unless I break the process down into multiple variables such as
first = FI.read()--> firsti = first -->firstim = bytearray(firsti) --> firstimgg = np.array(firstim)--->
def find_copies_folders(foldera,folderb):
start = time.time()
ar = []
arr =[]
imagefolder = []
foldercounter = 0
filecounter = 0
secondfilecounter = 0
try:
for subdir, dirs, files in os.walk(folderb):
for file in files:
path = os.path.join(subdir, file)
print(path)
print(time.time()-start)
for subdir,dirs,files in os.walk(foldera):
for fils in files:
FI = open(os.path.join(subdir,fils),'rb')
print("------")
first = FI.read()
firsti = first
firstim = bytearray(firsti)
firstimgg = np.array(firstim)
filecounter+=1
for subdirb,dirs,files in os.walk(folderb):
for sils in files:
SI = open(os.path.join(subdirb,sils),'rb')
SA = (os.path.join(subdirb,sils))
second = SI.read()
secondi = second
secondim = bytearray(secondi)
secondimgg = np.asarray(secondim)
print(fils+": "+str(firstimgg)+" "+sils+": "+str(secondimgg))
if np.array_equal(secondimgg, firstimgg) == True:
db = secondimgg
img = cv2.imdecode(db, cv2.IMREAD_COLOR)
arr.append(img)
ar.append(sils)
print(fils+" = "+ sils)
newfolder = SA
print(str(SA)+" lets see")
SI.close()
shutil.move(SA, Dest_folder)
continue
elif np.array_equal(secondimgg, firstimgg) == False:
#print(fils+" and "+sils+ " have no similarities")
continue
elif np.isclose(secondimgg, firstimgg,rtol=0.1) == True:
print(fils+" and "+sils+ " have slight similarities")
elif np.array_equiv(secondimgg, firstimgg) == True:
print(fils + " and " + sils + " have slight similarities2")
except FileNotFoundError:
print(str(files) +" has been moved")
except PermissionError:
print(folder+" Found it")
pass
except AttributeError:
print("Not an image")
pass
except shutil.Error:
print(newfolder +" Has been moved....Moving on.")
ap = len(ar)
print("Copies: "+str(ar))
#print(ar)
print(ap)
print(filecounter)

My python program can run in D drive,but it cannot run in C and other drives

import os
def delete(files):
os.system('del /f /s /q "%s\\*.*"' % files)
print("清理成功！")
users = os.path.expandvars('$HOMEPATH')
f = open(r'C:' + users + '\\AppData\\Roaming\\Tencent\\WeChat\\All Users\\config\\3ebffe94.ini')
if f.read() == 'MyDocument:':
location = 'C:' + users + '\Documents\WeChat Files'
else:
location = f.read() + "\WeChat Files"
list = os.listdir(location)
list.remove('All Users')
list.remove('Applet')
print("""
""")
print(list)
print("""
""")
while True:
temp = input("选择你要清理的微信号：")
try:
if 0<int(temp)<=len(list):
temp1 = int(temp) - 1
wxid = list[temp1]
break
else:
print("输入错误，请重新输入。")
except:
print("输入错误，请重新输入。")
print("""
-----------------------------Windows微信清理工具-------------------------------------
------------------------------【1.清理聊天记录】---------------------------------
-----------------------------【2.清理图片和视频】-----------------------------------
-----------------------------【3.清理接收到的文件】------------------------------
------------------------------【4.清理全部数据】-------------------------------
""")
while True:
choice = input("请输入要执行的操作所对应的代码：")
if choice == '1':
dialog = location + "\\" + wxid + '\Msg'
delete(dialog)
break
elif choice == '2':
pictures = location + "\\" + wxid + '\FileStorage\Image'
delete(pictures)
videos = location + "\\" + wxid + '\FileStorage\Video'
delete(videos)
break
elif choice == '3':
documents = location + "\\" + wxid + '\FileStorage\File'
delete(documents)
break
elif choice == '4':
delall = location + "\\" + wxid
delete(delall)
break
else:
print("输入错误，请重新输入。")
When it runs in D drive ,it works well.
But when it runs in other drives ,there is something wrong.
Traceback (most recent call last):
File "C:\Users\14932\Desktop\Windows微信清理工具.py", line 13, in <module>
list = os.listdir(location)
FileNotFoundError: [WinError 3] 系统找不到指定的路径。: '\\WeChat Files'
Access the result of variables：
>>> f
<_io.TextIOWrapper name='C:\\Users\\14932\\AppData\\Roaming\\Tencent\\WeChat\\All Users\\config\\3ebffe94.ini' mode='r' encoding='cp936'>
>>> f.read()
''
I packed an exe file on another computer and put it on my computer to work properly.
What is wrong with it?
Could you help me?
I do not know how to fix the bug.
I asked other bbs,did not find the best ways.

I believe the problem is here:
if f.read() == 'MyDocument:':
location = 'C:' + users + '\Documents\WeChat Files'
else:
location = f.read() + "\WeChat Files"
The first call to f.read() reads the entire contents of the file. There is nothing left to read on the second call, so it returns a blank string.

Executing Python Script in Terminal When Outside of Script's Directory

I have a Python script that prompts for text input, searches an online Korean dictionary, and then downloads MP3 audio files for the words found. I use the script to help me make Anki flashcards with audio. The script is originally from this post on reddit.
I can execute the script from the terminal while in the directory that the script is stored in. However, when I am in a different directory and execute the script by calling its full path, the script appears to run but does not find any words or download any MP3s. I cannot figure out why the script fails to execute correctly when I call it from a different directory.
The script is stored in the downloads folder on my Mac /Users/matt/Downloads
So, when I run the following commands, it works:
cd Downloads
python3 naver.py
However, when I run the following, the script executes, but doesn't download any MP3s:
python3 /Users/matt/Downloads/naver.py
The full Python script is here:
import urllib.request, json, codecs, math, time
def searchWords(koreanWords):
url = ('https://ko.dict.naver.com/api3/koko/search?' + urllib.parse.urlencode({'query': koreanWords}) + '&range=word&page=1')
response = urllib.request.urlopen(url)
reader = codecs.getreader("utf-8")
jsonInfo = json.load(reader(response))
pageCount = jsonInfo["pagerInfo"]["totalPages"]
searchData = jsonInfo["searchResultMap"]["searchResultListMap"]["WORD"]["items"]
for pageCountInc in range(0, pageCount):
if pageCountInc != 0:
url = ('https://ko.dict.naver.com/api3/koko/search?' + urllib.parse.urlencode({'query': koreanWords}) + '&range=word&page=' + str(pageCountInc+1))
response = urllib.request.urlopen(url)
reader = codecs.getreader("utf-8")
jsonInfo = json.load(reader(response))
searchData = jsonInfo["searchResultMap"]["searchResultListMap"]["WORD"]["items"]
for z in range (0, len(searchData)):
if searchData[z]["handleEntry"] in unchangedWordList:
if searchData[z]["searchPhoneticSymbolList"]:
if searchData[z]["searchPhoneticSymbolList"][0]["phoneticSymbolPath"] != "":
timesDownloaded[unchangedWordList.index(searchData[z]["handleEntry"])] += 1
mp3Link = searchData[z]["searchPhoneticSymbolList"][0]["phoneticSymbolPath"]
if mp3Link not in mp3Links:
mp3Links.append(mp3Link)
urllib.request.urlretrieve(mp3Link, searchData[z]["handleEntry"] + str(timesDownloaded[unchangedWordList.index(searchData[z]["handleEntry"])]) + ".mp3")
time.sleep(.3)
def parseWords(listOfWords):
for x in range(0, math.floor(len(listOfWords)/10)):
tempWords = []
for y in range(0, 10):
tempWords.append(listOfWords[x*10+y])
print("Searching: " + str(x+1) + "/" + str(math.ceil(len(listOfWords)/10)))
searchWords(tempWords)
tempWords = []
for y in range(math.floor(len(listOfWords)/10)*10+1, len(listOfWords)):
tempWords.append(listOfWords[y])
print("Searching: " + str((math.ceil(len(listOfWords)/10))) + "/" + str(math.ceil(len(listOfWords)/10)))
searchWords(tempWords)
unfoundWords = []
unchangedWordList = []
timesDownloaded = []
mp3Links = []
wordInputs = unchangedWordList = input('Enter Words: ').split()
timesDownloaded = [0] * len(unchangedWordList)
parseWords(wordInputs)
for z in range(0, len(timesDownloaded)):
if(timesDownloaded[z] == 0):
unfoundWords.append(unchangedWordList[z])
if unfoundWords:
print(",".join(str(x) for x in unfoundWords) + " could not be found.")
print("Rerunning individual searches for unfound words.")
print(unfoundWords)
oldUnfoundWords = unfoundWords
unfoundWords = []
for x in range(0, len(oldUnfoundWords)):
print("Searching: " + str(x+1) + "/" + str(len(oldUnfoundWords)))
searchWords(oldUnfoundWords[x])
for z in range(0, len(timesDownloaded)):
if(timesDownloaded[z] == 0):
unfoundWords.append(unchangedWordList[z])
if unfoundWords:
print(",".join(str(x) for x in unfoundWords) + " could not be found.")

To answer question of how to save to a specific folder use pathlib to construct the path to MP3 folder.
import os
from pathlib import Path
# Create parent folder
mp3DIR = os.path.join(Path.home(),'Music')
basename = searchData[z]["handleEntry"]
+ str(timesDownloaded[unchangedWordList.index(searchData[z]["handleEntry"])]) + ".mp3"
urllib.request.urlretrieve(mp3Link, os.path.join(mp3Dir, basename))

The reason is the following:
Your python file runs in your current directory. So, when you run this: python3 /Users/matt/Downloads/naver.py, it either runs and saves the mp3 files in the current directory, or it doesn't save anything at all if it doesn't have the permissions to.

read multiple files automatically no manual file naming

I have a directory contains 50 files I want to read them one by one and compare wit the other files - that is fixed. I am using glob.blob. But it didn't work.
Here how I am reading all files. Instead, path = '*.rbd' if I give the file name like path = run-01.rbd it works.
path = '*.rbd'
path = folder + path
files=sorted(glob.glob(path))
complete code
import glob
from itertools import islice
import linecache
num_lines_nonbram = 1891427
bits_perline = 32
total_bit_flips = 0
num_bit_diff_flip_zero = 0
num_bit_diff_flip_ones = 0
folder = "files/"
path = '*.rbd'
path = folder + path
files=sorted(glob.glob(path))
original=open('files/mull-original-readback.rbd','r')
#source1 = open(file1, "r")
for filename in files:
del_lines = 101
with open(filename,'r') as f:
i=1
while i <= del_lines:
line1 = f.readline()
lineoriginal=original.readline()
i+=1
i=0
num_bit_diff_flip_zero = 0
num_bit_diff_flip_ones = 0
num_lines_diff =0
i=0
j=0
k=0
a_write2 = ""
while i < (num_lines_nonbram-del_lines):
line1 = f.readline()
lineoriginal = original.readline()
while k < bits_perline:
if ((lineoriginal[k] == line1[k])):
a_write2 += " "
else:
if (lineoriginal[k]=="0"):
#if ((line1[k]=="0" and line1[k]=="1")):
num_bit_diff_flip_zero += 1
if (lineoriginal[k]=="1"):
#if ((line1[k]=="0" and line1[k]=="1")):
num_bit_diff_flip_ones += 1
#if ((line1[k]==1 and line1[k]==0)):
#a_write_file2 = str(i+1) + " " + str(31-k) + "\n" + a_write_file2
#a_write2 += "^"
#num_bit_diff_flip_one += 1
# else:
# a_write2 += " "
k+=1
total_bit_flips=num_bit_diff_flip_zero+num_bit_diff_flip_ones
i+=1
k=0
i = 0
print files
print "Number of bits flip zero= %d" %num_bit_diff_flip_zero +"\n" +"Number of bits flip one= %d" %num_bit_diff_flip_ones +"\n" "Total bit flips = %d " %total_bit_flips
f.close()
original.close()

You could use the os module to first list everything in a directory (both files and modules) then use a python generator to filter out only the files. You could then use a second python generator to filter out files with a specific extension. There is probably a more efficient way of doing it but this works:
import os
def main():
path = './' # The path to current directory
# Go through all items in the directory and filter out files
files = [file for file in os.listdir(path) if
os.path.isfile(os.path.join(path, file))]
# Go through all files and filter out files with .txt (for example)
specificExtensionFiles = [file for file in files if ".txt" in file]
# Now specificExtensionFiles is a generator for .txt files in current
# directory which you can use in a for loop
print (specificExtensionFiles)
if __name__ == '__main__':
main()
For further reference:
How do I list all files of a directory?

The problem is that you're not going back to the beginning of originalfile whenever you start comparing with the next file in the for filename in files: loop. The simplest solution is to put:
original.seek(0)
at the beginning of that loop.
You could also read the whole file into a list just once before the loop, and use that instead of reading the file repeatedly.
And if you only want to process part of the files, you can read the file into a list, and then use a list slice to get the lines you want.
You also shouldn't be setting num_bit_diff_flip_zero and num_bit_diff_flip_one to 0 each time through the loop, since these are supposed to be the total across all files.
with open('files/mull-original-readback.rbd','r') as original:
original_lines = list(original)[del_lines:num_lines_nonbram]
for filename in files:
with open(file, 'r') as f:
lines = list(f)[del_lines:num_lines_nonbram]
for lineoriginal, line1 in zip(original_lines, lines):
for k in range(bits_perline):
if lineoriginal[k] == line1[k]:
a_write2 += " "
elif lineoriginal[k] == "0"
num_bit_diff_flip_zero += 1
else:
num_bit_diff_flip_ones += 1
total_bit_flips = num_bit_diff_flip_zero + num_bit_diff_flip_ones

urllib.urlretrieve downloads empty zip files

I'm trying to download a export of space in a zip file. But somehow python downloads a empty and corrupted zip file. When you download the file manual by the browser everything is ok.
I use Python 2.7.13
#!/usr/bin/python
import xmlrpclib
import time
import urllib
confluencesite = "https://confluence.com"
server = xmlrpclib.ServerProxy(confluencesite + '/rpc/xmlrpc')
username = '*'
password = '*'
token = server.confluence2.login(username, password)
loginString = "?os_username=" + username + "&os_password=" + password
filelist = ""
start = True
spacesummary = server.confluence2.getSpaces(token)
for space in spacesummary:
#if space['name'] == "24-codING":
# start = True
# continue
if start:
if space['type'] == 'global':
print "Exporting space " + space['name']
spaceDownloadUrl = server.confluence2.exportSpace(token, space['key'],
"TYPE_XML",
exportAll['true'])
filename = spaceDownloadUrl.split('/')[-1].split('#')[0].split('?')[0]
time.sleep(0.5)
urllib.urlretrieve(spaceDownloadUrl + loginString, filename)
print filename + " saved."
f = open("exportedspaces.txt", 'a')
f.write(filename + "\n")
f.close()

It's solved by the answer of Coldspeed. Changing the following:
loginString = "?os_username=" + username + "&os_password=" + password
to
loginString = "?os_username=" + username + "&os_password=" + password