Develop Reference

I cannot delete a file with python

First off I am not too familiar with python and still currently learning also its my first time ever posting here so sorry if I am mess up with some details.
I am running experiments, and need to run multiple replicates. The issue arises when I need to start a new set of replicates, the program moves the already run experiments into a new folder and is then suppose to start the new replicates. However, what happens only on the last experiment, the environment.cfg folder is not transferred and the program crashes.
from os import listdir,chdir
import subprocess
from random import randrange,sample
from shutil import copy,copytree,rmtree
from os import mkdir,remove
import csv
import time
import shutil
test = input("Is this a test? (y/n)")
if test == "y":
text = input("What are you testing?")
testdoc=open("Testing Documentation.txt","a")
testdoc.write(text)
elif test != "y" and test != "n":
print("Please type in y or n")
test = input("Is this a test? (y/n)")
expnum = int(input("Number of Experiments: "))
exptype = int(input("Which experiment do you want to run? (1/2)"))
repnum= int(input("How many replicates do you want?"))
print(f"You want {repnum} replicates, each replicate will contain {expnum} for a total of {repnum*expnum}")
confirm = input("Is this correct? (y/n)")
if confirm == "y":
for rep in range(repnum):
if exptype == 1:
for ex in range (expnum):
num= ex + 1
mkdir('experiment_'+str(num)) #create a directory named cdir
copy('./default_files/environment.cfg','./experiment_'+str(num)+'/environment.cfg') #copy one file from one place to another
#WSIZE
env_file = open('./experiment_'+str(num)+'/environment.cfg','r')
env_file_content = []
for i in env_file:
env_file_content.append(i.split(' '))
#env_file_content = [['a','b'],['c','d']]
#access first line: env_file_content[0] ; note that index in python start from 0
#access first element in second line: env_file_content[1][0] ; note that index in python start from 0
n = num #number of resources
var0 = '100' #resource inflow
var1 = '0.01' #resource outflow
var3 = '1' # The minimum amount of resource required
reactiontype = ['not','nand','and','orn','or','andn','nor','xor','equ']
reward = ["1.0","1.0","2.0","2.0","4.0","4.0","8.0","8.0","16.0"]
#n = sample(range(10),1)[0]
out = open('./experiment_'+str(num)+'/environment.cfg','w')
for i in range(n):
out.write('RESOURCE res'+str(i)+':inflow='+var0+':outflow='+var1+'\n')
sc=0
for i in range(n):
out.write('REACTION reaction'+str(i)+' '+reactiontype[sc]+' process:resource=res'+str(i)+':value='+reward[sc]+':min='+var3+ '\n')
sc+=1
if sc==len(reactiontype):
sc = 0
out.close()
##RUN Avida from python
copy('./experiment_' + str(num) + '/environment.cfg', './')
print("starting experiment_" + str(num))
proc = subprocess.Popen(['./avida'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Wait for the subprocess to finish
output, error = proc.communicate()
# Close the subprocess
proc.terminate()
shutil.move('./data','./experiment_' +str(num))
#copytree('./data', './experiment_' + str(num) + '/data')
#rmtree('./data')
remove('./environment.cfg')
replicatenum = rep + 1
mkdir('replicate_'+str(replicatenum)) #create a directory named replicate_
for repl in range(expnum):
numb = repl + 1
source = './experiment_'+str(numb)
dest = './replicate_' + str(replicatenum) + '/experiment_' + str(numb)
shutil.move(source, dest)
I tried renaming the folder, however that also failed. The issue seems only to arise when the program is running as taking the same code after the program has crashed, will cause no problems.

Find and replace regex within text file (mac addresses)

This has been asked other places, but no joy when trying those solutions. I am trying to search and replace using open(file) instead of file input. Reason is I am printing a "x of y completed" message as it works (fileinput puts that in the file and not to terminal). My test file is 100 mac addresses separated by new lines.
All I would like to do is find the regex matching a mac address and replace it with "MAC ADDRESS WAS HERE". Below is what I have and it is only putting the replace string once at bottom of file.
#!/usr/bin/env python3
import sys
import getopt
import re
import socket
import os
import fileinput
import time
file = sys.argv[1]
regmac = re.compile("^(([a-fA-F0-9]{2}-){5}[a-fA-F0-9]{2}|([a-fA-F0-9]{2}:){5}[a-fA-F0-9]{2}|([0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})?$")
regmac1 = "^(([a-fA-F0-9]{2}-){5}[a-fA-F0-9]{2}|([a-fA-F0-9]{2}:){5}[a-fA-F0-9]{2}|([0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})?$"
regv4 = re.compile(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$')
regv41 = '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'
menu = {}
menu['1']="MAC"
menu['2']="IPV4"
menu['3']="IPV6"
menu['4']="STRING"
menu['5']="EXIT"
while True:
options=menu.keys()
sorted(options)
for entry in options:
print(entry, menu[entry])
selection = input("Please Select:")
if selection == '1':
print("MAC chosen...")
id = str('mac')
break
elif selection == '2':
print("IPV4 chosen")
id = str('ipv4')
break
elif selection == '3':
print("IPV6 chosen")
id = str('ipv6')
break
elif selection == '4':
print("String chosen")
id = str('string')
break
elif selection == '5':
print("Exiting...")
exit()
else:
print("Invalid selection!")
macmatch = 0
total = 0
while id == 'mac':
with open(file, 'r') as i:
for line in i.read().split('\n'):
matches = regmac.findall(line)
macmatch += 1
print("I found",macmatch,"MAC addresses")
print("Filtering found MAC addresses")
i.close()
with open(file, 'r+') as i:
text = i.readlines()
text = re.sub(regmac, "MAC ADDRESS WAS HERE", line)
i.write(text)
The above will put "MAC ADDRESS WAS HERE", at the end of the last line while not replacing any MAC addresses.
I am fundamentally missing something. If someone would please point me in right direction that would be great!
caveat, I have this working via fileinput, but cannot display progress from it, so trying using above. Thanks again!

All, I figured it out. Posting working code just in case someone happens upon this post.
#!/usr/bin/env python3
#Rewriting Sanitizer script from bash
#Import Modules, trying to not download any additional packages. Using regex to make this python2 compatible (does not have ipaddress module).
import sys
import getopt
import re
import socket
import os
import fileinput
import time
#Usage Statement sanitize.py /path/to/file, add help statement
#Test against calling entire directories, * usage
#Variables
file = sys.argv[1]
regmac = re.compile("^(([a-fA-F0-9]{2}-){5}[a-fA-F0-9]{2}|([a-fA-F0-9]{2}:){5}[a-fA-F0-9]{2}|([0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})?$")
regmac1 = "^(([a-fA-F0-9]{2}-){5}[a-fA-F0-9]{2}|([a-fA-F0-9]{2}:){5}[a-fA-F0-9]{2}|([0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4})?$"
regv4 = re.compile(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$')
regv41 = '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$'
#Functions
menu = {}
menu['1']="MAC"
menu['2']="IPV4"
menu['3']="IPV6"
menu['4']="STRING"
menu['5']="EXIT"
while True:
options=menu.keys()
sorted(options)
for entry in options:
print(entry, menu[entry])
selection = input("Please Select:")
if selection == '1':
print("MAC chosen...")
id = str('mac')
break
elif selection == '2':
print("IPV4 chosen")
id = str('ipv4')
break
elif selection == '3':
print("IPV6 chosen")
id = str('ipv6')
break
elif selection == '4':
print("String chosen")
id = str('string')
break
elif selection == '5':
print("Exiting...")
exit()
else:
print("Invalid selection!")
macmatch = 0
total = 0
while id == 'mac':
with open(file, 'r') as i:
for line in i.read().split('\n'):
matches = regmac.findall(line)
macmatch += 1
print("I found",macmatch,"MAC addresses")
print("Filtering found MAC addresses")
i.close()
with open(file, 'r') as i:
lines = i.readlines()
with open(file, 'w') as i:
for line in lines:
line = re.sub(regmac, "MAC ADDRESS WAS HERE", line)
i.write(line)
i.close()
break
The above overwrites the regex match (found MAC address) with "MAC ADDRESS WAS HERE". Hopefully this helps someone. Any suggestions to make this more efficient or another way to accomplish are welcomed. Will mark as answer once i am able to, 2days.

Getting Python code to call a PowerShell script and run it

I could really use some help on this python script that is to call and run existing PowerShell scripts that are in a specific folder. From want I can see on from many of the articles on this site I've read my code seems correct. First a little background I'm trying to write a python script that will take Powershell scripts in a targeted folder and create a menu that can be selected 1, 2, 3 etc. The use makes the selection and that corresponding Powershell script is run. Now the issue is when I place the code on a server and run it with some test PowerShell scripts I get the following error: The term "Filename.ps1" is not recognized as the name of a cmdlet, function, script file, or operable program. And of course the Powershell scripts won't run. A copy of my code is below. Can anyone see any issue.
## ENOC Text Menu Dynamic test
##version 1
## Created By MTDL Marcus Dixon
## Code produce in Notpad++ For python v3.4.4
import os, subprocess, time, pathlib, logging, fnmatch,
re, sys, io
## Directory Enumerator
fileFolderLocationFilter = fnmatch.filter(os.listdir
('C:\\Users\\MTDl\\Documents\\Automation_Scripts\
\ENScripts\\'), "*.ps1")
selectedFile=""
## Menu defined setting veriables
def ENOC_menu():
files = fileFolderLocationFilter
counter = 1
print (20 * "=" , "Enoc Quick Menu" , 20 * "=")
enumFiles = list(enumerate(files))
for counter, value in enumFiles:
str = repr(counter) + ") " + repr(value);
print(str)
str = repr(counter+1) + ") Exit";
print(str)
print (57 * "_")
str = "Enter your choice [1 - " + repr((counter+1))
+ "]:"
choice = int(input("Please Enter a Selection: "))
selectedFiles = enumFiles[choice]
return(selectedFiles[1])
if choice > counter :
choice = -1
elif choice != counter :
print("Please selecte a valid choice")
else:
selectedFiles = enumFiles[choice]
print(selectedFiles[1])
##selectedFiles = selectedFiles[1]
return choice
##initiating loop
loop = True
while loop:
try:
choice = ENOC_menu()
scriptToRun = choice
powershell = 'C:\\WINDOWS\\system32\
\WindowsPowerShell\\v1.0\\powershell.exe'
print ('\n' +'You selected '+ choice
+'\n')
subprocess.call(powershell + ' ' +
choice, shell=True)
except OSError as err:
logger.error(err) ##continue to work on
logging this renders but may be incorrect.
print ('OS error: {0}' .format(err))
except ValueError:
print ('Oops we had an issue try again')
else:
print ('---' + choice + '---' + '\n')

Python multithreading raw_input

I'm currently doing some work with multithreading and i'm trying to figure out why my program isn't working as intended.
def input_watcher():
while True:
input_file = os.path.abspath(raw_input('Input file name: '))
compiler = raw_input('Choose compiler: ')
if os.path.isfile(input_file):
obj = FileObject(input_file, compiler)
with file_lock:
files.append(obj)
print 'Adding %s with %s as compiler' % (obj.file_name, obj.compiler)
else:
print 'File does not exists'
This is running in one thread and it works fine until i start adding adding the second fileobject.
This is the output from the console:
Input file name: C:\Users\Victor\Dropbox\Private\multiFile\main.py
Choose compiler: aImport
Adding main.py with aImport as compiler
Input file name: main.py updated
C:\Users\Victor\Dropbox\Private\multiFile\main.py
Choose compiler: Input file name: Input file name: Input file name: Input file name:
The input filename keeps popping up the second i added the second filename and it ask for a compiler. The program keeps printing input file name until it crashes.'
I have other code running in a different thread, i don't think it has anything to do with the error, but tell me if you think you need to see it and i will post it.
the full code:
import multiprocessing
import threading
import os
import time
file_lock = threading.Lock()
update_interval = 0.1
class FileMethods(object):
def a_import(self):
self.mod_check()
class FileObject(FileMethods):
def __init__(self, full_name, compiler):
self.full_name = os.path.abspath(full_name)
self.file_name = os.path.basename(self.full_name)
self.path_name = os.path.dirname(self.full_name)
name, exstention = os.path.splitext(full_name)
self.concat_name = name + '-concat' + exstention
self.compiler = compiler
self.compiler_methods = {'aImport': self.a_import}
self.last_updated = os.path.getatime(self.full_name)
self.subfiles = []
self.last_subfiles_mod = {}
def exists(self):
return os.path.isfile(self.full_name)
def mod_check(self):
if self.last_updated < os.path.getmtime(self.full_name):
self.last_updated = os.path.getmtime(self.full_name)
print '%s updated' % self.file_name
return True
else:
return False
def sub_mod_check(self):
for s in self.subfiles:
if self.last_subfiles_mod.get(s) < os.path.getmtime(s):
self.last_subfiles_mod[s] = os.path.getmtime(s)
return True
return False
files = []
def input_watcher():
while True:
input_file = os.path.abspath(raw_input('Input file name: '))
compiler = raw_input('Choose compiler: ')
if os.path.isfile(input_file):
obj = FileObject(input_file, compiler)
with file_lock:
files.append(obj)
print 'Adding %s with %s as compiler' % (obj.file_name, obj.compiler)
else:
print 'File does not exists'
def file_manipulation():
if __name__ == '__main__':
for f in files:
p = multiprocessing.Process(target=f.compiler_methods.get(f.compiler)())
p.start()
#f.compiler_methods.get(f.compiler)()
def file_watcher():
while True:
with file_lock:
file_manipulation()
time.sleep(update_interval)
iw = threading.Thread(target=input_watcher)
fw = threading.Thread(target=file_watcher)
iw.start()
fw.start()

This is happening because you're not using an if __name__ == "__main__": guard, while also using multiprocessing.Process on Windows. Windows needs to re-import your module in the child processes it spawns, which means it will keep creating new threads to handle inputs and watch files. This, of course, is a recipe for disaster. Do this to fix the issue:
if __name__ == "__main__":
iw = threading.Thread(target=input_watcher)
fw = threading.Thread(target=file_watcher)
iw.start()
fw.start()
See the "Safe importing of the main module" section in the multiprocessing docs for more info.
I also have a feeling file_watcher isn't really doing what you want it to (it will keep re-spawning processes for files you've already processed), but that's not really related to the original question.

Creating log-files

i'm new here and a beginner in programming.
My problem is, my program should create a log-files during it runs.
It should look like this:
Start Copy "Log X" | Date-today | Time
Start Compress "Log X" | Date-today | Time | File sice
Ende Compress "Log X" | Date-today | Time | File sice
Start Delete "Log X" | Date-today | Time
End Delete "Log X" | Date-today | Time
...
' "Log X" means the name of the File
When i run the program again the "new log-file" should attachment to the "old file"
This is my program-code till now:
import os, datetime, zipfile
def showProgramInformation():
print " "
print "#######################################################"
print "Python Log-Packer.py Ver. 1.4"
print "Search for files, separate the .log fils, compress them"
print "and delete the origin file"
print "log-File = Files with '.log' in name"
print "#######################################################"
print " "
def conversationWithUser(talk):
print talk
return raw_input()
def doesPathExists(path):
if os.path.exists(path):
return True
return False
def isFileALogFile(filePath):
if filePath.find(".log") != -1:
return True
return False
def formatSeveralDateTime(dateTime):
return datetime.datetime.fromtimestamp(dateTime).strftime('%Y-%m-%d')
def isFileInDateRange(filePath, startDate, endDate):
fileDate = formatSeveralDateTime(os.path.getmtime(filePath))
if fileDate >= startDate and fileDate <= endDate:
return True
return False
def zipLogFile(zipFilePath, zipArchivContent):
myzip = zipfile.ZipFile(zipFilePath + '.zip', 'w', zipfile.ZIP_DEFLATED)
myzip.write(zipArchivContent)
def isValidDate(dateToBeChecked):
if len(dateToBeChecked) != 10:
return False
try:
datetime.datetime.strptime(dateToBeChecked, '%Y-%m-%d')
return True
except ValueError:
return False
def repeatUserInputUntilValidInput(aString):
userInsert = False
while userInsert == False:
newString = aString.upper()
if newString == "Y":
userInsert = True
return True
elif newString == "N":
userInsert = True
return False
else:
print errorMessage
aString = conversationWithUser("Please insert 'Y' or 'N'!")
def pathNameLongerThan0(path):
if len(path) > 0:
print "Path does not exist. Please try it again!"
############## here starts main Program ##############
showProgramInformation()
checkIfInofsAreOk = "N"
errorMessage = "Your input is invalid. Please try again!"
while repeatUserInputUntilValidInput(checkIfInofsAreOk) == False:
logFolder = ""
logArchivFolder = ""
validLogFiles = []
while not doesPathExists(logFolder):
pathNameLongerThan0(logFolder)
logFolder = conversationWithUser("Please enter a valid path: ")
userWanntDateRange = conversationWithUser("Do you want to define a Date Range? (Y/N): ")
if repeatUserInputUntilValidInput(userWanntDateRange):
dateRangeIsOk = False
beginDateIsOk = False
endDateIsOK = False
while not dateRangeIsOk:
while not beginDateIsOk:
userStartDate = conversationWithUser("Please enter the beginning date (e.g. 2014-05-23): ")
beginDateIsOk = isValidDate(userStartDate)
if beginDateIsOk == False:
print errorMessage
while not endDateIsOK:
userEndDate = conversationWithUser("Please enter the ending date (e.g. 2014-11-03): ")
endDateIsOK = isValidDate(userEndDate)
if endDateIsOK == False:
print errorMessage
if userStartDate <= userEndDate:
dateRangeIsOk = True
else:
print errorMessage + " \nDate out of Range. Begin again!"
beginDateIsOk = False
endDateIsOK = False
else:
userStartDate = '1900-01-01' # set as default a wide date to make all files
userEndDate = '2090-01-01' # set as default a wide date to make all files
userWanntALogArchivFolder = conversationWithUser("Do you want create a new folder or archive the files in another folder? (Y/N): ")
if repeatUserInputUntilValidInput(userWanntALogArchivFolder):
userWanntToCreatANewFolder = conversationWithUser("Do you want to create a new folder? (Y/N): ")
if repeatUserInputUntilValidInput(userWanntToCreatANewFolder):
logArchivFolder = conversationWithUser("Enter a new fullpath folder please:")
pathIsAbsolut = os.path.isabs(logArchivFolder)
while pathIsAbsolut == False:
print errorMessage
logArchivFolder = conversationWithUser("Enter a new fullpath folder please:")
pathIsAbsolut = os.path.isabs(logArchivFolder)
else:
logArchivFolder = conversationWithUser("Enter the fullpath folder please:")
while not doesPathExists(logArchivFolder):
pathNameLongerThan0(logArchivFolder)
logArchivFolder = conversationWithUser("Please enter a valid path: ")
else:
logArchivFolder = logFolder + "/" + logArchivFolder
print "#######################################################"
print "Informations "
print "Logfolder: " + logFolder
print "Stardate: " + userStartDate
print "Enddate: " + userEndDate
print "Destination: " + logArchivFolder
print "#######################################################"
checkIfInofsAreOk = conversationWithUser("Are those informations correct? (Y/N): ")
print "#######################################################"
############ here starts compress process ############
for logFolder, subFolders, files in os.walk(logFolder):
print "#######################################################"
for file in files:
absoluteLogFilePath = logFolder + '/' + file
if isFileALogFile(file) and isFileInDateRange(filePath=absoluteLogFilePath, startDate=userStartDate, endDate=userEndDate):
validLogFiles.append(absoluteLogFilePath)
userFolderPath = logFolder
if len(validLogFiles) > 0:
if len(logArchivFolder) > 0:
if not doesPathExists(logArchivFolder):
os.mkdir(logArchivFolder)
userFolderPath = logArchivFolder
for logFile in validLogFiles:
zipFilePath = userFolderPath + '/' + os.path.basename(logFile)
zipLogFile(zipFilePath, logFile)
print logFile
os.remove(logFile)
print "#######################################################"
print "finish"
print "#######################################################"
quit()
It would be nice if they could help me.
(Sorry if my english is no so good)
Yours truly
Johannes

The logging module defines functions and classes which implement a flexible event logging system for applications and libraries.
FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s'
logging.basicConfig(format=FORMAT)
d = {'clientip': '192.168.0.1', 'user': 'fbloggs'}
logger = logging.getLogger('tcpserver')
logger.warning('Protocol problem: %s', 'connection reset', extra=d)
would print something like
2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset
You can use a Formatter to define your own output format and use the different LogRecord attributes to merge data from the record into the format string.