i'm new here and a beginner in programming.
My problem is, my program should create a log-files during it runs.
It should look like this:
Start Copy "Log X" | Date-today | Time
Start Compress "Log X" | Date-today | Time | File sice
Ende Compress "Log X" | Date-today | Time | File sice
Start Delete "Log X" | Date-today | Time
End Delete "Log X" | Date-today | Time
...
' "Log X" means the name of the File
When i run the program again the "new log-file" should attachment to the "old file"
This is my program-code till now:
import os, datetime, zipfile
def showProgramInformation():
print " "
print "#######################################################"
print "Python Log-Packer.py Ver. 1.4"
print "Search for files, separate the .log fils, compress them"
print "and delete the origin file"
print "log-File = Files with '.log' in name"
print "#######################################################"
print " "
def conversationWithUser(talk):
print talk
return raw_input()
def doesPathExists(path):
if os.path.exists(path):
return True
return False
def isFileALogFile(filePath):
if filePath.find(".log") != -1:
return True
return False
def formatSeveralDateTime(dateTime):
return datetime.datetime.fromtimestamp(dateTime).strftime('%Y-%m-%d')
def isFileInDateRange(filePath, startDate, endDate):
fileDate = formatSeveralDateTime(os.path.getmtime(filePath))
if fileDate >= startDate and fileDate <= endDate:
return True
return False
def zipLogFile(zipFilePath, zipArchivContent):
myzip = zipfile.ZipFile(zipFilePath + '.zip', 'w', zipfile.ZIP_DEFLATED)
myzip.write(zipArchivContent)
def isValidDate(dateToBeChecked):
if len(dateToBeChecked) != 10:
return False
try:
datetime.datetime.strptime(dateToBeChecked, '%Y-%m-%d')
return True
except ValueError:
return False
def repeatUserInputUntilValidInput(aString):
userInsert = False
while userInsert == False:
newString = aString.upper()
if newString == "Y":
userInsert = True
return True
elif newString == "N":
userInsert = True
return False
else:
print errorMessage
aString = conversationWithUser("Please insert 'Y' or 'N'!")
def pathNameLongerThan0(path):
if len(path) > 0:
print "Path does not exist. Please try it again!"
############## here starts main Program ##############
showProgramInformation()
checkIfInofsAreOk = "N"
errorMessage = "Your input is invalid. Please try again!"
while repeatUserInputUntilValidInput(checkIfInofsAreOk) == False:
logFolder = ""
logArchivFolder = ""
validLogFiles = []
while not doesPathExists(logFolder):
pathNameLongerThan0(logFolder)
logFolder = conversationWithUser("Please enter a valid path: ")
userWanntDateRange = conversationWithUser("Do you want to define a Date Range? (Y/N): ")
if repeatUserInputUntilValidInput(userWanntDateRange):
dateRangeIsOk = False
beginDateIsOk = False
endDateIsOK = False
while not dateRangeIsOk:
while not beginDateIsOk:
userStartDate = conversationWithUser("Please enter the beginning date (e.g. 2014-05-23): ")
beginDateIsOk = isValidDate(userStartDate)
if beginDateIsOk == False:
print errorMessage
while not endDateIsOK:
userEndDate = conversationWithUser("Please enter the ending date (e.g. 2014-11-03): ")
endDateIsOK = isValidDate(userEndDate)
if endDateIsOK == False:
print errorMessage
if userStartDate <= userEndDate:
dateRangeIsOk = True
else:
print errorMessage + " \nDate out of Range. Begin again!"
beginDateIsOk = False
endDateIsOK = False
else:
userStartDate = '1900-01-01' # set as default a wide date to make all files
userEndDate = '2090-01-01' # set as default a wide date to make all files
userWanntALogArchivFolder = conversationWithUser("Do you want create a new folder or archive the files in another folder? (Y/N): ")
if repeatUserInputUntilValidInput(userWanntALogArchivFolder):
userWanntToCreatANewFolder = conversationWithUser("Do you want to create a new folder? (Y/N): ")
if repeatUserInputUntilValidInput(userWanntToCreatANewFolder):
logArchivFolder = conversationWithUser("Enter a new fullpath folder please:")
pathIsAbsolut = os.path.isabs(logArchivFolder)
while pathIsAbsolut == False:
print errorMessage
logArchivFolder = conversationWithUser("Enter a new fullpath folder please:")
pathIsAbsolut = os.path.isabs(logArchivFolder)
else:
logArchivFolder = conversationWithUser("Enter the fullpath folder please:")
while not doesPathExists(logArchivFolder):
pathNameLongerThan0(logArchivFolder)
logArchivFolder = conversationWithUser("Please enter a valid path: ")
else:
logArchivFolder = logFolder + "/" + logArchivFolder
print "#######################################################"
print "Informations "
print "Logfolder: " + logFolder
print "Stardate: " + userStartDate
print "Enddate: " + userEndDate
print "Destination: " + logArchivFolder
print "#######################################################"
checkIfInofsAreOk = conversationWithUser("Are those informations correct? (Y/N): ")
print "#######################################################"
############ here starts compress process ############
for logFolder, subFolders, files in os.walk(logFolder):
print "#######################################################"
for file in files:
absoluteLogFilePath = logFolder + '/' + file
if isFileALogFile(file) and isFileInDateRange(filePath=absoluteLogFilePath, startDate=userStartDate, endDate=userEndDate):
validLogFiles.append(absoluteLogFilePath)
userFolderPath = logFolder
if len(validLogFiles) > 0:
if len(logArchivFolder) > 0:
if not doesPathExists(logArchivFolder):
os.mkdir(logArchivFolder)
userFolderPath = logArchivFolder
for logFile in validLogFiles:
zipFilePath = userFolderPath + '/' + os.path.basename(logFile)
zipLogFile(zipFilePath, logFile)
print logFile
os.remove(logFile)
print "#######################################################"
print "finish"
print "#######################################################"
quit()
It would be nice if they could help me.
(Sorry if my english is no so good)
Yours truly
Johannes
The logging module defines functions and classes which implement a flexible event logging system for applications and libraries.
FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s'
logging.basicConfig(format=FORMAT)
d = {'clientip': '192.168.0.1', 'user': 'fbloggs'}
logger = logging.getLogger('tcpserver')
logger.warning('Protocol problem: %s', 'connection reset', extra=d)
would print something like
2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset
You can use a Formatter to define your own output format and use the different LogRecord attributes to merge data from the record into the format string.
Related
I've been using python for a little while and have made some improvements but this a new error to me. I'm trying to learn social media analysis for my career and that's why I am trying out this set of code here.
I've de bugged one error but this one, which appears at line 81, has got me stumped as I can't see why the function "def get_user_objects(follower_ids):" returns none and what i'd need to change it in accordance with previous advice on other questions here.
Here's script to that point for simplicity. All help appreciated.
The error, to repeat is TypeError: object of type 'NoneType' has no len()
from tweepy import OAuthHandler
from tweepy import API
from collections import Counter
from datetime import datetime, date, time, timedelta
import sys
import json
import os
import io
import re
import time
# Helper functions to load and save intermediate steps
def save_json(variable, filename):
with io.open(filename, "w", encoding="utf-8") as f:
f.write(str(json.dumps(variable, indent=4, ensure_ascii=False)))
def load_json(filename):
ret = None
if os.path.exists(filename):
try:
with io.open(filename, "r", encoding="utf-8") as f:
ret = json.load(f)
except:
pass
return ret
def try_load_or_process(filename, processor_fn, function_arg):
load_fn = None
save_fn = None
if filename.endswith("json"):
load_fn = load_json
save_fn = save_json
else:
load_fn = load_bin
save_fn = save_bin
if os.path.exists(filename):
print("Loading " + filename)
return load_fn(filename)
else:
ret = processor_fn(function_arg)
print("Saving " + filename)
save_fn(ret, filename)
return ret
# Some helper functions to convert between different time formats and
perform date calculations
def twitter_time_to_object(time_string):
twitter_format = "%a %b %d %H:%M:%S %Y"
match_expression = "^(.+)\s(\+[0-9][0-9][0-9][0-9])\s([0-9][0-9][0-9]
[09])$"
match = re.search(match_expression, time_string)
if match is not None:
first_bit = match.group(1)
second_bit = match.group(2)
last_bit = match.group(3)
new_string = first_bit + " " + last_bit
date_object = datetime.strptime(new_string, twitter_format)
return date_object
def time_object_to_unix(time_object):
return int(time_object.strftime("%s"))
def twitter_time_to_unix(time_string):
return time_object_to_unix(twitter_time_to_object(time_string))
def seconds_since_twitter_time(time_string):
input_time_unix = int(twitter_time_to_unix(time_string))
current_time_unix = int(get_utc_unix_time())
return current_time_unix - input_time_unix
def get_utc_unix_time():
dts = datetime.utcnow()
return time.mktime(dts.timetuple())
# Get a list of follower ids for the target account
def get_follower_ids(target):
return auth_api.followers_ids(target)
# Twitter API allows us to batch query 100 accounts at a time
# So we'll create batches of 100 follower ids and gather Twitter User
objects for each batch
def get_user_objects(follower_ids):
batch_len = 100
num_batches = len(follower_ids)/100
batches = (follower_ids[i:i+batch_len] for i in range(0,
len(follower_ids), batch_len))
all_data = []
for batch_count, batch in enumerate(batches):
sys.stdout.write("\r")
sys.stdout.flush()
sys.stdout.write("Fetching batch: " + str(batch_count) + "/" +
str(num_batches))
sys.stdout.flush()
users_list = auth_api.lookup_users(user_ids=batch)
users_json = (map(lambda t: t._json, users_list))
all_data += users_json
return all_data
# Creates one week length ranges and finds items that fit into those range
boundaries
def make_ranges(user_data, num_ranges=20):
range_max = 604800 * num_ranges
range_step = range_max/num_ranges
# We create ranges and labels first and then iterate these when going
through the whole list
# of user data, to speed things up
ranges = {}
labels = {}
for x in range(num_ranges):
start_range = x * range_step
end_range = x * range_step + range_step
label = "%02d" % x + " - " + "%02d" % (x+1) + " weeks"
labels[label] = []
ranges[label] = {}
ranges[label]["start"] = start_range
ranges[label]["end"] = end_range
for user in user_data:
if "created_at" in user:
account_age = seconds_since_twitter_time(user["created_at"])
for label, timestamps in ranges.iteritems():
if account_age > timestamps["start"] and account_age <
timestamps["end"]:
entry = {}
id_str = user["id_str"]
entry[id_str] = {}
fields = ["screen_name", "name", "created_at",
"friends_count", "followers_count", "favourites_count", "statuses_count"]
for f in fields:
if f in user:
entry[id_str][f] = user[f]
labels[label].append(entry)
return labels
if __name__ == "__main__":
account_list = []
if (len(sys.argv) > 1):
account_list = sys.argv[1:]
if len(account_list) < 1:
print("No parameters supplied. Exiting.")
sys.exit(0)
consumer_key="XXXXXXX"
consumer_secret="XXXXXX"
access_token="XXXXXXX"
access_token_secret="XXXXXXXX"
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
auth_api = API(auth)
for target in account_list:
print("Processing target: " + target)
# Get a list of Twitter ids for followers of target account and save it
filename = target + "_follower_ids.json"
follower_ids = try_load_or_process(filename, get_follower_ids,
target)
# Fetch Twitter User objects from each Twitter id found and save the data
filename = target + "_followers.json"
user_objects = try_load_or_process(filename, get_user_objects,
follower_ids)
total_objects = len(user_objects)
# Record a few details about each account that falls between specified age
ranges
ranges = make_ranges(user_objects)
filename = target + "_ranges.json"
save_json(ranges, filename)
# Print a few summaries
print
print("\t\tFollower age ranges")
print("\t\t===================")
total = 0
following_counter = Counter()
for label, entries in sorted(ranges.iteritems()):
print("\t\t" + str(len(entries)) + " accounts were created
within " + label)
total += len(entries)
for entry in entries:
for id_str, values in entry.iteritems():
if "friends_count" in values:
following_counter[values["friends_count"]] += 1
print("\t\tTotal: " + str(total) + "/" + str(total_objects))
print
print("\t\tMost common friends counts")
print("\t\t==========================")
total = 0
for num, count in following_counter.most_common(20):
total += count
print("\t\t" + str(count) + " accounts are following " +
str(num) + " accounts")
print("\t\tTotal: " + str(total) + "/" + str(total_objects))
print
print
The immediate problem is in load_json: you assume its return value is a list or dict, or something that can be passed to len. However, it can return None in a number of circumstances:
The file to read from isn't found
There is some error reading from the file
There is a problem decoding the contents of the file
The file contains just the JSON value null.
At no point after you call load_json do you check its return value.
Worse, you catch and ignore any exception that might occur in load_json, causing it to silently return None with no indication that something went wrong.
The function would be better written like
def load_json(filename):
with io.open(filename, "r", encoding="utf-8") as f:
return json.load(f)
At least now, any errors will raise an uncaught exception, making it more obvious that there was a problem and providing a clue as to what the problem was. The golden rule of exception handling is to only catch the exceptions you can do something about, and if you can't do anything about a caught exception, re-raise it.
You could check for the resultant value and follow accordingly:
# Fetch Twitter User objects from each Twitter id found and save the data
filename = target + "_followers.json"
res_get_user_objects = get_user_objects()
if res_get_user_objects is not None:
user_objects = try_load_or_process(filename, get_user_objects,
follower_ids)
total_objects = len(user_objects)
else:
# handle it otherwise
There are a log.txt.
"[25-Feb-2016 11:27:16 +0200]: Login failed .... 212.153.100.19 Get/.... emailaddress#email.com"........
How can i write a script which can grep or regex me only the dates/IP addresses and email addresses and write it out to an other .txt.
The most important thing is that i need dates and the corresponding IPs and emails.
I try it to with the next code, but it is segment all of the data ..
import os
import re
import datetime
filename = 'log.txt'
newfilename = 'output.txt'
if os.path.exists(filename):
data = open(filename,'r')
bulkemails = data.read()
else:
print "File not found."
raise SystemExit
r = re.compile(r'[\w\.-]+#[\w\.-]+\b')
results = r.findall(bulkemails)
emails = ""
for x in results:
emails += str(x)+"\n"
ip = re.compile(r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b')
result = ip.findall(bulkemails)
ip =""
for y in result:
ip += str(y)+"\n"
dt = re.compile(r'(\d{4})-(\d{2})-(\d{2})')
result = dt.findall(bulkemails)
dt =""
for z in result:
dt += str(z)+"\n"
def writefile():
f = open(newfilename, 'w')
f.write(emails + ip + dt)
f.close()
print "File written."
def overwrite_ok():
response = raw_input("Are you sure you want to overwrite "+str(newfilename)+"? Yes or No\n")
if response == "Yes":
writefile()
elif response == "No":
print "Aborted."
else:
print "Please enter Yes or No."
overwrite_ok()
if os.path.exists(newfilename):
overwrite_ok()
else:
writefile()
So i whould like to same output.txt what is included the next :
25-Feb-2016 11:27:16 +0200] -- 212.153.100.19 -- emailaddress#email.com"
25-Feb-2016 11:27:16 +0200] -- 212.153.100.10 -- emailaddress1#email.com"
25-Feb-2016 11:27:16 +0200] -- 212.153.100.11 -- emailaddress2#email.com"
Thanks for help, and have a nice day :)
You should make a regex with three groups, one for the time, one for the IP and one for the email.
import re
my_regex = re.compile(r".+?(\d{2}-\w+-\d{4}).+?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).+?\b([\w.\d]+#[\w.\d]+)(?:\b|$)")
with open("somefile") as f_logs:
logs = f_logs.readlines()
for line in logs:
my_regex.sub(r"[\1] -- \2 -- \3",line)
You can check it on regex101
This program works for me except the changes that are made to the input .srt (subtitle) file are displayed but NOT saved back in the input file or a new file. Can anyone please tell me what is missing? Thanks. Stuart.
code follows:
#! /usr/bin/python
import sys;
# Patching SRT files to make them readable on Samsung TVs
# It basically inserts blank subtitles when silence should occur.
seqNo=1
try:
subs = open(sys.argv[1])
except:
print "Please provide subtile file to process"
sys.exit(1)
while True:
srtSeqNo=subs.readline();
try:
begin,arrow,end=subs.readline().rstrip('\n\r').split(" ")
except:
break
srtText = subs.readline();
again = subs.readline();
while len(again.strip('\n\r')) > 0:
srtText = srtText + again
again = subs.readline()
print "%d\n%s --> %s\n%s" % (seqNo, begin, end, srtText)
seqNo = seqNo + 1
print "%d\n%s --> %s\n%s\n" % (seqNo, end, end, " ")
seqNo = seqNo + 1
I am using a simple python script to search and play songs on my laptop. The code goes as follows :-
import os
d_name = raw_input("enter drive name:-")
choice = raw_input("song or video(s/v):-")
if(choice == 's'):
s_name = raw_input("enter song name:- ")
flag = 1
elif(choice=='v'):
s_name = raw_input("enter video name:-")
flag = 2
if(flag == 1):
f_s_name = "start "+d_name+":/"+s_name+".mp3"
elif(flag == 2):
f_s_name = "start "+d_name+":/"+s_name+".mp4"
dir_list = os.listdir("d_name:/")
i=0
while(1):
if(not(os.system(f_s_name))):
break
else:
if(flag == 1):
f_s_name = "start "+d_name+":/"+dir_list[i]+"/"+s_name+".mp3"
elif(flag == 2):
f_s_name = "start "+d_name+":/"+dir_list[i]+"/"+s_name+".mp4"
i = i+1
the above program works fine but when one of the calls to the function os.system() fails until the required condition matches it pops out a dialog box claiming that the song is not there until it is found. How can i prevent popping up of that dialog box?
You'd use os.path.exists to test whether the file you're about to start actually exists; if it is not found, do not try to start that file:
import os
....
filename = '{}:/{}/{}.mp3'.format(d_name, dir_list[i], s_name)
if os.path.exists(filename):
system('start ' + filename)
else:
print "File {} was not found".format(filename)
I'm busy working on a tool for tagging documents. I can't seem to solve an issue im having. See the code below, these are 2 functions that are a small part of my code:
def runSearch():
time.sleep(1)
#checkt of de lengte van de list keywords groter is dan 0, zo niet, moet je eerst keywords opgeven
if len(keywords) > 0:
path_input = raw_input('Give path to check for documents(e.g. /Users/Frank/Desktop): ')
errCounter = 0
#hier word gekeken of de opgegeven directory bestaat
if os.path.isdir(path_input):
for root, dirs, files in os.walk(path_input):
for file in files:
fullFile = os.path.join(root, file)
if os.path.splitext(fullFile)[1].lower() == ('.docx') or \
os.path.splitext(file)[1].lower() == ('.doc') or \
os.path.splitext(file)[1].lower() == ('.pptx') or \
os.path.splitext(file)[1].lower() == ('.txt') or \
os.path.splitext(file)[1].lower() == ('.xlsx') or \
os.path.splitext(file)[1].lower() == ('.xls') or \
os.path.splitext(file)[1].lower() == ('.odt') or \
os.path.splitext(file)[1].lower() == ('.rtf') or \
os.path.splitext(file)[1].lower() == ('.csv') or \
os.path.splitext(file)[1].lower() == ('.html') or \
os.path.splitext(file)[1].lower() == ('.htm') or \
os.path.splitext(file)[1].lower() == ('.pdf'):
tagDocs(fullFile)
print ('\nSearch for keyword(s) completed.')
time.sleep(1)
else:
print("\nPlease enter a valid path, for example: '/Users/Frank/Documents'")
else:
print('\nNo keywords were defined, use menu option 2 first.')
raw_input("\nPress Enter to continue...")
menu()
def tagDocs(fullFile):
try:
content = textract.process(os.path.abspath(fullFile))
# contentList.append(content)
words = content.split()
words = [words.lower() for words in words]
tagCounter = 0
for item in enumerate(words):
for index, value in enumerate(keywords):
if value in item:
tagDocuments.append(fullFile)
tagCounter += 1
if tagCounter != 0:
print ("\nIn the file '" + os.path.abspath(fullFile) + "' the tag(s)" + " was found: " +
str(tagCounter) + " time(s).")
except Exception, e:
errorLog(e)
pass
Every time there's an error passed to errorLog() it print the error 4 times in the error.log. I know what's causing this, it's because it loops 4 times, because there are 4 files, if I add a file into the folder im scanning, it will print the error 5 times. So my question, how can I make it only print the error 1 time? Now it prints the error 1 time for every file it finds.
update #2
def errorLogOnce(err):
if not err in reported_errors:
reported_errors.append(err)
errorLog(err)
def errorLog(e):
errCounter = 0
errCounter += 1
logging.basicConfig(filename='error.log',level=logging.DEBUG)
logging.debug(e)
If you want less lines of code, without classes, you can also do this:
reported_errors = []
def errorLogOnce(err):
if not err in reported_errors:
reported_errors.append(err)
errorLog(err)
And then call errorLogOnce instead of calling errorLog
So you want a logger that never repeats a log of the same error?
If so, define a class
class MyLogger(object):
def __init__(self):
self.reported_errors = []
def log(self,err):
if not err in self.reported_errors:
self.repoted_errors.add(err)
errorLog(err)
And call this logger instead of directly calling errorLog(e) directly
def runSearch():
logger = MyLogger()
....
tagDocs(fullFile,logger)
....
def tagDocs(fullFile,logger):
try:
...
except Exception, e:
logger.log(e)
pass