Develop Reference

Python Try Except not Executing

I have written a script that is supposed to automate taking a data frame with pandas and interacting with it, then putting it to another part of the network as a reference, and then it interacts with SmartSheets. It uses a very basic API and most of the functions and logic work well, but the only issue is the try except block in the end that is supposed to be scheduled out.
There is no error thrown, it just sits in a blank terminal; this is odd because if I stack the functions I wrote one after another, they work with no issue.
This try except block is so that It can help write out errors on the machine and ultimately put into a scheduling function later in the script.
This script is on a VM that runs Ubuntu 18.04. It is written in Python 2.
I have researched online, and I cannot find a situation where the try except block does not error out and does not execute. I made sure my indentation on the IDE (Atom) is correct (4 spaces). I can line up the 4 functions themselves and they execute fine with no errors.
import os
import sys
import datetime
import tempfile
import smartsheet
import glob
import warnings
import platform
import pandas as pd
import math
from apscheduler.schedulers.blocking import BlockingScheduler
#variables
warnings.simplefilter(action='ignore', category=FutureWarning)
now = datetime.datetime.now()
PATH = "/mnt/cifs/rxlog.csv"
csvpath = "/home/ajskrilla/csvtest.csv"
End = '.csv'
today = now.strftime("%Y-%m-%d %H:%M")
path1 = "/mnt/cifs1"+"/Reports"+ now.strftime("%Y-%m-%d")+"/log Import"
path_glob = "/mnt/cifs1"+"/Reports"+ now.strftime("%Y-%m-%d")+"/log Import"+now.strftime("%Y-%m-%d")
Files_to_compare = glob.glob('/mnt/cifs1'+"/Reports"+ now.strftime("%Y-%m-%d")+'/log Import'+now.strftime("%Y-%m-%d")+'*.csv')
Fpath = path1 + now.strftime("%Y-%m-%d %H:%M") + End
SSName = 'Call Sheet/NDC ' + now.strftime("%Y-%m-%d %H:%M") + End
list_of_files = Files_to_compare
sched = BlockingScheduler()
#start of process
def Import_csv():
data_file = pd.read_csv(PATH, error_bad_lines=False, sep="|", parse_dates=True, low_memory=False, quotechar=None, quoting=3)
data_file.to_csv(csvpath)
def Clean_CSV():
file_path_directory = "/mnt/cifs1/"+"Reports" + now.strftime("%Y-%m-%d")
if not os.path.exists(file_path_directory):
os.makedirs(file_path_directory)
fields=['RXNBR', 'RX STOREID', 'FILLDATE', 'PATNAMELAST', 'PATNAMEFIRST', 'NH NBR', 'RX HOLD STATUS', 'RX HOLD REASON']
df = pd.read_csv(csvpath, skipinitialspace=True, usecols=fields, low_memory=False)
df.columns = ['RXNBR','RX_STOREID', 'FILLDATE', 'PATNAMELAST', 'PATNAMEFIRST', 'NH_NBR', 'RX_HOLD_STATUS', 'RX_HOLD_REASON']
nf = df[df.NH_NBR == 0][df.RX_HOLD_STATUS != 'Online Queued']
with tempfile.NamedTemporaryFile(delete=False) as temp:
nf.to_csv(Fpath, index=False)
def Compare_files():
if platform.system() == 'Linux':
if len(list_of_files) > 2:
latest_file = min(list_of_files, key=os.stat)
first_file = max(list_of_files, key=os.stat)
one_file= pd.read_csv(first_file)
two_file= pd.read_csv(latest_file)
out = one_file.append(two_file)
out.drop_duplicates('RXNBR', inplace=True)
with tempfile.NamedTemporaryFile(delete=False) as temp:
out.to_csv(Fpath, index=False)
for file in list_of_files:
if file != latest_file:
for files in list_of_files:
os.remove(files)
else:
pass
#delete the old file
def SS_import():
ss_client = smartsheet.Smartsheet("BANNERID#")
ss_client.errors_as_exceptions(True)
imported_sheet = ss_client.Workspaces.import_csv_sheet(
# need to change this based upon workspace ID
xxxxxxxxxxxxxxxx, #WS ID
Fpath,
SSName,
header_row_index=0
)
def SS_delete():
ss_client = smartsheet.Smartsheet("BANNNERID#")
action = ss_client.Sheets.list_sheets(include_all=True)
for single_sheet in action.data:
Sheetid= single_sheet.id
ss_client.Sheets.delete_sheet(
single_sheet.id)
######################################################################################################################################
#this is where the issue arises, during this function
#full fnct with error handling
def NDC_import():
try:
Import_csv()
except Exception as E:
Import_error_file = open('/mnt/Error Files/IE Error' + now.strftime("%Y-%m-%d %H:%M") + '.txt', 'w+')
for line in Import_error_file:
line.write(E)
line.close()
sys.exit()
try:
Clean_CSV()
except Exception as E:
Clean_CSV_error = open('/mnt/Error Files/CC Error' + now.strftime("%Y-%m-%d %H:%M") + '.txt', 'w+')
for line in Clean_CSV_error:
line.write(E)
line.close()
sys.exit()
try:
Compare_files()
except Exception as E:
Compare_files_error = open('/mnt/Error Files/CF Error' + now.strftime("%Y-%m-%d %H:%M") + '.txt', 'w+')
for line in Compare_files_error:
line.write(E)
line.close()
sys.exit()
try:
SS_import()
except Exception as E:
SS_import_error = open('/mnt/Error Files/SSI Error' + now.strftime("%Y-%m-%d %H:%M") + '.txt', 'w+')
for line in SS_import_error:
line.write(E)
line.close()
sys.exit()
NDC_import()
When the script is run for the NDC_import() function, it won't execute at all. if the 4 functions Import_csv(), Clean_CSV(), Compare_files(), and SS_import() are just by themselves, they execute normally.
Why can't it run in that try except block I wrote? It does not even throw an error and the terminal is blank.

I think the primary issue revolves around your exception handling. In particular, you're opening the files as w+ mode and then iterating through the opened file -- not exactly a syntax error but it's functionally broken. It's also worth noting that if there are no errors, you should not expect to see any printed output.
I would suggest to take a different direction with handling errors entirely and consider using the fabulous logging library.
An example of how you could use it would be
# ... other imports
import logging
# ... your function definitions
def NDC_Import():
try:
Import_csv()
except Exception as E:
logging.exception('error during Import_csv()')
return
# and so on ...
That logging.exception() line will print out the whole traceback to whatever logger is configured, and you can configure it to log to a file.

You're opening the file in w+ mode. Example:
SS_import_error = open('/mnt/Error Files/SSI Error' + now.strftime("%Y-%m-%d %H:%M") + '.txt', 'w+')
w+ means it will create the file if it doesn't exist, or overwrite it if it does. That means that SS_import_error will always point to an empty file after this line. Then you immediately try to loop over the lines of that file:
for line in SS_import_error:
line.write(E)
line.close()
sys.exit()
But attempting to loop over the lines of an empty file will end immediately, without ever executing the code within the block. The end result is that the except block runs, but it doesn't actually do anything (other than potentially wipe a file).
I don't know exactly what you intended the for loops to do, so I can't suggest an fix.

Unable to return the list that is read from a file(.csv)

I have been learning and practicing python and during which
I found one error in my program, but I'm unable to resolve. I want to return list of that is retrieved from a csv file. I tried the below code and it returns me an error.
import csv
def returnTheRowsInTheFile(fileName):
READ = 'r'
listOfRows = []
try:
with open(fileName, READ) as myFile:
listOfRows = csv.reader(myFile)
return listOfRows
except FileNotFoundError:
print('The file ' + fileName + ' is not found')
except:
print('Something went wrong')
finally:
#myFile.close()
print()
def main():
fullString = returnTheRowsInTheFile('ABBREVATIONS.CSV')
for eachRow in fullString:
print(eachRow)
return
main()
And the error is
Traceback (most recent call last): File
"C:\Users\santo\workspace\PyProject\hello\FinalChallenge.py", line 36,
in
main() File "C:\Users\santo\workspace\PyProject\hello\FinalChallenge.py", line 32,
in main
for eachRow in fullString: ValueError: I/O operation on closed file.

The easy way to solve this problem is to return a list from your function. I know you assigned listOfRows = [] but this was overwritten when you did listOfRows = csv.reader(myFile).
So, the easy solution is:
def returnTheRowsInTheFile(fileName):
READ = 'r'
try:
with open(fileName, READ) as myFile:
listOfRows = csv.reader(myFile)
return list(listOfRows) # convert to a list
except FileNotFoundError:
print('The file ' + fileName + ' is not found')
except:
print('Something went wrong')
You should also read pep8 which is the style guide for Python; in order to understand how to name your variables and functions.

When you use with open it closes the file when the context ends. Now listOfRows is of the return type of csv.Reader, and so is then fullString (not a list). You are trying to iterate on it, which seems to iterate over a file object, which is already closed.

As JulienD already pointed the file is alread closed when you try to read the rows from it. You can get rid of this exception using this for example:
with open(fileName, READ) as myFile:
listOfRows = csv.reader(myFile)
for row in listOfRows:
yield row
UPDATE
Btw the way you handle exceptions makes it pretty hard to debug. I'd suggest something like this.
except Exception as e:
print('Something went wrong: "%s"' e)
This way you can at least see the error message.

Directing print output to a .txt file

Is there a way to save all of the print output to a txt file in python? Lets say I have the these two lines in my code and I want to save the print output to a file named output.txt.
print ("Hello stackoverflow!")
print ("I have a question.")
I want the output.txt file to to contain
Hello stackoverflow!
I have a question.

Give print a file keyword argument, where the value of the argument is a file stream. The best practice is to open the file with the open function using a with block, which will ensure that the file gets closed for you at the end of the block:
with open("output.txt", "a") as f:
print("Hello stackoverflow!", file=f)
print("I have a question.", file=f)
From the Python documentation about print:
The file argument must be an object with a write(string) method; if it is not present or None, sys.stdout will be used.
And the documentation for open:
Open file and return a corresponding file object. If the file cannot be opened, an OSError is raised.
The "a" as the second argument of open means "append" - in other words, the existing contents of the file won't be overwritten. If you want the file to be overwritten instead at the beginning of the with block, use "w".
The with block is useful because, otherwise, you'd need to remember to close the file yourself like this:
f = open("output.txt", "a")
print("Hello stackoverflow!", file=f)
print("I have a question.", file=f)
f.close()

You can redirect stdout into a file "output.txt":
import sys
sys.stdout = open('output.txt','wt')
print ("Hello stackoverflow!")
print ("I have a question.")

Another method without having to update your Python code at all, would be to redirect via the console.
Basically, have your Python script print() as usual, then call the script from the command line and use command line redirection. Like this:
$ python ./myscript.py > output.txt
Your output.txt file will now contain all output from your Python script.
Edit:
To address the comment; for Windows, change the forward-slash to a backslash.
(i.e. .\myscript.py)

Use the logging module
def init_logging():
rootLogger = logging.getLogger('my_logger')
LOG_DIR = os.getcwd() + '/' + 'logs'
if not os.path.exists(LOG_DIR):
os.makedirs(LOG_DIR)
fileHandler = logging.FileHandler("{0}/{1}.log".format(LOG_DIR, "g2"))
rootLogger.addHandler(fileHandler)
rootLogger.setLevel(logging.DEBUG)
consoleHandler = logging.StreamHandler()
rootLogger.addHandler(consoleHandler)
return rootLogger
Get the logger:
logger = init_logging()
And start logging/output(ing):
logger.debug('Hi! :)')

Another Variation can be... Be sure to close the file afterwards
import sys
file = open('output.txt', 'a')
sys.stdout = file
print("Hello stackoverflow!")
print("I have a question.")
file.close()

Suppose my input file is "input.txt" and output file is "output.txt".
Let's consider the input file has details to read:
5
1 2 3 4 5
Code:
import sys
sys.stdin = open("input", "r")
sys.stdout = open("output", "w")
print("Reading from input File : ")
n = int(input())
print("Value of n is :", n)
arr = list(map(int, input().split()))
print(arr)
So this will read from input file and output will be displayed in output file.
For more details please see https://www.geeksforgeeks.org/inputoutput-external-file-cc-java-python-competitive-programming/

Be sure to import sys module. print whatever you want to write and want to save. In the sys module, we have stdout, which takes the output and stores it. Then close the sys.stdout . This will save the output.
import sys
print("Hello stackoverflow!" \
"I have a question.")
sys.stdout = open("/home/scilab/Desktop/test.txt", "a")
sys.stdout.close()

One can directly append the returned output of a function to a file.
print(output statement, file=open("filename", "a"))

Write line with timestamp+message to file

I want to create a logfile that adds every time an error occurs a new line to a textfile log.txt. I am pretty new to python, so maybe I miss something...but everytime an error occurs, the log.txt is overwritten and only the current error message is displayed although the error message is different every time (due to timestamp) and I added a \n.
Thats my code so far:
import os
import sys
import time
import datetime
try:
path = sys.argv[1]
ts = time.time()
sttime = datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d_%H:%M:%S - ')
#some more things but nothing else of interest for here
except:
error = "ERROR! No file 'bla' found!"
log = 'log.txt'
logfile = file(log, "w")
logfile.write(sttime + error + '\n')
logfile.close()
sys.exit(0)
Maybe you can help me out here. Do I need a loop somewhere? I tried to create an empty string (error = "") that adds the error message to log.txt with += each time an error occurs, but that didn't work at all :-/
Thank you!

Open the file in append mode as 'w' mode will truncate the file each time., i.e
logfile = open(log, "a")
And you should use with:
with open(log, 'a') as logfile:
logfile.write(sttime + error + '\n')
No need to close the file, this will happen automatically.
Note that if the exception is raised at path = sys.argv[1], the timestamp might not be set when you try to log. It would be better to get the timestamp in the logging code.
Also, you should not use a bare except clause, but at least catch the exception and report it.
from datetime import datetime
except Exception, exc:
sttime = datetime.now().strftime('%Y%m%d_%H:%M:%S - ')
error = "ERROR! {}".format(exc)
log = 'log.txt'
with open(log, 'a') as logfile:
logfile.write(sttime + error + '\n')
raise
# sys.exit(0)

When you do file(log, 'W'). The file log will become empty. If you want to add something you should use a instead of w:
open(log, "a")

class Logger(object):
def __init__(self, n):
self.n = n
self.count = 0
self.log = open('log.txt', 'a')
def write(self, message):
self.count+=1
if self.count<self.n:
self.log.write("%s %s"% (time,message))
self.log.flush()
import sys
sys.stdout= Logger()
time -- is time string formatted the way you want.
Now regular print function will write to file.

display an error message when file is empty - proper way?

hi im slowly trying to learn the correct way to write python code. suppose i have a text file which i want to check if empty, what i want to happen is that the program immediately terminates and the console window displays an error message if indeed empty. so far what ive done is written below. please teach me the proper method on how one ought to handle this case:
import os
def main():
f1name = 'f1.txt'
f1Cont = open(f1name,'r')
if not f1Cont:
print '%s is an empty file' %f1name
os.system ('pause')
#other code
if __name__ == '__main__':
main()

There is no need to open() the file, just use os.stat().
>>> #create an empty file
>>> f=open('testfile','w')
>>> f.close()
>>> #open the empty file in read mode to prove that it doesn't raise IOError
>>> f=open('testfile','r')
>>> f.close()
>>> #get the size of the file
>>> import os
>>> import stat
>>> os.stat('testfile')[stat.ST_SIZE]
0L
>>>

The pythonic way to do this is:
try:
f = open(f1name, 'r')
except IOError as e:
# you can print the error here, e.g.
print(str(e))

Maybe a duplicate of this.
From the original answer:
import os
if (os.stat(f1name).st_size == 0)
print 'File is empty!'

If file open succeeds the value of 'f1Cont` will be a file object and will not be False (even if the file is empty).One way you can check if the file is empty (after a successful open) is :
if f1Cont.readlines():
print 'File is not empty'
else:
print 'File is empty'

Assuming you are going to read the file if it has data in it, I'd recommend opening it in append-update mode and seeing if the file position is zero. If so, there's no data in the file. Otherwise, we can read it.
with open("filename", "a+") as f:
if f.tell():
f.seek(0)
for line in f: # read the file
print line.rstrip()
else:
print "no data in file"

one can create a custom exception and handle that using a try and except block as below
class ContentNotFoundError(Exception):
pass
with open('your_filename','r') as f:
try:
content=f.read()
if not content:
raise ContentNotFoundError()
except ContentNotFoundError:
print("the file you are trying to open has no contents in it")
else:
print("content found")
print(content)
This code will print the content of the file given if found otherwise will print the message
the file you are trying to open has no contents in it