Develop Reference

File operations - filtering data from txt file in python

This code works fine but is too big, i would like to know if there is any other way to write this code to make it shorter.
import openpyxl as excel
PATH = "/home/Fathima/workspace/training/"
ACCESS_LIST = []
def READ_CONFIG():
FROM_ZONE = ""
TO_ZONE = ""
POLICY_NAME = ""
SOURCE_ADDR = ""
DESTINATION_ADDR = ""
PORT = ""
global PATH
global ACCESS_LIST
count = 0
CONFIG_PATH=PATH+"hofwdcn05dcn_20210216(2).txt"
fh = open(CONFIG_PATH, 'r')
CONFIG_LINES=fh.readlines()
config_lines_cnt = len(CONFIG_LINES)
while count < config_lines_cnt:
line = CONFIG_LINES[count].strip()
if len(line) > 0:
line_to_array = line.split(' ')
if line.startswith('from-zone '):
FROM_ZONE = line_to_array[1]
TO_ZONE = line_to_array[3]
elif line.startswith('policy '):
POLICY_NAME = line_to_array[1]
elif line.startswith('source-address '):
SOURCE_ADDR = line_to_array[1].replace(";", "")
elif line.startswith('destination-address '):
DESTINATION_ADDR = line_to_array[1].replace(";", "")
elif line.startswith('application '):
PORT = line_to_array[1].replace(";", "")
elif line.startswith('then {'):
count = count+1
line = CONFIG_LINES[count].strip()
if line == "permit;":
dummy = { 'FROM_ZONE' : FROM_ZONE,'TO_ZONE' : TO_ZONE,'POLICY_NAME' : POLICY_NAME,'SOURCE_ADDR' : SOURCE_ADDR,'DESTINATION_ADDR' : DESTINATION_ADDR,'PORT' : PORT}
ACCESS_LIST.append(dummy)
FROM_ZONE = ""
TO_ZONE = ""
POLICY_NAME = ""
SOURCE_ADDR = ""
DESTINATION_ADDR = ""
PORT = ""
count +=1
#MAIN PROGRAM STARTS FROM HERE
READ_CONFIG()
print(ACCESS_LIST)
Here i have a huge file and need the output appearing as below format
[{
from-zone:
to-zone:
policy:
source-address:
destination-address:
application:
},{
from-zone:
to-zone:
policy:
source-address:
destination-address:
application:
}]

There is a separate related site for a review of working code i.e. StackExchange Code Review
That said, below is a more Pythonic code flow. I didn't change conditionals since they are easy to follow.
Main Changes
Eliminate globals (discouraged--only for special needs)
Use file context manager (i.e. use 'with block' on file open)
Iterate through file rather than read the entire file (allows processing arbitrary file size)
Use Python variable and function naming convention i.e. PEP 8
Remove import openpyxl (unused)
Code
def read_config(path):
from_zone, to_zone, policy_name, source_addr, destination_addr, port = [''] * 6
access_list = []
with open(path + "hofwdcn05dcn_20210216(2).txt", 'r') as fh:
for line in fh:
line = line.strip()
if line:
line_to_array = line.split(' ')
if line.startswith('from-zone '):
from_zone = line_to_array[1]
to_zone = line_to_array[3]
elif line.startswith('policy '):
policy_name = line_to_array[1]
elif line.startswith('source-address '):
source_addr = line_to_array[1].replace(";", "")
elif line.startswith('destination-address '):
destination_addr = line_to_array[1].replace(";", "")
elif line.startswith('application '):
port = line_to_array[1].replace(";", "")
elif line.startswith('then {'):
line = next(fh).strip() # Gets next line in file
if line == "permit;":
access_list.append({'FROM_ZONE': from_zone,
'TO_ZONE': to_zone,
'POLICY_NAME': policy_name,
'SOURCE_ADDR': source_addr,
'DESTINATION_ADDR': destination_addr,
'PORT': port})
from_zone, to_zone, policy_name, source_addr, destination_addr, port = [''] * 6
return access_list
access_list = read_config("/home/Fathima/workspace/training/")
print(access_list)

read a csv file containing data separated by a delimiter in Python Flask

I'm trying to read a csv file, using the separator ',' but when I do line [1] I have an indexing error: I go out of range. For what reason?
what follows is a part of the code that manages the split but does not work.
with open(fileLocation, newline='') as fileCsv:
reader = csv.reader(fileCsv, delimiter=',')
first = True
second = False
prev = fileCsv.readline()
reader= list(reader)
for line in reader:
if first:
first = False
second = True
continue
email = line[20].lower()
nameSurname = line[0]
msg=''
if nameSurname == "" and email == "":
return render_template('LoadCSV.html', login=loginStr(), avviso='err'+msg, fileLocation=fileLocation, category=app.category)
data = line[3][0:10].replace("/","-")
group = line[1]
startingTime = line[16][11:]
EndingTime = line[3][11:]
language = line[10]
This instead is an example of the data contained in the file:
Name Surname,Student,"Assessment timed out","2019/10/31
12:10:08",19,72,26,Fail,000000000,000000000000000,"English
B1",Stefano,"2019/10/29
08:45:15",IDNUMBER,10.30.6.5,157.27.1.13,"2019/10/31
11:24:56",2706,,,xxxx.xxxx#gmail.com,,,,,,,,,"English B1"
Name2 Surname2,Student,"Assessment timed out","2019/10/31
12:10:08",19,72,26,Fail,000000000,000000000000000,"Enghlish
B1",Stefano,"2019/10/29
08:45:15",IDNUMBER,10.30.6.5,157.27.1.13,"2019/10/31
11:24:56",2706,,,xxxx.xxxx#gmail.com,,,,,,,,,"English B1"

How to import every Nth row of CSV file using python into script

I am using a script in fusion360 called importsplinecsv
I was wondering if it was possible to modify the script so that it would import one row every 10th row?
as the amount of rows that are being imported are very large and bloating.
if I could get some help that would be awesome.
here is the text
Author-Autodesk Inc.
Description-Import spline from csv file
import adsk.core, adsk.fusion, traceback
import io
def run(context):
ui = None
try:
app = adsk.core.Application.get()
ui = app.userInterface
# Get all components in the active design.
product = app.activeProduct
design = adsk.fusion.Design.cast(product)
title = 'Import Spline csv'
if not design:
ui.messageBox('No active Fusion design', title)
return
dlg = ui.createFileDialog()
dlg.title = 'Open CSV File'
dlg.filter = 'Comma Separated Values (*.csv);;All Files (*.*)'
if dlg.showOpen() != adsk.core.DialogResults.DialogOK :
return
filename = dlg.filename
with io.open(filename, 'r', encoding='utf-8-sig') as f:
points = adsk.core.ObjectCollection.create()
line = f.readline()
data = []
while line:
pntStrArr = line.split(',')
for pntStr in pntStrArr:
try:
data.append(float(pntStr))
except:
break
if len(data) >= 3 :
point = adsk.core.Point3D.create(data[0], data[1], data[2])
points.add(point)
line = f.readline()
data.clear()
if points.count:
root = design.rootComponent
sketch = root.sketches.add(root.xYConstructionPlane)
sketch.sketchCurves.sketchFittedSplines.add(points)
else:
ui.messageBox('No valid points', title)
except:
if ui:
ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

I have not used this library before but try:
for i, line in enumerate(f):
if i%10==0:
then your import command here
f is your filepointer
i will be the linenumber and line will be your line
dlg = ui.createFileDialog()
dlg.title = 'Open CSV File'
dlg.filter = 'Comma Separated Values (*.csv);;All Files (*.*)'
if dlg.showOpen() != adsk.core.DialogResults.DialogOK :
return
filename = dlg.filename
with io.open(filename, 'r', encoding='utf-8-sig') as f:
points = adsk.core.ObjectCollection.create()
for i, line in enumerate(f):
if i%10==0:
while line:
pntStrArr = line.split(',')
for pntStr in pntStrArr:
try:
data.append(float(pntStr))
except:
break
if len(data) >= 3 :
point = adsk.core.Point3D.create(data[0], data[1], data[2])
points.add(point)
line = f.readline()
data.clear()
if points.count:
root = design.rootComponent
sketch = root.sketches.add(root.xYConstructionPlane)
sketch.sketchCurves.sketchFittedSplines.add(points)
else:
ui.messageBox('No valid points', title)
except:
if ui:
ui.messageBox('Failed:\n{}'.format(traceback.format_exc()))

blob.noun_phrases fails with ConllExtractor()

I'm importing the yaml file and while calling blob.noun_phrases is giving me the error. I tried for a simple example by specifying unicode and it works but in this case it doesn't. Can somebody help me? to figure this out. Thanks in advance.
class TatvamNPExtractService:
text_column_no = -1
def Read_config(self, config_file, cust_id):
path = {}
config_file_obj = open(self.config_file, "r")
consts = YAML.load(config_file_obj)
if consts:
log_out_path = consts['log_file_dir']
path['log_out_path'] = log_out_path
self.text_column_no = consts['comment_column_no'] # column contains reviews
return path
def NounPhraseExtrctor(self, file):
word_chars_pattern = re.compile('\W')
try:
in_file_path = self.paths['in_dir_path'] + file
with open(in_file_path, 'rb') as csvfile:
reader = csv.reader(csvfile, delimiter="|")
extractor = ConllExtractor()
for row in reader:
if len(row) > 2:
mod_text = row[self.text_column_no-1]
print " the text is %s" % ", ".join(mod_text)
tBlob = TextBlob(mod_text, np_extractor=extractor)
tlist = tBlob.noun_phrases
return True
The above code is giving me below error:
AttributeError: 'str' object has no attribute 'raw'
EDIT
Main Program:
import TatvamNPExtractService
def main():
cmd_args =[]
config_file = ""
cmd_args.extend(sys.argv)
if len(cmd_args) < 3:
print "Expecting Config File and Customer ID!!! Terminating..."
sys.exit()
config_file = sys.argv[1]
cust_id = sys.argv[2]
if not (os.path.exists(config_file)):
print "config File not exists, Terminating"
sys.exit()
ps = TatvamNPExtractService.TatvamNPExtractService(cust_id, config_file)
ps.doProcess()

There's no value in my output file

my file contains "Name" and 5 eye movement values (TFF, TFD, TVD, FB, FC). I want to sum up each eye movement values if the rows under Name column are the same. It seems like the code is working, there's no error happened, but my output files stayed empty. Could anyone give me some pointers where went wrong? Here's the code:
import csv
file = open("P01_All.csv", "r") #Open CSV File in Read Mode
reader = csv.reader(file) #Create reader object which iterates over lines
outfile = open("Name.csv","w")
outfile2 = open("TFF.csv","w")
outfile3 = open("TFD.csv","w")
outfile4 = open("TVD.csv","w")
outfile5 = open("FB.csv","w")
outfile6 = open("FC.csv","w")
class Object: #Object to store unique data
def __init__(self, Name, TFF, TFD, TVD, FB, FC):
self.Name = Name
self.TFF = TFF
self.TFD = TFD
self.TVD = TVD
self.FB = FB
self.FC = FC
rownum = 0 #Row Number currently iterating over
list = [] #List to store objects
def checkList(Name, TFF, TFD, TVD, FB, FC):
for object in list: #Iterate through list
if object.Name == Name:
object.TFF += float(TFF)
object.TFD += float(TFD)
object.TVD += float(TVD)
object.FB += float(FB)
object.FC += float(FC)
return
newObject = Object(Name, float(TFF),float(TFD), float(TVD), float(FB), float(FC)) #Create a new object with new eye and TFF
list.append(newObject) #Add to list and break out
for row in reader: #Iterate through all the rows
if rownum == 0: #Store header row seperately to not get confused
header = row
else:
Name = row[0]
TFF = row[1]
TFD = row[2]
TVD = row[3]
FB = row[4]
FC = row[5]
if len(list) == 0: #Default case if list = 0
newObject = Object(Name, float(TFF),float(TFD), float(TVD), float(FB), float(FC))
list.append(newObject)
else: #If not...
checkList(Name, TFF, TFD, TVD, FB, FC)
rownum += 1
for each in list: #Print out result
# print(each.Name, each.TFF, each.TFD, each.TVD, each.FB, each.FC)
outfile.write(each.Name + "\n" )
outfile2.write(str(each.TFF)+ "\n" )
outfile3.write(str(each.TFD)+ "\n" )
outfile4.write(str(each.TVD)+ "\n" )
outfile5.write(str(each.FB)+ "\n" )
outfile6.write(str(each.FC)+ "\n" )
file.close() #Close file
outfile.close()
outfile2.close()
outfile3.close()
outfile4.close()
outfile5.close()
outfile6.close()

Like #zwer said, the reason why you have nothing in your output file is because you don't increment rownum while you are iterating the rows from your input file. By indenting the line rownum += 1 you put it inside your loop where you read each row. So with minimal modification it would look
import csv
file = open("P01_All.csv", "r") #Open CSV File in Read Mode
reader = csv.reader(file) #Create reader object which iterates over lines
outfile = open("Name.csv","w")
outfile2 = open("TFF.csv","w")
outfile3 = open("TFD.csv","w")
outfile4 = open("TVD.csv","w")
outfile5 = open("FB.csv","w")
outfile6 = open("FC.csv","w")
class Movement_value: #Object to store unique data
def __init__(self, Name, TFF, TFD, TVD, FB, FC):
self.Name = Name
self.TFF = TFF
self.TFD = TFD
self.TVD = TVD
self.FB = FB
self.FC = FC
rownum = 0 #Row Number currently iterating over
notebook = [] #List to store objects
def checkList(Name, TFF, TFD, TVD, FB, FC):
for value in notebook: #Iterate through list
if value.Name == Name:
value.TFF += float(TFF)
value.TFD += float(TFD)
value.TVD += float(TVD)
value.FB += float(FB)
value.FC += float(FC)
return
newObject = Movement_value(Name, float(TFF),float(TFD), float(TVD), float(FB), float(FC)) #Create a new object with new eye and TFF
notebook.append(newObject) #Add to list and break out
for row in reader: #Iterate through all the rows
if rownum == 0: #Store header row seperately to not get confused
header = row
else:
Name = row[0]
TFF = row[1]
TFD = row[2]
TVD = row[3]
FB = row[4]
FC = row[5]
if len(notebook) == 0: #Default case if list = 0
newObject = Movement_value(Name, float(TFF),float(TFD), float(TVD), float(FB), float(FC))
notebook.append(newObject)
else: #If not...
checkList(Name, TFF, TFD, TVD, FB, FC)
rownum += 1
for each in notebook: #Print out result
# print(each.Name, each.TFF, each.TFD, each.TVD, each.FB, each.FC)
outfile.write(each.Name + "\n" )
outfile2.write(str(each.TFF)+ "\n" )
outfile3.write(str(each.TFD)+ "\n" )
outfile4.write(str(each.TVD)+ "\n" )
outfile5.write(str(each.FB)+ "\n" )
outfile6.write(str(each.FC)+ "\n" )
file.close() #Close file
outfile.close()
outfile2.close()
outfile3.close()
outfile4.close()
outfile5.close()
outfile6.close()
I have made some additional change: It's better that you don't use list or object as variable names because they are already used in Python and by doing so you'll override their meaning. You could have a bad surprise eventually.
But we can do more.
We don't need to create a class to hold the values
We can work with files using context managers to make sure that our file is not kept open for not relevant reasons.
Here's a version that is shorter than yours:
import csv
import pathlib
input_filepath = pathlib.Path("Input.csv")
output_filepath = pathlib.Path("")
with open(input_filepath, newline="") as input_file:
# Where our data will be kept
input_data = {}
csv_reader = csv.reader(input_file)
# Skip the first line
next(csv_reader)
for (Name, *rest_of_data) in csv_reader:
if Name in input_data:
for (index_of_data_to_update, data_to_update) in enumerate(rest_of_data):
input_data[Name][index_of_data_to_update] += float(data_to_update)
else:
input_data[Name] = [float(x) for x in rest_of_data]
output_rows = ([name] + list(data) for (name, data) in input_data.items())
output_filenames = [
"Name.csv",
"TFF.csv",
"TFD.csv",
"TVD.csv",
"FB.csv",
"FC.csv"
]
output_files = [open(output_filepath / filename, "w") for filename in output_filenames]
# Open all the files
with output_files[0], output_files[1], output_files[2], output_files[3], \
output_files[4], output_files[5]:
for row in output_rows:
for (output_file, data) in zip(output_files, row):
output_file.write("{}\n".format(data))