i have a text file with the below contents
url1,user1,xxxxxxxxx
url2,user2,yyyyyyyyy
I have a block of code that is supposed to get the value xxxxxxxxx or yyyyyyyyy based on the env value passed(prod or test)
#!/usr/bin/python
import os
class test:
def __init__(self, env):
self.env = env
def func(self):
res = []
try:
if os.path.exists("file.txt"):
try:
with open("file.txt", 'r') as fp:
for line in fp:
print("line is " +line)
line_api = line.split(',')[2]
print(line_api)
res.append(line_api)
print(res)
if self.env == "prod":
self.api = res[0]
print(self.api)
else:
self.api = res[1]
print(self.api)
except Exception as e:
print(e)
except Exception as e:
print(e)
value when else part is executed
list index out of range
Now when the env passed is prod the function works but when the value is test and the else part is executed the value of list res is only xxxxxx, there is only one value in list and the code self.api = res[1] fails. print(res) only prints ['xxxxxxxxxxxx\n'] but when url passed is url1 print(res) only prints both ['xxxxxxxxxxx\n', 'yyyyyyyyy \n']
What is wrong with my code?
The issue with your code is that you split each line that you read in with line_api = line.split(',')[2] and the [2] is referencing the third element that exists in that list which is xxxxxxxxx or yyyyyyyyy, then when you call self.api = res[0] you reference the first (and only) element in that list.
self.api = res[1] will always throw an error because it will never exist in the 1 element list. I'm not sure what the goal was for this else statement, but I would suggest using DirtyBit's elegant solution
Since it is hard to debug your already excessive code, Here is a shorter snippet:
Using startswith():
list.txt:
url1,user1,xxxxxxxxx
url2,user2,yyyyyyyyy
Hence:
logFile = "list.txt"
def getUrlValue(url):
with open(logFile) as f:
content = f.readlines()
# you may also want to remove empty lines
content = [l.strip() for l in content if l.strip()]
for line in content:
if line.startswith(url):
print(line.split(',')[2])
getUrlValue("url1")
getUrlValue("url2")
OUTPUT:
xxxxxxxxx
yyyyyyyyy
Related
I'm using this function to search a site to see if a particular item i'm interested in s on sale. It first grabs the html from the page, then searches for an item i'm interested. When it finds the item it adds a number of the following lines (dictated by the rangenumber) to the variable 'endresult'. It then searches for the keyword ("sale") in endresult, at which point I'd like it to notify me if the keyword is present or not.
When I print endresult the output contains the keyword, but the if statement at the very end of the function always returns "keyword is missing" despite this and I can't work out why.
def bargainscraper(self, website, item, keyword,rangenum):
request = urllib.request.Request(website)
response = urllib.request.urlopen(request)
data = response.read()
html = str(data)
data1 = html2text.html2text(html)
fw = open('result1.txt', 'w')
fw.write(str(data1))
fw.close()
with open('result1.txt', 'r') as f:
for line in f:
if item in line:
for x in range(rangenum):
endresult = str(f.readline())
print (endresult)
if keyword in endresult:
print("keyword is present")
else:
print("keyword is missing")
Possibly need to concatenate the endresult instead of overwriting it with something like: endresult += str(f.readline()) notice the "+" before the "=".
I found writing the endresult to a file within the for loop then searching that file for the keyword outside of the for loop was the answer I was looking for:
def bargainscraper(self, website, item, keyword,rangenum):
request = urllib.request.Request(website)
response = urllib.request.urlopen(request)
data = response.read()
html = str(data)
data1 = html2text.html2text(html)
fw = open('result1.txt', 'w')
fw.write(str(data1))
fw.close()
with open('result1.txt', 'r') as f:
for line in f:
if item in line:
for x in range(rangenum):
endresult = str(f.readline())
# the 'a' switch is used to append
with open('result2.txt', 'a') as n:
n.write(endresult)
# This is outside of the for loop as otherwise it will iterate for each line of the rangenum
if keyword in open('result2.txt').read():
print ("keyword is present")
else:
print ("keyword is missing")
I have a text file named 'triple_response.txt' which contain the some text as :
(1,(db_name,string),DSP)
(1,(rel, id),2)
(2,(rel_name, string),DataSource)
(2,(tuple, id),201)
(2,(tuple, id),202)
(2,(tuple, id),203)
(201,(src_id,varchar),Pos201510070)
(201,(src_name,varchar),Postgres)
(201,(password,varchar),root)
(201,(host,varchar),localhost)
(201,(created_date,date),2015-10-07)
(201,(user_name,varchar),postgres)
(201,(src_type,varchar),Structured)
(201,(db_name,varchar),postgres)
(201,(port,numeric),None)
(202,(src_id,varchar),pos201510060)
(202,(src_name,varchar),Postgres)
(202,(password,varchar),root)
(202,(host,varchar),localhost)
(202,(created_date,date),2015-10-06)
(202,(user_name,varchar),postgres)
(202,(src_type,varchar),Structured)
(202,(db_name,varchar),DSP)
(202,(port,numeric),5432)
(203,(src_id,varchar),pos201510060)
(203,(src_name,varchar),Postgres)
(203,(password,varchar),root)
(203,(host,varchar),localhost)
(203,(created_date,date),2015-10-06)
(203,(user_name,varchar),postgres)
(203,(src_type,varchar),Structured)
(203,(db_name,varchar),maindb)
(203,(port,numeric),5432)
I am trying to convert these contents into JSON using a python script:
import re
import collections
import json, jsonpickle
def convertToJSON(File):
word_list=[]
row_list = []
try:
with open(File,'r') as f:
for word in f:
word_list.append(word)
with open(File,'r+') as f:
for row in f:
print row
row_list.append(row.split())
column_list = zip(*row_list)
except IOError:
print "Error in opening file.."
triple =""
for t in word_list:
triple+=t
tripleList = re.findall(r"\([^\(^\)]*\)",triple)
idList = re.split(r"\([^\(^\)]*\)",triple)
i =0
jsonDummy = []
jsonData = {}
for trip in tripleList:
nameAndType = re.split(r",|:",trip)
if(i==0):
key = re.compile("[^\w']|_").sub("",idList[i])
else:
try:
key = re.compile("[^\w']|_").sub("",idList[i].split("(")[1])
except IndexError:
pass
i = i+1
if(idList[i].find('(')!=-1):
try:
content = re.compile("[^\w']|_").sub("",idList[i].split(")")[0])
except IndexError:
pass
else:
content = re.compile("[^\w']|_").sub("",idList[i])
try:
trip = trip[1:-1]
tripKey = trip[1]
except IndexError:
tripKey = ''
name = re.compile("[^\w']").sub("",nameAndType[0])
try:
typeName = re.compile("[^\w']|_").sub("",nameAndType[1])
except IndexError:
typeName = 'String'
tripDict = dict()
value = dict()
value[name] = content
tripDict[key]=value
jsonDummy.append(tripDict)
for j in jsonDummy:
for k,v in j.iteritems():
jsonData.setdefault(k, []).append(v)
data = dict()
data['data'] = jsonData
obj = {}
obj=jsonpickle.encode(data, unpicklable=False)
return obj
pass
I am calling this function convertToJSON() within the same file as:
print convertToJSON("triple_response.txt")
I am getting the output as i expect like:
{"data": {"1": [{"db_name": "DSP"}, {"rel": "2"}], "201": [{"src_id": "Pos201510070"}, {"src_name": "Postgres"}, {"password": "root"}, {"host": "localhost"}, {"created_date": "20151007"}, {"user_name": "postgres"}, {"src_type": "Structured"}, {"db_name": "postgres"}, {"port": "None"}], "203": [{"src_id": "pos201510060"}, {"src_name": "Postgres"}, {"password": "root"}, {"host": "localhost"}, {"created_date": "20151006"}, {"user_name": "postgres"}, {"src_type": "Structured"}, {"db_name": "maindb"}, {"port": "5432"}], "2": [{"rel_name": "DataSource"}, {"tuple": "201"}, {"tuple": "202"}, {"tuple": "203"}], "202": [{"src_id": "pos201510060"}, {"src_name": "Postgres"}, {"password": "root"}, {"host": "localhost"}, {"created_date": "20151006"}, {"user_name": "postgres"}, {"src_type": "Structured"}, {"db_name": "DSP"}, {"port": "5432"}]}}
Now the problem which i am facing, which i am calling this from the outside the class as:
def extractConvertData(self):
triple_response = SPO(source, db_name, table_name, response)
try:
_triple_file = open('triple_response.txt','w+')
_triple_file.write(triple_response)
print "written data in file.."
with open('triple_response.txt','r+') as f:
for word in f:
print word
jsonData = convertToJSON(str('triple_response.txt'))
except IOError:
print "Not able to open a file"
print "Converted into JSON"
print jsonData
pass
The same code of convertToJSON() is not working...
It neither giving any output nor giving any error, it is not able to read the content from the 'triple_response.txt' file in the line.
with open('triple_response.txt','r+') as f:
for word in f:
print word
Any one can tell me solution to this problem..
_triple_file is never closed (except implicitly when you end the Python process, which is a terrible practice).
You can get platform-specific behavior when you have dangling filehandles like that (what is your platform? Unix? Windows?). Probably the write to _triple_file is not getting flushed.
So don't leave it dangling. Make sure to close it after you write it: (_triple_file.write(triple_response)). And in fact then assert that that file length is non-zero, using os.stat(), otherwise raise an Exception.
Also, you only have one big try...except clause to catch all errors, this is too much in one bite. Break it into two separate try...except clauses for writing _triple_file, and then reading it back. (Btw you might like to use tempfile library instead, to sidestep needing to know your intermediate file's pathname).
Something like the following untested pseudocode:
triple_response = SPO(source, db_name, table_name, response)
try:
_triple_file = open('triple_response.txt','w+')
_triple_file.write(triple_response)
_triple_file.close()
except IOError:
print "Not able to write intermediate JSON file"
raise
assert [suitable expression involving os.stat('triple_response.txt') to test size > 0 ], "Error: intermediate JSON file was empty"
try:
with open('triple_response.txt','r+') as f:
for word in f:
print word
jsonData = convertToJSON(str('triple_response.txt'))
except IOError:
print "Not able to read back intermediate JSON file"
#raise # if you want to reraise the exception
...
I'm not sure why this function is returning "List index out of range" Since it seens ok. What the function does is basically work the content of the file.txt to create a dictionary with the values.
Resuming, the function will transform the content of the file.txt:
access_1;test_group_1,test_group_2
to:
[{'groups': 'test_group_1,test_group_2\n', 'acl': 'access_1'}
The current code:
def prepare_list():
try:
input_list = []
ins = open("file.txt", "r")
for line in ins:
values = dict()
values['acl'] = line.split(";")[0]
values['groups'] = line.split(";")[1]
input_list.append(values)
except Exception, e:
print "Error : %s" % e
finally:
return input_list
Output:
Error : list index out of range
Process finished with exit code 0
I'm running the script on the folder where the file exists.
data = "a;b"
print data.split(";") # Prints ['a', 'b']
data = "ab"
print data.split(";") # Prints ['ab']
print data.split(";")[1] # raises IndexError: list index out of range
So, if ; is not there, it will return only one element. And if you try to access the second element, it will fail with that error.
If you know for sure that, all lines will have ; in them, your whole program can be written like this
keys = ("acl", "groups")
with open("Input.txt", "r") as ins:
input_list = [{k:v for k, v in zip(keys,l.rstrip().split(";"))} for l in ins]
print input_list
Based on the answer given by thefourtheye I was able to solve the problem testing if the file have lines with ; and striping new lines:
I could use the following correction:
def prepare_list():
try:
input_list = []
ins = open("file.txt", "r")
for line in ins:
if line.strip() and ';' in line:
values = dict()
values['acl'] = line.split(";")[0]
values['groups'] = line.split(";")[1]
input_list.append(values)
print "Correct : ",line
else:
print "line not correct. please fix it : ", line
return False
except Exception, e:
print "Error : %s" % e
But it would print " line not correct, please fix it " for each new line, example with file with 4 new lines:
line not correct
line not correct
line not correct
line not correct
So, using the following i can remove the new lines, and clean the output to just show what is really wrong:
file content:
access_1;test_group_1,test_group_2
access_2test_group_1,test_group_2
<new line>
<new line>
<new line>
<new line>
function:
def prepare_list():
try:
input_list = []
ins = open("file.txt", "r")
for line in ins:
# go away new lines, GO AWAY
if line.strip()
# test for valid char in line
if ';' in line:
values = dict()
values['acl'] = line.split(";")[0]
values['groups'] = line.split(";")[1]
input_list.append(values)
print "Correct : ",line
else:
print "line not correct. please fix it : ", line
except Exception, e:
print "Error : %s" % e
output:
Correct : access_1;test_group_1,test_group_2
line not correct. please fix it : access_2test_group_1,test_group_2
Thanks for the help thefourtheye :)
Simple question. It is possible to make configobj to not put a space before and after the '=' in a configuration entry ?
I'm using configobj to read and write a file that is later processed by a bash script, so putting an antry like:
VARIABLE = "value"
breaks the bash script, it needs to always be:
VARIABLE="value"
Or if someone has another suggestion about how to read and write a file with this kind of entries (and restrictions) is fine too.
Thanks
I was looking into same and modified configobj.py by changing line 1980 in:
def _write_line(self, indent_string, entry, this_entry, comment)
from:
self._a_to_u(' = ')
to:
self._a_to_u('=')
After the change the output is without the space before and after equal sign.
Configobj is for reading and writing ini-style config files. You are apparently trying to use it to write bash scripts. That's not something that is likely to work.
Just write the bash-script like you want it to be, perhaps using a template or something instead.
To make ConfigParses not write the spaces around the = probably requires that you subclass it. I would guess that you have to modify the write method, but only reading the code can help there. :-)
Well, as suggested, I ended up writing my own parser for this that can be used exactly in the same way as ConfigObj:
config = MyConfigParser("configuration_file")
print config["CONFIG_OPTION_1"]
config["CONFIG_OPTION_1"]= "Value 1"
print config["CONFIG_OPTION_1
config.write()
This is the code if someone is interested or wants to give suggestions (I started coding in python not so long ago so probably there are lots of room for improvement). It respects the comments and the order of the options in the file, and correctly scapes and adds double quotes where needed:
import os
import sys
class MyConfigParser:
name = 'MyConfigParser'
debug = False
fileName = None
fileContents = None
configOptions = dict()
def __init__(self, fileName, debug=False):
self.fileName = fileName
self.debug = debug
self._open()
def _open(self):
try:
with open(self.fileName, 'r') as file:
for line in file:
#If it isn't a comment get the variable and value and put it on a dict
if not line.startswith("#") and len(line) > 1:
(key, val) = line.rstrip('\n').split('=')
val = val.strip()
val = val.strip('\"')
val = val.strip('\'')
self.configOptions[key.strip()] = val
except:
print "ERROR: File " + self.fileName + " Not Found\n"
def write(self):
try:
#Write the file contents
with open(self.fileName, 'r+') as file:
lines = file.readlines()
#Truncate file so we don't need to close it and open it again
#for writing
file.seek(0)
file.truncate()
i = 0
#Loop through the file to change with new values in dict
for line in lines:
if not line.startswith("#") and len(line) > 1:
(key, val) = line.rstrip('\n').split('=')
try:
if key in line:
newVal = self.configOptions[key]
#Only update if the variable value has changed
if val != newVal:
newLine = key + "=\"" + newVal + "\"\n"
line = newLine
except:
continue
i +=1
file.write(line)
except IOError as e:
print "ERROR opening file " + self.fileName + ": " + e.strerror + "\n"
#Redefinition of __getitem__ and __setitem__
def __getitem__(self, key):
try:
return self.configOptions.__getitem__(key)
except KeyError as e:
if isinstance(key,int):
keys = self.configOptions.keys()
return self.configOptions[keys[key]]
else:
raise KeyError("Key " +key+ " doesn't exist")
def __setitem__(self,key,value):
self.configOptions[key] = value
As suggested above, it is possible to remove the spaces either side of the equals sign by making a small change to the _write_line method. This can be done conveniently by subclassing ConfigObj and overwriting _write_line as follows -
from configobj import ConfigObj
class MyConfigObj(ConfigObj):
def __init__(self, *args, **kwargs):
ConfigObj.__init__(self, *args, **kwargs)
def _write_line(self, indent_string, entry, this_entry, comment):
"""Write an individual line, for the write method"""
# NOTE: the calls to self._quote here handles non-StringType values.
if not self.unrepr:
val = self._decode_element(self._quote(this_entry))
else:
val = repr(this_entry)
return '%s%s%s%s%s' % (indent_string,
self._decode_element(self._quote(entry, multiline=False)),
self._a_to_u('='),
val,
self._decode_element(comment))
Then just use MyConfigObj in place of ConfigObj and all the functionality of ConfigObj is maintained
As Lennart suggests, configobj is probably not the right tool for the job: how about:
>>> import pipes
>>> def dict2bash(d):
... for k, v in d.iteritems():
... print "%s=%s" % (k, pipes.quote(v))
...
>>> dict2bash({'foo': "bar baz quux"})
foo='bar baz quux'
since configobj returns something that looks a lot like a dict, you could probably still use it to read the data you are trying to process.
First of all, thanks Juancho. That's what i was looking for. But i edited the ConfigParser a little bit. Now it can handle bash script arrays in form of:
# Network interfaces to be configured
ifaces=( "eth0" "eth1" "eth2" "eth3" )
If you set a value it just proves if an value is a list and if, it sets the quotes correctly. So you can set values still the same way, even it is a list:
ifaces = ['eth0', 'eth1', 'eth2', 'eth3']
conf['ifaces'] = ifaces
Here's the code:
import os
import sys
class MyConfigParser:
name = 'MyConfigParser'
debug = False
fileName = None
fileContents = None
configOptions = dict()
qouteOptions = dict()
def __init__(self, fileName, debug=False):
self.fileName = fileName
self.debug = debug
self._open()
def _open(self):
try:
with open(self.fileName, 'r') as file:
for line in file:
#If it isn't a comment get the variable and value and put it on a dict
if not line.startswith("#") and len(line) > 1:
(key, val) = line.rstrip('\n').split('=')
val = val.strip()
val = val.strip('\"')
val = val.strip('\'')
self.configOptions[key.strip()] = val
if val.startswith("("):
self.qouteOptions[key.strip()] = ''
else:
self.qouteOptions[key.strip()] = '\"'
except:
print "ERROR: File " + self.fileName + " Not Found\n"
def write(self):
try:
#Write the file contents
with open(self.fileName, 'r+') as file:
lines = file.readlines()
#Truncate file so we don't need to close it and open it again
#for writing
file.seek(0)
file.truncate()
#Loop through the file to change with new values in dict
for line in lines:
if not line.startswith("#") and len(line) > 1:
(key, val) = line.rstrip('\n').split('=')
try:
if key in line:
quotes = self.qouteOptions[key]
newVal = quotes + self.configOptions[key] + quotes
#Only update if the variable value has changed
if val != newVal:
newLine = key + "=" + newVal + "\n"
line = newLine
except:
continue
file.write(line)
except IOError as e:
print "ERROR opening file " + self.fileName + ": " + e.strerror + "\n"
#Redefinition of __getitem__ and __setitem__
def __getitem__(self, key):
try:
return self.configOptions.__getitem__(key)
except KeyError as e:
if isinstance(key,int):
keys = self.configOptions.keys()
return self.configOptions[keys[key]]
else:
raise KeyError("Key " + key + " doesn't exist")
def __setitem__(self, key, value):
if isinstance(value, list):
self.qouteOptions[key] = ''
value_list = '('
for item in value:
value_list += ' \"' + item + '\"'
value_list += ' )'
self.configOptions[key] = value_list
else:
self.qouteOptions[key] = '\"'
self.configOptions[key] = value
I have a function which takes a list of custom objects, conforms some values then writes them to a CSV file. Something really strange is happening in that when the list only contains a few objects, the resulting CSV file is always blank. When the list is longer, the function works fine. Is it some kind of weird anomaly with the temporary file perhaps?
I have to point out that this function returns the temporary file to a web server allowing the user to download the CSV. The web server function is below the main function.
def makeCSV(things):
from tempfile import NamedTemporaryFile
# make the csv headers from an object
headers = [h for h in dir(things[0]) if not h.startswith('_')]
# this just pretties up the object and returns it as a dict
def cleanVals(item):
new_item = {}
for h in headers:
try:
new_item[h] = getattr(item, h)
except:
new_item[h] = ''
if isinstance(new_item[h], list):
if new_item[h]:
new_item[h] = [z.__str__() for z in new_item[h]]
new_item[h] = ', '.join(new_item[h])
else:
new_item[h] = ''
new_item[h] = new_item[h].__str__()
return new_item
things = map(cleanVals, things)
f = NamedTemporaryFile(delete=True)
dw = csv.DictWriter(f,sorted(headers),restval='',extrasaction='ignore')
dw.writer.writerow(dw.fieldnames)
for t in things:
try:
dw.writerow(t)
# I can always see the dicts here...
print t
except Exception as e:
# and there are no exceptions
print e
return f
Web server function:
f = makeCSV(search_results)
response = FileResponse(f.name)
response.headers['Content-Disposition'] = (
"attachment; filename=export_%s.csv" % collection)
return response
Any help or advice greatly appreciated!
Summarizing eumiro's answer: the file needs to be flushed. Call f.flush() at the end of makeCSV().