Develop Reference

Data gets mixed up while trying to transfer it to arangodb

I'm trying to transfer ca. 10GB of json data (tweets in my case) to a collection in arangodb. I'm also trying to use joblib for it:
from ArangoConn import ArangoConn
import Userdata as U
import encodings
from joblib import Parallel,delayed
import json
from glob import glob
import time
def progress(total, prog, start, stri = ""):
if(prog == 0):
print("")
prog = 1;
perc = prog / total
diff = time.time() - start
rem = (diff / prog) * (total - prog)
bar = ""
for i in range(0,int(perc*20)):
bar = bar + "|"
for i in range(int(perc*20),20):
bar = bar + " "
print("\r"+"progress: " + "[" + bar + "] " + str(prog) + " of " +
str(total) + ": {0:.1f}% ".format(perc * 100) + "- " +
time.strftime("%H:%M:%S", time.gmtime(rem)) + " " + stri, end="")
def processfile(filepath):
file = open(filepath,encoding='utf-8')
s = file.read()
file.close()
data = json.loads(s)
Parallel(n_jobs=12, verbose=0, backend="threading"
(map(delayed(ArangoConn.createDocFromObject), data))
files = glob(U.path+'/*.json')
i = 1
j = len(files)
starttime = time.time()
for f in files:
progress(j,i,starttime,f)
i = i+1
processfile(f)
and
from pyArango.connection import Connection
import Userdata as U
import time
class ArangoConn:
def __init__(self,server,user,pw,db,collectionname):
self.server = server
self.user = user
self.pw = pw
self.db = db
self.collectionname = collectionname
self.connection = None
self.dbHandle = self.connect()
if not self.dbHandle.hasCollection(name=self.collectionname):
coll = self.dbHandle.createCollection(name=collectionname)
else:
coll = self.dbHandle.collections[collectionname]
self.collection = coll
def db_createDocFromObject(self, obj):
data = obj.__dict__()
doc = self.collection.createDocument()
for key,value in data.items():
doc[key] = value
doc._key= str(int(round(time.time() * 1000)))
doc.save()
def connect(self):
self.connection = Connection(arangoURL=self.server + ":8529",
username=self.user, password=self.pw)
if not self.connection.hasDatabase(self.db):
db = self.connection.createDatabase(name=self.db)
else:
db = self.connection.databases.get(self.db)
return db
def disconnect(self):
self.connection.disconnectSession()
def getAllData(self):
docs = []
for doc in self.collection.fetchAll():
docs.append(self.doc_to_result(doc))
return docs
def addData(self,obj):
self.db_createDocFromObject(obj)
def search(self,collection,search,prop):
docs = []
aql = """FOR q IN """+collection+""" FILTER q."""+prop+""" LIKE
"%"""+search+"""%" RETURN q"""
results = self.dbHandle.AQLQuery(aql, rawResults=False, batchSize=1)
for doc in results:
docs.append(self.doc_to_result(doc))
return docs
def doc_to_result(self,arangodoc):
modstore = arangodoc.getStore()
modstore["_key"] = arangodoc._key
return modstore
def db_createDocFromJson(self,json):
for d in json:
doc = self.collection.createDocument()
for key,value in d.items():
doc[key] = value
doc._key = str(int(round(time.time() * 1000)))
doc.save()
#staticmethod
def createDocFromObject(obj):
c = ArangoConn(U.url, U.user, U.pw, U.db, U.collection)
data = obj
doc = c.collection.createDocument()
for key, value in data.items():
doc[key] = value
doc._key = doc["id"]
doc.save()
c.connection.disconnectSession()
It kinda works like that. My problem is that the data that lands in the database is somehow mixed up.
as you can see in the screenshot "id" and "id_str" are not the same - as they should be.
what i investigated so far:
I thought that at some points the default keys in the databese may "collide"
because of the threading so I set the key to the tweet id.
I tried to do it without multiple threads. the threading doesn't seem to be
the problem
I looked at the data I send to the database... everything seems to be fine
But as soon as I communicate with the db the data mixes up.
My professor thought that maybe something in pyarango isn't threadsafe and it messes up the data but I don't think so as threading doesn't seem to be the problem.
I have no ideas left where this behavior could come from...
Any ideas?

The screenshot shows the following values:
id : 892886691937214500
id_str : 892886691937214465
It looks like somewhere along the way the value is converted to an IEEE754 double, which cannot safely represent the latter value. So there is potentially some precision loss due to conversion.
A quick example in node.js (JavaScript is using IEEE754 doubles for any number values greater than 0xffffffff) shows that this is likely the problem cause:
$ node
> 892886691937214500
892886691937214500
> 892886691937214465
892886691937214500
So the question is where the conversion does happen. Can you check whether the python client program is correctly sending the expected values to ArangoDB, or does it already send the converted/truncated values?
In general, any integer number that exceeds 0x7fffffffffffffff will be truncated when stored in ArangoDB, or converted to an IEEE754 double. This can be avoided by storing the number values inside a string, but of course comparing two number strings will produce different results than comparing two numbers (e.g. "10" < "9" vs. 10 > 9).

Python script to print nslookup

import os
ipRange = []
for i in range(1, 254):
ipRange.append('192.168.5' + '.' + str(i))
for e in ipRange:
print os.system('nslookup ' + str(e))
This outputs the full output of nslookup for each ip - is there a way to discard the empty results and make the output look more like this?
192.168.5.5 testbox4
192.168.5.6 box3
192.168.5.8 hellobox
192.168.5.9 server2012
192.168.5.18 dnsbox
192.168.5.19 sallysbox
192.168.5.20 bobsbox
192.168.5.21 serverx

Do you need to use system? This would do without system calls:
import socket
for i in range(0, 255):
ipa = "130.233.192." + str(i)
try:
a = socket.gethostbyaddr(ipa)
print (ipa, a[0])
except socket.herror:
pass
EDIT: change 255 to 256 if you want to query .255 as well but in class C networks this is the broadcast address and not in DNS. If you are trawling through class A or B networks, then .255 could be valid as well

Find / Slicing Method issue

I was trying to use the find and string slicing method to extract the number at the end of the line. but I get this mismatch error because I came back with position 18 but from what I have read and research this position is suppose to be 18 am I missing something here?
str = ('X-DSPAM-Confidence:0.8475')
atpos = str.find(':')
print atpos
sppos = str.find(' ',atpos)
print sppos
host = float(str[atpos + 1:sppos])
print host

str = ('X-DSPAM-Confidence:0.8475')
atpos = str.find(':')
sppos = str.find(' ',atpos)
host = float(str[atpos + 1:])
print (host)

You should rather use the string.split method than looking for the specific position of the : delimiter,
_str = 'X-DSPAM-Confidence:0.8475'
host = float(_str.split(':')[1])
print(host)

str = ('X-DSPAM-Confidence:0.8475')
atpos = str.find(':')
sppos = str.find(' ',atpos)
host = float(str[atpos + 1:])
print host

text = "X-DSPAM-Confidence: 0.8475";
spacePos = text.find(" ")
number = text[spacePos::1]
#not really necessary but since we are just learning and playing
strippedNumber = number.lstrip();
result = float(strippedNumber)
def reprint(printed):
print printed
reprint(result)

Python program crashing

So I've designed a program that runs on a computer, looks for particular aspects of files that have been plaguing us, and deletes the files if a flag is passed. Unfortunately the program seems to be almost-randomly shutting down/crashing. I say almost-randomly, because the program always exits after it deletes a file, though it will commonly stay up after a success.
I've run a parallel Python program that counts upwards in the same intervals, but does nothing else. This program does not crash/exit, and stays open.
Is there perhaps a R/W access issue? I am running the program as administrator, so I'm not sure why that would be the case.
Here's the code:
import glob
import os
import time
import stat
#logging
import logging
logging.basicConfig(filename='disabledBots.log')
import datetime
runTimes = 0
currentPhp = 0
output = 0
output2 = 0
while runTimes >= 0:
#Cycles through .php files
openedProg = glob.glob('*.php')
openedProg = openedProg[currentPhp:currentPhp+1]
progInput = ''.join(openedProg)
if progInput != '':
theBot = open(progInput,'r')
#Singles out "$output" on this particular line and closes the process
readLines = theBot.readlines()
wholeLine = (readLines[-4])
output = wholeLine[4:11]
#Singles out "set_time_limit(0)"
wholeLine2 = (readLines[0])
output2 = wholeLine2[6:23]
theBot.close()
if progInput == '':
currentPhp = -1
#Kills the program if it matches the code
currentTime = datetime.datetime.now()
if output == '$output':
os.chmod(progInput, stat.S_IWRITE)
os.remove(progInput)
logging.warning(str(currentTime) +' ' + progInput + ' has been deleted. Please search for a faux httpd.exe process and kill it.')
currentPhp = 0
if output2 == 'set_time_limit(0)':
os.chmod(progInput, stat.S_IWRITE)
os.remove(progInput)
logging.warning(str(currentTime) +' ' + progInput + ' has been deleted. Please search for a faux httpd.exe process and kill it.')
currentPhp = 0
else:
currentPhp = currentPhp + 1
time.sleep(30)
#Prints the number of cycles
runTimes = runTimes + 1
logging.warning((str(currentTime) + ' botKiller2.0 has scanned '+ str(runTimes) + ' times.'))
print('botKiller3.0 has scanned ' + str(runTimes) + ' times.')

Firstly, it'll be hell of a lot easier to work out what's going on if you base your code around something like this...
for fname in glob.glob('*.php'):
with open(fname) as fin:
lines = fin.readlines()
if '$output' in lines[-4] or 'set_time_limit(0)' in lines[0]:
try:
os.remove(fname)
except IOError as e:
print "Couldn't remove:", fname
And err, that's not actually a secondly at the moment, your existing code is just too tricky to follow fullstop, let alone all the bits that could cause a strange error that we don't know yet!

if os.path.exists(progInput):
os.chmod(progInput, stat.S_IWRITE)
os.remove(progInput)
ALSO:
You never reset the output or output2 variables in the loop?
is this on purpose?

Userfriendly way of handling config files in python?

I want to write a program that sends an e-mail to one or more specified recipients when a certain event occurs. For this I need the user to write the parameters for the mail server into a config. Possible values are for example: serveradress, ports, ssl(true/false) and a list of desired recipients.
Whats the user-friendliest/best-practice way to do this?
I could of course use a python file with the correct parameters and the user has to fill it out, but I wouldn't consider this user friendly. I also read about the 'config' module in python, but it seems to me that it's made for creating config files on its own, and not to have users fill the files out themselves.

Are you saying that the fact that the config file would need to be valid Python makes it unfriendly? It seems like having lines in a file like:
server = 'mail.domain.com'
port = 25
...etc would be intuitive enough while still being valid Python. If you don't want the user to have to know that they have to quote strings, though, you might go the YAML route. I use YAML pretty much exclusively for config files and find it very intuitive, and it would also be intuitive for an end user I think (though it requires a third-party module - PyYAML):
server: mail.domain.com
port: 25
Having pyyaml load it is simple:
>>> import yaml
>>> yaml.load("""a: 1
... b: foo
... """)
{'a': 1, 'b': 'foo'}
With a file it's easy too.
>>> with open('myconfig.yaml', 'r') as cfile:
... config = yaml.load(cfile)
...
config now contains all of the parameters.

I doesn't matter technically proficient your users are; you can count on them to screw up editing a text file. (They'll save it in the wrong place. They'll use MS Word to edit a text file. They'll make typos.) I suggest making a gui that validates the input and creates the configuration file in the correct format and location. A simple gui created in Tkinter would probably fit your needs.

I've been using ConfigParser. It's designed to read .ini style files that have:
[section]
option = value
It's quite easy to use and the documentation is pretty easy to read. Basically you just load the whole file into a ConfigParser object:
import ConfigParser
config = ConfigParser.ConfigParser()
config.read('configfile.txt')
Then you can make sure the users haven't messed anything up by checking the options. I do so with a list:
OPTIONS =
['section,option,defaultvalue',
.
.
.
]
for opt in OPTIONS:
section,option,defaultval = opt.split(',')
if not config.has_option(section,option):
print "Missing option %s in section %s" % (option,section)
Getting the values out is easy too.
val = config.get('section','option')
And I also wrote a function that creates a sample config file using that OPTIONS list.
new_config = ConfigParser.ConfigParser()
for opt in OPTIONS:
section,option,defaultval = opt.split(',')
if not new_config.has_section(section):
new_config.add_section(section)
new_config.set(section, option, defaultval)
with open("sample_configfile.txt", 'wb') as newconfigfile:
new_config.write(newconfigfile)
print "Generated file: sample_configfile.txt"

What are the drawbacks of such a solution:
ch = 'serveradress = %s\nport = %s\nssl = %s'
a = raw_input("Enter the server's address : ")
b = 'a'
bla = "\nEnter the port : "
while not all(x.isdigit() for x in b):
b = raw_input(bla)
bla = "Take care: you must enter digits exclusively\n"\
+" Re-enter the port (digits only) : "
c = ''
bla = "\nChoose the ssl option (t or f) : "
while c not in ('t','f'):
c = raw_input(bla)
bla = "Take care: you must type f or t exclusively\n"\
+" Re-choose the ssl option : "
with open('configfile.txt','w') as f:
f.write(ch % (a,b,c))
.
PS
I've read in the jonesy's post that the value in a config file may have to be quoted. If so, and you want the user not to have to write him/her-self the quotes, you simply add
a = a.join('""')
b = b.join('""')
c = c.join('""')
.
EDIT
ch = 'serveradress = %s\nport = %s\nssl = %s'
d = {0:('',
"Enter the server's address : "),
1:("Take care: you must enter digits exclusively",
"Enter the port : "),
2:("Take care: you must type f or t exclusively",
"Choose the ssl option (t or f) : ") }
def func(i,x):
if x is None:
return False
if i==0:
return True
elif i==1:
try:
ess = int(x)
return True
except:
return False
elif i==2:
if x in ('t','f'):
return True
else:
return False
li = len(d)*[None]
L = range(len(d))
while True:
for n in sorted(L):
bla = d[n][1]
val = None
while not func(n,val):
val = raw_input(bla)
bla = '\n '.join(d[n])
li[n] = val.join('""')
decision = ''
disp = "\n====== If you choose to process, =============="\
+"\n the content of the file will be :\n\n" \
+ ch % tuple(li) \
+ "\n==============================================="\
+ "\n\nDo you want to process (type y) or to correct (type c) : "
while decision not in ('y','c'):
decision = raw_input(disp)
disp = "Do you want to process (type y) or to correct (type c) ? : "
if decision=='y':
break
else:
diag = False
while not diag:
vi = '\nWhat lines do you want to correct ?\n'\
+'\n'.join(str(j)+' - '+line for j,line in enumerate((ch % tuple(li)).splitlines()))\
+'\nType numbers of lines belonging to range(0,'+str(len(d))+') separated by spaces) :\n'
to_modify = raw_input(vi)
try:
diag = all(int(entry) in xrange(len(d)) for entry in to_modify.split())
L = [int(entry) for entry in to_modify.split()]
except:
diag = False
with open('configfile.txt','w') as f:
f.write(ch % tuple(li))
print '-*- Recording of the config file : done -*-'