Develop Reference

Porting Node.js script to Python

I want to port this Node.js script to control a Sky box over into Python, https://github.com/dalhundal/sky-remote/blob/master/sky-remote.js
I've gone through and done the best that I can do, code is below;
import time, math, socket, struct, time
from array import array
#sky q port 5900
class remote:
commands={"power": 0, "select": 1, "backup": 2, "dismiss": 2, "channelup": 6, "channeldown": 7, "interactive": 8, "sidebar": 8, "help": 9, "services": 10, "search": 10, "tvguide": 11, "home": 11, "i": 14, "text": 15, "up": 16, "down": 17, "left": 18, "right": 19, "red": 32, "green": 33, "yellow": 34, "blue": 35, 0: 48, 1: 49, 2: 50, 3: 51, 4: 52, 5: 53, 6: 54, 7: 55, 8: 56, 9: 57, "play": 64, "pause": 65, "stop": 66, "record": 67, "fastforward": 69, "rewind": 71, "boxoffice": 240, "sky": 241}
connectTimeout = 1000;
def __init__(self, ip, port=49160):
self.ip=ip
self.port=port
def showCommands(self):
for command, value in self.commands.iteritems():
print str(command)+ " : "+str(value)
def getCommand(self, code):
try:
return self.commands[code]
except:
print "Error: command '"+code+"' is not valid"
return False
def press (self, sequence):
if isinstance(sequence, list):
for item in sequence:
toSend=self.getCommand(item)
if toSend:
self.sendCommand(toSend)
time.sleep(0.5)
else:
toSend=self.getCommand(sequence)
if toSend:
self.sendCommand(toSend)
def sendCommand(self, code):
commandBytes = array('l', [4,1,0,0,0,0, int(math.floor(224 + (code/16))), code % 16])
try:
client=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error, msg:
print 'Failed to create socket. Error code: ' + str(msg[0]) + ' , Error message : ' + msg[1]
return
try:
client.connect((self.ip, self.port))
except:
print "Failed to connect to client"
return
l=12
timeout=time.time()+self.connectTimeout
while 1:
data=client.recv(1024)
data=data
if len(data)<24:
client.sendall(data[0:l])
l=1
else:
client.sendall(buffer(commandBytes))
commandBytes[1]=0
client.sendall(buffer(commandBytes))
client.close()
break
if time.time() > timeout:
print "timeout error"
break
I think the issue is how I form the buffers? I'm not entirely sure as this is the first time I've dealt with buffers.
Having read through the Node.js documentation on new Buffer, it looks like it creates an array of Octets, whereas what I have is an array of ints, I may be wrong but an Octet is 8bits whilst an int is 4bits, I've tried changing the array to long and double, but this doesn't seem to resolve the issue

Having had the time to have a quick read through on how Node.js handles buffers and buffers in general and it looks like what I thought was right.
Change;
commandBytes = array('l', [4,1,0,0,0,0, int(math.floor(224 + (code/16))), code % 16])
to;
commandBytes = bytearray([4,1,0,0,0,0, int(math.floor(224 + (code/16))), code % 16])
Also, then just pass the bytearray;
client.sendall(commandBytes)
instead of;
client.sendall(buffer(commandBytes))

Add a timestamp to data simulator

I am simulating time series data using Python TestData and trying to add a new key value (event_time) that includes a time stamp when the record is generated. The issue is that the field is not incrementing as the script runs, just at first execution. Is there a simple way to do this?
import testdata
import datetime
EVENT_TYPES = ["USER_DISCONNECT", "USER_CONNECTED", "USER_LOGIN", "USER_LOGOUT"]
class EventsFactory(testdata.DictFactory):
event_time = testdata.DateIntervalFactory(datetime.datetime.now(), datetime.timedelta(minutes=0))
start_time = testdata.DateIntervalFactory(datetime.datetime.now(), datetime.timedelta(minutes=12))
end_time = testdata.RelativeToDatetimeField("start_time", datetime.timedelta(minutes=20))
event_code = testdata.RandomSelection(EVENT_TYPES)
for event in EventsFactory().generate(100):
print event
Outputs:
{'start_time': datetime.datetime(2016, 6, 21, 17, 47, 50, 422020), 'event_code': 'USER_CONNECTED', 'event_time': datetime.datetime(2016, 6, 21, 17, 47, 50, 422006), 'end_time': datetime.datetime(2016, 6, 21, 18, 7, 50, 422020)}
{'start_time': datetime.datetime(2016, 6, 21, 17, 59, 50, 422020), 'event_code': 'USER_CONNECTED', 'event_time': datetime.datetime(2016, 6, 21, 17, 47, 50, 422006), 'end_time': datetime.datetime(2016, 6, 21, 18, 19, 50, 422020)}
{'start_time': datetime.datetime(2016, 6, 21, 18, 11, 50, 422020), 'event_code': 'USER_LOGOUT', 'event_time': datetime.datetime(2016, 6, 21, 17, 47, 50, 422006), 'end_time': datetime.datetime(2016, 6, 21, 18, 31, 50, 422020)}

So the timedelta() is how much into the future you want the event to happen. Notice that the timedelta(minutes=12) causes the time between each start_time generated to be 12 minutes from datetime.datetime.now() from the previous iteration of the for-loop (not the execution of the script). Similarly, the end_time is a relative timedelta(minutes=20) to start_time so it will always be 20 minutes in front of start_time. Your event_time is not incrementing because it has no delta (change) value for any time the code is run, and it will always use the datetime.datetime.now() from the time the script is run.
It if is test data, I think you would be looking for something like
import testdata
import datetime
EVENT_TYPES = ["USER_DISCONNECT", "USER_CONNECTED", "USER_LOGIN", "USER_LOGOUT"]
class EventsFactory(testdata.DictFactory):
start_time = testdata.DateIntervalFactory(datetime.datetime.now(), datetime.timedelta(minutes=12))
event_time = testdata.RelativeToDatetimeField("start_time", datetime.timedelta(minutes=10))
end_time = testdata.RelativeToDatetimeField("start_time", datetime.timedelta(minutes=20))
event_code = testdata.RandomSelection(EVENT_TYPES)
for event in EventsFactory().generate(100):
print event
Edit: if it doesn't have to do with the data provided:
So the testdata.DictFactory that you are passing in just creates a dictionary based on the instance variables you create as far as I can see.
You want an event_time instance variable that gets the time for every iteration of the for-loop, to do that it would look like:
import testdata
import datetime
EVENT_TYPES = ["USER_DISCONNECT", "USER_CONNECTED", "USER_LOGIN", "USER_LOGOUT"]
class EventsFactory(testdata.DictFactory):
start_time = testdata.DateIntervalFactory(datetime.datetime.now(), datetime.timedelta(minutes=12))
end_time = testdata.RelativeToDatetimeField("start_time", datetime.timedelta(minutes=20))
event_time = datetime.datetime.now()
event_code = testdata.RandomSelection(EVENT_TYPES)
for event in EventsFactory().generate(100):
print event
If I am understanding what you are wanting correctly, this should achieve it in the output.
Edit 2:
After looking at this again this may not achieve what you are wanting because EventsFactory().generate(100) seems to instantiate all 100 at the same time, and to get a dictionary key of event_time you would have to use the testdata.RelativeToDatetimeField() method to change the time

for event in EventsFactory().generate(10):
event["event_time"] = datetime.datetime.now()
print event

SHA 512 crypt output written with Python code is different from mkpasswd

Running mkpasswd -m sha-512 -S salt1234 password results in the following:
$6$salt1234$Zr07alHmuONZlfKILiGKKULQZaBG6Qmf5smHCNH35KnciTapZ7dItwaCv5SKZ1xH9ydG59SCgkdtsTqVWGhk81
I have this snippet of Python code that I thought would output the same, but isn't:
import hashlib, base64
print(base64.b64encode(hashlib.sha512('password' + 'salt1234').digest()))
It instead results in:
nOkBUt6l7zlKAfjtk1EfB0TmckXfDiA4FPLcpywOLORZ1PWQK4+PZVEiT4+9rFjqR3xnaruZBiRjDGcDpxxTig==
Not sure what I am doing wrong.
Another question I have is, how do I tell sha512 function to do custom rounds. It seems to take only 1 argument.

mkpasswd is a front-end to the crypt() function. I don't think it is a straight-forward SHA512 hash here.
A little research points to the specification for SHA256-crypt and SHA512-crypt, which shows the hash is applied a default 5000 times. You can specify a different number of rounds using the -R switch to mkpasswd; -R 5000 indeed gives you the same output:
$ mkpasswd -m sha-512 -S salt1234 -R 5000 password
$6$rounds=5000$salt1234$Zr07alHmuONZlfKILiGKKULQZaBG6Qmf5smHCNH35KnciTapZ7dItwaCv5SKZ1xH9ydG59SCgkdtsTqVWGhk81
The minimum number of rounds offered by the command-line tool is 1000:
$ mkpasswd -m sha-512 -S salt1234 -R 999 password
$6$rounds=1000$salt1234$SVDFHbJXYrzjGi2fA1k3ws01/D9q0ZTAh1KfRF5.ehgjVBqfHUaKqfynXefJ4DxIWxkMAITYq9mmcBl938YQ//
$ mkpasswd -m sha-512 -S salt1234 -R 1 password
$6$rounds=1000$salt1234$SVDFHbJXYrzjGi2fA1k3ws01/D9q0ZTAh1KfRF5.ehgjVBqfHUaKqfynXefJ4DxIWxkMAITYq9mmcBl938YQ//
The algorithm is a bit more involved, requiring you to create several digests. You could instead access the C crypt() function through the crypt.crypt() function, and drive it the same way the mkpasswd commandline does.
It depends on your platform if the SHA512-crypt method is available; the Python 3 version of the crypt module offers a crypt.methods list that tells you what methods your platform supports. Since this use the exact same library mkpasswd uses, your OS obviously does support SHA512-crypt and Python will have access too.
You need to prefix the salt with '$6$ to specify the different method. You can specify the number of rounds by adding a 'rounds=<N>$' string between the '$6$' string and your salt:
import crypt
import os
import string
try: # 3.6 or above
from secrets import choice as randchoice
except ImportError:
from random import SystemRandom
randchoice = SystemRandom().choice
def sha512_crypt(password, salt=None, rounds=None):
if salt is None:
salt = ''.join([randchoice(string.ascii_letters + string.digits)
for _ in range(8)])
prefix = '$6$'
if rounds is not None:
rounds = max(1000, min(999999999, rounds or 5000))
prefix += 'rounds={0}$'.format(rounds)
return crypt.crypt(password, prefix + salt)
This then produces the same output as the mkpasswd command line:
>>> sha512_crypt('password', 'salt1234')
'$6$salt1234$Zr07alHmuONZlfKILiGKKULQZaBG6Qmf5smHCNH35KnciTapZ7dItwaCv5SKZ1xH9ydG59SCgkdtsTqVWGhk81'
>>> sha512_crypt('password', 'salt1234', rounds=1000)
'$6$rounds=1000$salt1234$SVDFHbJXYrzjGi2fA1k3ws01/D9q0ZTAh1KfRF5.ehgjVBqfHUaKqfynXefJ4DxIWxkMAITYq9mmcBl938YQ//'

You need to use crypt.crypt:
>>> import crypt
>>> crypt.crypt('password', '$6$' + 'salt1234')
'$6$salt1234$Zr07alHmuONZlfKILiGKKULQZaBG6Qmf5smHCNH35KnciTapZ7dItwaCv5SKZ1xH9ydG59SCgkdtsTqVWGhk81'

Here is a pure python3 implementation of the sha512_crypt function based on the specification. This is for illustration only, always use crypt.crypt instead!
import hashlib, base64
SHUFFLE_SHA512_INDICES = [
42, 21, 0, 1, 43, 22, 23, 2, 44, 45, 24, 3, 4, 46, 25,
26, 5, 47, 48, 27, 6, 7, 49, 28, 29, 8, 50, 51, 30, 9,
10, 52, 31, 32, 11, 53, 54, 33, 12, 13, 55, 34, 35, 14, 56,
57, 36, 15, 16, 58, 37, 38, 17, 59, 60, 39, 18, 19, 61, 40,
41, 20, 62, 63
]
def shuffle_sha512(data):
return bytes(data[i] for i in SHUFFLE_SHA512_INDICES)
def extend_by_repeat(data, length):
return (data * (length // len(data) + 1))[:length]
CUSTOM_ALPHABET = './0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
''' Base64 encode based on SECTION 22.e)
'''
def custom_b64encode(data, alphabet = CUSTOM_ALPHABET):
buffer,count,result = 0,0,[]
for byte in data:
buffer |= byte << count
count += 8
while count >= 6:
result.append(buffer & 0x3f)
buffer >>= 6
count -= 6
if count > 0:
result.append(buffer)
return ''.join(alphabet[idx] for idx in result)
''' From http://www.akkadia.org/drepper/SHA-crypt.txt
'''
def sha512_crypt(password, salt, rounds_in = None):
rounds,rounds_defined = 5000, False
if rounds_in is not None:
rounds,rounds_defined = rounds_in, True
assert 1000 <= rounds <= 999999999
hash = hashlib.sha512
salt_prefix = '$6$'
password = password.encode('utf8')
salt = salt.encode('ascii')[:16]
A = hash() # SECTION 1.
A.update(password) # SECTION 2.
A.update(salt) # SECTION 3.
B = hash() # SECTION 4.
B.update(password) # SECTION 5.
B.update(salt) # SECTION 6.
B.update(password) # SECTION 7.
digestB = B.digest(); # SECTION 8.
A.update(extend_by_repeat(digestB, len(password))) # SECTION 9., 10.
# SECTION 11.
i = len(password)
while i > 0:
if i & 1:
A.update(digestB) # SECTION 11.a)
else:
A.update(password) # SECTION 11.b)
i = i >> 1
digestA = A.digest() # SECTION 12.
DP = hash() # SECTION 13.
# SECTION 14.
for _ in range(len(password)):
DP.update(password)
digestDP = DP.digest() # SECTION 15.
P = extend_by_repeat(digestDP, len(password)) # SECTION 16.a), 16.b)
DS = hash() # SECTION 17.
# SECTION 18.
for _ in range(16 + digestA[0]):
DS.update(salt)
digestDS = DS.digest() # SECTION 19.
S = extend_by_repeat(digestDS, len(salt)) # SECTION 20.a), 20.b)
# SECTION 21.
digest_iteration_AC = digestA
for i in range(rounds):
C = hash() # SECTION 21.a)
if i % 2:
C.update(P) # SECTION 21.b)
else:
C.update(digest_iteration_AC) # SECTION 21.c)
if i % 3:
C.update(S) # SECTION 21.d)
if i % 7:
C.update(P) # SECTION 21.e)
if i % 2:
C.update(digest_iteration_AC) # SECTION 21.f)
else:
C.update(P) # SECTION 21.g)
digest_iteration_AC = C.digest() # SECTION 21.h)
shuffled_digest = shuffle_sha512(digest_iteration_AC)
prefix = salt_prefix # SECTION 22.a)
# SECTION 22.b)
if rounds_defined:
prefix += 'rounds={0}$'.format(rounds_in)
return (prefix
+ salt.decode('ascii') # SECTION 22.c)
+ '$' # SECTION 22.d)
+ custom_b64encode(shuffled_digest) # SECTION 22.e)
)
actual = sha512_crypt('password', 'salt1234')
expected = '$6$salt1234$Zr07alHmuONZlfKILiGKKULQZaBG6Qmf5smHCNH35KnciTapZ7dItwaCv5SKZ1xH9ydG59SCgkdtsTqVWGhk81'
print(actual)
print(expected)
assert actual == expected

Turning list of lists into objects

I currently have a list of lists where every list consists of the same kind of information, say:
[['Planet Name', 16, 19, 27, 11], ['Planet Name 2', 12, 22, 11, 42], ....]
and I would like to use a class to make this into a list of objects with the same information, where index 0 is self.name, index 1 is self.distance and so on for every seperate list.
I know that I need to use some kind of a for loop, but have no idea how to go about and do this.
I would really appreciate some help, trying to learn Python and currently classes!

You can use namedtuple like this, to create an object dynamically, with the list of field names. *item in this code is called, unpacking of arguments list
from collections import namedtuple
Planet = namedtuple("Planet", ["name", "distance", "a", "b", "c"])
data = [['Planet Name', 16, 19, 27, 11],['Planet Name 2', 12, 22, 11, 42]]
for item in data:
planet = Planet(*item)
print planet.name, planet.distance, planet
Output
Planet Name 16 Planet(name='Planet Name', distance=16, a=19, b=27, c=11)
Planet Name 2 12 Planet(name='Planet Name 2', distance=12, a=22, b=11, c=42)
Note: namedtuple is a subclass of tuple. So, all the objects created with namedtuple are immutable. It means that, once the object is created, data in the member variables cannot be changed.

Well... To make a class like you want you can do something like this:
class Planet(object):
def __init__(self, *args, **kwargs):
self.name = args[0]
self.distance = args[1]
# ... etc ...
Or something like this:
class Planet(object):
def __init__(self, name, distance, ...):
self.name = name
self.distance = distance
# ... etc ...
And then you call it like this:
p = Planet(*['Planet Name', 16, 19, 27, 11])
In a loop that would be:
l = [['Planet Name', 16, 19, 27, 11], ['Planet Name 2', 12, 22, 11, 42], ....]
planets = [Planet(*data) for data in l]

I'm confused. Have you created the Planet constructor yet?
The code would be something like:
class Planet(object):
def __init__(self, ....):
....
planets = [['Planet Name', 16, 19, 27, 11]['Planet Name 2', 12, 22, 11, 42]....]
planet_list = [Planet(*p) for p in planets]

If you don't want to have a constructor (__init__) which knows about the specifics of your lists, you could do it like this
lists = [['Planet Name', 16, 19, 27, 11], ['Planet Name 2', 12, 22, 11, 42]]
class Planet(object):
pass
for l in lists:
planet = Planet()
setattr(planet, 'name', l[0])
setattr(planet, 'distance', l[1])
setattr(planet, 'size', l[2])
print planet.name, planet.distance, planet.size

python json database corrupted

I got a bug in a huge database with thousand of lines, so i tried to remove thousands of them (obviously got a backup file) to lock out the problem and i end up with a problem. I'll provide a working without problem list in the database in case you want to compare:
#Working
["#saelyth", 8, 40, 4, "000", "000", "0", 11, "!Bot, me lees?", "legionanimenet", 0, "primermensajitodeesapersona"]
#Not working
["!anon7002", 545, 3166, 7, "000", "000", "0", 13, "\u2014\u00a1 Hijo! Estas calificaciones merecen una golpiza. \u2014\u00bf Verdad que si mam ...Vamos que yo se donde vive la maestra.", "legionanimenet", 0, "primermensajitodeesapersona"]
Causing this error:
ValueError: Extra data: line 1 column 240 - line 2 column 1 (char 239 - 366)
My question is: what is wrong there? i have no idea and all efforts to find what is the problem for json to give me such error are unsuccessful.
So i completely deleted that line and try to load the full database without that line but i also get now a new error:
ValueError: Expecting ',' delimiter: line 1 column 62 (char 61)
With so maaany and maaany and maaany records like:
["tyjyu", 59, 302, 19, "000", "000", "0", 13, "holas", "legionanimenet", 0, "primermensajitodeesapersona"]
["inuyacha64", 15944, 79401, 3496, "000", "F00", "0", 16, "cuidence chau", "legionanimenet", 2, "primermensajitodeesapersona"]
["!anon3573", 24, 140, "1", "nada", "nada", "nada", "nada", "nada", "legionanimenet", 0, "primermensajitodeesapersona"]
["eldiaoscuro", 74, 446, 16, "603", "369", "4", 13, "nada", "legionanimenet", 0, "primermensajitodeesapersona"]
What would be an efficient way to FIND the missing , giving me that error? And if possible i'd like to know also if json got a maximum number of items to load or something like that.
EDIT
The code to load info is:
data = []
with open('listas\Estadisticas.txt', 'r+') as f:
for line in f:
data_line = json.loads(line)
if data_line[0] == user.name and data_line[9] == "legionanimenet":
data_line[1] = int(data_line[1])+int(palabrasdelafrase)
data_line[2] = int(data_line[2])+int(letrasdelafrase)
data_line[3] = int(data_line[3])+1
data_line[4] = user.nameColor
data_line[5] = user.fontColor
data_line[6] = user.fontFace
data_line[7] = user.fontSize
data_line[11] = data_line[8]
data_line[8] = message.body
data_line[9] = "legionanimenet"
data.append(data_line)
f.seek(0)
f.writelines(["%s\n" % json.dumps(i) for i in data])
f.truncate()
I hope anyone can help me on that.
EDIT2: Python version is 3.3.2 IDLE

print(repr(data_line))
before loading the file will print it UNTIL find the error. I thank #nneonneo for that