I have the next value
value = bytearray(b'\x85\x13\xbd|\xfb\xbc\xc3\x95\xbeL6L\xfa\xbf0U_`$]\xca\xee]z\xef\xa0\xd6(\x15\x8b\xca\x0e\x1f7\xa9\xf0\xa4\x98\xc5\xdf\xcdM5\xef\xc2\x052`\xeb\x13\xd9\x99B.\x95\xb2\xbd\x96\xd9\x14\xe6F\x9e\xfd\xd8\x00')
when I try to convert in python3.x it works well.
>>> int.from_bytes(value, byteorder='little')
2909369579440607969688280064437289348250138784421305732473112318543540722321676649649580720015118044118243611774710427666475769804427735898727217762490192773
How to convert it in python2.7? I already read the convert a string of bytes into an int (python)
struct.unpack(fmt, value)[0]
But don't know what to do with fmt.
You can just write your own from_bytes function in Python 2:
def from_bytes (data, big_endian = False):
if isinstance(data, str):
data = bytearray(data)
if big_endian:
data = reversed(data)
num = 0
for offset, byte in enumerate(data):
num += byte << (offset * 8)
return num
Used like this:
>>> data = b'\x85\x13\xbd|\xfb\xbc\xc3\x95\xbeL6L\xfa\xbf0U_`$]\xca\xee]z\xef\xa0\xd6(\x15\x8b\xca\x0e\x1f7\xa9\xf0\xa4\x98\xc5\xdf\xcdM5\xef\xc2\x052`\xeb\x13\xd9\x99B.\x95\xb2\xbd\x96\xd9\x14\xe6F\x9e\xfd\xd8\x00'
>>> from_bytes(data)
2909369579440607969688280064437289348250138784421305732473112318543540722321676649649580720015118044118243611774710427666475769804427735898727217762490192773L
As for struct, you cannot really use this, as it only supports unpacking elements of a certain kind, up to 8 byte integers. But since you want to handle arbitrary byte strings, you will have to use something else.
You can use a combination of .encode('hex') and int(x, 16):
num = int(str(value).encode('hex'), 16)
Note that you need to use something like
int(''.join(reversed(value)).encode('hex'), 16)
in order to parse it as little endian.
reference: https://stackoverflow.com/a/444814/8747
Related
Iam receiving single bytes via serial and I know, that every 4 of them are a float. F.e. I receive b'\x3c' and b'\xff' and I want it to be b'\x3c\xff'.
What is the best way to convert it?
You can use join() as you do with strings.
byte_1 = b'\x3c'
byte_2 = b'\xff'
joined_bytes = b''.join([byte_1, byte_2]) #b'\x3c\xff'
You can use it along the struct module to obtain your decoded float, be aware it returns a tuple even if it has only one element inside.
import struct
byte_1 = b'\x3c'
byte_2 = b'\xff'
byte_3 = b'\x20'
byte_4 = b'\xff'
joined_bytes = b''.join([byte_1, byte_2, byte_3, byte_4])
result = struct.unpack('f', joined_bytes)
print(result[0])
TensorFlow provides 3 different formats for data to be stored in a tf.train.Feature. These are:
tf.train.BytesList
tf.train.FloatList
tf.train.Int64List
I often struggle to choose between tf.train.Int64List / tf.train.FloatList and tf.train.BytesList.
I see some examples online where they convert ints/floats into bytes and then store them in a tf.train.BytesList. Is this preferable to using one of the other formats? If so, why does TensorFlow even provide tf.train.Int64List and tf.train.FloatList as optional formats when you could just convert them to bytes and use tf.train.BytesList?
Thank you.
Because bytes list will require more memory. It's designed to store string data, or for example numpy arrays converted to single bytestring. Consider example:
def int64_feature(value):
if type(value) != list:
value = [value]
return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
def float_feature(value):
if type(value) != list:
value = [value]
return tf.train.Feature(float_list=tf.train.FloatList(value=value))
def bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
writer = tf.python_io.TFRecordWriter('file.tfrecords')
bytes = np.array(1.1).tostring()
int = 1
float = 1.1
example = tf.train.Example(features=tf.train.Features(feature={'1': float_feature(float)}))
writer.write(example.SerializeToString())
writer.close()
for str_rec in tf.python_io.tf_record_iterator('file.tfrecords'):
example = tf.train.Example()
example.ParseFromString(str_rec)
str = (example.features.feature['1'].float_list.value[0])
print(getsizeof(str))
For dtype float it will output 24 bytes, the lowest value. However, you can't pass int to a tf.train.FloatList. int dtype will occupy 28 bytes in this case, while bytes will be 41 undecoded(before applying np.fromstring) and even more after.
I am attempting to read my players health. I have been on a roll but have run into a problem. I am able to read what type of information is at a certain address but can't read what the actual value is, for example here is the response I receive.
<ctypes.c_char_Array_64 object at 0x0000000002EBF9C8>
I am looking for what information is held in the c_char_Array_64 object but have no idea how I would go about it.
Here is my code:
class User:
ctypes.wintypes.DWORD = "Entity"
ctypes.wintypes.c_int = "Team"
ctypes.wintypes.c_int = "Health"
ctypes.wintypes.c_int = "Player"
def getSelfInfo(self):
adr1 = clientdll + dw_LocalPlayer
adr2 = ctypes.create_string_buffer(64)
bytes_read = ctypes.c_size_t()
(rPM(PROCESS.handle, adr1, adr2, sys.getsizeof(ctypes.wintypes.DWORD), ctypes.byref(bytes_read)))
print adr2
t = User()
t.getSelfInfo()
You need to get the value:
print(ar2.value)
From the docs:
If you need mutable memory blocks, ctypes has a create_string_buffer()
function which creates these in various ways. The current memory block
contents can be accessed (or changed) with the raw property; if you
want to access it as NUL terminated string, use the value property:
>>> from ctypes import *
>>> p = create_string_buffer(3) # create a 3 byte buffer, initialized to NUL bytes
>>> print sizeof(p), repr(p.raw)
3 '\x00\x00\x00'
>>> p = create_string_buffer("Hello") # create a buffer containing a NUL terminated string
>>> print sizeof(p), repr(p.raw)
6 'Hello\x00'
>>> print repr(p.value)
'Hello'
>>> p = create_string_buffer("Hello", 10) # create a 10 byte buffer
>>> print sizeof(p), repr(p.raw)
10 'Hello\x00\x00\x00\x00\x00'
>>> p.value = "Hi"
>>> print sizeof(p), repr(p.raw)
10 'Hi\x00lo\x00\x00\x00\x00\x00'
>>>
The empty slice of most ctypes array types will return the Python equivalent type. So to convert your 64 byte buffer to a str (in Py3 bytes), you can do:
print ar2[:]
That will read the full raw 64 bytes mind you. If you want to read it as a C-style string (so the first NUL byte terminates the Python equivalent str), you'd use .value:
print ar2.value
I get data over TCP and try to compare it with a known value (0xAD):
b, addr = sock.recvfrom(1)
h = "".join(hex(ord(i)) for i in b)
print h
if h == str(0xad):
print "Work"
data = bytearray()
data.append(observer.OBSERVER_VALIDATION_BYTE)
sock.sendto(data, 0, addr)
I tried to compare them like strings as it shown above and tried to compare them like bytes in two ways:
b[0] == 0xAD
or
b2 = bytearray()
b2.append(0xAD)
b2[0] == b[0]
And all of the comparisons failed, though. print h gives me 0xad.
I have a set of bytes defined like BYTE = 0xAD.
I need to send them over TCP and compare the read result.
If I define them like strings (BYTE = '0xAD'), it provides an ability to compare, but I can't put them in the bytearray to send because bytearr.append(BYTE) reasonably returns an error. So I can't redefine them as strings. So what is the way to compare bytes got from sock.recvfrom and value declared in the way I have?
If your problem is casting, you can cast variable BYTES to a bytearray this way:
>>> BYTE = '0xAD'
>>> ba = bytearray([int(BYTE, 16)])
Then compare bytearrays using ==.
I am using the Python construct parser to process some binary data but am not managing to obtain strings in the way I expected.
Note that in the simplified example below I could use unpack or even just a slice, but the real data I am parsing does not align neatly to byte boundaries.
Some example code:
from construct import BitStruct, BitField, Padding, String
struct = BitStruct("foo",
BitField("bar", 8),
BitField("baz", 16),
Padding(4),
BitField("bat", 4)
)
struct2 = BitStruct("foo",
BitField("bar", 8),
String("baz", 16),
Padding(4),
BitField("bat", 4)
)
data = "\x01AB\xCD"
print struct.parse(data)
print struct2.parse(data)
This prints the output:
Container:
bar = 1
baz = 16706
bat = 13
Container:
bar = 1
baz = '\x00\x01\x00\x00\x00\x00\x00\x01\x00\x01\x00\x00\x00\x00\x01\x00'
bat = 13
I was expecting that String would give me back AB as an actual string. However it is returning the equivalent binary string instead.
How can I persuade construct to return me the actual ASCII string?
I solved this by creating an Adapter. The original ASCII values are parsed into a list of integers which can then be converted into a string representation.
It's not the most elegant way but due to BitStruct operating only on bit values it seems to be the easiest workaround. An improved version would parse different length strings (e.g. 7-bit ASCII).
from binascii import hexlify
from construct import BitStruct, BitField, Padding, Array, Octet, Adapter
class BitStringAdapter(Adapter):
def _encode(self, obj, context):
return list(ord(b) for b in obj)
def _decode(self, obj, context):
return "".join(chr(b) for b in obj)
struct = BitStruct("foo",
BitField("bar", 8),
BitStringAdapter(Array(2, Octet("baz"))),
Padding(4),
BitField("bat", 4)
)
data = "\x01AB\xCD"
out = struct.parse(data)
print hexlify(struct.build(out))
This outputs:
Container:
bar = 1
baz = 16706
bat = 13
0141420d
Which is correct - the C byte is discarded because it's marked as padding, this is fine.
The python module bitstruct can also be used to parse bit fields. It uses format strings, just like the standard library struct module.
The format specifier 't' is for text.
>>> from bitstruct import unpack
>>> data = b'\x01AB\xCD'
>>> unpack("u8u16p4u4", data)
(1, 16706, 13)
>>> unpack("u8t16p4u4", data)
(1, u'AB', 13)