Develop Reference

Python script taking too much memory [duplicate]

I've recently become interested in algorithms and have begun exploring them by writing a naive implementation and then optimizing it in various ways.
I'm already familiar with the standard Python module for profiling runtime (for most things I've found the timeit magic function in IPython to be sufficient), but I'm also interested in memory usage so I can explore those tradeoffs as well (e.g. the cost of caching a table of previously computed values versus recomputing them as needed). Is there a module that will profile the memory usage of a given function for me?

Python 3.4 includes a new module: tracemalloc. It provides detailed statistics about which code is allocating the most memory. Here's an example that displays the top three lines allocating memory.
from collections import Counter
import linecache
import os
import tracemalloc
def display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
tracemalloc.start()
counts = Counter()
fname = '/usr/share/dict/american-english'
with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1
print('Top prefixes:', counts.most_common(3))
snapshot = tracemalloc.take_snapshot()
display_top(snapshot)
And here are the results:
Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]
Top 3 lines
#1: scratches/memory_test.py:37: 6527.1 KiB
words = list(words)
#2: scratches/memory_test.py:39: 247.7 KiB
prefix = word[:3]
#3: scratches/memory_test.py:40: 193.0 KiB
counts[prefix] += 1
4 other: 4.3 KiB
Total allocated size: 6972.1 KiB
When is a memory leak not a leak?
That example is great when the memory is still being held at the end of the calculation, but sometimes you have code that allocates a lot of memory and then releases it all. It's not technically a memory leak, but it's using more memory than you think it should. How can you track memory usage when it all gets released? If it's your code, you can probably add some debugging code to take snapshots while it's running. If not, you can start a background thread to monitor memory usage while the main thread runs.
Here's the previous example where the code has all been moved into the count_prefixes() function. When that function returns, all the memory is released. I also added some sleep() calls to simulate a long-running calculation.
from collections import Counter
import linecache
import os
import tracemalloc
from time import sleep
def count_prefixes():
sleep(2) # Start up time.
counts = Counter()
fname = '/usr/share/dict/american-english'
with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1
sleep(0.0001)
most_common = counts.most_common(3)
sleep(3) # Shut down time.
return most_common
def main():
tracemalloc.start()
most_common = count_prefixes()
print('Top prefixes:', most_common)
snapshot = tracemalloc.take_snapshot()
display_top(snapshot)
def display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
main()
When I run that version, the memory usage has gone from 6MB down to 4KB, because the function released all its memory when it finished.
Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]
Top 3 lines
#1: collections/__init__.py:537: 0.7 KiB
self.update(*args, **kwds)
#2: collections/__init__.py:555: 0.6 KiB
return _heapq.nlargest(n, self.items(), key=_itemgetter(1))
#3: python3.6/heapq.py:569: 0.5 KiB
result = [(key(elem), i, elem) for i, elem in zip(range(0, -n, -1), it)]
10 other: 2.2 KiB
Total allocated size: 4.0 KiB
Now here's a version inspired by another answer that starts a second thread to monitor memory usage.
from collections import Counter
import linecache
import os
import tracemalloc
from datetime import datetime
from queue import Queue, Empty
from resource import getrusage, RUSAGE_SELF
from threading import Thread
from time import sleep
def memory_monitor(command_queue: Queue, poll_interval=1):
tracemalloc.start()
old_max = 0
snapshot = None
while True:
try:
command_queue.get(timeout=poll_interval)
if snapshot is not None:
print(datetime.now())
display_top(snapshot)
return
except Empty:
max_rss = getrusage(RUSAGE_SELF).ru_maxrss
if max_rss > old_max:
old_max = max_rss
snapshot = tracemalloc.take_snapshot()
print(datetime.now(), 'max RSS', max_rss)
def count_prefixes():
sleep(2) # Start up time.
counts = Counter()
fname = '/usr/share/dict/american-english'
with open(fname) as words:
words = list(words)
for word in words:
prefix = word[:3]
counts[prefix] += 1
sleep(0.0001)
most_common = counts.most_common(3)
sleep(3) # Shut down time.
return most_common
def main():
queue = Queue()
poll_interval = 0.1
monitor_thread = Thread(target=memory_monitor, args=(queue, poll_interval))
monitor_thread.start()
try:
most_common = count_prefixes()
print('Top prefixes:', most_common)
finally:
queue.put('stop')
monitor_thread.join()
def display_top(snapshot, key_type='lineno', limit=3):
snapshot = snapshot.filter_traces((
tracemalloc.Filter(False, "<frozen importlib._bootstrap>"),
tracemalloc.Filter(False, "<unknown>"),
))
top_stats = snapshot.statistics(key_type)
print("Top %s lines" % limit)
for index, stat in enumerate(top_stats[:limit], 1):
frame = stat.traceback[0]
# replace "/path/to/module/file.py" with "module/file.py"
filename = os.sep.join(frame.filename.split(os.sep)[-2:])
print("#%s: %s:%s: %.1f KiB"
% (index, filename, frame.lineno, stat.size / 1024))
line = linecache.getline(frame.filename, frame.lineno).strip()
if line:
print(' %s' % line)
other = top_stats[limit:]
if other:
size = sum(stat.size for stat in other)
print("%s other: %.1f KiB" % (len(other), size / 1024))
total = sum(stat.size for stat in top_stats)
print("Total allocated size: %.1f KiB" % (total / 1024))
main()
The resource module lets you check the current memory usage, and save the snapshot from the peak memory usage. The queue lets the main thread tell the memory monitor thread when to print its report and shut down. When it runs, it shows the memory being used by the list() call:
2018-05-29 10:34:34.441334 max RSS 10188
2018-05-29 10:34:36.475707 max RSS 23588
2018-05-29 10:34:36.616524 max RSS 38104
2018-05-29 10:34:36.772978 max RSS 45924
2018-05-29 10:34:36.929688 max RSS 46824
2018-05-29 10:34:37.087554 max RSS 46852
Top prefixes: [('con', 1220), ('dis', 1002), ('pro', 809)]
2018-05-29 10:34:56.281262
Top 3 lines
#1: scratches/scratch.py:36: 6527.0 KiB
words = list(words)
#2: scratches/scratch.py:38: 16.4 KiB
prefix = word[:3]
#3: scratches/scratch.py:39: 10.1 KiB
counts[prefix] += 1
19 other: 10.8 KiB
Total allocated size: 6564.3 KiB
If you're on Linux, you may find /proc/self/statm more useful than the resource module.

This one has been answered already here: Python memory profiler
Basically you do something like that (cited from Guppy-PE):
>>> from guppy import hpy; h=hpy()
>>> h.heap()
Partition of a set of 48477 objects. Total size = 3265516 bytes.
Index Count % Size % Cumulative % Kind (class / dict of class)
0 25773 53 1612820 49 1612820 49 str
1 11699 24 483960 15 2096780 64 tuple
2 174 0 241584 7 2338364 72 dict of module
3 3478 7 222592 7 2560956 78 types.CodeType
4 3296 7 184576 6 2745532 84 function
5 401 1 175112 5 2920644 89 dict of class
6 108 0 81888 3 3002532 92 dict (no owner)
7 114 0 79632 2 3082164 94 dict of type
8 117 0 51336 2 3133500 96 type
9 667 1 24012 1 3157512 97 __builtin__.wrapper_descriptor
<76 more rows. Type e.g. '_.more' to view.>
>>> h.iso(1,[],{})
Partition of a set of 3 objects. Total size = 176 bytes.
Index Count % Size % Cumulative % Kind (class / dict of class)
0 1 33 136 77 136 77 dict (no owner)
1 1 33 28 16 164 93 list
2 1 33 12 7 176 100 int
>>> x=[]
>>> h.iso(x).sp
0: h.Root.i0_modules['__main__'].__dict__['x']
>>>

If you only want to look at the memory usage of an object, (answer to other question)
There is a module called Pympler which contains the asizeof
module.
Use as follows:
from pympler import asizeof
asizeof.asizeof(my_object)
Unlike sys.getsizeof, it works for your self-created objects.
>>> asizeof.asizeof(tuple('bcd'))
200
>>> asizeof.asizeof({'foo': 'bar', 'baz': 'bar'})
400
>>> asizeof.asizeof({})
280
>>> asizeof.asizeof({'foo':'bar'})
360
>>> asizeof.asizeof('foo')
40
>>> asizeof.asizeof(Bar())
352
>>> asizeof.asizeof(Bar().__dict__)
280
>>> help(asizeof.asizeof)
Help on function asizeof in module pympler.asizeof:
asizeof(*objs, **opts)
Return the combined size in bytes of all objects passed as positional arguments.

Disclosure:
Applicable on Linux only
Reports memory used by the current process as a whole, not individual functions within
But nice because of its simplicity:
import resource
def using(point=""):
usage=resource.getrusage(resource.RUSAGE_SELF)
return '''%s: usertime=%s systime=%s mem=%s mb
'''%(point,usage[0],usage[1],
usage[2]/1024.0 )
Just insert using("Label") where you want to see what's going on. For example
print(using("before"))
wrk = ["wasting mem"] * 1000000
print(using("after"))
>>> before: usertime=2.117053 systime=1.703466 mem=53.97265625 mb
>>> after: usertime=2.12023 systime=1.70708 mem=60.8828125 mb

Below is a simple function decorator which allows to track how much memory the process consumed before the function call, after the function call, and what is the difference:
import time
import os
import psutil
def elapsed_since(start):
return time.strftime("%H:%M:%S", time.gmtime(time.time() - start))
def get_process_memory():
process = psutil.Process(os.getpid())
mem_info = process.memory_info()
return mem_info.rss
def profile(func):
def wrapper(*args, **kwargs):
mem_before = get_process_memory()
start = time.time()
result = func(*args, **kwargs)
elapsed_time = elapsed_since(start)
mem_after = get_process_memory()
print("{}: memory before: {:,}, after: {:,}, consumed: {:,}; exec time: {}".format(
func.__name__,
mem_before, mem_after, mem_after - mem_before,
elapsed_time))
return result
return wrapper
Here is my blog which describes all the details. (archived link)

Since the accepted answer and also the next highest voted answer have, in my opinion, some problems, I'd like to offer one more answer that is based closely on Ihor B.'s answer with some small but important modifications.
This solution allows you to run profiling on either by wrapping a function call with the profile function and calling it, or by decorating your function/method with the #profile decorator.
The first technique is useful when you want to profile some third-party code without messing with its source, whereas the second technique is a bit "cleaner" and works better when you are don't mind modifying the source of the function/method you want to profile.
I've also modified the output, so that you get RSS, VMS, and shared memory. I don't care much about the "before" and "after" values, but only the delta, so I removed those (if you're comparing to Ihor B.'s answer).
Profiling code
# profile.py
import time
import os
import psutil
import inspect
def elapsed_since(start):
#return time.strftime("%H:%M:%S", time.gmtime(time.time() - start))
elapsed = time.time() - start
if elapsed < 1:
return str(round(elapsed*1000,2)) + "ms"
if elapsed < 60:
return str(round(elapsed, 2)) + "s"
if elapsed < 3600:
return str(round(elapsed/60, 2)) + "min"
else:
return str(round(elapsed / 3600, 2)) + "hrs"
def get_process_memory():
process = psutil.Process(os.getpid())
mi = process.memory_info()
return mi.rss, mi.vms, mi.shared
def format_bytes(bytes):
if abs(bytes) < 1000:
return str(bytes)+"B"
elif abs(bytes) < 1e6:
return str(round(bytes/1e3,2)) + "kB"
elif abs(bytes) < 1e9:
return str(round(bytes / 1e6, 2)) + "MB"
else:
return str(round(bytes / 1e9, 2)) + "GB"
def profile(func, *args, **kwargs):
def wrapper(*args, **kwargs):
rss_before, vms_before, shared_before = get_process_memory()
start = time.time()
result = func(*args, **kwargs)
elapsed_time = elapsed_since(start)
rss_after, vms_after, shared_after = get_process_memory()
print("Profiling: {:>20} RSS: {:>8} | VMS: {:>8} | SHR {"
":>8} | time: {:>8}"
.format("<" + func.__name__ + ">",
format_bytes(rss_after - rss_before),
format_bytes(vms_after - vms_before),
format_bytes(shared_after - shared_before),
elapsed_time))
return result
if inspect.isfunction(func):
return wrapper
elif inspect.ismethod(func):
return wrapper(*args,**kwargs)
Example usage, assuming the above code is saved as profile.py:
from profile import profile
from time import sleep
from sklearn import datasets # Just an example of 3rd party function call
# Method 1
run_profiling = profile(datasets.load_digits)
data = run_profiling()
# Method 2
#profile
def my_function():
# do some stuff
a_list = []
for i in range(1,100000):
a_list.append(i)
return a_list
res = my_function()
This should result in output similar to the below:
Profiling: <load_digits> RSS: 5.07MB | VMS: 4.91MB | SHR 73.73kB | time: 89.99ms
Profiling: <my_function> RSS: 1.06MB | VMS: 1.35MB | SHR 0B | time: 8.43ms
A couple of important final notes:
Keep in mind, this method of profiling is only going to be approximate, since lots of other stuff might be happening on the machine. Due to garbage collection and other factors, the deltas might even be zero.
For some unknown reason, very short function calls (e.g. 1 or 2 ms)
show up with zero memory usage. I suspect this is some limitation of
the hardware/OS (tested on basic laptop with Linux) on how often
memory statistics are updated.
To keep the examples simple, I didn't use any function arguments, but they should work as one would expect, i.e.
profile(my_function, arg) to profile my_function(arg)

A simple example to calculate the memory usage of a block of codes / function using memory_profile, while returning result of the function:
import memory_profiler as mp
def fun(n):
tmp = []
for i in range(n):
tmp.extend(list(range(i*i)))
return "XXXXX"
calculate memory usage before running the code then calculate max usage during the code:
start_mem = mp.memory_usage(max_usage=True)
res = mp.memory_usage(proc=(fun, [100]), max_usage=True, retval=True)
print('start mem', start_mem)
print('max mem', res[0][0])
print('used mem', res[0][0]-start_mem)
print('fun output', res[1])
calculate usage in sampling points while running function:
res = mp.memory_usage((fun, [100]), interval=.001, retval=True)
print('min mem', min(res[0]))
print('max mem', max(res[0]))
print('used mem', max(res[0])-min(res[0]))
print('fun output', res[1])
Credits: #skeept

maybe it help:
<see additional>
pip install gprof2dot
sudo apt-get install graphviz
gprof2dot -f pstats profile_for_func1_001 | dot -Tpng -o profile.png
def profileit(name):
"""
#profileit("profile_for_func1_001")
"""
def inner(func):
def wrapper(*args, **kwargs):
prof = cProfile.Profile()
retval = prof.runcall(func, *args, **kwargs)
# Note use of name from outer scope
prof.dump_stats(name)
return retval
return wrapper
return inner
#profileit("profile_for_func1_001")
def func1(...)

Cipher program losing accuracy

I have a program in python that takes two strings. One is the plain text string, another is the cipher key. what it does is go over each of the characters and xors the bits with the cipher characters. But when going back and forth a few of the letter do not seem to change properly. Here is the code:
//turns int into bin string length 8
def bitString(n):
bin_string = bin(n)[2:]
bin_string = ("0" * (8 - len(bin_string))) + bin_string
return bin_string
//xors the bits
def bitXOR(b0, b1):
nb = ""
for x in range(min(len(b0), len(b1))):
nb += "0" if b0[x] == b1[x] else "1"
return nb
//takes 2 chars, turns them into bin strings, xors them, then returns the new char
def cypherChar(c0, c1):
return chr(int(bitXOR(bitString(ord(c0)), bitString(ord(c1))), 2))
//takes s0 (the plaintext) and encrypts it using the cipher key (s1)
def cypherString(s0, s1):
ns = ""
for x in range(len(s0)):
ns += cypherChar(s0[x], s1[x%len(s1)])
return ns
For example sometimes in a long string the word 'test' will cipher back into 'eest', and stuff like that
I have checked over the code a dozen times and I can't figure out whats causing some of the characters to change. Is it possible some characters just behave strangely?
EDIT:
example:
This is a test
Due to the fact that in the last test
Some symbols: !##$%^&*()
were not changed properly
I am retesting
END
using the cipher key : 'cypher key'
translates back to :
This is a test
Due to toe aact that in the last sest
Some symbols: !##$%^&*()
were not changed properly
I am retestiig
END

Sorry it its a little messy, I put it together real quick
from binascii import hexlify, unhexlify
from sys import version_info
def bit_string(string):
if version_info >= (3, 0):
return bin(int.from_bytes(string.encode(), 'big'))
else:
return bin(int(hexlify(string), 16))
def bitXOR_encrypt(plain_text, key):
encrypted_list = []
for j in range(2, len(plain_text)):
encrypted_list.append(int(plain_text[j]) ^ int(key[j])) #Assume the key and string are the same length
return encrypted_list
def decrypt(cipher_text, key):
decrypted_list = []
for j in range(2, len(cipher_text)): #or xrange
decrypted_list.append(int(cipher_text[j]) ^ int(key[j])) #Again assumes key is the same length as the string
decrypted_list = [str(i) for i in decrypted_list]
add_binary = "0b" + "".join(decrypted_list)
decrypted_string = int(add_binary, 2)
if version_info >= (3, 0):
message = decrypted_string.to_bytes((decrypted_string.bit_length() + 7) // 8, 'big').decode()
else:
message = unhexlify('%x' % decrypted_string)
return message
def main():
plain_text = "Hello"
plain_text_to_bits = bit_string(plain_text)
key_to_bits = bit_string("candy")
#Encrypt
cipher_text = bitXOR_encrypt(plain_text_to_bits, key_to_bits)
#make Strings
cipher_text_string = "".join([str(i) for i in cipher_text])
key_string = "".join([str(i) for i in key_to_bits])
#Decrypt
decrypted_string = decrypt("0B"+cipher_text_string, key_string)
print("plain text: %s" % plain_text)
print("plain text to bits: % s" % plain_text_to_bits)
print("key string in bits: %s" % key_string)
print("Ciphered Message: %s" %cipher_text_string)
print("Decrypted String: %s" % decrypted_string)
main()
for more details or example code you can visit my repository either on github
https://github.com/marcsantiago/one_time_pad_encryption
Also, I know that in this example the key is the same length as the string. If you want to use a string that is smaller than the string try wrapping it like in a vigenere cipher (http://en.wikipedia.org/wiki/Vigenère_cipher)

I think you are overcomplicating things:
def charxor(s1, s2):
return chr(ord(s1) ^ ord(s2))
def wordxor(w1, w2):
return ''.join(charxor(w1[i], w2[i]) for i in range(min(len(w1), len(w2))))
word = 'test'
key = 'what'
cyphered = wordxor(word, key)
uncyphered = wordxor(cyphered, key)
print(repr(cyphered))
print(uncyphered)
You get
'\x03\r\x12\x00'
test
There is a fairly good explanation of Python's bit arithmetic in How do you get the logical xor of two variables in Python?

I could find nothing wrong with the results of your functions when testing with your input data and key. To demonstrate, you could try this test code which should not fail:
import random
def random_string(n):
return ''.join(chr(random.getrandbits(8)) for _ in range(n))
for i in range(1000):
plaintext = random_string(500)
key = random_string(random.randrange(1,100))
ciphertext = cypherString(plaintext, key)
assert cypherString(ciphertext, key) == plaintext
If you can provide a definitive sample of plain text, key, and cipher text that fails, I can look further.

Reading a binary file into a struct

I have a binary file with a known format/structure.
How do I read all the binary data in to an array of the structure?
Something like (in pseudo code)
bytes = read_file(filename)
struct = {'int','int','float','byte[255]'}
data = read_as_struct(bytes, struct)
data[1]
>>> 10,11,10.1,Arr[255]
My solution so far is:
data = []
fmt = '=iiiii256i'
fmt_s = '=iiiii'
fmt_spec = '256i'
struct_size = struct.calcsize(fmt)
for i in range(struct_size, len(bytes)-struct_size, struct_size):
dat1= list(struct.unpack(fmt_s, bytes[i-struct_size:i-1024]))
dat2= list(struct.unpack(fmt_spec, bytes[i-1024:i]))
dat1.append(dat2)
data.append(dat1)

Actually it looks like you're trying to read a list (or array) of structures from the file. The idiomatic way to do this in Python is use the struct module and call struct.unpack() in a loop—either a fixed number of times if you know the number of them in advance, or until end-of-file is reached—and store the results in a list. Here's an example of the latter:
import struct
struct_fmt = '=5if255s' # int[5], float, byte[255]
struct_len = struct.calcsize(struct_fmt)
struct_unpack = struct.Struct(struct_fmt).unpack_from
results = []
with open(filename, "rb") as f:
while True:
data = f.read(struct_len)
if not data: break
s = struct_unpack(data)
results.append(s)
The same results can be also obtained slightly more concisely using a list comprehension along with a short generator function helper (i.e. read_chunks() below):
def read_chunks(f, length):
while True:
data = f.read(length)
if not data: break
yield data
with open(filename, "rb") as f:
results = [struct_unpack(chunk) for chunk in read_chunks(f, struct_len)]
Update
You don't, in fact, need to explicitly define a helper function as shown above because you can use Python's built-in iter() function to dynamically create the needed iterator object in the list comprehension itself like so:
from functools import partial
with open(filename, "rb") as f:
results = [struct_unpack(chunk) for chunk in iter(partial(f.read, struct_len), b'')]

Use the struct module; you need to define the types in a string format documented with that library:
struct.unpack('=HHf255s', bytes)
The above example expects native byte-order, two unsigned shorts, a float and a string of 255 characters.
To loop over an already fully read bytes string, I'd use itertools; there is a handy grouper recipe that I've adapted here:
from itertools import izip_longest, imap
from struct import unpack, calcsize
fmt_s = '=5i'
fmt_spec = '=256i'
size_s = calcsize(fmt_s)
size = size_s + calcsize(fmt_spec)
def chunked(iterable, n, fillvalue=''):
args = [iter(iterable)] * n
return imap(''.join, izip_longest(*args, fillvalue=fillvalue))
data = [unpack(fmt_s, section[:size_s]) + (unpack(fmt_spec, section[size_s:]),)
for section in chunked(bytes, size)]
This produces tuples rather than lists, but it's easy enough to adjust if you have to:
data = [list(unpack(fmt_s, section[:size_s])) + [list(unpack(fmt_spec, section[size_s:]))]
for section in chunked(bytes, size)]

Add comments
import struct
First just read the binary into an array
mbr = file('mbrcontent', 'rb').read()
So you can just fetch some piece of the the array
partition_table = mbr[446:510]
and then unpack it as an integer
signature = struct.unpack('<H', mbr[510:512])[0]
a more complex example
little_endian = (signature == 0xaa55) # should be True
print "Little endian:", little_endian
PART_FMT = (little_endian and '<' or '>') + (
"B" # status (0x80 = bootable (active), 0x00 = non-bootable)
# CHS of first block
"B" # Head
"B" # Sector is in bits 5; bits 9 of cylinder are in bits 7-6
"B" # bits 7-0 of cylinder
"B" # partition type
# CHS of last block
"B" # Head
"B" # Sector is in bits 5; bits 9 of cylinder are in bits 7-6
"B" # bits 7-0 of cylinder
"L" # LBA of first sector in the partition
"L" # number of blocks in partition, in little-endian format
)
PART_SIZE = 16
fmt_size = struct.calcsize(PART_FMT)
# sanity check expectations
assert fmt_size == PART_SIZE, "Partition format string is %i bytes, not %i" % (fmt_size, PART_SIZE)
def cyl_sector(sector_cyl, cylinder7_0):
sector = sector_cyl & 0x1F # bits 5-0
# bits 7-6 of sector_cyl contain bits 9-8 of the cylinder
cyl_high = (sector_cyl >> 5) & 0x03
cyl = (cyl_high << 8) | cylinder7_0
return sector, cyl
#I have corrected the indentation, but the change is refused because less than 6 characters, so I am adding this useful comment.
for partition in range(4):
print "Partition #%i" % partition,
offset = PART_SIZE * partition
(status, start_head, start_sector_cyl, start_cyl7_0, part_type, end_head, end_sector_cyl, end_cyl7_0,
lba, blocks ) = struct.unpack( PART_FMT,partition_table[offset:offset + PART_SIZE])
if status == 0x80:
print "Bootable",
elif status:
print "Unknown status [%s]" % hex(status),
print "Type=0x%x" % part_type
start = (start_head,) + cyl_sector(start_sector_cyl, start_cyl7_0)
end = (end_head,) + cyl_sector(end_sector_cyl, end_cyl7_0)
print " (Start: Heads:%i\tCyl:%i\tSect:%i)" % start
print " (End: Heads:%i\tCyl:%i\tSect:%i)" % end
print " LBA:", lba
print " Blocks:", blocks

import os, re
import functools
import ctypes
from ctypes import string_at, byref, sizeof, cast, POINTER, pointer, create_string_buffer, memmove
import numpy as np
import pandas as pd
class _StructBase(ctypes.Structure):
__type__ = 0
_fields_ = []
#classmethod
def Offsetof(cls, field):
pattern = '(?P<field>\w+)\[(?P<idx>\d+)\]'
mat = re.match(pattern, field)
if mat:
fields = dict(cls.Fields())
f = mat.groupdict()['field']
idx = mat.groupdict()['idx']
return cls.Offsetof(f) + int(idx) * ctypes.sizeof(fields[field])
else:
return getattr(cls, field).offset
#classmethod
def DType(cls):
map = {
ctypes.c_byte: np.byte,
ctypes.c_ubyte: np.ubyte,
ctypes.c_char: np.ubyte,
ctypes.c_int8: np.int8,
ctypes.c_int16: np.int16,
ctypes.c_int32: np.int32,
ctypes.c_int64: np.int64,
ctypes.c_uint8: np.uint8,
ctypes.c_uint16: np.uint16,
ctypes.c_uint32: np.uint32,
ctypes.c_uint64: np.uint64,
ctypes.c_float: np.float32,
ctypes.c_double: np.float64,
}
res = []
for k, v in cls.Fields():
if hasattr(v, '_length_'):
if v._type_ != ctypes.c_char:
for i in range(v._length):
res.append((k, map[v], cls.Offsetof(k)))
else:
res.append((k, 'S%d' % v._length_, cls.Offsetof(k)))
else:
res.append((k, map[v], cls.Offsetof(k)))
res = pd.DataFrame(res, columns=['name', 'format', 'offset'])
return np.dtype({
'names': res['name'],
'formats': res['format'],
'offsets': res['offset'],
})
#classmethod
def Attr(cls):
fields = cls._fields_
res = []
for attr, tp in fields:
if str(tp).find('_Array_') > 0 and str(tp).find('char_Array_') < 0:
for i in range(tp._length_):
res.append((attr + '[%s]' % str(i), tp._type_))
else:
res.append((attr, tp))
return res
#classmethod
def Fields(cls, notype=False):
res = [cls.Attr()]
cur_cls = cls
while True:
cur_cls = cur_cls.__bases__[0]
if cur_cls == ctypes.Structure:
break
res.append(cur_cls.Attr())
if notype:
return [k for k, v in functools.reduce(list.__add__, reversed(res), [])]
else:
return functools.reduce(list.__add__, reversed(res), [])
#classmethod
def size(cls):
return sizeof(cls)
#classmethod
def from_struct_binary(cls, path, max_count=2 ** 32, decode=True):
print(os.path.getsize(path), cls.size())
assert os.path.getsize(path) % cls.size() == 0
size = os.path.getsize(path) // cls.size()
size = min(size, max_count)
index = range(size)
array = np.fromfile(path, dtype=cls.DType(), count=size)
df = pd.DataFrame(array, index=index)
for attr, tp in eval(str(cls.DType())):
if re.match('S\d+', tp) is not None and decode:
try:
df[attr] = df[attr].map(lambda x: x.decode("utf-8"))
except:
df[attr] = df[attr].map(lambda x: x.decode("gbk"))
return df
class StructBase(_StructBase):
_fields_ = [
('Type', ctypes.c_uint32),
]
class IndexStruct(StructBase):
_fields_ = [
('Seq', ctypes.c_uint32),
('ExID', ctypes.c_char * 8),
('SecID', ctypes.c_char * 8),
('SecName', ctypes.c_char * 16),
('SourceID', ctypes.c_int32),
('Time', ctypes.c_uint32),
('PreClose', ctypes.c_uint32),
('Open', ctypes.c_uint32),
('High', ctypes.c_uint32),
('Low', ctypes.c_uint32),
('Match', ctypes.c_uint32),
]
df = IndexStruct.from_struct_binary('your path')
print(df)

Parsing srt subtitles

I want to parse srt subtitles:
1
00:00:12,815 --> 00:00:14,509
Chlapi, jak to jde s
těma pracovníma světlama?.
2
00:00:14,815 --> 00:00:16,498
Trochu je zesilujeme.
3
00:00:16,934 --> 00:00:17,814
Jo, sleduj.
Every item into structure. With this regexs:
A:
RE_ITEM = re.compile(r'(?P<index>\d+).'
r'(?P<start>\d{2}:\d{2}:\d{2},\d{3}) --> '
r'(?P<end>\d{2}:\d{2}:\d{2},\d{3}).'
r'(?P<text>.*?)', re.DOTALL)
B:
RE_ITEM = re.compile(r'(?P<index>\d+).'
r'(?P<start>\d{2}:\d{2}:\d{2},\d{3}) --> '
r'(?P<end>\d{2}:\d{2}:\d{2},\d{3}).'
r'(?P<text>.*)', re.DOTALL)
And this code:
for i in Subtitles.RE_ITEM.finditer(text):
result.append((i.group('index'), i.group('start'),
i.group('end'), i.group('text')))
With code B I have only one item in array (because of greedy .*) and with code A I have empty 'text' because of no-greedy .*?
How to cure this?
Thanks

Why not use pysrt?

I became quite frustrated with srt libraries available for Python (often because they were heavyweight and eschewed language-standard types in favour of custom classes), so I've spent the last year or so working on my own srt library. You can get it at https://github.com/cdown/srt.
I tried to keep it simple and light on classes (except for the core Subtitle class, which more or less just stores the SRT block data). It can read and write SRT files, and turn noncompliant SRT files into compliant ones.
Here's a usage example with your sample input:
>>> import srt, pprint
>>> gen = srt.parse('''\
... 1
... 00:00:12,815 --> 00:00:14,509
... Chlapi, jak to jde s
... těma pracovníma světlama?.
...
... 2
... 00:00:14,815 --> 00:00:16,498
... Trochu je zesilujeme.
...
... 3
... 00:00:16,934 --> 00:00:17,814
... Jo, sleduj.
...
... ''')
>>> pprint.pprint(list(gen))
[Subtitle(start=datetime.timedelta(0, 12, 815000), end=datetime.timedelta(0, 14, 509000), index=1, proprietary='', content='Chlapi, jak to jde s\ntěma pracovníma světlama?.'),
Subtitle(start=datetime.timedelta(0, 14, 815000), end=datetime.timedelta(0, 16, 498000), index=2, proprietary='', content='Trochu je zesilujeme.'),
Subtitle(start=datetime.timedelta(0, 16, 934000), end=datetime.timedelta(0, 17, 814000), index=3, proprietary='', content='Jo, sleduj.')]

The text is followed by an empty line, or the end of file. So you can use:
r' .... (?P<text>.*?)(\n\n|$)'

Here's some code I had lying around to parse SRT files:
from __future__ import division
import datetime
class Srt_entry(object):
def __init__(self, lines):
def parsetime(string):
hours, minutes, seconds = string.split(u':')
hours = int(hours)
minutes = int(minutes)
seconds = float(u'.'.join(seconds.split(u',')))
return datetime.timedelta(0, seconds, 0, 0, minutes, hours)
self.index = int(lines[0])
start, arrow, end = lines[1].split()
self.start = parsetime(start)
if arrow != u"-->":
raise ValueError
self.end = parsetime(end)
self.lines = lines[2:]
if not self.lines[-1]:
del self.lines[-1]
def __unicode__(self):
def delta_to_string(d):
hours = (d.days * 24) \
+ (d.seconds // (60 * 60))
minutes = (d.seconds // 60) % 60
seconds = d.seconds % 60 + d.microseconds / 1000000
return u','.join((u"%02d:%02d:%06.3f"
% (hours, minutes, seconds)).split(u'.'))
return (unicode(self.index) + u'\n'
+ delta_to_string(self.start)
+ ' --> '
+ delta_to_string(self.end) + u'\n'
+ u''.join(self.lines))
srt_file = open("foo.srt")
entries = []
entry = []
for line in srt_file:
if options.decode:
line = line.decode(options.decode)
if line == u'\n':
entries.append(Srt_entry(entry))
entry = []
else:
entry.append(line)
srt_file.close()

splits = [s.strip() for s in re.split(r'\n\s*\n', text) if s.strip()]
regex = re.compile(r'''(?P<index>\d+).*?(?P<start>\d{2}:\d{2}:\d{2},\d{3}) --> (?P<end>\d{2}:\d{2}:\d{2},\d{3})\s*.*?\s*(?P<text>.*)''', re.DOTALL)
for s in splits:
r = regex.search(s)
print r.groups()

Here's a snippet I wrote which converts SRT files into dictionaries:
import re
def srt_time_to_seconds(time):
split_time=time.split(',')
major, minor = (split_time[0].split(':'), split_time[1])
return int(major[0])*1440 + int(major[1])*60 + int(major[2]) + float(minor)/1000
def srt_to_dict(srtText):
subs=[]
for s in re.sub('\r\n', '\n', srtText).split('\n\n'):
st = s.split('\n')
if len(st)>=3:
split = st[1].split(' --> ')
subs.append({'start': srt_time_to_seconds(split[0].strip()),
'end': srt_time_to_seconds(split[1].strip()),
'text': '<br />'.join(j for j in st[2:len(st)])
})
return subs
Usage:
import srt_to_dict
with open('test.srt', "r") as f:
srtText = f.read()
print srt_to_dict(srtText)

Python: File formatting

I have a for loop which references a dictionary and prints out the value associated with the key. Code is below:
for i in data:
if i in dict:
print dict[i],
How would i format the output so a new line is created every 60 characters? and with the character count along the side for example:
0001
MRQLLLISDLDNTWVGDQQALEHLQEYLGDRRGNFYLAYATGRSYHSARELQKQVGLMEP
0061
DYWLTAVGSEIYHPEGLDQHWADYLSEHWQRDILQAIADGFEALKPQSPLEQNPWKISYH
0121 LDPQACPTVIDQLTEMLKETGIPVQVIFSSGKDVDLLPQRSNKGNATQYLQQHLAMEPSQ

It's a finicky formatting problem, but I think the following code:
import sys
class EveryN(object):
def __init__(self, n, outs):
self.n = n # chars/line
self.outs = outs # output stream
self.numo = 1 # next tag to write
self.tll = 0 # tot chars on this line
def write(self, s):
while True:
if self.tll == 0: # start of line: emit tag
self.outs.write('%4.4d ' % self.numo)
self.numo += self.n
# wite up to N chars/line, no more
numw = min(len(s), self.n - self.tll)
self.outs.write(s[:numw])
self.tll += numw
if self.tll >= self.n:
self.tll = 0
self.outs.write('\n')
s = s[numw:]
if not s: break
if __name__ == '__main__':
sys.stdout = EveryN(60, sys.stdout)
for i, a in enumerate('abcdefgh'):
print a*(5+ i*5),
shows how to do it -- the output when running for demonstration purposes as the main script (five a's, ten b's, etc, with spaces in-between) is:
0001 aaaaa bbbbbbbbbb ccccccccccccccc dddddddddddddddddddd eeeeee
0061 eeeeeeeeeeeeeeeeeee ffffffffffffffffffffffffffffff ggggggggg
0121 gggggggggggggggggggggggggg hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh
0181 hhhhhhh

# test data
data = range(10)
the_dict = dict((i, str(i)*200) for i in range( 10 ))
# your loops as a generator
lines = ( the_dict[i] for i in data if i in the_dict )
def format( line ):
def splitter():
k = 0
while True:
r = line[k:k+60] # take a 60 char block
if r: # if there are any chars left
yield "%04d %s" % (k+1, r) # format them
else:
break
k += 60
return '\n'.join(splitter()) # join all the numbered blocks
for line in lines:
print format(line)

I haven't tested it on actual data, but I believe the code below would do the job. It first builds up the whole string, then outputs it a 60-character line at a time. It uses the three-argument version of range() to count by 60.
s = ''.join(dict[i] for i in data if i in dict)
for i in range(0, len(s), 60):
print '%04d %s' % (i+1, s[i:i+60])

It seems like you're looking for textwrap
The textwrap module provides two convenience functions, wrap() and
fill(), as well as TextWrapper, the class that does all the work, and
a utility function dedent(). If you’re just wrapping or filling one or
two text strings, the convenience functions should be good enough;
otherwise, you should use an instance of TextWrapper for efficiency.