Related
I created ten threads to process item in a global list
but I do not why only the first worker do this work
and also, the main thread finished before sub-threads finished, even though I had used the thread.join().
here is the code, I think the problem may because I use while loop in myThread.run . but I do not know how to tell these threads keep working before the global list is empty.
# coding=utf-8
import threading
import numpy as np
dfs = ['units' + str(i).zfill(5) for i in range(250)]
units = dfs.copy()
k = [str(i).zfill(5) for i in range(500, 21800000)]
units.extend(k)
np.random.shuffle(units)
marker = []
def working_fun(df, unit):
global marker
if unit in df:
threadlock.acquire()
marker.append(int(unit[5:]))
class myThread(threading.Thread):
def __init__(self, name):
threading.Thread.__init__(self)
self.name = name
self.work_load = []
def run(self):
global dfs
print("start thread" + self.name)
while True:
threadlock.acquire()
if units != []:
unit = units.pop()
else:
unit = None
threadlock.release()
if unit is not None:
self.work_load.append(unit)
working_fun(dfs, unit)
else:
print('------', self.name, '--finish---', len(self.work_load), '--------')
break
threadlock = threading.RLock()
thds = []
for i in range(10):
thd = myThread(name='thd' + str(i))
thds.append(thd)
for thd in thds:
thd.start()
thd.join()
print('output:', marker)
Try it this way:
import numpy
import multiprocessing
# Same as before
dfs = ['units' + str(i).zfill(5) for i in range(250)]
units = dfs.copy()
k = [str(i).zfill(5) for i in range(500, 21800000)]
units.extend(k)
numpy.random.shuffle(units)
# Almost the same as before
def working_fun(inp):
df, unit = inp
if unit in df:
return int(unit[5:])
# This is needed for multiprocessing/threading
if __name__ == "__main__":
# Create a pool of workers (10 in this case)
with multiprocessing.Pool(10) as pool:
# Map some (global) iterable on the pool of workers
result = pool.map(working_fun, [(dfs, unit) for unit in units])
# Show the results (note that the function returns None if the unit is not in df)
print([r for r in result if r is not None])
Output:
$ python test.py
[1, 75, 139, 24, 101, 72, 156, 55, 58, 235, 14, 123, 177, 112, 168, 178, 173, 162, 104, 226, 230, 205, 69, 100, 246, 18, 117, 149, 37, 214, 206, 26, 136, 87, 144, 79, 50, 222, 7, 133, 36, 41, 30, 163, 103, 187, 6, 225, 15, 223, 234, 138, 126, 19, 64, 224, 39, 145, 130, 42, 11, 221, 128, 213, 204, 2, 45, 220, 242, 109, 59, 238, 232, 68, 152, 107, 148, 83, 197, 241, 118, 32, 90, 99, 22, 119, 0, 67, 48, 181, 71, 193, 95, 29, 113, 40, 134, 218, 141, 27, 121, 8, 207, 110, 60, 237, 47, 94, 73, 157, 184, 78, 159, 49, 202, 239, 124, 215, 127, 209, 62, 4, 52, 82, 74, 9, 199, 158, 188, 3, 61, 180, 57, 219, 245, 38, 16, 190, 12, 17, 175, 46, 196, 125, 194, 76, 129, 161, 81, 93, 137, 155, 174, 54, 35, 25, 115, 140, 216, 23, 21, 233, 77, 33, 92, 208, 120, 86, 165, 70, 135, 28, 91, 66, 85, 169, 203, 211, 114, 154, 122, 217, 247, 31, 147, 96, 142, 191, 10, 183, 80, 179, 189, 56, 105, 160, 228, 185, 132, 5, 53, 106, 13, 210, 182, 89, 192, 153, 170, 111, 65, 212, 186, 151, 200, 248, 229, 102, 240, 198, 176, 43, 131, 166, 236, 231, 116, 172, 146, 88, 44, 98, 227, 20, 34, 164, 108, 171, 244, 243, 195, 150, 249, 97, 167, 51, 201, 84, 63, 143]
I am trying to sign a byte array in python by the same way as it happens on the crypto library with the secp256k1 from NodeJS
This is the code on NodeJS/Browser:
const secp256k1 = require('secp256k1')
var message = [2, 118, 145, 101, 166, 249, 149, 13, 2, 58, 65, 94, 230, 104, 184, 11, 185, 107, 92, 154, 226, 3, 93, 151, 189, 251, 68, 243, 86, 23, 90, 68, 255, 111, 3, 0, 0, 0, 0, 0, 0, 187, 226, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 84, 101, 115, 116, 105, 0, 0, 0, 0, 0, 0, 0];
var private_key_buffer = [122, 241, 114, 103, 51, 227, 157, 149, 221, 126, 157, 173, 31, 111, 43, 118, 208, 71, 123, 59, 96, 68, 57, 177, 53, 59, 151, 188, 36, 167, 40, 68]
const signature = secp256k1.sign(SHA3BUF(message), private_key_buffer)
This is my implementation in python:
import hashlib
import ecdsa
message = bytearray([2, 118, 145, 101, 166, 249, 149, 13, 2, 58, 65, 94, 230, 104, 184, 11, 185, 107, 92, 154, 226, 3, 93, 151, 189, 251, 68, 243, 86, 23, 90, 68, 255, 111, 3, 0, 0, 0, 0, 0, 0, 187, 226, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 0, 84, 101, 115, 116, 105, 0, 0, 0, 0, 0, 0, 0])
private_key_buffer = bytearray([122, 241, 114, 103, 51, 227, 157, 149, 221, 126, 157, 173, 31, 111, 43, 118, 208, 71, 123, 59, 96, 68, 57, 177, 53, 59, 151, 188, 36, 167, 40, 68])
signinKey = ecdsa.SigningKey.from_string(private_key_buffer, curve=ecdsa.SECP256k1)
signature = signinKey.sign_deterministic(message, hashfunc=hashlib.sha3_256)
but by some reason, the signature I get in the javascript code is diferent from the python code:
java script signature: [23, 54, 64, 151, 95, 33, 200, 66, 246, 166, 144, 182, 81, 179, 124, 223, 250, 50, 137, 169, 45, 181, 197, 74, 225, 207, 116, 125, 50, 241, 38, 52, 118, 215, 252, 94, 191, 154, 200, 195, 152, 73, 1, 197, 158, 24, 72, 177, 118, 39, 241, 82, 114, 107, 25, 106, 67, 205, 202, 4, 7, 57, 82, 237]
python script signature: [213, 69, 97, 237, 85, 226, 217, 201, 51, 14, 220, 92, 105, 59, 54, 92, 87, 88, 233, 147, 191, 15, 21, 86, 134, 202, 205, 223, 83, 134, 70, 39, 10, 19, 147, 20, 181, 180, 88, 103, 79, 55, 144, 98, 84, 2, 224, 127, 192, 200, 200, 250, 170, 129, 67, 99, 163, 72, 92, 253, 109, 108, 104, 206]
So how can I make the python code output the same signature of the JS code?
For the deterministic ECDSA as described in RFC6979, a hash algorithm is used in two places: One algorithm (H1) is used for hashing the message, another (H2) for determining the k-value. k is a parameter within the signature algorithm, whose role is described e.g. in RFC6979, section 2.4 or also here. For the non-deterministic variant k is determined randomly, for the deterministic variant as described in RFC6979.
RFC6979 doesn't specify that H1 and H2 must be different, see RFC6979, section 3.6. Nevertheless, it makes sense that an implementation offers the possibility to define both hash algorithms separately.
The ECDSA implementation of Python generally allows two different hash algorithms to be applied. Before this is shown in the 2nd case, the following variant, which correponds to the posted Python-code, applies the same hash algorithm H1 = H2 = SHA3-256. The hash algorithm specified in the sign_deterministic-method defines both H1 and H2:
import hashlib
import ecdsa
message = b'Everything should be made as simple as possible, but not simpler.'
private_key_buffer = bytearray.fromhex('0000000000000000000000000000000000000000000000000000000000000001')
sk = ecdsa.SigningKey.from_string(private_key_buffer, curve=ecdsa.SECP256k1)
signature = sk.sign_deterministic(message, hashfunc=hashlib.sha3_256)
print(signature.hex())
The signature is:
r = 88ecdbc6a2762e7ad1160f7c984cd61385ff07982280538dd7d2103be2dce720
s = c1487df9feab7afda6e6115bdd4d9c5316e3f917a3235a5e47aee09624491304
The next variant uses H1 = SHA3-256 for hashing the message and H2 = SHA256 for k-determination. This is possible by replacing the sign_deterministic-method with the sign_digest_deterministic-method, which allows separate hashing of the message with H1. The hash algorithm specified in the sign_digest_deterministic-method then only defines H2:
import hashlib
import ecdsa
message = b'Everything should be made as simple as possible, but not simpler.'
private_key_buffer = bytearray.fromhex('0000000000000000000000000000000000000000000000000000000000000001')
digest = hashlib.sha3_256()
digest.update(message)
hash = digest.digest()
sk = ecdsa.SigningKey.from_string(private_key_buffer, curve=ecdsa.SECP256k1)
signature = sk.sign_digest_deterministic(hash, hashfunc=hashlib.sha256)
print(signature.hex())
The signature is:
r = 64b10395957b78d3bd3db279e5fa4ebee36b58dd1becace4bc2d7e3a04cf6259
s = 19f1eee7495064ac679d7b64ab7213b921b650c0a3746f2938ffeede0ff1f2e8
The following code is functionally identical to the posted NodeJS-code:
const secp256k1 = require('secp256k1')
const sha3 = require('js-sha3')
message = 'Everything should be made as simple as possible, but not simpler.'
private_key_buffer = Buffer.from('0000000000000000000000000000000000000000000000000000000000000001','hex')
digest = sha3.sha3_256;
hash = Buffer.from(digest(message), 'hex')
signature = secp256k1.sign(hash, private_key_buffer)
console.log(signature.signature.toString('hex'))
and generates the same signature as in the 2nd case, i.e. apparently H2 = SHA256. I didn't find a way to change this to SHA3-256 without much effort. However, according to the documentation it is possible to replace the default generator that implements RFC6979. This should also change H2, but could be more expensive.
In summary: The simplest way to fix the incompatibility of both codes
is to change the Python-code as described above in the 2nd case, i.e. to use the sign_digest_deterministic-method. The message is then hashed with SHA3-256, the k-generation takes place with SHA256. A more expensive alternative would be to implement an own generator to enable k-generation with SHA3-256 in the NodeJS-code. Or of course, you try to find another ECDSA-library for the NodeJS-code that allows you to define H1 and H2 separately, analogous to the Python-code.
Update:
Canonical signature: If (r,s) is a signature, then (r, -s mod n) = (r, n - s) is also a valid signature. Here n is the order of the base point. If in case s > n/2 the part -s mod n = n - s is used instead of s, then the result for the signature is unambiguous and is limited to the area below n/2. This is called canonical signature, which is particularly relevant for the Bitcoin topic and also frequently used for test vectors.
I am using OpenCV 2.4.9 Python knnMatch where the query descriptors come directly from detectAndCompute and are formatted correctly, but the train descriptors will come from a list I made in a different program.
When I get the descriptors from my other program, they look like:
[array([ 14, 21, 234, 147, 215, 115, 190, 215, 94, 231, 31, 34, 200,
124, 127, 104, 255, 123, 179, 147, 180, 240, 61, 226, 111, 95,
159, 131, 151, 127, 253, 231], dtype=uint8), array([162, 150, 101, 219, 117, 151, 173, 113, 93, 29, 81, 23, 232,
13, 60, 133, 221, 2, 147, 165, 242, 188, 120, 221, 39, 26,
154, 194, 87, 140, 245, 252], dtype=uint8)]
That would be 2 descriptors.
How can I format these so I do not get the "OpenCV Error: Unsupported format or combination of formats" error when matching these descriptors with those coming straight out of detectAndCompute? I have tried using np.asarray(list, np.float32) to no avail. If I do:
[[d for d in des] for des in list] with list as the train descriptors then the two lists will LOOK the same but I get the same error!
list = [[d for d in des] for des in list]
list = np.asarray(list, np.uint8)
for d in list:
for x in d:
x = x.astype(np.uint8)
I'm trying to make a smooth fit to some unregular data points in 3D.
Data points for which I have z-values are not lying on a grid, however they do follow some pattern. There is a limited set of integer y values (say, 10, 14, 30, 41, etc) and for each of these I've got a set of ~100 x float values ranging from 0 to 1 in some random non-uniform way.
For my case it is important to have the fit pass through all of the initial data points (or at least to have a maximal allowance if not so).
So, given that data my first try was to use griddata from scipy.interpolate. If I use linear fit, I get a picture which I expect:
However, cubic interpolation doesn't work good for me. Firstly, it hangs forever apparently because I have many aligned points, but that can be fixed by adding some tiny random number to each of y values which I did (see this question: Alternatives to scipy.interpolate.griddata that don't hang on aligned points). But it produces result with wild oscillations:
So, the question is how can I tweak the data and / or the algorithm to deal with it? If it is not possible, which other interpolation method could I use to create a smoothed surface close to the original one and if Python has an implementation for it?
Some sample data to try it out:
x = [0.95568, 0.94055, 0.91823, 0.88796, 0.84862, 0.79112, 0.7164900000000001, 0.6240399999999999, 0.5173399999999999, 0.40369, 0.29454, 0.20756, 0.14302, 0.09365, 0.06564, 0.04701, 0.95658, 0.95192, 0.94714, 0.94223, 0.93716, 0.93034, 0.92336, 0.91618, 0.90876, 0.89963, 0.89024, 0.87918, 0.86782, 0.85608, 0.84268, 0.82767, 0.81219, 0.79511, 0.77554, 0.75548, 0.73397, 0.71104, 0.68674, 0.66057, 0.6332599999999999, 0.6045, 0.57483, 0.5442199999999999, 0.51271, 0.48081999999999997, 0.44866999999999996, 0.41712, 0.38525, 0.35475, 0.32392, 0.29431999999999997, 0.26863000000000004, 0.24311, 0.22016999999999998, 0.19732, 0.17751, 0.15778, 0.14337, 0.1293, 0.11547, 0.10378, 0.09437000000000001, 0.08304, 0.0762, 0.06959, 0.06316000000000001, 0.056870000000000004, 0.05305, 0.04938, 0.04583, 0.04239, 0.04141, 0.95758, 0.95418, 0.9507099999999999, 0.94718, 0.9424, 0.93871, 0.93379, 0.92769, 0.9226, 0.91635, 0.91001, 0.90355, 0.89695, 0.88925, 0.8814, 0.8725, 0.86346, 0.85424, 0.844, 0.83358, 0.82218, 0.81057, 0.79801, 0.78522, 0.77151, 0.75695, 0.74212, 0.7259599999999999, 0.7095400000000001, 0.6928000000000001, 0.67492, 0.65671, 0.6375500000000001, 0.61807, 0.59798, 0.57735, 0.5562, 0.5346, 0.51262, 0.49051999999999996, 0.46813, 0.44582, 0.42352, 0.40166999999999997, 0.37961, 0.35773, 0.33666, 0.31537, 0.29589, 0.27712, 0.25829, 0.24111999999999997, 0.22393000000000002, 0.2087, 0.19242, 0.1783, 0.16417, 0.15253, 0.14231, 0.13223, 0.12222999999999999, 0.11374000000000001, 0.10535, 0.09702000000000001, 0.09031, 0.08367999999999999, 0.07057999999999999, 0.0658, 0.06283, 0.0582, 0.055389999999999995, 0.05265, 0.049980000000000004, 0.047369999999999995, 0.04481, 0.95916, 0.95644, 0.95368, 0.94994, 0.94709, 0.94328, 0.93942, 0.9355100000000001, 0.93155, 0.92752, 0.92258, 0.91841, 0.91333, 0.90739, 0.89607, 0.88363, 0.86937, 0.86207, 0.85465, 0.84645, 0.8375, 0.82845, 0.81925, 0.80935, 0.79878, 0.7885800000000001, 0.7772300000000001, 0.7540800000000001, 0.7418199999999999, 0.72936, 0.71632, 0.70275, 0.68897, 0.6744300000000001, 0.6597, 0.64455, 0.6290100000000001, 0.61327, 0.58075, 0.54713, 0.52998, 0.5126999999999999, 0.47773999999999994, 0.46008999999999994, 0.44242, 0.42504, 0.4075, 0.38974000000000003, 0.37248000000000003, 0.35545, 0.33824, 0.32234, 0.30695, 0.27663000000000004, 0.26236, 0.24877, 0.2223, 0.19730999999999999, 0.18517, 0.17392, 0.15335, 0.14518, 0.13709000000000002, 0.1222, 0.11542000000000001, 0.10869000000000001, 0.09658, 0.08592000000000001, 0.07537999999999999, 0.0677, 0.060160000000000005, 0.05566, 0.04986, 0.04565, 0.04155, 0.9578, 0.9522, 0.94897, 0.94571, 0.93909, 0.93572, 0.93157, 0.92387, 0.9146, 0.90512, 0.89475, 0.88291, 0.8708, 0.8572500000000001, 0.85036, 0.84285, 0.82759, 0.81933, 0.81098, 0.8025100000000001, 0.7935, 0.77511, 0.76534, 0.75507, 0.74468, 0.73414, 0.72314, 0.7117100000000001, 0.70014, 0.6883900000000001, 0.67624, 0.66372, 0.6508499999999999, 0.63783, 0.62449, 0.61087, 0.59709, 0.58306, 0.56881, 0.5543899999999999, 0.53978, 0.52502, 0.51024, 0.49533, 0.4804, 0.46531000000000006, 0.45023, 0.43518999999999997, 0.42025, 0.40544, 0.3905, 0.37573, 0.3608, 0.34648, 0.33249, 0.31934, 0.30618, 0.29296, 0.28021999999999997, 0.268, 0.25575, 0.24408000000000002, 0.23304, 0.22129000000000001, 0.21091, 0.20052, 0.19009, 0.18043, 0.17073, 0.16276, 0.14783, 0.13495, 0.12225, 0.11645, 0.11068, 0.10494, 0.10029, 0.09568, 0.0911, 0.08655, 0.08202000000000001, 0.07867, 0.07418, 0.07089, 0.06762, 0.06116, 0.056029999999999996, 0.05414, 0.05099, 0.047330000000000004, 0.04023, 0.95692, 0.94198, 0.92747, 0.9099, 0.9, 0.88993, 0.87805, 0.86599, 0.8532299999999999, 0.83973, 0.8251, 0.80977, 0.77629, 0.75789, 0.74871, 0.71942, 0.69878, 0.67726, 0.66622, 0.65489, 0.63163, 0.61971, 0.6075999999999999, 0.57047, 0.55782, 0.54503, 0.5321199999999999, 0.48023999999999994, 0.46718000000000004, 0.45416999999999996, 0.42828, 0.40247, 0.37711, 0.35202, 0.28007, 0.25742, 0.23735, 0.18382, 0.12552, 0.11413, 0.1047, 0.09735, 0.08913, 0.08307, 0.07607, 0.05196, 0.95684, 0.95154, 0.94005, 0.92637, 0.91001, 0.88141, 0.87061, 0.82046, 0.79141, 0.75937, 0.74222, 0.7059500000000001, 0.68668, 0.66683, 0.6566000000000001, 0.6462600000000001, 0.61404, 0.60303, 0.54618, 0.48743000000000003, 0.46386000000000005, 0.45203000000000004, 0.41678000000000004, 0.30339, 0.26345999999999997, 0.21106, 0.18027, 0.14079, 0.12022000000000001, 0.09763, 0.06905, 0.04882]
y = [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 34, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 63, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 96, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 157, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187, 187]
z = [0.22435679999999997, 0.2142728, 0.2057014, 0.1968387, 0.1865643, 0.1796222, 0.17312629999999998, 0.16746, 0.16087820000000003, 0.15840410000000002, 0.1555075, 0.15833699999999998, 0.162905, 0.1655981, 0.1740023, 0.1830651, 0.3173607, 0.3141693, 0.3103343, 0.3059325, 0.3010236, 0.29817009999999994, 0.2945795, 0.2903363, 0.2855052, 0.28205630000000004, 0.27788779999999996, 0.2747509, 0.27082059999999997, 0.2661676, 0.26225640000000006, 0.2589043, 0.254695, 0.25088649999999996, 0.2484922, 0.24507220000000002, 0.24173200000000003, 0.23839229999999997, 0.23498379999999996, 0.2323808, 0.2295386, 0.227307, 0.22472129999999998, 0.22174899999999997, 0.21921300000000002, 0.2179324, 0.21616149999999998, 0.2156397, 0.2137493, 0.2131757, 0.21121109999999998, 0.2097163, 0.21087199999999998, 0.210788, 0.2116495, 0.21128049999999998, 0.21214299999999997, 0.211811, 0.2146356, 0.2166123, 0.2176525, 0.21949929999999998, 0.2224562, 0.22256820000000002, 0.22624740000000002, 0.2294154, 0.23202870000000003, 0.2340288, 0.23844570000000004, 0.24255429999999997, 0.2463399, 0.2497837, 0.256654, 0.3265555, 0.3239183, 0.32090609999999997, 0.3175562, 0.3160885, 0.3120466, 0.30971770000000004, 0.3087986, 0.30556690000000003, 0.303637, 0.30122990000000005, 0.2983908, 0.295156, 0.2928952, 0.2901738, 0.2882425, 0.28581090000000003, 0.2829183, 0.2806566, 0.2779074, 0.2756775, 0.2729446, 0.27064109999999997, 0.2678258, 0.2653658, 0.26320079999999996, 0.260491, 0.2587889, 0.25649970000000005, 0.2536468, 0.2516623, 0.2490801, 0.2472737, 0.2448443, 0.24246020000000004, 0.2400991, 0.2377411, 0.2360094, 0.2336024, 0.23242979999999996, 0.2305617, 0.22927460000000002, 0.2279277, 0.22717960000000004, 0.22572289999999998, 0.2242105, 0.2233436, 0.22174970000000002, 0.22159279999999998, 0.22150509999999998, 0.22073520000000002, 0.2208658, 0.2203379, 0.22086889999999998, 0.21986309999999998, 0.2200442, 0.2195909, 0.22059220000000002, 0.2221877, 0.2233259, 0.22397229999999999, 0.22539499999999998, 0.22638089999999997, 0.2268911, 0.2284279, 0.22957579999999997, 0.23055279999999997, 0.232251, 0.2356695, 0.23681100000000002, 0.23983890000000002, 0.24270160000000002, 0.2453939, 0.2479096, 0.250241, 0.3320691, 0.32972009999999996, 0.327113, 0.32625570000000004, 0.3231124, 0.3215746, 0.3196973, 0.3175134, 0.3150502, 0.31233059999999996, 0.3107871, 0.3075553, 0.3054195, 0.3042283, 0.299627, 0.2949814, 0.2911661, 0.28878190000000004, 0.286107, 0.2840244, 0.2824596, 0.2805393, 0.2782853, 0.276471, 0.2750419, 0.2725318, 0.2710885, 0.26709390000000005, 0.2652112, 0.262967, 0.260981, 0.25922399999999995, 0.2570881, 0.25572670000000003, 0.2539644, 0.25236420000000004, 0.2509092, 0.24904690000000002, 0.24570100000000003, 0.2422638, 0.2409471, 0.2397154, 0.2369755, 0.23546060000000002, 0.2340183, 0.2331739, 0.2318828, 0.2301334, 0.22899679999999997, 0.2279498, 0.2264395, 0.2261656, 0.22604139999999998, 0.22510679999999997, 0.2249176, 0.22495259999999997, 0.22446420000000003, 0.22370749999999998, 0.2230614, 0.2227643, 0.22260990000000003, 0.22373089999999998, 0.22456220000000002, 0.2262969, 0.22724429999999998, 0.22791370000000002, 0.22953600000000002, 0.23131880000000002, 0.23203249999999997, 0.2346393, 0.23642559999999999, 0.24096170000000003, 0.2431212, 0.24664070000000002, 0.2496571, 0.33568990000000004, 0.3323899, 0.3312173, 0.3297465, 0.3260212, 0.3238114, 0.3227546, 0.31851399999999996, 0.3157475, 0.31193000000000004, 0.3081958, 0.30537109999999995, 0.3015, 0.2983052, 0.29632179999999997, 0.2948481, 0.29110759999999997, 0.2895523, 0.2877298, 0.2856533, 0.28396409999999994, 0.2798044, 0.27794009999999997, 0.2763865, 0.2745558, 0.2724577, 0.270638, 0.2690729, 0.2672236, 0.2650978, 0.2632022, 0.261519, 0.2600321, 0.2582472, 0.256644, 0.25521020000000005, 0.2534704, 0.2518887, 0.2504563, 0.24871, 0.24710429999999997, 0.24563320000000002, 0.2447436, 0.243529, 0.24243969999999998, 0.24102220000000002, 0.2397272, 0.23855549999999998, 0.2375092, 0.23659229999999998, 0.235341, 0.23422110000000002, 0.23275860000000004, 0.23191660000000003, 0.23123139999999998, 0.2312197, 0.2308987, 0.2302589, 0.22982170000000002, 0.22960830000000002, 0.2290843, 0.22880830000000002, 0.228809, 0.22791129999999998, 0.2279124, 0.2276229, 0.2270263, 0.2267863, 0.2262448, 0.22684320000000002, 0.2280863, 0.2300344, 0.2311638, 0.23185279999999997, 0.2323363, 0.23260229999999996, 0.23365740000000002, 0.2345442, 0.2352545, 0.23577889999999999, 0.23610570000000003, 0.23744459999999998, 0.237389, 0.2384175, 0.23930059999999997, 0.2405984, 0.24279650000000003, 0.24451889999999998, 0.24447689999999997, 0.24740340000000002, 0.2521253, 0.3245058, 0.3180594, 0.3126934, 0.3080352, 0.3057295, 0.3025071, 0.3008695, 0.2982377, 0.2954341, 0.2924375, 0.28985259999999996, 0.28697940000000005, 0.2814263, 0.2791592, 0.2774252, 0.2728833, 0.2698403, 0.26680290000000007, 0.2651647, 0.2637353, 0.2610313, 0.2597459, 0.25863800000000003, 0.2546754, 0.25341060000000004, 0.2518993, 0.2505452, 0.247048, 0.2460449, 0.24519210000000002, 0.24353870000000002, 0.24168850000000003, 0.2400663, 0.23827089999999998, 0.23288040000000002, 0.231019, 0.2306532, 0.2300663, 0.2283889, 0.22863720000000004, 0.22986700000000002, 0.2323381, 0.23337829999999998, 0.23591220000000002, 0.23694320000000002, 0.24616210000000002, 0.3323315, 0.33019509999999996, 0.3247211, 0.31962840000000003, 0.315317, 0.30788730000000003, 0.3053588, 0.29530209999999996, 0.2897451, 0.2842533, 0.2815665, 0.2757268, 0.27296190000000004, 0.2698501, 0.2684138, 0.2667857, 0.2630827, 0.26183429999999996, 0.2552114, 0.25014339999999996, 0.2485022, 0.2475529, 0.24493779999999998, 0.2358408, 0.2340855, 0.2329842, 0.2322835, 0.2317147, 0.2329453, 0.23787809999999998, 0.24544570000000004, 0.25218320000000005]
I have a problem with this algorithm:
function crc16 = crc16eval(D)
% CRC16EVAL CRC-CCITT check with the polynomial: x^16+x^12+x^5+1
D = uint16(D);
crchi = 255;
crclo = 255;
t = '00102030405060708191a1b1c1d1e1f112023222524272629383b3a3d3c3f3e32434041464744454a5b58595e5f5c5d53626160676665646b7a79787f7e7d7c74858687808182838c9d9e9f98999a9b95a4a7a6a1a0a3a2adbcbfbeb9b8bbbab6c7c4c5c2c3c0c1cedfdcdddadbd8d9d7e6e5e4e3e2e1e0effefdfcfbfaf9f8f9181b1a1d1c1f1e110003020504070608393a3b3c3d3e3f30212223242526272b5a59585f5e5d5c53424140474645444a7b78797e7f7c7d72636061666764656d9c9f9e99989b9a95848786818083828cbdbebfb8b9babbb4a5a6a7a0a1a2a3afdedddcdbdad9d8d7c6c5c4c3c2c1c0cefffcfdfafbf8f9f6e7e4e5e2e3e0e1e';
crc16htab = hex2dec(reshape(t,2,length(t)/2)');
t = '0021426384a5c6e708294a6b8cadceef31107352b594f7d639187b5abd9cffde62432001e6c7a4856a4b2809eecfac8d53721130d7f695b45b7a1938dffe9dbcc4e586a740610223cced8eaf48690a2bf5d4b79671503312fddcbf9e79583b1aa687e4c522036041ae8feccd2a0b684997b6d5f4133251709fbeddfc1b3a597888a9caeb0c2d4e6f80a1c2e304254667b998fbda3d1c7f5eb190f3d235147756eacba8896e4f2c0de2c3a08166472405dbfa99b85f7e1d3cd3f291b0577615344c6d0e2fc8e98aab44650627c0e182a37d5c3f1ef9d8bb9a75543716f1d0b3922e0f6c4daa8be8c926076445a283e0c11f3e5d7c9bbad9f81736557493b2d1f0';
crc16ltab = hex2dec(reshape(t,2,length(t)/2)');
for k = 1:length(D)
ix = double(bitxor(crchi,D(k)))+1;
crchi = bitxor(crclo,crc16htab(ix));
crclo = crc16ltab(ix);
end
crc16 = crchi*256+crclo;
end
I need translate that code to Python, and I have done the next:
def crc16eval(D):
crchi = 255
crclo = 255
t = '00102030405060708191a1b1c1d1e1f112023222524272629383b3a3d3c3f3e32434041464744454a5b58595e5f5c5d53626160676665646b7a79787f7e7d7c74858687808182838c9d9e9f98999a9b95a4a7a6a1a0a3a2adbcbfbeb9b8bbbab6c7c4c5c2c3c0c1cedfdcdddadbd8d9d7e6e5e4e3e2e1e0effefdfcfbfaf9f8f9181b1a1d1c1f1e110003020504070608393a3b3c3d3e3f30212223242526272b5a59585f5e5d5c53424140474645444a7b78797e7f7c7d72636061666764656d9c9f9e99989b9a95848786818083828cbdbebfb8b9babbb4a5a6a7a0a1a2a3afdedddcdbdad9d8d7c6c5c4c3c2c1c0cefffcfdfafbf8f9f6e7e4e5e2e3e0e1e'
# crc16htab = hex2dec(reshape(t,2,length(t)/2)');
tarray = [int(n, 16) for n in t] # Recorro el string t, y por cada caracter creo un nuevo entero en el array
crc16htab = reshape(tarray, (2, (len(t)/2) )).transpose()
#print crc16htab
t = '0021426384a5c6e708294a6b8cadceef31107352b594f7d639187b5abd9cffde62432001e6c7a4856a4b2809eecfac8d53721130d7f695b45b7a1938dffe9dbcc4e586a740610223cced8eaf48690a2bf5d4b79671503312fddcbf9e79583b1aa687e4c522036041ae8feccd2a0b684997b6d5f4133251709fbeddfc1b3a597888a9caeb0c2d4e6f80a1c2e304254667b998fbda3d1c7f5eb190f3d235147756eacba8896e4f2c0de2c3a08166472405dbfa99b85f7e1d3cd3f291b0577615344c6d0e2fc8e98aab44650627c0e182a37d5c3f1ef9d8bb9a75543716f1d0b3922e0f6c4daa8be8c926076445a283e0c11f3e5d7c9bbad9f81736557493b2d1f0'
# crc16ltab = hex2dec(reshape(t,2,length(t)/2)');
tarray = [int(n, 16) for n in t] # Recorro el string t, y por cada caracter creo un nuevo entero en el array
crc16ltab = reshape(tarray, (2, (len(t)/2))).transpose()
#print crc16ltab
for k in range(len(D)):
ix = crchi ^ D[k]
crchi = crclo ^ crc16htab[ix]
crclo = crc16ltab[ix]
return crchi*256+crclo
My problem is the next:
When I execute de Python code, It's take a loooong time to calculate de xor, I think the the problem is that
crclo = crc16ltab[ix]
is a matrix and that's take a long time to calculate. Which is the problem?
The pseudo-code of this algorithm is the next:
The algorithm for the CRC-CCITT is below described. Note that all operations are on bytes.
A = new byte
B = temp byte
CRCHI = High byte (most significant) of the 16-bit CRC
CRCLO = Low byte (least significant) of the 16-bit CRC
START:
FOR A = FIRST_BYTE TO LAST_BYTE IN BLOCK DO:
A = A XOR CRCHI
CRCHI = A
SHIFT A RIGHT FOUR TIMES (ZERO FILL)
A = A XOR CRCHI {IJKLMNOP}
CRCHI = CRCLO { swap CRCHI, CRCLO }
CRCLO = A
ROTATE A LEFT 4 TIMES {MNOPIJKL}
B=A { temp save }
ROTATE A LEFT ONCE {NOPIJKLM}
A = A AND $1F {000IJLLM}
CRCHI = A XOR CRCHI
A = B AND $F0 {MNOP0000}
CRCHI = A XOR CRCHI { CRCHI complete }
ROTATE B LEFT ONCE {NOP0000M}
B = B AND $ E0 {NOP00000}
CRCLO = B XOR CRCLO { CRCLO complete }
DOEND;
FINISH.
My question is: Why my python code take long time to execute? What is wrong? The problem I think is in
for k in range(len(D)):
ix = crchi ^ D[k]
crchi = crclo ^ crc16htab[ix]
crclo = crc16ltab[ix]
Thanks a lot!
I recommend a read of Ross Williams, A painless guide to CRC algorigthms which will teach you everything you ever wanted to know about CRC's and how to calculate them quickly.
Here is a conversion of the CCITT CRC algorithm used in the linux kernel. It may or may not be the same as what you are calculating as (if you read the above) you'll realise that there are quite a lot of knobs to twiddle when calculating CRCs.
# This mysterious table is just the CRC of each possible byte. It can be
# computed using the standard bit-at-a-time methods. The polynomial can
# be seen in entry 128, 0x8408. This corresponds to x^0 + x^5 + x^12.
# Add the implicit x^16, and you have the standard CRC-CCITT.
# From linux kernel lib/crc-ccitt.c
_crc_table = (
0x0000, 0x1189, 0x2312, 0x329b, 0x4624, 0x57ad, 0x6536, 0x74bf,
0x8c48, 0x9dc1, 0xaf5a, 0xbed3, 0xca6c, 0xdbe5, 0xe97e, 0xf8f7,
0x1081, 0x0108, 0x3393, 0x221a, 0x56a5, 0x472c, 0x75b7, 0x643e,
0x9cc9, 0x8d40, 0xbfdb, 0xae52, 0xdaed, 0xcb64, 0xf9ff, 0xe876,
0x2102, 0x308b, 0x0210, 0x1399, 0x6726, 0x76af, 0x4434, 0x55bd,
0xad4a, 0xbcc3, 0x8e58, 0x9fd1, 0xeb6e, 0xfae7, 0xc87c, 0xd9f5,
0x3183, 0x200a, 0x1291, 0x0318, 0x77a7, 0x662e, 0x54b5, 0x453c,
0xbdcb, 0xac42, 0x9ed9, 0x8f50, 0xfbef, 0xea66, 0xd8fd, 0xc974,
0x4204, 0x538d, 0x6116, 0x709f, 0x0420, 0x15a9, 0x2732, 0x36bb,
0xce4c, 0xdfc5, 0xed5e, 0xfcd7, 0x8868, 0x99e1, 0xab7a, 0xbaf3,
0x5285, 0x430c, 0x7197, 0x601e, 0x14a1, 0x0528, 0x37b3, 0x263a,
0xdecd, 0xcf44, 0xfddf, 0xec56, 0x98e9, 0x8960, 0xbbfb, 0xaa72,
0x6306, 0x728f, 0x4014, 0x519d, 0x2522, 0x34ab, 0x0630, 0x17b9,
0xef4e, 0xfec7, 0xcc5c, 0xddd5, 0xa96a, 0xb8e3, 0x8a78, 0x9bf1,
0x7387, 0x620e, 0x5095, 0x411c, 0x35a3, 0x242a, 0x16b1, 0x0738,
0xffcf, 0xee46, 0xdcdd, 0xcd54, 0xb9eb, 0xa862, 0x9af9, 0x8b70,
0x8408, 0x9581, 0xa71a, 0xb693, 0xc22c, 0xd3a5, 0xe13e, 0xf0b7,
0x0840, 0x19c9, 0x2b52, 0x3adb, 0x4e64, 0x5fed, 0x6d76, 0x7cff,
0x9489, 0x8500, 0xb79b, 0xa612, 0xd2ad, 0xc324, 0xf1bf, 0xe036,
0x18c1, 0x0948, 0x3bd3, 0x2a5a, 0x5ee5, 0x4f6c, 0x7df7, 0x6c7e,
0xa50a, 0xb483, 0x8618, 0x9791, 0xe32e, 0xf2a7, 0xc03c, 0xd1b5,
0x2942, 0x38cb, 0x0a50, 0x1bd9, 0x6f66, 0x7eef, 0x4c74, 0x5dfd,
0xb58b, 0xa402, 0x9699, 0x8710, 0xf3af, 0xe226, 0xd0bd, 0xc134,
0x39c3, 0x284a, 0x1ad1, 0x0b58, 0x7fe7, 0x6e6e, 0x5cf5, 0x4d7c,
0xc60c, 0xd785, 0xe51e, 0xf497, 0x8028, 0x91a1, 0xa33a, 0xb2b3,
0x4a44, 0x5bcd, 0x6956, 0x78df, 0x0c60, 0x1de9, 0x2f72, 0x3efb,
0xd68d, 0xc704, 0xf59f, 0xe416, 0x90a9, 0x8120, 0xb3bb, 0xa232,
0x5ac5, 0x4b4c, 0x79d7, 0x685e, 0x1ce1, 0x0d68, 0x3ff3, 0x2e7a,
0xe70e, 0xf687, 0xc41c, 0xd595, 0xa12a, 0xb0a3, 0x8238, 0x93b1,
0x6b46, 0x7acf, 0x4854, 0x59dd, 0x2d62, 0x3ceb, 0x0e70, 0x1ff9,
0xf78f, 0xe606, 0xd49d, 0xc514, 0xb1ab, 0xa022, 0x92b9, 0x8330,
0x7bc7, 0x6a4e, 0x58d5, 0x495c, 0x3de3, 0x2c6a, 0x1ef1, 0x0f78
)
def update_crc(data, crc, table=_crc_table):
"""
Add a byte to the crc calculation
"""
return (crc >> 8) ^ table[(crc ^ data) & 0xff]
def calculate_crc(data, crc=0xFFFF, table=_crc_table):
"""
Calculates the CRC for the data string passed in
"""
for c in data:
crc = update_crc(ord(c), crc, table)
return crc
print "%04X" % calculate_crc("Hello")
If you need to calculate CRC16 only (just result, not code), you could use PyCRC or CRC-16.
This was the final solution, too late to post here maybe... But I think that maybe It's useful for someone.
# CRC16EVAL CRC-CCITT check with the polynomial: x^16+x^12+x^5+1
def crc16eval(D):
crchi = 255
crclo = 255
crc16htab = [0, 16, 32, 48, 64, 80, 96, 112, 129, 145, 161, 177, 193, 209, 225, 241, 18, 2, 50, 34, 82, 66, 114, 98, 147, 131, 179, 163, 211, 195, 243, 227, 36, 52, 4, 20, 100, 116, 68, 84, 165, 181, 133, 149, 229, 245, 197, 213, 54, 38, 22, 6, 118, 102, 86, 70, 183, 167, 151, 135, 247, 231, 215, 199, 72, 88, 104, 120, 8, 24, 40, 56, 201, 217, 233, 249, 137, 153, 169, 185, 90, 74, 122, 106, 26, 10, 58, 42, 219, 203, 251, 235, 155, 139, 187, 171, 108, 124, 76, 92, 44, 60, 12, 28, 237, 253, 205, 221, 173, 189, 141, 157, 126, 110, 94, 78, 62, 46, 30, 14, 255, 239, 223, 207, 191, 175, 159, 143, 145, 129, 177, 161, 209, 193, 241, 225, 16, 0, 48, 32, 80, 64, 112, 96, 131, 147, 163, 179, 195, 211, 227, 243, 2, 18, 34, 50, 66, 82, 98, 114, 181, 165, 149, 133, 245, 229, 213, 197, 52, 36, 20, 4, 116, 100, 84, 68, 167, 183, 135, 151, 231, 247, 199, 215, 38, 54, 6, 22, 102, 118, 70, 86, 217, 201, 249, 233, 153, 137, 185, 169, 88, 72, 120, 104, 24, 8, 56, 40, 203, 219, 235, 251, 139, 155, 171, 187, 74, 90, 106, 122, 10, 26, 42, 58, 253, 237, 221, 205, 189, 173, 157, 141, 124, 108, 92, 76, 60, 44, 28, 12, 239, 255, 207, 223, 175, 191, 143, 159, 110, 126, 78, 94, 46, 62, 14, 30]
crc16ltab = [0, 33, 66, 99, 132, 165, 198, 231, 8, 41, 74, 107, 140, 173, 206, 239, 49, 16, 115, 82, 181, 148, 247, 214, 57, 24, 123, 90, 189, 156, 255, 222, 98, 67, 32, 1, 230, 199, 164, 133, 106, 75, 40, 9, 238, 207, 172, 141, 83, 114, 17, 48, 215, 246, 149, 180, 91, 122, 25, 56, 223, 254, 157, 188, 196, 229, 134, 167, 64, 97, 2, 35, 204, 237, 142, 175, 72, 105, 10, 43, 245, 212, 183, 150, 113, 80, 51, 18, 253, 220, 191, 158, 121, 88, 59, 26, 166, 135, 228, 197, 34, 3, 96, 65, 174, 143, 236, 205, 42, 11, 104, 73, 151, 182, 213, 244, 19, 50, 81, 112, 159, 190, 221, 252, 27, 58, 89, 120, 136, 169, 202, 235, 12, 45, 78, 111, 128, 161, 194, 227, 4, 37, 70, 103, 185, 152, 251, 218, 61, 28, 127, 94, 177, 144, 243, 210, 53, 20, 119, 86, 234, 203, 168, 137, 110, 79, 44, 13, 226, 195, 160, 129, 102, 71, 36, 5, 219, 250, 153, 184, 95, 126, 29, 60, 211, 242, 145, 176, 87, 118, 21, 52, 76, 109, 14, 47, 200, 233, 138, 171, 68, 101, 6, 39, 192, 225, 130, 163, 125, 92, 63, 30, 249, 216, 187, 154, 117, 84, 55, 22, 241, 208, 179, 146, 46, 15, 108, 77, 170, 139, 232, 201, 38, 7, 100, 69, 162, 131, 224, 193, 31, 62, 93, 124, 155, 186, 217, 248, 23, 54, 85, 116, 147, 178, 209, 240]
for k in range(len(D)):
ix = crchi ^ D[k]
crchi = crclo ^ crc16htab[ix]
crclo = crc16ltab[ix]
return crchi*256+crclo
Antonio.