Related
I’m currently trying to adapt a PyTorch Wave-U-Net implementation (https://github.com/f90/Wave-U-Net-Pytorch) so that it’ll work for audio mixing rather than source separation. I’m still quite new to PyTorch so I’m not exactly sure how one would go about structuring a forward pass for multiple inputs (8 audio tracks, each corresponding to a separate stem or instrument) and a single output (a mixture track, produced from the inputs).
The network is below, I don’t think the functions in the middle are relevant or need changing. The output is a dictionary with a key for each instrument matching the estimated audio source, however this is obviously not necessary for my task, I think the only changes needed are in __init__ and forward, but I’m unclear what changes are necessary and how to implement them.
class Waveunet(nn.Module):
def __init__(self, num_inputs, num_channels, num_outputs, instruments, kernel_size, target_output_size, conv_type, res, separate=False, depth=1, strides=2):
super(Waveunet, self).__init__()
self.num_levels = len(num_channels)
self.strides = strides
self.kernel_size = kernel_size
self.num_inputs = num_inputs
self.num_outputs = num_outputs
self.depth = depth
self.instruments = instruments
self.separate = separate
# Only odd filter kernels allowed
assert(kernel_size % 2 == 1)
self.waveunets = nn.ModuleDict()
model_list = instruments if separate else ["ALL"]
# Create a model for each source if we separate sources separately, otherwise only one (model_list=["ALL"])
for instrument in model_list:
module = nn.Module()
module.downsampling_blocks = nn.ModuleList()
module.upsampling_blocks = nn.ModuleList()
for i in range(self.num_levels - 1):
in_ch = num_inputs if i == 0 else num_channels[i]
module.downsampling_blocks.append(
DownsamplingBlock(in_ch, num_channels[i], num_channels[i+1], kernel_size, strides, depth, conv_type, res))
for i in range(0, self.num_levels - 1):
module.upsampling_blocks.append(
UpsamplingBlock(num_channels[-1-i], num_channels[-2-i], num_channels[-2-i], kernel_size, strides, depth, conv_type, res))
module.bottlenecks = nn.ModuleList(
[ConvLayer(num_channels[-1], num_channels[-1], kernel_size, 1, conv_type) for _ in range(depth)])
# Output conv
outputs = num_outputs if separate else num_outputs * len(instruments)
module.output_conv = nn.Conv1d(num_channels[0], outputs, 1)
self.waveunets[instrument] = module
self.set_output_size(target_output_size)
def set_output_size(self, target_output_size):
self.target_output_size = target_output_size
self.input_size, self.output_size = self.check_padding(target_output_size)
print("Using valid convolutions with " + str(self.input_size) + " inputs and " + str(self.output_size) + " outputs")
assert((self.input_size - self.output_size) % 2 == 0)
self.shapes = {"output_start_frame" : (self.input_size - self.output_size) // 2,
"output_end_frame" : (self.input_size - self.output_size) // 2 + self.output_size,
"output_frames" : self.output_size,
"input_frames" : self.input_size}
def check_padding(self, target_output_size):
# Ensure number of outputs covers a whole number of cycles so each output in the cycle is weighted equally during training
bottleneck = 1
while True:
out = self.check_padding_for_bottleneck(bottleneck, target_output_size)
if out is not False:
return out
bottleneck += 1
def check_padding_for_bottleneck(self, bottleneck, target_output_size):
module = self.waveunets[[k for k in self.waveunets.keys()][0]]
try:
curr_size = bottleneck
for idx, block in enumerate(module.upsampling_blocks):
curr_size = block.get_output_size(curr_size)
output_size = curr_size
# Bottleneck-Conv
curr_size = bottleneck
for block in reversed(module.bottlenecks):
curr_size = block.get_input_size(curr_size)
for idx, block in enumerate(reversed(module.downsampling_blocks)):
curr_size = block.get_input_size(curr_size)
assert(output_size >= target_output_size)
return curr_size, output_size
except AssertionError as e:
return False
def forward_module(self, x, module):
'''
A forward pass through a single Wave-U-Net (multiple Wave-U-Nets might be used, one for each source)
:param x: Input mix
:param module: Network module to be used for prediction
:return: Source estimates
'''
shortcuts = []
out = x
# DOWNSAMPLING BLOCKS
for block in module.downsampling_blocks:
out, short = block(out)
shortcuts.append(short)
# BOTTLENECK CONVOLUTION
for conv in module.bottlenecks:
out = conv(out)
# UPSAMPLING BLOCKS
for idx, block in enumerate(module.upsampling_blocks):
out = block(out, shortcuts[-1 - idx])
# OUTPUT CONV
out = module.output_conv(out)
if not self.training: # At test time clip predictions to valid amplitude range
out = out.clamp(min=-1.0, max=1.0)
return out
def forward(self, x, inst=None):
curr_input_size = x.shape[-1]
assert(curr_input_size == self.input_size) # User promises to feed the proper input himself, to get the pre-calculated (NOT the originally desired) output size
if self.separate:
return {inst : self.forward_module(x, self.waveunets[inst])}
else:
assert(len(self.waveunets) == 1)
out = self.forward_module(x, self.waveunets["ALL"])
out_dict = {}
for idx, inst in enumerate(self.instruments):
out_dict[inst] = out[:, idx * self.num_outputs:(idx + 1) * self.num_outputs]
return out_dict'
The callback is called when specific events occur in an environment (e.g. at the beginning/end of a reset and beginning/end of a step).
I have written a stub of a GEM-Callback and added it to the environment.
That can be utilized to log data at every step.
Now, which function can be called to get data from that class in List Format / .CSV Format.
Code of callback class is attached below:
class CurrentLoggingCallback(gem.core.Callback):
def __init__(self):
super().__init__ ()
self._i_sd_idx = None
self._i_sq_idx = None
self._i_sd_max = None
self._i_sq_max = None
def set_env(self, env):
super().set_env(env)
assert 'i_sd'in env.state_names, 'the environment has no state "i_sd".'
assert 'i_sq'in env.state_names, 'the environment has no state "i_sq".'
self._i_sd_idx = env.state_names.index('i_sd')
self._i_sq_idx = env.state_names.index('i_sq')
#Environments Observations are normalized by their limits.
#for current values in Amperes, the observations have to be multiplied with their limits.
self._i_sd_max = env.limits[self._i_sd_idx]
self._i_sq_max = env.limits[self._i_sq_idx]
def on_step_end(self, k, state, reference, reward, done):
""" Gets called at the end of each step"""
i_sd = state[self._i_sd_idx] * self._i_sd_max
i_sq = state[self._i_sq_idx] * self._i_sq_max
i_sd_ref = reference[self._i_sd_idx] * self._i_sd_max
i_sq_ref = reference[self._i_sq_idx] * self._i_sq_max
# Append to list or store to file...
def on_reset_end(self, state, reference):
"""Gets called at the end of each reset """
i_sd = state[self._i_sd_idx] * self._i_sd_max
i_sq = state[self._i_sq_idx] * self._i_sq_max
i_sd_ref = reference[self._i_sd_idx] * self._i_sd_max
i_sq_ref = reference[self._i_sq_idx] * self._i_sq_max
# Append to list or store to file...
current_logger = CurrentLoggingCallback()
Now, i want to log values of i_sd and i_sq in list/.CSV .
This is how the dataset was created
def createDataset1(
outputPath,
imagePathList,
labelList,
lexiconList=None,
validset_percent=10,
testset_percent=0,
random_seed=1111,
checkValid=True,
):
"""
Create LMDB dataset for CRNN training.
ARGS:
outputPath : LMDB output path
imagePathList : list of image path
labelList : list of corresponding groundtruth texts
lexiconList : (optional) list of lexicon lists
checkValid : if true, check the validity of every image
"""
train_path = os.path.join(outputPath, "training", "9M")
valid_path = os.path.join(outputPath, "validation", "9M")
# CAUTION: if train_path (lmdb) already exists, this function add dataset
# into it. so remove former one and re-create lmdb.
if os.path.exists(train_path):
os.system(f"rm -r {train_path}")
if os.path.exists(valid_path):
os.system(f"rm -r {valid_path}")
os.makedirs(train_path, exist_ok=True)
os.makedirs(valid_path, exist_ok=True)
gt_train_path = gt_file.replace(".txt", "_train.txt")
gt_valid_path = gt_file.replace(".txt", "_valid.txt")
data_log = open(gt_train_path, "w", encoding="utf-8")
if testset_percent != 0:
test_path = os.path.join(outputPath, "evaluation", dataset_name)
if os.path.exists(test_path):
os.system(f"rm -r {test_path}")
os.makedirs(test_path, exist_ok=True)
gt_test_path = gtFile.replace(".txt", "_test.txt")
assert(len(imagePathList) == len(labelList))
nSamples = len(imagePathList)
num_valid_dataset = int(nSamples * validset_percent / 100.0)
num_test_dataset = int(nSamples * testset_percent / 100.0)
num_train_dataset = nSamples - num_valid_dataset - num_test_dataset
print("validation datasets: ",num_valid_dataset,"\n", "test datasets: ", num_test_dataset, " \n training datasets: ", num_train_dataset)
env = lmdb.open(outputPath, map_size=1099511627776)
cache = {}
cnt = 1
random.seed(random_seed)
random.shuffle(imagePathList)
for i in tqdm(range(nSamples)):
data_log.write(imagePathList[i])
imagePath = imagePathList[i]
label = labelList[i]
if len(label) == 0:
continue
if not os.path.exists(imagePath):
print('%s does not exist' % imagePath)
continue
with open(imagePath, 'rb') as f:
imageBin = f.read()
if checkValid:
if not checkImageIsValid(imageBin):
print('%s is not a valid image' % imagePath)
continue
embed_vec = fasttext_model[label]
imageKey = 'image-%09d' % cnt
labelKey = 'label-%09d' % cnt
embedKey = 'embed-%09d' % cnt
cache[imageKey] = imageBin
cache[labelKey] = label.encode()
cache[embedKey] = ' '.join(str(v) for v in embed_vec.tolist()).encode()
if lexiconList:
lexiconKey = 'lexicon-%09d' % cnt
cache[lexiconKey] = ' '.join(lexiconList[i])
if cnt % 1000 == 0:
writeCache(env, cache)
cache = {}
print('Written %d / %d' % (cnt, nSamples))
#finish train dataset and start validation dataset
if i + 1 == num_train_dataset:
print(f"# Train dataset: {num_train_dataset} is finished")
cache["num-samples".encode()] = str(num_train_dataset).encode()
writeCache(env, cache)
data_log.close()
#start validation set
env = lmdb.open(valid_path, map_size=30 * 2 ** 30)
cache = {}
cnt = 0
data_log = open(gt_valid_path, "w", encoding="utf-8")
# Finish train/valid dataset and Start test dataset
if (i + 1 == num_train_dataset + num_valid_dataset) and num_test_dataset != 0:
print(f"# Valid dataset: {num_valid_dataset} is finished")
cache["num-samples".encode()] = str(num_valid_dataset).encode()
writeCache(env, cache)
data_log.close()
# start test set
env = lmdb.open(test_path, map_size=30 * 2 ** 30)
cache = {}
cnt = 0 # not 1 at this time
data_log = open(gt_test_path, "w", encoding="utf-8")
cnt += 1
if testset_percent == 0:
cache["num-samples".encode()] = str(num_valid_dataset).encode()
writeCache(env, cache)
print(f"# Valid datast: {num_valid_dataset} is finished")
else:
cache["num-samples".encode()] = str(num_test_dataset).encode()
writeCache(env, cache)
print(f"# Test datast: {num_test_dataset} is finished")
This is how i am trying to retrieve the data
class LmdbDataset(data.Dataset):
def __init__(self, root, voc_type, max_len, num_samples, transform=None):
super(LmdbDataset, self).__init__()
if global_args.run_on_remote:
dataset_name = os.path.basename(root)
data_cache_url = "/cache/%s" % dataset_name
if not os.path.exists(data_cache_url):
os.makedirs(data_cache_url)
if mox.file.exists(root):
mox.file.copy_parallel(root, data_cache_url)
else:
raise ValueError("%s not exists!" % root)
self.env = lmdb.open(data_cache_url, max_readers=32, readonly=True)
else:
self.env = lmdb.open(root, max_readers=32, readonly=True)
assert self.env is not None, "cannot create lmdb from %s" % root
self.txn = self.env.begin()
self.voc_type = voc_type
self.transform = transform
self.max_len = max_len
# nums = b"num-samples"
# print('NUM SAMPLES ------ \n',nums)
nSamples = self.txn.get('num-samples'.encode())
print("STRING nSamples :", nSamples)
self.nSamples = int(self.txn.get(b"num-samples"))
self.nSamples = min(self.nSamples, num_samples)
assert voc_type in ['LOWERCASE', 'ALLCASES', 'ALLCASES_SYMBOLS']
self.EOS = 'EOS'
self.PADDING = 'PADDING'
self.UNKNOWN = 'UNKNOWN'
self.voc = get_vocabulary(voc_type, EOS=self.EOS, PADDING=self.PADDING, UNKNOWN=self.UNKNOWN)
self.char2id = dict(zip(self.voc, range(len(self.voc))))
self.id2char = dict(zip(range(len(self.voc)), self.voc))
self.rec_num_classes = len(self.voc)
self.lowercase = (voc_type == 'LOWERCASE')
I am getting the error below whenever the code tries to call elf.txn.get(b"num-samples")
Traceback (most recent call last):
File "main.py", line 268, in <module>
main(args)
File "main.py", line 157, in main
train_dataset, train_loader = get_data_lmdb(args.synthetic_train_data_dir, args.voc_type, args.max_len, args.num_train,
File "main.py", line 66, in get_data_lmdb
dataset_list.append(LmdbDataset(data_dir_, voc_type, max_len, num_samples))
File "/Users/SEED/lib/datasets/dataset.py", line 189, in __init__
self.nSamples = int(self.txn.get(b"num-samples"))
TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'
I have tried many different suggestions online and some stackoverflow threads but could not figure out what is wrong.
What is causing this error and how can I fix this?
Your code is dense, convoluted and hard to follow the flow of data. It mixes lots of computation with IO, side effects, and even system calls (os.system(f"rm -r {test_path}"), you should use shutil.rmtree instead for example).
Try breaking up each action that you wish to perform so that it primarily does one specific thing:
logical operations but no side effects
input (read from file or network)
output (write to file, generate a result)
file system operations/cleanup
At each stage, you should perform verifications, and use the rule of least power. If you expect self.txn to always have 'num-samples, then you should use self.txn[b'num-samples']rather.get, which defaults to None`. This makes it easier to catch errors earlier in the chain.
Also I have no idea what the lmbd module is. Is that a library, or another file in your codebase? You should link to any libraries you are using if they aren't well known. I see several python packages pertaining to lmbd.
From the code example, I can dissect for you. May be it will help you to close the issue.
The log says..
self.nSamples = int(self.txn.get(b"num-samples"))
TypeError: int() argument must be a string, a bytes-like object or a number, not 'NoneType'
self.txn should be a dictionary to use get method there. As you are trying to get it, there can be two scenarios
num-samples can be '' in the data means there is no value
there is no variable called num-samples in the data.
Both of them can cause a NoneType which cannot be handled by type conversion(to int). So, you have to check whether that field is present in the data or not, by using self.txn.keys() and see if that key is present
Furthermore, if
self.nSamples = int(self.txn.get(b"num-samples"))
is failing and not the above statement
nSamples = self.txn.get('num-samples'.encode())
then you can simply decode the variable and use it there
like
Samples = nSamples.decode("utf-8")
try:
int_samples = int(Samples)
except TypeError:
print(Samples)
Currently i try to verify the Bitcoin Block 77504 by my own. But from the satoshi whitepaper it seems i have more questions than answer to do so.
First information from the previous block:
### What we know from last block ###
# height = 77503
# id = 00000000000447829abff59b3208a08ff28b3eb184b1298929abe6dd65c3578a
# version = 1
# timestamp = 1283325019
# bits = 459874456
# nonce = 1839166754
# difficulty = 623.3869598689275
# merkle_root = f18107935e8853011e477244241b5d786966495f8c59be46c92ac323c9cc8cde
# tx_count = 6
# size = 1438
# weight = 5752
Then the information from the block i want to verify
### What we now want to mine ###
# height = 77504
# id = 00000000004582246e63ff7e0760c6f009e5ef5ce1eb5397be6f3eb9d698bda5
# version = 1
# timestamp = 1283326637
# bits = 459874456
# nonce = 191169021
# difficulty = 623.3869598689275
# merkle_root = 59c77dabd9f005c771b23b846c79c7741dc0e70d912f9470eace886b42a0d601
# tx_count = 44
# size = 11052
# weight = 44208
# txids = ["b899c55adb5a9604b72643c0f6cd5bf6c2447bb0fc035c50e13d2e471cbf5aa5","05180e3252c48a54d4d0abe9359621f54f3031fd318a812be96da0f13bfa8bf3","29d641bd4a5d4b01ceee1126af920513d52e088bad500fad1358c96962e25e28","40d52b5aa4be889739410f82f36c71fdda554b999fb14fc12aeab5bb2e6498cb","62d5e84500cc674a5172bea5755a223da974f90f614deb45c160478a8974419c","78de7a104617f58620ae9e7cf58bcd875d8043ee5046d93c9d69224c2ae39a1e","8831ad38deb23e1fbea6d376f1805aec194760b0f334a3c4b623aa0751445c9b","8a6bd0c2d74ea785d886bd6d87b6a4eb4cd35af5fb7ae3a364eb1f76b114c375","90d6da6a4b48e7330ae926cd00623fa8d94fd0a2b9a001475da22cbc49435ff9","d002da9953844c767cf7d42092b81e8c5bb03baf520d79028013fd3400bc8651","d1f8573148126e8d17641276f22ece33b8276311d93794ed2975ebb802b98fc8","d22ed765adba9c7f5fef19ff15cb89559b4148d571fcb40ee2889231ac1b8dea","f32b000adf9ab6d7a66593cb20cba4d3a3e0cbb3453608ce11a780fab532add5","32d2ff811677a8dbed4f317c9fcae4796b491bde944cca4a993734be787b4e79","4b806d44d9aff762601f21ad541c0e99a77d0a14b730774a2d7721dd094d9030","8c5258a8e3f60c9edfa55b86780a9832c8cd5f407dbe25948cd2fd87910ca4c4","bc4fcea23cd93bac13ab75bad8d23576be88a89e72f2c455932f096d6dd2a2da","ca5c53ef34ff5a2f816daf648c8dafb01680502c2c0c98b82b9527392f707e70","f9db6e9a62502dfe8057e7b1c0f3b8f145d354ee4e341233bfe8861fff143822","fc3730bbfa443558c677da6898f106ee7d5516b14e21bf369def7cb6a5bf6b8b","1cfa85d94ebfb9206ad49f421319a6ee99b339e4e8d292b866459bb742731d83","80fa7f38cc02b05b765675adba589d426e6122b1e8158726df0c55cf44c937eb","8c72683585901ff96edd14bde9c87ee91a9d54c187a15aa333e3d6b916399fd2","905e015afa4df7d9dc4a1a80a029e469258045fe9288071b16af49a2f458c2cb","bd8fab0ca0072cd230a4bb0a6efff5964756a023ca53d1f06c3fa22800fe044c","464280d62b8965255c286f1c4c5c457f594db64bdef1c8aaa7ddf776fc4d320e","625b8ec5af9ad2c1506aca8ad61670ce3acf7070fe5aabc2dec06dcda119503a","a2e06f6b0ea68cc2c9bf44d09e54832c830971961ed8ea5ec553918ab7eb48d2","a4de41f56f0970d9b1948f1e386a124860891d790f506c2e3bbe71dd289031d4","11475d2fbbc5e3aee2eff54aa9bf2f83d5f33fffce528cc9804f820e0f6a76e7","5dc019a6397c25d0e7db56f3ed2ccdc1db5642701224d56fb9ad1d1017279e7b","e5d1e0e5a2309cb07ec522a1eb56da5aa5e58ecaea6d49e278a52c1c24230dae","21d192ea46007dbeef7c9673ac158c0f9dbf80e0785380ae562a1fbb10430ae7","8fafe7a8168563c4c186d792b49fc0fa4368c6b2e5a1217f2f98b127ff1cdf87","d2410a45bcc0e4f5b7a8d84e730ffd9744e0dd0d9fb2d7e93fb71e590bf0f1fb","6103334a35171bc5a153b51dd7c94977c62822b1cec2fcac20ea9d0a959129d7","6551831774420989df2d9deeab196e14025f2e5fd502feb86dfc7ccedb917ce0","7c1a188e0c94c7d61aea1ebddb359f508c99fdd0e028887bbf3a3036a1b5bf8a","8b9c989cee69c107697b13aebd677879db48275c089ae206c85eb8db45acf50f","4195c5abf97adb2108de8aeee99cb751e2b4f9698607f60e326b9a67b9127a31","800b308f49fe86ff3323dd6240190212626d052a017dd1cad01540790604c00f","1d2fb37bab59d6f3f83f7596fde128a0b7b0f7ccd8fabc8d2a929923a268a847","8a8149d58791ace6cefd803021b4e870acca5b2c40e2e1415f423e6ec4333e32","7a1eb6b8ee1ff52648cd9a099c7658be53627732b226aa93f56d430c85a52991"]
I have prepared a small script that should calculate it for me but no matter what i am not able to get to the target hash 00000000004582246e63ff7e0760c6f009e5ef5ce1eb5397be6f3eb9d698bda5 to verify the block mined. What is also unclear to me where would one have to add his own Bitcoin wallet to get the reward of the transaction.
from hashlib import sha256
def SHA256(text):
return sha256(text.encode("ascii")).hexdigest()
def mine(block_number, transactions, previous_hash, prefix_zeros):
prefix_str = '0'*prefix_zeros
text = str(block_number) + str(transactions) + str(previous_hash) + str(nonce_to_verify)
new_hash = SHA256(text)
if new_hash.startswith(prefix_str):
print(f"Jipiiii! Successfully mined bitcoins with nonce value:{nonce_to_verify}")
return new_hash
else:
new_hash = None
return new_hash
### normally this is unknown, would be somethinge like range(0,100000000000), i just want to verify a block ###
nonce_to_verify = 191169021
### In what format are transactions presented ? ###
transactions = ["b899c55adb5a9604b72643c0f6cd5bf6c2447bb0fc035c50e13d2e471cbf5aa5","05180e3252c48a54d4d0abe9359621f54f3031fd318a812be96da0f13bfa8bf3","29d641bd4a5d4b01ceee1126af920513d52e088bad500fad1358c96962e25e28","40d52b5aa4be889739410f82f36c71fdda554b999fb14fc12aeab5bb2e6498cb","62d5e84500cc674a5172bea5755a223da974f90f614deb45c160478a8974419c","78de7a104617f58620ae9e7cf58bcd875d8043ee5046d93c9d69224c2ae39a1e","8831ad38deb23e1fbea6d376f1805aec194760b0f334a3c4b623aa0751445c9b","8a6bd0c2d74ea785d886bd6d87b6a4eb4cd35af5fb7ae3a364eb1f76b114c375","90d6da6a4b48e7330ae926cd00623fa8d94fd0a2b9a001475da22cbc49435ff9","d002da9953844c767cf7d42092b81e8c5bb03baf520d79028013fd3400bc8651","d1f8573148126e8d17641276f22ece33b8276311d93794ed2975ebb802b98fc8","d22ed765adba9c7f5fef19ff15cb89559b4148d571fcb40ee2889231ac1b8dea","f32b000adf9ab6d7a66593cb20cba4d3a3e0cbb3453608ce11a780fab532add5","32d2ff811677a8dbed4f317c9fcae4796b491bde944cca4a993734be787b4e79","4b806d44d9aff762601f21ad541c0e99a77d0a14b730774a2d7721dd094d9030","8c5258a8e3f60c9edfa55b86780a9832c8cd5f407dbe25948cd2fd87910ca4c4","bc4fcea23cd93bac13ab75bad8d23576be88a89e72f2c455932f096d6dd2a2da","ca5c53ef34ff5a2f816daf648c8dafb01680502c2c0c98b82b9527392f707e70","f9db6e9a62502dfe8057e7b1c0f3b8f145d354ee4e341233bfe8861fff143822","fc3730bbfa443558c677da6898f106ee7d5516b14e21bf369def7cb6a5bf6b8b","1cfa85d94ebfb9206ad49f421319a6ee99b339e4e8d292b866459bb742731d83","80fa7f38cc02b05b765675adba589d426e6122b1e8158726df0c55cf44c937eb","8c72683585901ff96edd14bde9c87ee91a9d54c187a15aa333e3d6b916399fd2","905e015afa4df7d9dc4a1a80a029e469258045fe9288071b16af49a2f458c2cb","bd8fab0ca0072cd230a4bb0a6efff5964756a023ca53d1f06c3fa22800fe044c","464280d62b8965255c286f1c4c5c457f594db64bdef1c8aaa7ddf776fc4d320e","625b8ec5af9ad2c1506aca8ad61670ce3acf7070fe5aabc2dec06dcda119503a","a2e06f6b0ea68cc2c9bf44d09e54832c830971961ed8ea5ec553918ab7eb48d2","a4de41f56f0970d9b1948f1e386a124860891d790f506c2e3bbe71dd289031d4","11475d2fbbc5e3aee2eff54aa9bf2f83d5f33fffce528cc9804f820e0f6a76e7","5dc019a6397c25d0e7db56f3ed2ccdc1db5642701224d56fb9ad1d1017279e7b","e5d1e0e5a2309cb07ec522a1eb56da5aa5e58ecaea6d49e278a52c1c24230dae","21d192ea46007dbeef7c9673ac158c0f9dbf80e0785380ae562a1fbb10430ae7","8fafe7a8168563c4c186d792b49fc0fa4368c6b2e5a1217f2f98b127ff1cdf87","d2410a45bcc0e4f5b7a8d84e730ffd9744e0dd0d9fb2d7e93fb71e590bf0f1fb","6103334a35171bc5a153b51dd7c94977c62822b1cec2fcac20ea9d0a959129d7","6551831774420989df2d9deeab196e14025f2e5fd502feb86dfc7ccedb917ce0","7c1a188e0c94c7d61aea1ebddb359f508c99fdd0e028887bbf3a3036a1b5bf8a","8b9c989cee69c107697b13aebd677879db48275c089ae206c85eb8db45acf50f","4195c5abf97adb2108de8aeee99cb751e2b4f9698607f60e326b9a67b9127a31","800b308f49fe86ff3323dd6240190212626d052a017dd1cad01540790604c00f","1d2fb37bab59d6f3f83f7596fde128a0b7b0f7ccd8fabc8d2a929923a268a847","8a8149d58791ace6cefd803021b4e870acca5b2c40e2e1415f423e6ec4333e32","7a1eb6b8ee1ff52648cd9a099c7658be53627732b226aa93f56d430c85a52991"]
### Just a check of 5 leading zeros... but why the difficulty 623.3869598689275 how to get to the 11 zeros? ###
difficulty=11
### Last Block (77503) found ###
lastfoundblock = "00000000000447829abff59b3208a08ff28b3eb184b1298929abe6dd65c3578a"
print("start mining")
new_hash = mine(77504,transactions,lastfoundblock, difficulty)
print("finnished mining.")
print(f"Found block is: {new_hash} should be the same as 00000000004582246e63ff7e0760c6f009e5ef5ce1eb5397be6f3eb9d698bda5")
Help would be appreciated so that i can verify a single block. Already pointing in the right directions would be appreciated so that i can solve my problem.
As no one was able to answer it... here is the code to verify a block's nonce:
import hashlib, struct, binascii
from time import time
def get_target_str(bits):
# https://en.bitcoin.it/wiki/Difficulty
exp = bits >> 24
mant = bits & 0xffffff
target_hexstr = '%064x' % (mant * (1<<(8*(exp - 3))))
print(f'T: {target_hexstr}')
target_str = bytes.fromhex(target_hexstr)
return target_str
def verify_nonce(version, prev_block, mrkl_root,
timestamp, bits_difficulty,nonce):
target_str = get_target_str(bits_difficulty)
header = ( struct.pack("<L", version) +
bytes.fromhex(prev_block)[::-1] +
bytes.fromhex(mrkl_root)[::-1] +
struct.pack("<LLL", timestamp, bits_difficulty, nonce))
hash_result = hashlib.sha256(hashlib.sha256(header).digest()).digest()
return bytes.hex(hash_result[::-1])
#nonce += 1
test1_version = 0x3fff0000
test1_prev_block = "0000000000000000000140ac4688aea45aacbe7caf6aaca46f16acd93e1064c3"
test1_merkle_root = "422458fced12693312058f6ee4ada19f6df8b29d8cac425c12f4722e0dc4aafd"
test1_timestamp = 0x5E664C76
test1_bits_diff = 0x17110119
test1_nonce1 = 538463288 #(0x20184C38)
test1_block_hash = "0000000000000000000d493c3c1b91c8059c6b0838e7e68fbcf8f8382606b82c"
test1_calc_block_hash = verify_nonce(test1_version,
test1_prev_block,
test1_merkle_root,
test1_timestamp,
test1_bits_diff,
test1_nonce1)
print(f'S: {test1_block_hash}')
print(f'R: {test1_calc_block_hash}')
if test1_block_hash == test1_calc_block_hash:
print("hashing is correct")
Thanks to https://github.com/razvancazacu/bitcoin-mining-crypto
I am new to python and trying to get Kauffman's NK model work in it...I found the code online and was hoping I could make some changes over time but I am not able to run the code...It is giving error on line 31...'f = open(options.in_filenames)'....I am sure I am missing something really small....any help would be appreciated...
import csv
from numpy import *
import Gnuplot
import time
from optparse import OptionParser
from pylab import *
# set up and read command line options
parser = OptionParser()
parser.add_option("-f", "--file", dest="in_filenames",
help="read data from FILE - enclose comma-separated file list in quotes e.g. \"FILE1, FILE2\"", metavar="FILE")
(options, args) = parser.parse_args()
# set up constants
# column titles
columnvar_titles = (["A", "N", "K"])
series_titles = (["Average Fitness", "Maximum Fitness","Minimum Fitness",
"Average Wait Before Move","Maximum Wait Before Move",
"Minimum Wait Before Move","Average Number of Fitter Neighbours",
"Maximum Number of Fitter Neighbours",
"Minimum Number of Fitter Neighbours"])
f = open(options.in_filenames)
reader = csv.reader(f)
floats = []
options_dict={}
# start from the first line in the file
# read lines until we hit a blank
# lines will be in form "Key: Value"
# so split them and build a dictionary
while (1):
readstring = reader.next()
if len(readstring)==0:
break
dict_entry = readstring[0].split(': ', 1)
options_dict[dict_entry[0]] = dict_entry[1]
#print readstring
#print reader.line_num
#print options_dict
#print len(options_dict)
#print options_dict['Fitness_method']
# after the model parameters, we have blank line(s) before the data headers
# keep skipping blanks, then grab the first non-blank line
# then read the first line into a list of strings.
while (1):
readstring = reader.next()
if len(readstring) > 0:
column_headers = readstring;
break
#print column_headers
# need to check if we have a 'run' column
# single run gui output doesn't produce one, so need to add to column headers
# First six cols are "run, tick, A_size_of, RNGseed, [N & K]_size_of"
# "tick is already contained in the data, but is overwritten with K_size_of
# as it needs to be moved
# set add_run_data - flag to insert corresponding columns into the numeric data
add_run_data = 0
if column_headers[0]!="run":
column_headers[0] = "K_size_of"
column_headers = ["run", "tick", "A_size_of", "RngSeed", "N_size_of"] + column_headers
add_run_data = 1
print "Processing one run GUI output format..."
else:
print "Processing batch mode output..."
#print column_headers
# read lines from the data until we hit a blank.
# if data is numeric, put it into out 2d list of floats
countlines=0
while (1):
try:
readstring=reader.next()
countlines = countlines + 1
if len(readstring)==0:
print "Stopped reading"
break
try:
floats.append(map(float, readstring))
except:
print "Bad data - not adding"
except StopIteration:
#print "Read:", countlines
break
print "Read", countlines, "lines of data from file"
#print floats
xdata=array(floats)
#print xdata
# if we needed to add column headers before, we are dealing with single run
# output. If so, we need to add columns at the left of the data.
# First six cols are "run, tick, A_size_of, RNGseed, [N & K]_size_of"
# We add five (not six) cols, since "tick" is currently in the data already
# Other values are run = 1 (by def), RNGSeed (doesn't matter)
# A, N, K values are taken from the dictionary made from the header data in the file
if add_run_data == 1:
newcol = ones((xdata.shape[0],5), dtype="float")
xdata = concatenate((newcol, xdata), axis=1)
A_val = float(options_dict['A_size_of'])
N_val = float(options_dict['N_size_of'])
K_val = float(options_dict['K_size_of'])
#print A_val, N_val, K_val
xdata[:,1] = xdata[:,5] # ticks - already there but needs to move
xdata[:,2] = A_val
xdata[:,3] = 0 #RNG seed doesn't matter
xdata[:,4] = N_val
xdata[:,5] = K_val #overwrites original tick column
#print column_headers
#print xdata[1]
A_uniques = unique(xdata[:,2])
N_uniques = unique(xdata[:,4])
K_uniques = unique(xdata[:,5])
series_total = len(A_uniques) * len(N_uniques) * len(K_uniques)
# set up an array to hold averages (no columns for run number or rng seed)
# needs to move to handle multiple variables
maxticks = xdata[:,1].max()
#print series_total
#print maxticks
#print xdata.shape[1]-2
averages = zeros((series_total, maxticks, xdata.shape[1]-2), float)
# three loop setup for varying A / N / K values
series_counter = 0
Aseries_name=""
Nseries_name=""
Kseries_name=""
series_keys=[]
for A_value in A_uniques:
if len(A_uniques)>1:
Aseries_name = "A=" + str(A_value) + ", "
dataA = compress(xdata[:,2]==A_value, xdata, axis=0)
for N_value in N_uniques:
#if len(N_uniques)>1:
Nseries_name = "N=" + str(N_value)
dataAN = compress(dataA[:,4]==N_value, dataA, axis = 0)
for K_value in K_uniques:
#if len(K_uniques)>1:
Kseries_name = ", " + "K=" + str(K_value)
dataANK = compress(dataAN[:,5]==K_value, dataAN, axis = 0)
series_keys.append(Aseries_name + Nseries_name + Kseries_name)
# when multiple variables are used, run values continue to count from the
# previous variable value (e.g. A=2 (runs 1-100) A=3 (runs 101-200))
# we need to number the runs in ascending order from 1.
firstrun=dataANK[:,0].min()
lastrun=dataANK[:,0].max()
totalruns = 1 + lastrun - firstrun
#print firstrun, lastrun, totalruns
# for each run, find the last actual tick data
last_tick_array = zeros((totalruns, dataANK.shape[1]), float)
#print last_tick_array.shape
for k in arange(totalruns): # for each run get the data for the last tick
this_run=compress(dataANK[:,0]==k+firstrun, dataANK, axis=0)
last_tick_array[k]=this_run[-1]
#print "Last tick array"
#print last_tick_array[-1]
print "Processing simulation " + str(series_counter+1) + "/" + str(series_total)
for i in arange(maxticks): # for each tick value up to the maximum
# array to hold one tick from each run for averaging
# will contain either actual or extrapolated data
selected_ticks = zeros((totalruns, dataANK.shape[1]), float)
#print selected_ticks
# get dataANK for this tick from all runs. May be empty.
this_tick = compress(dataANK[:,1]==i+1, dataANK, axis=0)
#print "this tick"
#print this_tick
for j in arange(totalruns): # for each run
if (i+1) < last_tick_array[j,1]:# do we have actual data?
#print "Using real data"
# if so, get it
selected_ticks[j] = compress(this_tick[:,0]==j+firstrun, this_tick, axis=0)
#print selected_ticks
else:
# if not, use the last tick we do have
#print "Using last tick"
selected_ticks[j] = last_tick_array[j]
#print "selected_ticks"
#print selected_ticks[0]
averages[series_counter][i][0]=i+1 # tick number
averages[series_counter][i][1]=selected_ticks[:,2].max() #A_size_of
averages[series_counter][i][2]=selected_ticks[:,4].min() #N_size_of
averages[series_counter][i][3]=selected_ticks[:,5].min() #K_size_of
for m in xrange(6,16):
#print m
averages[series_counter][i][m-2]=selected_ticks[:,m].mean()
# increment to fill next index
series_counter = series_counter + 1
# matplotlib plots
print "Plotting graphs..."
matplotlib.use('Agg')
for graph_num in (4,5,6,7,8,9,10,11,12,13):
ylabel(column_headers[graph_num+2])
for ser in range(series_total):
plot(averages[ser][:,0], averages[ser][:,graph_num], label=series_keys[ser])
legend(loc='center right').draw_frame(0)
show()
savefig('nk' + column_headers[graph_num+2] + '.png')
clf()
print "Writing CSV files"
f1 = open("nk_allticks.csv","wt")
csv1 = csv.writer(f1)
f2 = open("nk_finaltick.csv","wt")
csv2 = csv.writer(f2)
column_headers.remove('RngSeed')
# replace spaces for underscores in column headers for better file compatability
for i in range(len(column_headers)):
column_headers[i]=column_headers[i].replace(' ','_')
try:
csv1.writerow(column_headers[1:])
csv2.writerow(column_headers[1:])
for series in range(series_total):
csv1.writerows(averages[series])
csv2.writerow(averages[series][-1])
finally:
f1.close
f2.close
f3 = open("nk_allticks_crosstab.csv","wt")
csv3 = csv.writer(f3)
out_array = zeros((averages[0].shape[0], 1 +(series_total * 10)), float)
out_array[:,0] = averages[0][:,0]
headers=[1000]#[8 * series_total]
headers[0]="tick"
datacol = 1
for column_num in (3, 4,5,6,7,8,9,10,11,12):
for ser in range (series_total):
headers.append((column_headers[column_num+2] + " " + series_keys[ser]).replace(' ','_'))
out_array[:,datacol] = averages[ser][:,column_num+1]
#print averages[ser][0]
datacol = datacol + 1
#print headers
#print out_array.shape
try:
csv3.writerow(headers)
csv3.writerows(out_array)
finally:
f3.close
You need to tell it which file to open. When you run the script you'd need to do something like -
python myscript.py -f myfile
Otherwise, you have not given it a file. Use the interpreter, it's great for catching these things.
>>> from optparse import OptionParser
>>> parser = OptionParser()
>>> parser.add_option("-f", "--file", dest="in_filenames",
... help="read data from FILE - enclose comma-separated file list in quotes e.g. \"FILE1, FILE2\"", metavar="FILE")
<Option at 0x29f47c8: -f/--file>
>>> (options, args) = parser.parse_args()
>>> options # Notice that in_filenames is None here
<Values at 0x29f6648: {'in_filenames': None}>
>>> import sys
>>> sys.argv
['']
>>> sys.argv = ['','-f','myfile'] # Let's explicitly set the argument
>>> (options, args) = parser.parse_args()
>>> options # Now it works...
<Values at 0x29fd848: {'in_filenames': 'myfile'}>