Develop Reference

Order of operations for cryto nodes/algorithms?

#I am trying to make a compiler with dummy data via a JSON. I have to encrypt it as a .bin and #then hash it and export it as a .blob file. Is there a certain order of operations when it comes #to all of theses cryptography nodes/algorithms? Anyways I am able to encrypt the json and then #turn it into a .bin. All I have to do now is hash it & export it as a .blob. Is there a certain #order I need to do this process? Thank you and have a good day.
#! /usr/bin/env python3
import os
import json
import hashlib #justin import
# import rsa
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
from cryptography.hazmat.primitives.asymmetric import ec
from base64 import b64encode, b64decode
key = os.urandom(16)
iv = os.urandom(16)
output_list = []
def encrypt_input(elements, num, type):
for i in range(1, num + 1):
if str(i) in elements:
if type == 'uint32_t':
value = int(elements[str(i)])
plaintext = value.to_bytes(4, 'big', signed=False)
elif type == 'hex':
value = (elements[str(i)])
plaintext = bytes.fromhex(value)
else:
value = 0
if type == 'uint32_t':
plaintext = value.to_bytes(4, 'big', signed=False)
elif type == 'hex':
plaintext = value.to_bytes(32, 'big', signed=False)
cipher = Cipher(algorithms.AES(key), modes.CTR(iv))
encryptor = cipher.encryptor()
output_list.append(encryptor.update(plaintext) + encryptor.finalize())
with open(filename, mode='rb') as input_file:
file_contents = input_file.read()
cipher = Cipher(algorithms.AES(key), modes.CTR(iv))
encryptor = cipher.encryptor()
file_length = len(file_contents).to_bytes(4, 'big', signed=False)
output_list.append(encryptor.update(file_length) + encryptor.finalize)
encryptor = cipher.encryptor()
ciphertext - encryptor.update(file_contents) + encryptor.finalize()
output_list.append(ciphertext)
with open('input.json') as json_file:
data = json.load(json_file)
encrypt_input(data['parameter'], 4 , 'uint_32_t')
encrypt_input(data['user_data'], 128 , 'hex')
encrypt_input(data['internal_data'], 32 , 'hex')
encrypt_file_input(data['user_binary']['file'])
data = b''.join(output_list)
private_key = ec.generate_private_key(ec.SECP256R1())
signature = private_key.sign(data, ec.ECDSA(hashes.SHA256))
print(signature)
with open('sip.bin','w') as output:
output.write(signature+data)
# justin's code
# this example is taking our file and turning it into BLOB format
# after its encrypted, you are using hash function,
#using the cryptography library located here https://cryptography.io/en/latest/hazmat/primitives/asymmetric/ec/#elliptic-curve-signature-algorithms
# https://github.com/pyca/cryptography/blob/main/src/cryptography/hazmat/primitives/hashes.py#L136-L139
# from cryptography.hazmat.primitives.asymmetric import utils
# chosen_hash = hashes.SHA256()
# hasher = hashes.Hash(chosen_hash)
# hasher.update(b"data & ")
# hasher.update(b"more data")
# digest = hasher.finalize()
# sig = private_key.sign(
# digest,
# ec.ECDSA(utils.Prehashed(chosen_hash))
# )
# #SHA256 Secure Hash Algorithm

Decrypt a encrypted secret using PyCrypto AES & sha256

I have a encrypted message in a file, encrypted by the following code.
I wrote a function to decrypt this message. I know the password used to encrypt it.
But I got the following error:
python3 decrypt.py enim_msg.txt
Traceback (most recent call last):
File "decrypt.py", line 45, in <module>
print(":: Decrypted: \n" + bytes.decode(decrypted))
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x88 in position 2: invalid start byte
How can I fix this problem pls ?
Is My decrypt function wrong ?
My code :
Encrypt function
import os
from Crypto import Random
from Crypto.Cipher import AES
from Crypto.Hash import SHA256
def encrypt(key, filename):
chunksize = 64*1024
outputFile = "en" + filename
filesize = str(os.path.getsize(filename)).zfill(16)
IV = Random.new().read(16)
encryptor = AES.new(key, AES.MODE_CBC, IV)
with open(filename, 'rb') as infile:
with open(outputFile, 'wb') as outfile:
outfile.write(filesize.encode('utf-8'))
outfile.write(IV)
while True:
chunk = infile.read(chunksize)
if len(chunk) == 0:
break
elif len(chunk) % 16 != 0:
chunk += b' ' * (16 - (len(chunk) % 16))
outfile.write(encryptor.encrypt(chunk))
def getKey(password):
hasher = SHA256.new(password.encode('utf-8'))
return hasher.digest()
Decrypt function I wrote
def decrypt(enc, password):
#print(":: enc => " + enc)
private_key = hashlib.sha256(password.encode("utf-8")).digest()
iv = enc[:16]
cipher = AES.new(private_key, AES.MODE_CBC, iv)
return cipher.decrypt(enc[16:])
How I call this function
password = "azerty123"
secret_file_path = sys.argv[1]
the_file = open(secret_file_path, "rb")
encrypted = the_file.read()
decrypted = decrypt(encrypted, password)
the_file.close()
print(":: Decrypted: \n" + bytes.decode(decrypted))

The bytes.decrypt() function by default expects an UTF-8 encoded string. But not every sequence of bytes is a valid UTF-8 sequence. In your case cipher.decrypt() (which may return any sequence of bytes) returned a byte-sequence, which is not a valid UTF-8 sequence. Thus the bytes.decode() function raised an error.
The actual reason why cipher.decrypt() returned a non-UTF-8 string is a bug in your code:
Your encrypted file format contains non-utf-8 data. Its format is like:
16 bytes len info (unencrypted, UTF-8 encoded)
16 bytes IV (unencrypted, binary i.e. non-UTF-8 encoded)
n bytes payload (encrypted, UTF-8 encoded)
You have to ensure that on decryption you only decode parts of your file, that are UTF-8 encoded. Furthermore you have to ensure that you decrypt only encrypted parts of your file (as mentioned in your comments)

pycrypto: unable to decrypt file

I am using PKCS1_OAEP crypto algorithm to encrypt a file. The file is encrypted successfully but unable to decrypt file, getting the error "Ciphertext with incorrect length."
Encryption Algorithm is here:
#!/usr/bin/python
from Crypto.Cipher import PKCS1_OAEP
from Crypto.PublicKey import RSA
import zlib
import base64
fd = open('test.doc', 'rb')
message = fd.read()
fd.close()
print "[*] Original File Size: %d" % len(message)
#message = 'To be encrypted'
key = RSA.importKey(open('pubkey.der').read())
cipher = PKCS1_OAEP.new(key)
compressed = zlib.compress(message)
print "[*] Compressed File Size: %d" % len(compressed)
chunk_size = 128
ciphertext = ""
offset = 0
while offset < len(compressed):
chunk = compressed[offset:offset+chunk_size]
if len(chunk) % chunk_size != 0:
chunk += " " * (chunk_size - len(chunk)) # Padding with spaces
ciphertext += cipher.encrypt(chunk)
offset += chunk_size
print "[*] Encrypted File Size: %d" % len(ciphertext)
encoded = ciphertext.encode("base64")
print "[*] Encoded file size: %d" % len(encoded)
fd = open("enc.data", 'wb')
fd.write(encoded)
fd.close()
print "[+] File saved successfully!"
Decryption Algorithm is here:
#!/usr/bin/python
from Crypto.Cipher import PKCS1_OAEP
from Crypto.PublicKey import RSA
import zlib
import base64
key = RSA.importKey(open('privkey.der').read())
cipher = PKCS1_OAEP.new(key)
fd = open('enc.data', 'rb')
encoded = fd.read().strip('\n')
fd.close()
decoded = encoded.decode("base64")
chunk_size = 128
offset = 0
plaintext = ""
while offset < len(decoded):
plaintext += cipher.decrypt(decoded[offset:offset+chunk_size])
offset += chunk_size
#plaintext = cipher.decrypt(decoded)
decompress = zlib.decompress(plaintext)
fd = open('decr.doc', 'wb')
fd.write(decompress)
fd.close()
Using the following script to generate key
from Crypto.PublicKey import RSA
new_key = RSA.generate(2048, e=65537)
public_key = new_key.publickey().exportKey("PEM")
private_key = new_key.exportKey("PEM")
fileWrite(fileName, data):
fd = open(fileName, 'wb')
fd.write(data)
fd.close()
fileWrite('privkey.der', private_key)
fileWrite('pubkey.der', public_key)
Here is the Error Message

You encrypt with a 2048 bit RSA key, which gives encrypted blocks of 2048 bites (256 bytes). Your decrypt implementation assumes the encrypted blocks are 128 bytes where they are actually 256 bytes, and thus you get the 'incorrect length' error. Notice how your encrypted files size (64512) is more than double of the compressed file size (32223).
In general you would not use RSA for bulk encryption (as it's quite slow) but would instead combine it with a symmetric encryption like AES. You would then encrypt the data with a random AES key, and then encrypt the AES key with the RSA key. This way you get the speed of AES and the two keys of RSA. This is known as Hybrid Encryption.

Decrypting a file to a stream and reading the stream into pandas (hdf or stata)

Overview of what I'm trying to do. I have encrypted versions of files that I need to read into pandas. For a couple of reasons it is much better to decrypt into a stream rather than a file, so that's my interest below although I also attempt to decrypt to a file just as an intermediate step (but this also isn't working).
I'm able to get this working for a csv, but not for either hdf or stata (I'd accept an answer that works for either hdf or stata, though the answer might be the same for both, which is why I'm combining in one question).
The code for encrypting/decrypting files is taken from another stackoverflow question (which I can't find at the moment).
import pandas as pd
import io
from Crypto import Random
from Crypto.Cipher import AES
def pad(s):
return s + b"\0" * (AES.block_size - len(s) % AES.block_size)
def encrypt(message, key, key_size=256):
message = pad(message)
iv = Random.new().read(AES.block_size)
cipher = AES.new(key, AES.MODE_CBC, iv)
return iv + cipher.encrypt(message)
def decrypt(ciphertext, key):
iv = ciphertext[:AES.block_size]
cipher = AES.new(key, AES.MODE_CBC, iv)
plaintext = cipher.decrypt(ciphertext[AES.block_size:])
return plaintext.rstrip(b"\0")
def encrypt_file(file_name, key):
with open(file_name, 'rb') as fo:
plaintext = fo.read()
enc = encrypt(plaintext, key)
with open(file_name + ".enc", 'wb') as fo:
fo.write(enc)
def decrypt_file(file_name, key):
with open(file_name, 'rb') as fo:
ciphertext = fo.read()
dec = decrypt(ciphertext, key)
with open(file_name[:-4], 'wb') as fo:
fo.write(dec)
And here's my attempt to extend the code to decrypt to a stream rather than a file.
def decrypt_stream(file_name, key):
with open(file_name, 'rb') as fo:
ciphertext = fo.read()
dec = decrypt(ciphertext, key)
cipherbyte = io.BytesIO()
cipherbyte.write(dec)
cipherbyte.seek(0)
return cipherbyte
Finally, here's the sample program with sample data attempting to make this work:
key = 'this is an example key'[:16]
df = pd.DataFrame({ 'x':[1,2], 'y':[3,4] })
df.to_csv('test.csv',index=False)
df.to_hdf('test.h5','test',mode='w')
df.to_stata('test.dta')
encrypt_file('test.csv',key)
encrypt_file('test.h5',key)
encrypt_file('test.dta',key)
decrypt_file('test.csv.enc',key)
decrypt_file('test.h5.enc',key)
decrypt_file('test.dta.enc',key)
# csv works here but hdf and stata don't
# I'm less interested in this part but include it for completeness
df_from_file = pd.read_csv('test.csv')
df_from_file = pd.read_hdf('test.h5','test')
df_from_file = pd.read_stata('test.dta')
# csv works here but hdf and stata don't
# the hdf and stata lines below are what I really need to get working
df_from_stream = pd.read_csv( decrypt_stream('test.csv.enc',key) )
df_from_stream = pd.read_hdf( decrypt_stream('test.h5.enc',key), 'test' )
df_from_stream = pd.read_stata( decrypt_stream('test.dta.enc',key) )
Unfortunately I don't think I can shrink this code anymore and still have a complete example.
Again, my hope would be to have all 4 non-working lines above working (file and stream for hdf and stata) but I'm happy to accept an answer that works for either the hdf stream alone or the stata stream alone.
Also, I'm open to other encryption alternatives, I just used some existing pycrypto-based code that I found here on SO. My work explicitly requires 256-bit AES but beyond that I'm open so this solution needn't be based specifically on the pycrypto library or the specific code example above.
Info on my setup:
python: 3.4.3
pandas: 0.17.0 (anaconda 2.3.0 distribution)
mac os: 10.11.3

The biggest issue is the padding/unpadding method. It assumes that the null character can't be part of the actual content. Since stata/hdf files are binary, it's safer to pad using the number of extra bytes we use, encoded as a character. This number will be used during unpadding.
Also for this time being, read_hdf doesn't support reading from a file like object, even if the API documentation claims so. If we restrict ourselves to the stata format, the following code will perform what you need:
import pandas as pd
import io
from Crypto import Random
from Crypto.Cipher import AES
def pad(s):
n = AES.block_size - len(s) % AES.block_size
return s + n * chr(n)
def unpad(s):
return s[:-ord(s[-1])]
def encrypt(message, key, key_size=256):
message = pad(message)
iv = Random.new().read(AES.block_size)
cipher = AES.new(key, AES.MODE_CBC, iv)
return iv + cipher.encrypt(message)
def decrypt(ciphertext, key):
iv = ciphertext[:AES.block_size]
cipher = AES.new(key, AES.MODE_CBC, iv)
plaintext = cipher.decrypt(ciphertext[AES.block_size:])
return unpad(plaintext)
def encrypt_file(file_name, key):
with open(file_name, 'rb') as fo:
plaintext = fo.read()
enc = encrypt(plaintext, key)
with open(file_name + ".enc", 'wb') as fo:
fo.write(enc)
def decrypt_stream(file_name, key):
with open(file_name, 'rb') as fo:
ciphertext = fo.read()
dec = decrypt(ciphertext, key)
cipherbyte = io.BytesIO()
cipherbyte.write(dec)
cipherbyte.seek(0)
return cipherbyte
key = 'this is an example key'[:16]
df = pd.DataFrame({
'x': [1,2],
'y': [3,4]
})
df.to_stata('test.dta')
encrypt_file('test.dta', key)
print pd.read_stata(decrypt_stream('test.dta.enc', key))
Output:
index x y
0 0 1 3
1 1 2 4
In python 3 you can use the following pad, unpad versions:
def pad(s):
n = AES.block_size - len(s) % AES.block_size
return s + bytearray([n] * n)
def unpad(s):
return s[:-s[-1]]

What worked for me in the case of .h5 format and the cryptography library was:
from cryptography.fernet import Fernet
def read_h5_file(new_file:str, decrypted: bytes, verbose=False):
with open(new_file, 'wb') as f:
f.write(decrypted)
print(f'Created {new_file}') if verbose else ''
df = pd.read_hdf(new_file)
os.remove(new_file)
print(f'Deleted {new_file}') if verbose else ''
return df
with open(path_to_file, 'rb') as f:
data = f.read()
fernet = Fernet(key)
decrypted = fernet.decrypt(data)
new_file = './example_path/example.h5'
df = read_h5_file(new_file, decrypted, verbose=verbose)
So I created a .h5 file. Read its content. Return it with the function. Delete the decrypted file again.
Maybe this approach helps, as I didn't find any other or similar solution on this online.

Python Encrypting with PyCrypto AES

I just found pycrypto today, and I've been working on my AES encryption class. Unfortunately it only half-works. self.h.md5 outputs md5 hash in hex format, and is 32byte.
This is the output. It seems to decrypt the message, but it puts random characters after decryption, in this case \n\n\n... I think I have a problem with block size of self.data, anyone know how to fix this?
Jans-MacBook-Pro:test2 jan$ ../../bin/python3 data.py
b'RLfGmn5jf5WTJphnmW0hXG7IaIYcCRpjaTTqwXR6yiJCUytnDib+GQYlFORm+jIctest
1 2 3 4 5 endtest\n\n\n\n\n\n\n\n\n\n'
from Crypto.Cipher import AES
from base64 import b64encode, b64decode
from os import urandom
class Encryption():
def __init__(self):
self.h = Hash()
def values(self, data, key):
self.data = data
self.key = key
self.mode = AES.MODE_CBC
self.iv = urandom(16)
if not self.key:
self.key = Cfg_Encrypt_Key
self.key = self.h.md5(self.key, True)
def encrypt(self, data, key):
self.values(data, key)
return b64encode(self.iv + AES.new(self.key, self.mode, self.iv).encrypt(self.data))
def decrypt(self, data, key):
self.values(data, key)
self.iv = b64decode(self.data)[:16]
return AES.new(self.key, self.mode, self.iv).decrypt(b64decode(self.data)[16:])

To be honest, the characters "\n\n\n\n\n\n\n\n\n\n" don't look that random to me. ;-)
You are using AES in CBC mode. That requires length of plaintext and ciphertext to be always a multiple of 16 bytes. With the code you show, you should actually see an exception being raised when data passed to encrypt() does not fulfill such condition. It looks like you added enough new line characters ('\n' to whatever the input is until the plaintext happened to be aligned.
Apart from that, there are two common ways to solve the alignment issue:
Switch from CBC (AES.MODE_CBC) to CFB (AES.MODE_CFB). With the default segment_size used by PyCrypto, you will not have any restriction on plaintext and ciphertext lengths.
Keep CBC and use a padding scheme like PKCS#7, that is:
before encrypting a plaintext of X bytes, append to the back as many bytes you need to to reach the next 16 byte boundary. All padding bytes have the same value: the number of bytes that you are adding:
length = 16 - (len(data) % 16)
data += bytes([length])*length
That's Python 3 style. In Python 2, you would have:
length = 16 - (len(data) % 16)
data += chr(length)*length
after decrypting, remove from the back of the plaintext as many bytes as indicated by padding:
data = data[:-data[-1]]
Even though I understand in your case it is just a class exercise, I would like to point out that it is insecure to send data without any form of authentication (e.g. a MAC).

You can use a fix character as long as you remember the length of your initial payload, so you don't "throw" useful end bytes away.
Try this:
import base64
from Crypto.Cipher import AES
def encrypt(payload, salt, key):
return AES.new(key, AES.MODE_CBC, salt).encrypt(r_pad(payload))
def decrypt(payload, salt, key, length):
return AES.new(key, AES.MODE_CBC, salt).decrypt(payload)[:length]
def r_pad(payload, block_size=16):
length = block_size - (len(payload) % block_size)
return payload + chr(length) * length
print(decrypt(encrypt("some cyphertext", "b" * 16, "b" * 16), "b" * 16, "b" * 16, len("some cyphertext")))

from hashlib import md5
from Crypto.Cipher import AES
from Crypto import Random
import base64
def derive_key_and_iv(password, salt, key_length, iv_length):
d = d_i = ''
while len(d) < key_length + iv_length:
d_i = md5(d_i + password + salt).digest()
d += d_i
return d[:key_length], d[key_length:key_length+iv_length]
def encrypt(in_file, out_file, password, key_length=32):
bs = AES.block_size
salt = Random.new().read(bs - len('Salted__'))
key, iv = derive_key_and_iv(password, salt, key_length, bs)
cipher = AES.new(key, AES.MODE_CBC, iv)
#print in_file
in_file = file(in_file, 'rb')
out_file = file(out_file, 'wb')
out_file.write('Salted__' + salt)
finished = False
while not finished:
chunk = in_file.read(1024 * bs)
if len(chunk) == 0 or len(chunk) % bs != 0:
padding_length = bs - (len(chunk) % bs)
chunk += padding_length * chr(padding_length)
finished = True
out_file.write(cipher.encrypt(chunk))
in_file.close()
out_file.close()
def decrypt(in_file, out_file, password, key_length=32):
bs = AES.block_size
in_file = file(in_file, 'rb')
out_file = file(out_file, 'wb')
salt = in_file.read(bs)[len('Salted__'):]
key, iv = derive_key_and_iv(password, salt, key_length, bs)
cipher = AES.new(key, AES.MODE_CBC, iv)
next_chunk = ''
finished = False
while not finished:
chunk, next_chunk = next_chunk, cipher.decrypt(in_file.read(1024 * bs))
if len(next_chunk) == 0:
padding_length = ord(chunk[-1])
if padding_length < 1 or padding_length > bs:
raise ValueError("bad decrypt pad (%d)" % padding_length)
# all the pad-bytes must be the same
if chunk[-padding_length:] != (padding_length * chr(padding_length)):
# this is similar to the bad decrypt:evp_enc.c from openssl program
raise ValueError("bad decrypt")
chunk = chunk[:-padding_length]
finished = True
out_file.write(chunk)
in_file.close()
out_file.close()
def encode(in_file, out_file):
in_file = file(in_file, 'rb')
out_file = file(out_file, 'wb')
data = in_file.read()
out_file.write(base64.b64encode(data))
in_file.close()
out_file.close()
def decode(in_file, out_file):
in_file = file(in_file, 'rb')
out_file = file(out_file, 'wb')
data = in_file.read()
out_file.write(base64.b64decode(data))
in_file.close()
out_file.close()

AES.new().encrypt() and .decrypt() take as both input and output strings whose length is a multiple of 16. You have to fix it in one way or another. For example you can store the real length at the start, and use that to truncate the decrypted string.
Note also that while it's the only restriction for AES, other modules (notably in Crypto.PublicKey) have additional restrictions that comes from their mathematical implementation and that shouldn't (in my opinion) be visible to the end user, but are. For example Crypto.PublicKey.ElGamal will encrypt any short string, but if it starts with null characters, they are lost upon decryption.