We Keep Coding

AES encryption and padding across multiple blocks

I am encrypting a large (100GB+) file with Python using PyCryptodome using AES-256 in CBC mode.
Rather than read the entire file into memory and encrypt it in one fell swoop, I would like to read the input file a 'chunk' at a time and append to the output file with the results of encrypting each 'chunk.'
Regrettably, the documentation for PyCryptodome is lacking in that I can't find any examples of how to encrypt a long plaintext with multiple calls to encrypt(). All the examples use a short plaintext and encrypt the entire plaintext in a single call to encrypt().
I had assumed that if my input 'chunk' is a multiple of 16 bytes (the block size of AES in CBC mode) I wouldn't need to add padding to any 'chunk' but the last one. However, I wasn't able to get that to work. (I got padding errors while decrypting.)
I'm finding that in order to successfully decrypt the file, I need to add padding to every 'chunk' when encrypting, and decrypt in units of the input chunk size plus 16 bytes. This means the decrypting process needs to know the 'chunk size' used for encryption, which makes me believe that this is probably an incorrect implementation.
While I do have my encryption/decryption working as described, I wonder if this is the 'correct' way to do it. (I suspect it is not.) I've read inconsistent claims on whether or not every such 'chunk' needs padding. If not, I'd like some handholding to get Pycryptodome to encrypt and then decrypt a large plaintext across multiple calls to encrypt() and decrypt().
EDIT: This code throws a ValueError, "Padding is incorrect," when decrpyting the first 'chunk'.
def encrypt_file(infile, outfile, aeskey, iv):
cipher = AES.new(aeskey, AES.MODE_CBC, iv)
with open(infile, "rb") as fin:
with open(outfile, "wb") as fout:
while True:
data = fin.read(16 * 32)
if len(data) ==0:
break
insize = len(data)
if insize == (16 * 32):
padded_data = data
else:
padded_data = pad(data, AES.block_size)
fout.write(cipher.encrypt(padded_data))
def decrypt_file(infile, outfile, aeskey, iv):
cipher = AES.new(aeskey, AES.MODE_CBC, iv)
with open (infile, "rb") as fin:
with open(outfile, "wb") as fout:
while True:
data = fin.read(16 * 32)
if len(data) == 0:
break
fout.write(unpad(cipher.decrypt(data), AES.block_size))

My problem was related to the PAD of the last block. It is necessary to detect which is the last fragment read in bytes in order to add the PAD.
def decrypt_file(
self, filename: str, output_file: str, save_path: str, key, iv
):
cipher_aes = AES.new(key, AES.MODE_CBC, iv)
log.info(f'Decrypting file: {filename} output: {output_file}')
count = 0
previous_data = None
with open(filename, "rb") as f, open(
f"{save_path}/{output_file}", "wb"
) as f2:
while True:
count+=1
data = f.read(self.block_size)
if data == b"":
decrypted = cipher_aes.decrypt(previous_data)
log.info(f'Last block UnPadding Count: {count} BlockSize: {self.block_size}')
decrypted = unpad(decrypted, AES.block_size, style="pkcs7")
f2.write(decrypted)
break
if previous_data:
decrypted = cipher_aes.decrypt(previous_data)
f2.write(decrypted)
previous_data = data
And apply the decrypt:
def decrypt_file(
self, filename: str, output_file: str, save_path: str, key, iv
):
cipher_aes = AES.new(key, AES.MODE_CBC, iv)
log.info(f'Decrypting file: {filename} output: {output_file}')
count = 0
previous_data = None
with open(filename, "rb") as f, open(
f"{save_path}/{output_file}", "wb"
) as f2:
while True:
count+=1
data = f.read(self.block_size)
if data == b"":
decrypted = cipher_aes.decrypt(previous_data)
log.info(f'Last block UnPadding Count: {count} BlockSize: {self.block_size}')
decrypted = unpad(decrypted, AES.block_size, style="pkcs7")
f2.write(decrypted)
break
if previous_data:
decrypted = cipher_aes.decrypt(previous_data)
f2.write(decrypted)
previous_data = data

It looks like the fix is to do similar chunksize/padding comparison in the decrypt function as I used in the encrypt function:
def decrypt_file(infile, outfile, aeskey, iv):
cipher = AES.new(aeskey, AES.MODE_CBC, iv)
with open (infile, "rb") as fin:
with open(outfile, "wb") as fout:
while True:
data = fin.read(16 * 32)
if len(data) == 0:
break
if len(data) == (16 * 32):
decrypted_data = cipher.decrypt(data)
else:
decrypted_data = unpad(cipher.decrypt(data), AES.block_size)
fout.write(decrypted_data)

AES: Input strings must be a multiple of 16 in length

I want to make a script to decrypt my filess, but when I try to run my script then show me this message , how can I fix it?
Traceback (most recent call last): File "F:\bug_bounty\decrypt.py",
line 46, in File "F:\bug_bounty\decrypt.py", line 24, in
decrypt File
"C:\Python27\lib\site-packages\Crypto\Cipher\blockalgo.py", line 295,
in decrypt
return self._cipher.decrypt(ciphertext) ValueError: Input strings must be a multiple of 16 in length
from Crypto.Hash import SHA256
from Crypto.Cipher import AES
import os
import random
import sys
def decrypt(key, filename):
outFile = os.path.join(os.path.dirname(filename),
os.path.basename(filename[11:]))
chunksize = 64 * 1024
with open(filename, 'rb') as infile:
filesize = infile.read(16)
IV = infile.read(16)
decryptor = AES.new(key, AES.MODE_CBC, IV)
with open(outFile, 'wb') as outfile:
while True:
chunk = infile.read(chunksize)
if len(chunk) == 0:
break
outfile.write(decryptor.decrypt(chunk))
outfile.truncate(int(filesize))
def allfiles():
allFiles = []
for (root, subfiles, files) in os.walk(os.getcwd()):
for names in files:
allFiles.append(os.path.join(root, names))
return allFiles
password = 'M4st3rRul3zs'
files = allfiles();
for filename in files:
if os.path.basename(filename).startswith("(encrypted)"):
print "%s is already encrypted" %filename
pass
else:
decrypt(SHA256.new(password).digest(), filename)
print "Done decrypting %s" %filename
"""os.remove(filename)"""

Here is the small trick you could use while encrypting the data if your data size is not large.
plaintext = "some text"
encryptor = AES.new(key, AES.MODE_CBC, iv)
ciphertext = encryptor.encrypt(plaintext*16)
This will ensure that your input data is a multiple of 16. And of course, you would like to get the original data back when decrypting.
cipher = AES.new(key, AES.MODE_CBC, iv)
decrypttext = cipher.decrypt(ciphertext)
decrypttext = decrypttext[0:len(plaintext)]
Now, decrpyttext has your original plaintext.

From Crypto++ wiki.
The block size is determined by AES::BLOCKSIZE. For AES, this is
always 16 bytes
AES is a block cipher, it works on 16-byte (128-bit) blocks. It can't work with data smaller or bigger than 16 bytes. Smaller data needs to be padded until they're 16 bytes, and larger data needs to be split into 16-byte blocks.
Also there are algorithms that help you achieve just that (work on data larger than the cipher's block size), they're called block cipher modes of operation.
Have a look at this How to encrypt more than 16 bytes using AES?

ValueError: Input strings must be a multiple of 16 in length
That is because AES works with blocks of 128 bits (16 chars). You can consider adding padding to fix this.

AES works with blocks of 16 chars. This how you can add extra padding
import random
import string
plaintext = "Encrypt me"
encryptor = AES.new(key, AES.MODE_CBC, iv)
while len(bytes(plaintext, encoding='utf-8')) % 16 != 0:
plaintext = plaintext + random.choice(string.ascii_letters)
ciphertext = encryptor.encrypt(plaintext)

PyCrypto in Python, cannot choose what file to open

I am fairly new to python and pycrypto. I read many guides and manuals on pycrypto and I have come up with a small bit of code. I don't understand how to choose which files I want to encrypt. Is it supposed to be defined somewhere? My code:
import os, random, struct
from Crypto.Cipher import AES
def encrypt_file(key, in_filename, out_filename=None, chunksize=64*1024):
if not out_filename:
out_filename = in_filename + '.enc'
iv = ''.join(chr(random.randint(0, 0xFF)) for i in range(16))
encryptor = AES.new(key, AES.MODE_CBC, iv)
filesize = os.path.getsize(in_filename)
with open(in_filename, 'rb') as infile:
with open(out_filename, 'wb') as outfile:
outfile.write(struct.pack('<Q', filesize))
outfile.write(iv)
while True:
chunk = infile.read(chunksize)
if len(chunk) == 0:
break
elif len(chunk) % 16 != 0:
chunk += ' ' * (16 - len(chunk) % 16)
outfile.write(encryptor.encrypt(chunk))
I just want to know how to choose which specific file to encrypt. I tried to write he file path in with open but that didn't seem to work.

Pycrypto - Encrypt on Linux / decrypt on Windows

I've got a encryption/decryption class that I'm using cross platform. I'm using the same class on both server and client. I encrypt a file on a Linux server, then decrypt on either a Linux or Windows client. I have no problems when decrypting on Linux, but when I transfer the file to Windows and try to decrypt, I get the following exception:
ValueError: Input strings must be a multiple of 16 in length
My first thought is that it is caused by the different filesystems, and any characters that are used to create the padding. Here is my class code:
class FileSec:
def __init__(self):
# File chunk size
self.chunk_size = 64*1024
# Encrypt file with OpenSSL
def encrypt(self, infile, outfile, key):
if not infile or not os.path.isfile(infile):
return False
if not outfile or os.path.isfile(outfile):
return False
if not key:
return False
# Encrypt the file
iv = ''.join(chr(random.randint(0, 0xFF)) for i in range(16))
encryptor = AES.new(key, AES.MODE_CBC, iv)
filesize = os.path.getsize(infile)
with open(infile, 'rb') as ifh:
with open(outfile, 'wb') as ofh:
ofh.write(struct.pack('<Q', filesize))
ofh.write(iv)
while True:
chunk = ifh.read(self.chunk_size)
if len(chunk) == 0:
break
elif len(chunk) % 16 != 0:
chunk += ' ' * (16 - len(chunk) % 16)
ofh.write(encryptor.encrypt(chunk))
return True
# Decrypt file with OpenSSL
def decrypt(self, infile, outfile, key):
if not infile or not os.path.isfile(infile):
return False
if not outfile or os.path.isfile(outfile):
return False
if not key:
return False
# Decrypt the file
with open(infile, 'rb') as ifh:
origsize = struct.unpack('<Q', ifh.read(struct.calcsize('Q')))[0]
iv = ifh.read(16)
decryptor = AES.new(key, AES.MODE_CBC, iv)
with open(outfile, 'wb') as ofh:
while True:
chunk = ifh.read(self.chunk_size)
if len(chunk) == 0:
break
ofh.write(decryptor.decrypt(chunk))
ofh.truncate(origsize)
return True
http://pastebin.com/Dvf6nUxH
I'm using code adapted from here: http://eli.thegreenplace.net/2010/06/25/aes-encryption-of-files-in-python-with-pycrypto/
Anyone have any suggestions on how I can modify this class to work cross-platform?

myfile.read(x) reads any amount up to x bytes; it is not guaranteed to return all x.
Note that it will always return at least one until the file is empty, so it is possible to wrap this in a loop, and then join the returned strings.

Closing this one. Turns out the problem has nothing to do with the encryption/decryption function, but with an extra byte being tacked on to the encrypted file when I transfer it to the Windows machine, causing the exception.

How can I encrypt .docx files with AES & pycrypto without corrupting the files

I've got this bit of python code that I want to use to encrypt various kinds of files with AES 256. I am using the pycrypto module. It works fine for most files (exe, deb, jpg, pdf, txt) but when it comes to office files (docx, xlsx, ppt etc) the file is corrupted upon decryption and will no open (nor can it be repaired) in LibreOffice. I am using Linux mint, python 2.7.6, pycrypto 2.6.1. I'm still a bit of a noob so I'd appreciate it if you could give me code examples of the corrections you'd recommend.
Thanks
from Crypto import Random
from Crypto.Cipher import AES
import os
def pad(s):
return s + b"\0" * (AES.block_size - len(s) % AES.block_size)
def encrypt(message, key, key_size=256):
message = pad(message)
iv = Random.new().read(AES.block_size)
cipher = AES.new(key, AES.MODE_CBC, iv)
return iv + cipher.encrypt(message)
def decrypt(ciphertext, key):
iv = ciphertext[:AES.block_size]
cipher = AES.new(key, AES.MODE_CBC, iv)
plaintext = cipher.decrypt(ciphertext[AES.block_size:])
return plaintext.rstrip(b"\0")
def encrypt_file(file_name, key):
with open(file_name, 'rb') as fo:
plaintext = fo.read()
enc = encrypt(plaintext, key)
with open(file_name + ".enc", 'wb') as fo:
fo.write(enc)
def decrypt_file(file_name, key):
with open(file_name, 'rb') as fo:
ciphertext = fo.read()
dec = decrypt(ciphertext, key)
with open(file_name[:-4], 'wb') as fo:
fo.write(dec)
key = b'\xbf\xc0\x85)\x10nc\x94\x02)j\xdf\xcb\xc4\x94\x9d(\x9e[EX\xc8\xd5\xbfI{\xa2$\x05(\xd5\x18'
encrypt_file('file.docx', key)

The problem is here
plaintext.rstrip(b"\0")
I have run the program and see the reason is:
There was a bug in here that caused the last bytes of the original file to be discarded if they happened to have the same value as the padding bytes!
To fix this issue, we have to store how many padding bytes were used during encryption, then remove them during decryption. Here is my code, it works for me (tested with word and excel 2013 files, pdf, jpg). Let me know if still some bugs.
from Crypto import Random
from Crypto.Cipher import AES
import hashlib
def pad(s):
padding_size = AES.block_size - len(s) % AES.block_size
return s + b"\0" * padding_size, padding_size
def encrypt(message, key, key_size=256):
message, padding_size = pad(message)
iv = Random.new().read(AES.block_size)
cipher = AES.new(key, AES.MODE_CFB, iv)
enc_bytes = iv + cipher.encrypt(message) + bytes([padding_size])
return enc_bytes
def decrypt(ciphertext, key):
iv = ciphertext[:AES.block_size]
cipher = AES.new(key, AES.MODE_CFB, iv)
plaintext = cipher.decrypt(ciphertext[AES.block_size:-1])
padding_size = ciphertext[-1] * (-1)
return plaintext[:padding_size]
def encrypt_file(file_name, key):
with open(file_name, 'rb') as fo:
plaintext = fo.read()
enc = encrypt(plaintext, key)
with open(file_name + ".enc", 'wb') as fo:
fo.write(enc)
def decrypt_file(file_name, key):
with open(file_name, 'rb') as fo:
ciphertext = fo.read()
dec = decrypt(ciphertext, key)
with open('processed_' + file_name[:-4], 'wb') as fo:
fo.write(dec)
key = 'Quan'
hash_object = hashlib.md5(key.encode())
while True:
filename = input('File: ')
en_de = input('En or De?')
if en_de.upper() == 'EN':
encrypt_file(filename, hash_object.hexdigest())
elif en_de.upper() == 'DE':
decrypt_file(filename, hash_object.hexdigest())
else:
print('Did not pick either en or de!')
cont = input('Continue?')
if cont.upper() == 'N':
break

If you need to add padding to make the plaintext a multiple of 16 bytes, the extra bytes need to be stripped before you write the decrypted data. This means you will need to somehow include the number of pad bytes added with the padding before you encrypt it. See PKCS#7 for one possible technique. There are a number of other schemes as well.