February 06, 2019
Hi All,

I have tried encrypting a message using public key and sending message to client, who decrypts using private key. I have tried with openssl d package. Below is my code. However it always fails in creating the context and exits by throwing exception. Key that i am using is RSA key.

Please help me on this. I have also tried secured and crypto packages, but i see crash in these packages also.

import std.array;
import std.string;
import std.algorithm;
import std.stdio;

import core.cpuid;
import core.memory;
import deimos.openssl.evp;
import deimos.openssl.rand;
import deimos.openssl.pem;
import deimos.openssl.bio;
import deimos.openssl.rsa;
import deimos.openssl.engine;

ubyte[] encrypt(const ubyte[] inMessage, EVP_PKEY* keypair)
{
    ENGINE* eng = null; // Use default RSA implementation
    ubyte* out2;
    const ubyte* in2 = inMessage.ptr;
    size_t outlen;
    size_t inlen = inMessage.length;

    auto ctx = EVP_PKEY_CTX_new(keypair, eng);
    if (ctx == null)
    {
        throw new Exception("EVP_PKEY_CTX_new.");
    }
    scope (exit)
    {
        if (ctx !is null)
        {
            EVP_PKEY_CTX_free(ctx);
        }
    }

    if (EVP_PKEY_encrypt_init(ctx) <= 0)
    {
        if (!ctx)
        {
            throw new Exception("EVP_PKEY_CTX_new.");
        }
    }
    scope (exit)
    {
        if (ctx !is null)
        {
            EVP_PKEY_CTX_free(ctx);
        }
    }

    if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) <= 0)
    {
        throw new Exception("EVP_PKEY_CTX_set_rsa_padding failed.");
    }

    if (EVP_PKEY_encrypt(ctx, null, &outlen, in2, inlen) <= 0)
    {
        throw new Exception("EVP_PKEY_encrypt failed.");
    }

    out2 = cast(ubyte*) GC.malloc(outlen);
    if (out2 == null)
    {
        throw new Exception("Malloc failed.");
    }

    if (EVP_PKEY_encrypt(ctx, out2, &outlen, in2, inlen) <= 0)
    {
        throw new Exception("EVP_PKEY_encrypt failed.");
    }

    return (out2)[0 .. outlen];
}

void main()
{
    const(ubyte)[] message = cast(const(ubyte)[]) "Hello, this is a binary message!";
    import core.stdc.stdlib : calloc, free;
    import std.base64;
    auto keyStr = "AAACANhbLxG1cezzbb/NsLQQSFseuOSKe2x7xcjuJmiiBC6tlbhT0RV6Y9Zad0tMp3OvS7etq0XRmeKK2QcP6gcqsVnOEX9qtPJMcGoyZxNo7nwU7DEyu9qEBhKMcI+jtRHEqcD0hGHgQg/KJ0bZsIFikKdezDSXQz8uVaJeypwkFChN
    auto keyBytes = Base64.decode(keyStr);
    // auto keyBytes = cast(ubyte[]) keyStr;

    auto bio = BIO_new_mem_buf(keyBytes.ptr, cast(int) keyBytes.length);
    auto keypair = PEM_read_bio_PUBKEY(bio, null, null, null);
    // auto rsaKey = PEM_read_bio_RSAPrivateKey(bio, null, null, null);

    BIO_free_all(bio);

    encrypt(message, keypair);
}