Skip to content

Refactor rsa/add modern hashs in OAEP encrypt-decrypt #833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
noiser7 wants to merge 34 commits into
base: main
Choose a base branch
from
Draft

Refactor rsa/add modern hashs in OAEP encrypt-decrypt #833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
36c5c16
Add OAEP parameter passing
Dec 24, 2025
5bd133f
Add Asymmetric encrypt/decrypt parameter
Dec 24, 2025
73e5fe2
Refactor RSA/ add OAEP encrypt/decrypt parameters
Dec 24, 2025
31b359a
Fix RSA encrypt-decrypt tests
Dec 24, 2025
9edab6c
Fix memory leaks and messages
Dec 25, 2025
9a03665
Add BuildRSAOAEPParam fuction
Dec 25, 2025
eb02aa2
Add OpenSSL 1 support
Dec 26, 2025
bb16236
Remove legacy code
Dec 26, 2025
69d3118
Fix RSA_set_method call
Dec 26, 2025
6c1e1d0
Add PSS salt length setting in VerifyFinal
Dec 26, 2025
530cba7
Add salt length setting in one-stage verify
Dec 26, 2025
65c0423
Add OAEP encrypt-decrypt parameters for Botan
Dec 30, 2025
dd25216
Add asymmetric encrypt-decrypt parameters passing
Dec 30, 2025
2f850fc
Fix sourceData pointer check
Dec 30, 2025
51450ed
Fix maximum length validation for RSA encryption
Dec 30, 2025
49df263
Revert codestyle changes
Jan 29, 2026
477e16d
Revert code style changes
Jan 29, 2026
e3f93b7
Merge branch 'main' into Refactor-RSA/Add-modern-hash-on-OAEP
noiser7 Jan 29, 2026
dfddb57
fix pointer calculation
Jan 29, 2026
80f9daa
Fix memory leak on *parameters
Feb 2, 2026
deadbc1
Fix OSSL_PARAM_BLD_to_param calling on error
Feb 2, 2026
139e2c2
Fix inRSA NULL check before dereference
Feb 2, 2026
c7bbcf7
Fix bug wth pointer check
Feb 2, 2026
f85d283
Fix maximum key length checking
Feb 2, 2026
934ce4d
Check OAEP label length to prevent buffer overflow
Feb 2, 2026
65ec656
Check input parameter in setFromOSSL
Feb 2, 2026
bc008af
Fix BN free if RSA_new fails
Feb 3, 2026
1574ae4
Text format changed
Mar 10, 2026
8c8f9a3
Text format changed, debug artifacts removed
Mar 10, 2026
254c779
Merge branch 'main' into Refactor-RSA/Add-modern-hash-on-OAEP
noiser7 Mar 17, 2026
7153b77
Fix OSSLMLDSA encrypt/decrypt
Mar 17, 2026
9a8fd9d
Fixed NITs
Mar 25, 2026
b1173b3
Refactor RSA OAEP to *MechanismParam parameters tranfer
Mar 26, 2026
ebb28c2
Refactor RSA OAEP to *MechanismParam parameters tranfer for Botan
Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
224 changes: 179 additions & 45 deletions src/lib/SoftHSM.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
#include "DESKey.h"
#include "RNG.h"
#include "RSAParameters.h"
#include "RSAMechanismParam.h"
#include "RSAPublicKey.h"
#include "RSAPrivateKey.h"
#include "DSAParameters.h"
Expand Down Expand Up @@ -2521,6 +2522,7 @@ CK_RV SoftHSM::AsymEncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
// Get the asymmetric algorithm matching the mechanism
AsymMech::Type mechanism;
bool isRSA = false;
MechanismParam* mechanismParam = NULL;
switch(pMechanism->mechanism) {
case CKM_RSA_PKCS:
if (keyType != CKK_RSA)
Expand Down Expand Up @@ -2571,16 +2573,35 @@ CK_RV SoftHSM::AsymEncryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
}
else
{
return CKR_MECHANISM_INVALID;
}
return CKR_MECHANISM_INVALID;
}
// set mechanism parameters
if (pMechanism->mechanism == CKM_RSA_PKCS_OAEP)
{
RSAOaepMechanismParam* rsaOaepMechanismParam = new RSAOaepMechanismParam;
rv = BuildRSAOAEPParam((CK_RSA_PKCS_OAEP_PARAMS *)pMechanism->pParameter,
rsaOaepMechanismParam);
if (rv != CKR_OK)
{
delete rsaOaepMechanismParam;
asymCrypto->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(asymCrypto);
return rv;
}
mechanismParam = rsaOaepMechanismParam;
}

session->setOpType(SESSION_OP_ENCRYPT);
session->setAsymmetricCryptoOp(asymCrypto);
session->setMechanism(mechanism);
session->setAllowMultiPartOp(false);
session->setAllowSinglePartOp(true);
session->setPublicKey(publicKey);

if (mechanismParam != NULL)
{
session->setMechanismParam(mechanismParam);
delete mechanismParam;
}
return CKR_OK;
}

Expand Down Expand Up @@ -2676,7 +2697,9 @@ static CK_RV AsymEncrypt(Session* session, CK_BYTE_PTR pData, CK_ULONG ulDataLen
{
AsymmetricAlgorithm* asymCrypto = session->getAsymmetricCryptoOp();
AsymMech::Type mechanism = session->getMechanism();
MechanismParam* mechanismParam = session->getMechanismParam();
PublicKey* publicKey = session->getPublicKey();

if (asymCrypto == NULL || !session->getAllowSinglePartOp() || publicKey == NULL)
{
session->resetOp();
Expand Down Expand Up @@ -2711,7 +2734,7 @@ static CK_RV AsymEncrypt(Session* session, CK_BYTE_PTR pData, CK_ULONG ulDataLen
data += ByteString(pData, ulDataLen);

// Encrypt the data
if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism))
if (!asymCrypto->encrypt(publicKey,data,encryptedData,mechanism, mechanismParam))
{
session->resetOp();
return CKR_GENERAL_ERROR;
Expand Down Expand Up @@ -3254,6 +3277,7 @@ CK_RV SoftHSM::AsymDecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec

// Get the asymmetric algorithm matching the mechanism
AsymMech::Type mechanism = AsymMech::Unknown;
MechanismParam* mechanismParam = NULL;
bool isRSA = false;
switch(pMechanism->mechanism) {
case CKM_RSA_PKCS:
Expand Down Expand Up @@ -3313,14 +3337,32 @@ CK_RV SoftHSM::AsymDecryptInit(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pMec
{
session->setReAuthentication(true);
}

// set mechanism parameters
if (pMechanism->mechanism == CKM_RSA_PKCS_OAEP)
{
RSAOaepMechanismParam* rsaOaepMechanismParam = new RSAOaepMechanismParam;
rv = BuildRSAOAEPParam((CK_RSA_PKCS_OAEP_PARAMS *)pMechanism->pParameter,
rsaOaepMechanismParam);
if (rv != CKR_OK)
{
delete rsaOaepMechanismParam;
asymCrypto->recyclePrivateKey(privateKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(asymCrypto);
return rv;
}
mechanismParam = rsaOaepMechanismParam;
}
session->setOpType(SESSION_OP_DECRYPT);
session->setAsymmetricCryptoOp(asymCrypto);
session->setMechanism(mechanism);
session->setAllowMultiPartOp(false);
session->setAllowSinglePartOp(true);
session->setPrivateKey(privateKey);

if (mechanismParam != NULL)
{
session->setMechanismParam(mechanismParam);
delete mechanismParam;
}
return CKR_OK;
}

Expand Down Expand Up @@ -3408,7 +3450,9 @@ static CK_RV AsymDecrypt(Session* session, CK_BYTE_PTR pEncryptedData, CK_ULONG
{
AsymmetricAlgorithm* asymCrypto = session->getAsymmetricCryptoOp();
AsymMech::Type mechanism = session->getMechanism();
MechanismParam* mechanismParam = session->getMechanismParam();
PrivateKey* privateKey = session->getPrivateKey();

if (asymCrypto == NULL || !session->getAllowSinglePartOp() || privateKey == NULL)
{
session->resetOp();
Expand Down Expand Up @@ -3442,7 +3486,7 @@ static CK_RV AsymDecrypt(Session* session, CK_BYTE_PTR pEncryptedData, CK_ULONG
ByteString data;

// Decrypt the data
if (!asymCrypto->decrypt(privateKey,encryptedData,data,mechanism))
if (!asymCrypto->decrypt(privateKey,encryptedData,data,mechanism,mechanismParam))
{
session->resetOp();
return CKR_ENCRYPTED_DATA_INVALID;
Expand Down Expand Up @@ -6668,9 +6712,12 @@ CK_RV SoftHSM::WrapKeyAsym
ByteString& wrapped
)
{
CK_RV rv = CKR_OK;
const size_t bb = 8;
AsymAlgo::Type algo = AsymAlgo::Unknown;
AsymMech::Type mech = AsymMech::Unknown;
MechanismParam* mechanismParam = NULL;
size_t hashLen = 0;

CK_ULONG modulus_length;
switch(pMechanism->mechanism) {
Expand Down Expand Up @@ -6698,10 +6745,8 @@ CK_RV SoftHSM::WrapKeyAsym

case CKM_RSA_PKCS_OAEP:
mech = AsymMech::RSA_PKCS_OAEP;
// SHA-1 is the only supported option
// PKCS#11 2.40 draft 2 section 2.1.8: input length <= k-2-2hashLen
if (keydata.size() > modulus_length - 2 - 2 * 160 / 8)
return CKR_KEY_SIZE_RANGE;
// key length will be check later
break;

default:
Expand Down Expand Up @@ -6730,20 +6775,39 @@ CK_RV SoftHSM::WrapKeyAsym
break;

default:
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return CKR_MECHANISM_INVALID;
}
// Wrap the key
if (!cipher->wrapKey(publicKey, keydata, wrapped, mech))
if (pMechanism->mechanism == CKM_RSA_PKCS_OAEP)
{
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return CKR_GENERAL_ERROR;
RSAOaepMechanismParam* rsaOaepMechanismParam = new RSAOaepMechanismParam;
rv = BuildRSAOAEPParam((CK_RSA_PKCS_OAEP_PARAMS *)pMechanism->pParameter,
rsaOaepMechanismParam,&hashLen);
if (rv != CKR_OK)
{
delete rsaOaepMechanismParam;
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return rv;
}
if (keydata.size() > modulus_length - 2 - (2 * hashLen))
{
delete rsaOaepMechanismParam;
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return CKR_KEY_SIZE_RANGE;
}
mechanismParam = rsaOaepMechanismParam;
}

// Wrap the key
if (!cipher->wrapKey(publicKey, keydata, wrapped, mech, mechanismParam))
rv = CKR_GENERAL_ERROR;
delete mechanismParam;
cipher->recyclePublicKey(publicKey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);

return CKR_OK;
return rv;
}

// Internal: Wrap with mechanism RSA_AES_KEY_WRAP
Expand Down Expand Up @@ -7269,6 +7333,8 @@ CK_RV SoftHSM::UnwrapKeyAsym
// Get the symmetric algorithm matching the mechanism
AsymAlgo::Type algo = AsymAlgo::Unknown;
AsymMech::Type mode = AsymMech::Unknown;
MechanismParam* mechanismParam = NULL;

switch(pMechanism->mechanism) {
case CKM_RSA_PKCS:
algo = AsymAlgo::RSA;
Expand Down Expand Up @@ -7305,12 +7371,29 @@ CK_RV SoftHSM::UnwrapKeyAsym
break;

default:
cipher->recyclePrivateKey(unwrappingkey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return CKR_MECHANISM_INVALID;
}


if (pMechanism->mechanism == CKM_RSA_PKCS_OAEP)
{
RSAOaepMechanismParam* rsaOaepMechanismParam = new RSAOaepMechanismParam;
rv = BuildRSAOAEPParam((CK_RSA_PKCS_OAEP_PARAMS *)pMechanism->pParameter,
rsaOaepMechanismParam);
if (rv != CKR_OK)
{
delete rsaOaepMechanismParam;
cipher->recyclePrivateKey(unwrappingkey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return rv;
}
mechanismParam = rsaOaepMechanismParam;
}
// Unwrap the key
if (!cipher->unwrapKey(unwrappingkey, wrapped, keydata, mode))
if (!cipher->unwrapKey(unwrappingkey, wrapped, keydata, mode, mechanismParam ))
rv = CKR_GENERAL_ERROR;
delete mechanismParam;
cipher->recyclePrivateKey(unwrappingkey);
CryptoFactory::i()->recycleAsymmetricAlgorithm(cipher);
return rv;
Expand Down Expand Up @@ -13676,31 +13759,16 @@ CK_RV SoftHSM::MechParamCheckRSAPKCSOAEP(CK_MECHANISM_PTR pMechanism)
ERROR_MSG("pParameter must be of type CK_RSA_PKCS_OAEP_PARAMS");
return CKR_ARGUMENTS_BAD;
}

CK_RSA_PKCS_OAEP_PARAMS_PTR params = (CK_RSA_PKCS_OAEP_PARAMS_PTR)pMechanism->pParameter;
if (params->hashAlg != CKM_SHA_1)
{
ERROR_MSG("hashAlg must be CKM_SHA_1");
return CKR_ARGUMENTS_BAD;
}
if (params->mgf != CKG_MGF1_SHA1)
{
ERROR_MSG("mgf must be CKG_MGF1_SHA1");
return CKR_ARGUMENTS_BAD;
}

if (params->source != CKZ_DATA_SPECIFIED)
{
ERROR_MSG("source must be CKZ_DATA_SPECIFIED");
return CKR_ARGUMENTS_BAD;
}
if (params->pSourceData != NULL)
{
ERROR_MSG("pSourceData must be NULL");
return CKR_ARGUMENTS_BAD;
}
if (params->ulSourceDataLen != 0)
if ((params-> pSourceData == NULL)&&(params->ulSourceDataLen != 0))
{
ERROR_MSG("ulSourceDataLen must be 0");
ERROR_MSG("pSourceData is NULL");
return CKR_ARGUMENTS_BAD;
}
return CKR_OK;
Expand Down Expand Up @@ -13732,27 +13800,93 @@ CK_RV SoftHSM::MechParamCheckRSAAESKEYWRAP(CK_MECHANISM_PTR pMechanism)
ERROR_MSG("pOAEPParams must be of type CK_RSA_PKCS_OAEP_PARAMS");
return CKR_ARGUMENTS_BAD;
}
if (params->pOAEPParams->mgf < 1UL || params->pOAEPParams->mgf > 5UL)
if (params->pOAEPParams->source != CKZ_DATA_SPECIFIED)
{
ERROR_MSG("mgf not supported");
ERROR_MSG("source must be CKZ_DATA_SPECIFIED");
return CKR_ARGUMENTS_BAD;
}
if (params->pOAEPParams->source != CKZ_DATA_SPECIFIED)
if ((params->pOAEPParams->pSourceData == NULL) && (params->pOAEPParams->ulSourceDataLen != 0))
{
ERROR_MSG("pSourceData is NULL");
return CKR_ARGUMENTS_BAD;
}
return CKR_OK;
}

CK_RV SoftHSM::BuildRSAOAEPParam(const CK_RSA_PKCS_OAEP_PARAMS *params,
RSAOaepMechanismParam* mechanismParam,
size_t* hashLen)
{
if (params == NULL)
{
ERROR_MSG("parameters is NULL for RSA OAEP encryption");
return CKR_ARGUMENTS_BAD;
}
if (params->source != CKZ_DATA_SPECIFIED)
{
ERROR_MSG("source must be CKZ_DATA_SPECIFIED");
return CKR_ARGUMENTS_BAD;
}
if ((params->pSourceData == NULL) && (params->ulSourceDataLen != 0))
{
ERROR_MSG("pSourceData is NULL");
return CKR_ARGUMENTS_BAD;
}
if (params->pOAEPParams->pSourceData != NULL)
switch (params->hashAlg)
{
ERROR_MSG("pSourceData must be NULL");
case CKM_SHA_1:
mechanismParam->hashAlg = HashAlgo::SHA1;
if (hashLen) *hashLen = 20;
break;
case CKM_SHA224:
mechanismParam->hashAlg = HashAlgo::SHA224;
if (hashLen) *hashLen = 28;
break;
case CKM_SHA256:
mechanismParam->hashAlg = HashAlgo::SHA256;
if (hashLen) *hashLen = 32;
break;
case CKM_SHA384:
mechanismParam->hashAlg = HashAlgo::SHA384;
if (hashLen) *hashLen = 48;
break;
case CKM_SHA512:
mechanismParam->hashAlg = HashAlgo::SHA512;
if (hashLen) *hashLen = 64;
break;
default:
ERROR_MSG("hash algorithm not supported for OAEP");
return CKR_ARGUMENTS_BAD;
}
switch (params->mgf)
{
case CKG_MGF1_SHA1:
mechanismParam->mgfAlg = AsymRSAMGF::MGF1_SHA1;
break;
case CKG_MGF1_SHA224:
mechanismParam->mgfAlg = AsymRSAMGF::MGF1_SHA224;
break;
case CKG_MGF1_SHA256:
mechanismParam->mgfAlg = AsymRSAMGF::MGF1_SHA256;
break;
case CKG_MGF1_SHA384:
mechanismParam->mgfAlg = AsymRSAMGF::MGF1_SHA384;
break;
case CKG_MGF1_SHA512:
mechanismParam->mgfAlg = AsymRSAMGF::MGF1_SHA512;
break;
default:
ERROR_MSG("mgf algorithm not supported for OAEP");
return CKR_ARGUMENTS_BAD;
}
if (params->pOAEPParams->ulSourceDataLen != 0)
if (params->ulSourceDataLen > 0xffffffff - sizeof(RSA_PKCS_OAEP_PARAMS))
{
ERROR_MSG("ulSourceDataLen must be 0");
ERROR_MSG("OAEP Label too large");
return CKR_ARGUMENTS_BAD;
}

// copy label data to mechanismParam
mechanismParam->label = ByteString(reinterpret_cast<const unsigned char*>(params->pSourceData),params->ulSourceDataLen);

return CKR_OK;
}

Expand Down
Loading