Incorrect decrypting AESGCM with 16 bytes nonce

[tplsqd]

Hi!

Fully correct decrypting with default 12-bytes nonce, but if I try to decrypt data with 16-bytes nonce - fail.
But OpenSSL's method works well with 100% same input data. I think problem in incorrect GHASH, or something else in AESGCMHandleNonce.

<title>Document</title>

	Имя	Значение	Тип
▶	GCMNonce	0x006ff50c "¶@ћ‹ѓе?jяоO5\x2*\x13€...	unsigned char[16]
▶	pTemp	0x00a64a88 "ээээ"	unsigned char *
▶	this	0x006ff3e0 {Nk=8 Nr=14 key=0x006ff3e8 {1913689351, 3283579358, 2506403086, 2165086380, 1511688941, 2491306669, ...} ...}	CAES *

Thanks!

`#include <Windows.h>
#include <bcrypt.h>
#include
#include
#include
#include
#include
#include "iAes.h"

#pragma comment(lib, "bcrypt.lib")
#pragma comment(lib, "crypt32.lib")

class AlgorithmProvider {
public:
explicit AlgorithmProvider(PCWSTR pszAlgId, ULONG dwFlags) {
if (BCryptOpenAlgorithmProvider(&hAlgorithm, pszAlgId, NULL, dwFlags) != ERROR_SUCCESS) {
hAlgorithm = nullptr;
}
}
~AlgorithmProvider() {
if (hAlgorithm != nullptr) {
BCryptCloseAlgorithmProvider(hAlgorithm, 0);
}
}
BCRYPT_ALG_HANDLE get() const {
return hAlgorithm;
}
private:
BCRYPT_ALG_HANDLE hAlgorithm = nullptr;
};

class KeyHandle {
public:
KeyHandle(BCRYPT_ALG_HANDLE hAlgorithm, const std::vector& key) : hKey(nullptr), pbKeyObject(nullptr) {
DWORD cbKeyObject = 0, cbData = 0;
if (BCryptGetProperty(hAlgorithm, BCRYPT_OBJECT_LENGTH, reinterpret_cast(&cbKeyObject), sizeof(DWORD), &cbData, 0) != ERROR_SUCCESS) {
return;
}
pbKeyObject.reset(new unsigned char[cbKeyObject]);
if (BCryptGenerateSymmetricKey(hAlgorithm, &hKey, pbKeyObject.get(), cbKeyObject, const_cast(key.data()), static_cast(key.size()), 0) != ERROR_SUCCESS) {
hKey = nullptr;
}
}
~KeyHandle() {
if (hKey != nullptr) {
BCryptDestroyKey(hKey);
}
}
BCRYPT_KEY_HANDLE get() const {
return hKey;
}
private:
BCRYPT_KEY_HANDLE hKey = nullptr;
std::unique_ptr<unsigned char[]> pbKeyObject;
};

static std::vector WideStringToBytes(const std::wstring& input) {
if (input.empty()) {
return std::vector();
}
const int inputLength = static_cast(input.length());
int sizeNeeded = WideCharToMultiByte(CP_UTF8, 0, input.c_str(), inputLength, NULL, 0, NULL, NULL);
std::vector charBytesArray(sizeNeeded, 0);
WideCharToMultiByte(CP_UTF8, 0, input.c_str(), inputLength, reinterpret_cast<char*>(&charBytesArray[0]), sizeNeeded, NULL, NULL);
return charBytesArray;
}

static std::vector WideBase64ToBytes(const std::wstring& wideBase64String) {
DWORD binarySize = 0;
if (!CryptStringToBinaryW(wideBase64String.c_str(), 0, CRYPT_STRING_BASE64, nullptr, &binarySize, nullptr, nullptr)) {
return std::vector();
}
std::vector binaryData(binarySize);
if (!CryptStringToBinaryW(wideBase64String.c_str(), 0, CRYPT_STRING_BASE64, binaryData.data(), &binarySize, nullptr, nullptr)) {
return std::vector();
}
return binaryData;
}

static std::wstring BytesToWideString(const std::vector& input) {
if (input.empty()) {
return std::wstring();
}
int sizeNeeded = MultiByteToWideChar(CP_UTF8, 0, reinterpret_cast<const char*>(input.data()), static_cast(input.size()), NULL, 0);
std::wstring result(sizeNeeded, 0);
MultiByteToWideChar(CP_UTF8, 0, reinterpret_cast<const char*>(input.data()), static_cast(input.size()), &result[0], sizeNeeded);
return result;
}

std::vector ComputePBKDF2(const std::vector& passwordBytes, const std::vector& salt, int iterations, int outputBytes) {
try {
AlgorithmProvider algorithm(BCRYPT_SHA256_ALGORITHM, BCRYPT_ALG_HANDLE_HMAC_FLAG);
std::vector key(outputBytes, 0);
NTSTATUS status = BCryptDeriveKeyPBKDF2(algorithm.get(), const_cast(passwordBytes.data()), passwordBytes.size(), const_cast(salt.data()), salt.size(), iterations, key.data(), key.size(), 0);
if (status != ERROR_SUCCESS) {
return std::vector();
}
return key;
}
catch (...) {
return std::vector();
}
}

std::vector Decrypt(const std::vector& encryptedData,
const std::vector& key,
const std::vector& iv)
{
CAES aes;

if (!aes.AesInit(const_cast<unsigned char*>(key.data()), key.size(), const_cast<unsigned char*>(iv.data()), iv.size(), aes.AES_GCM))
{
    throw std::runtime_error("Failed to initialize AES.");
}

std::vector<unsigned char> decryptedData(encryptedData.size());
std::vector<unsigned char> tag(16);
if (!aes.Decrypt_GCM(const_cast<unsigned char*>(encryptedData.data()), encryptedData.size(),
    decryptedData.data(), decryptedData.size(), nullptr, 0,
    tag.data(), tag.size()))
{
    throw std::runtime_error("Decryption failed.");
}

return decryptedData;

}

std::wstring password = L"Qweasd123!";
std::vector passwordBytes = WideStringToBytes(password);
int iterations = 600000;
std::vector dataBytes = WideBase64ToBytes(L"FnMj1gnYPeDA6s7nf6XMQ+VxxV8Y+C33yvPobG3ukZuNLJb6ZqmzHPZ2GHxa9iTxtd/q+ACSjlqcZu9g+J87IxtnhvXwF8oYp4FD6WvpALw8mHJ9eK7qrmMBfmR4AYdNmk9plb68eITjpsQZK46jWSYoI4J/3/y0n/SDUejnIqlqa/3P6RHszhyGON4bJkaxPmU8Z5p+dYUs2L6wLnZ5XhPe+Se3OsjQYlHXaH8sfiaOHpNypagzvF/rZr0ywMI0zORgJKMexrKyYrjMkWh01gvzYjtXA99VyB/oUpvC3FnMDMG5gv/c+Z56WueJO3ysAyJmYGmwLRTdg6wVy4uHv6vCcHndqQtD1yiAkfmDhZBeQ3XipPUuB+kVZeImGIqW+vCVvqcRoA7xwn0aOJltSWXLAjP8ZvBpCk0w89Rjm+C8LWaFwF/cJH3A0R4fFPiZDvKfD8sLw2UKbbbRjFz7CUMBXRqwoQN7Wo5BXeZ4b28evgn3omuTwsJIq8N3lgnQciWWnWIzhz8=","iv":"olxCbjuq3V19RICOp0CpAA==");
std::vector saltBytes = WideBase64ToBytes(L"Dz2sNjFUQXQo1nabPz9DQwWl8F27AEIr8kak4zmurS0=");
std::vector ivBytes = WideBase64ToBytes(L"olxCbjuq3V19RICOp0CpAA==");
std::vector key = ComputePBKDF2(passwordBytes, saltBytes, iterations, 32);

int main() {
std::vector decryptedDataBytesOpenSsl = Decrypt(dataBytes, key, ivBytes);
std::wstring decryptedDataWideStringOpenSsl = BytesToWideString(decryptedDataBytesOpenSsl);
std::wcout << L"Decrypted Data: " << decryptedDataWideStringOpenSsl << std::endl;
}`