Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Several bug fixes for serialization or nested/repeated messages #386

Closed
wants to merge 3 commits into from
Closed

Several bug fixes for serialization or nested/repeated messages #386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
775f881
Convert tunnel to shift register to avoid zero iteration for loop issue.
jasonmreding Sep 13, 2024
09b8a06
Resaving with source version as 2019.
jasonmreding Sep 13, 2024
afa127f
Several bug fixes for serializing messages with repeated/nested messa…
jasonmreding Sep 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified labview source/gRPC lv Support/Server API/Server/Register Message Metadata.vi
View file
Open in desktop
Binary file not shown.
25 changes: 16 additions & 9 deletions src/cluster_copier.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,13 +95,13 @@ namespace grpc_labview {
}
}

// second pass to fill the oneof selected_index. We can do this in one pass when we push the selected_field to the end of the oneof cluster!
// second pass to fill the oneof selected_index. We can do this in one pass when we push the selected_field to the end of the oneof cluster!
// TODO: Skip the entire loop if the message has no oneof. It's a bool in the metadata.
for (auto val : message._metadata->_mappedElements)
{
auto fieldMetadata = val.second;
auto fieldMetadata = val.second;
if (fieldMetadata->isInOneof&& fieldMetadata->protobufIndex < 0)
{
{
// This field is the selected_index field of a oneof
if (oneof_containerToSelectedIndexMap.find(fieldMetadata->oneofContainerName) != oneof_containerToSelectedIndexMap.end())
{
Expand All @@ -119,7 +119,7 @@ namespace grpc_labview {
message._values.clear();
std::map<std::string, int> oneof_containerToSelectedIndexMap; // Needed to serialize only the field related to the selected_index
for (auto val : message._metadata->_mappedElements)
{
{
auto fieldMetadata = val.second;
if (fieldMetadata->isInOneof)
{
Expand Down Expand Up @@ -287,7 +287,7 @@ namespace grpc_labview {
auto repeatedString = static_cast<const LVRepeatedMessageValue<std::string>&>(*value);
if (repeatedString._value.size() != 0)
{
NumericArrayResize(0x08, 1, start, repeatedString._value.size());
NumericArrayResize(GetTypeCodeForSize(sizeof(char*)), 1, start, repeatedString._value.size());
auto array = *(LV1DArrayHandle*)start;
(*array)->cnt = repeatedString._value.size();
int x = 0;
Expand Down Expand Up @@ -330,14 +330,21 @@ namespace grpc_labview {
{
auto nestedMetadata = repeatedNested->_value.front()->_metadata;
auto clusterSize = nestedMetadata->clusterSize;
auto byteSize = repeatedNested->_value.size() * clusterSize;
auto alignment = nestedMetadata->alignmentRequirement;
auto alignedElementSize = byteSize / alignment;
if (byteSize % alignment != 0)
{
alignedElementSize++;
}

NumericArrayResize(0x08, 1, start, repeatedNested->_value.size() * clusterSize);
NumericArrayResize(GetTypeCodeForSize(alignment), 1, start, alignedElementSize);
auto array = *(LV1DArrayHandle*)start;
(*array)->cnt = repeatedNested->_value.size();
int x = 0;
for (auto str : repeatedNested->_value)
{
auto lvCluster = (LVCluster**)(*array)->bytes(x * clusterSize, nestedMetadata->alignmentRequirement);
auto lvCluster = (LVCluster**)(*array)->bytes(x * clusterSize, alignment);
*lvCluster = nullptr;
CopyToCluster(*str, (int8_t*)lvCluster);
x += 1;
Expand Down Expand Up @@ -419,7 +426,7 @@ namespace grpc_labview {
auto byteCount = count * sizeof(int32_t);
memcpy((*array)->bytes<int32_t>(), mappedArray, byteCount);
}

free(mappedArray);
}
else
Expand Down Expand Up @@ -810,7 +817,7 @@ namespace grpc_labview {
repeatedValue->_value.Reserve(count);
auto dest = repeatedValue->_value.AddNAlreadyReserved(count);
memcpy(dest, mappedArray, count * sizeof(int32_t));

free(mappedArray);
}
}
Expand Down
24 changes: 20 additions & 4 deletions src/lv_interop.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,15 +131,15 @@ namespace grpc_labview
//---------------------------------------------------------------------
//---------------------------------------------------------------------
int NumericArrayResize(int32_t typeCode, int32_t numDims, void* handle, size_t size)
{
{
return NumericArrayResizeImp(typeCode, numDims, handle, size);
}

//---------------------------------------------------------------------
//---------------------------------------------------------------------
int PostUserEvent(LVUserEventRef ref, void *data)
{
return PostLVUserEvent(ref, data);
return PostLVUserEvent(ref, data);
}

//---------------------------------------------------------------------
Expand Down Expand Up @@ -167,7 +167,7 @@ namespace grpc_labview
//---------------------------------------------------------------------
void SetLVString(LStrHandle* lvString, std::string str)
{
auto length = str.length();
auto length = str.length();
auto error = NumericArrayResize(0x01, 1, lvString, length);
memcpy((**lvString)->str, str.c_str(), length);
(**lvString)->cnt = (int)length;
Expand All @@ -176,7 +176,7 @@ namespace grpc_labview
//---------------------------------------------------------------------
//---------------------------------------------------------------------
std::string GetLVString(LStrHandle lvString)
{
{
if (lvString == nullptr || *lvString == nullptr)
{
return std::string();
Expand Down Expand Up @@ -216,4 +216,20 @@ namespace grpc_labview
return clusterOffset;
#endif
}

int32_t GetTypeCodeForSize(int byteSize)
{
switch (byteSize)
{
case 1:
return 0x5; // u8
case 2:
return 0x6; // uW
case 4:
return 0x7; // uL
case 8:
default:
return 0x8; // uQ
}
}
}
7 changes: 5 additions & 2 deletions src/lv_interop.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
#define LIBRARY_EXPORT extern "C" __attribute__((visibility("default")))
#endif

namespace grpc_labview
namespace grpc_labview
{
class gRPCid;
extern PointerManager<gRPCid> gPointerManager;
Expand All @@ -50,6 +50,9 @@ namespace grpc_labview

int AlignClusterOffset(int clusterOffset, int alignmentRequirement);

// Provides type code for use with NumericArrayResize function for various sizes of data types.
int32_t GetTypeCodeForSize(int byteSize);

//---------------------------------------------------------------------
// LabVIEW definitions
//---------------------------------------------------------------------
Expand Down Expand Up @@ -99,7 +102,7 @@ namespace grpc_labview
#pragma pack (push, 1)
#endif
struct AnyCluster
{
{
LStrHandle TypeUrl;
LV1DArrayHandle Bytes;
};
Expand Down
104 changes: 52 additions & 52 deletions src/lv_message_efficient.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,24 +68,16 @@ namespace grpc_labview
//---------------------------------------------------------------------
//---------------------------------------------------------------------
const char *LVMessageEfficient::ParseString(google::protobuf::uint32 tag, const MessageElementMetadata& fieldInfo, uint32_t index, const char *protobuf_ptr, ParseContext *ctx)
{
{
const char* lv_ptr = (this->GetLVClusterHandleSharedPtr()) + fieldInfo.clusterOffset;

if (fieldInfo.isRepeated)
{
std::string key = fieldInfo.fieldName + std::to_string(_currentIndexForRepeatedMessageValue);

// Get the _repeatedMessageValues vector from the map
auto _repeatedStringValuesIt = _repeatedStringValuesMap.find(key);
if (_repeatedStringValuesIt == _repeatedStringValuesMap.end())
{
_repeatedStringValuesIt = _repeatedStringValuesMap.emplace(key, google::protobuf::RepeatedField<std::string>()).first;
}

{
auto repeatedString = google::protobuf::RepeatedField<std::string>();
protobuf_ptr -= 1;
do {
protobuf_ptr += 1;
auto str = _repeatedStringValuesIt->second.Add();
auto str = repeatedString.Add();
protobuf_ptr = InlineGreedyStringParser(str, protobuf_ptr, ctx);
if (!ctx->DataAvailable(protobuf_ptr))
{
Expand All @@ -94,20 +86,21 @@ namespace grpc_labview
} while (ExpectTag(tag, protobuf_ptr));


NumericArrayResize(0x08, 1, reinterpret_cast<void*>(const_cast<char*>(lv_ptr)), _repeatedStringValuesIt->second.size());
NumericArrayResize(GetTypeCodeForSize(sizeof(char*)), 1, reinterpret_cast<void*>(const_cast<char*>(lv_ptr)), repeatedString.size());
auto arrayHandle = *(LV1DArrayHandle*)lv_ptr;
(*arrayHandle)->cnt = _repeatedStringValuesIt->second.size();
(*arrayHandle)->cnt = repeatedString.size();

// Copy the repeated string values into the LabVIEW array
auto lvStringPtr = (*arrayHandle)->bytes<LStrHandle>();
for (auto str:_repeatedStringValuesIt->second)
for (auto str:repeatedString)
{
*lvStringPtr = nullptr;
SetLVString(lvStringPtr, str);
lvStringPtr++;
}
}
else {
else
{
auto str = std::string();
protobuf_ptr = InlineGreedyStringParser(&str, protobuf_ptr, ctx);
SetLVString((LStrHandle*)lv_ptr, str);
Expand All @@ -119,80 +112,76 @@ namespace grpc_labview
//---------------------------------------------------------------------
const char *LVMessageEfficient::ParseBytes(google::protobuf::uint32 tag, const MessageElementMetadata& fieldInfo, uint32_t index, const char *ptr, ParseContext *ctx)
{
return ParseString(tag, fieldInfo, index, ptr, ctx);
return ParseString(tag, fieldInfo, index, ptr, ctx);
}

//---------------------------------------------------------------------
//---------------------------------------------------------------------
const char *LVMessageEfficient::ParseNestedMessage(google::protobuf::uint32 tag, const MessageElementMetadata& fieldInfo, uint32_t index, const char *protobuf_ptr, ParseContext *ctx)
{
{
auto metadata = fieldInfo._owner->FindMetadata(fieldInfo.embeddedMessageName);

LVMessageEfficient nestedMessage(metadata);

if (fieldInfo.isRepeated) {
if (fieldInfo.isRepeated)
{
// if the array is not big enough, resize it to 2x the size
auto numElements = 128;
auto elementIndex = 0;
auto clusterSize = metadata->clusterSize;
auto alignment = metadata->alignmentRequirement;
auto arraySize = numElements * clusterSize;
char _fillData = '\0';

// Get the _repeatedMessageValues vector from the map
auto _repeatedMessageValuesIt = _repeatedMessageValuesMap.find(metadata->messageName);
auto _repeatedMessageValuesIt = _repeatedMessageValuesMap.find(fieldInfo.fieldName);
if (_repeatedMessageValuesIt == _repeatedMessageValuesMap.end())
{
auto m_val = std::make_shared<RepeatedMessageValue>(fieldInfo, google::protobuf::RepeatedField<char>());
_repeatedMessageValuesIt = _repeatedMessageValuesMap.emplace(metadata->messageName, m_val).first;
_repeatedMessageValuesIt = _repeatedMessageValuesMap.emplace(fieldInfo.fieldName, m_val).first;
_repeatedMessageValuesIt->second.get()->_buffer.Resize(arraySize, _fillData);
}
else
{
// Zero out data from the previous use of the buffer.
auto bytesUsed = _repeatedMessageValuesIt->second.get()->_numElements * clusterSize;
auto bufferPtr = _repeatedMessageValuesIt->second.get()->_buffer.data();
memset(reinterpret_cast<void*>(const_cast<char*>(bufferPtr)), _fillData, bytesUsed);

// There are situations where the protobuf message is not complete, and we need to continue from the last index.
// This function returns to _internalParse, and then gets back to this function.
// If we are continuing from a previous parse, then we need to continue from the last index
auto _continueFromIndex = _repeatedField_continueIndex.find(metadata->messageName);
if (_continueFromIndex != _repeatedField_continueIndex.end()) {
elementIndex = _continueFromIndex->second;
_repeatedField_continueIndex.erase(_continueFromIndex);
// find next largest power of 2, as we assume that we previously resized it to a power of 2
auto _size = (int)ceil(log2(elementIndex));
numElements = ((1 << _size) > 128) ? (1 << _size) : 128;
}
else {
// occurs on the first time this function is called
_repeatedMessageValuesIt->second.get()->_buffer.Resize(arraySize, _fillData);
// Recalculate number of cluster elements that can fit in the buffer based on its current capacity.
numElements = _repeatedMessageValuesIt->second.get()->_buffer.Capacity() / clusterSize;
}
_repeatedMessageValuesIt->second.get()->_validData = true;

protobuf_ptr -= 1;
do
{
protobuf_ptr += 1;

// Resize the vector if we need more memory
if (elementIndex >= numElements - 1) {
if (elementIndex >= numElements - 1)
{
numElements *= 2;
arraySize = numElements * clusterSize;
auto s = _repeatedMessageValuesIt->second.get()->_buffer.size();
_repeatedMessageValuesIt->second.get()->_buffer.Resize(arraySize, _fillData);
}

auto _vectorPtr = _repeatedMessageValuesIt->second.get()->_buffer.data();
_vectorPtr = _vectorPtr + (elementIndex * clusterSize);
nestedMessage.SetLVClusterHandle(_vectorPtr);
nestedMessage._currentIndexForRepeatedMessageValue = elementIndex;
protobuf_ptr = ctx->ParseMessage(&nestedMessage, protobuf_ptr);

elementIndex++;

if (!ctx->DataAvailable(protobuf_ptr)) {
if (!ctx->DataAvailable(protobuf_ptr))
{
break;
}
} while (ExpectTag(tag, protobuf_ptr));

_repeatedMessageValuesIt->second.get()->_numElements = elementIndex;
_repeatedField_continueIndex.emplace(metadata->messageName, elementIndex);
}
else {
else
{
const char* lv_ptr = (this->GetLVClusterHandleSharedPtr()) + fieldInfo.clusterOffset;
nestedMessage.SetLVClusterHandle(lv_ptr);
protobuf_ptr = ctx->ParseMessage(&nestedMessage, protobuf_ptr);
Expand All @@ -204,27 +193,38 @@ namespace grpc_labview
{
for (auto nestedMessage : _repeatedMessageValuesMap)
{
if (!nestedMessage.second.get()->_validData)
{
continue;
}

auto fieldInfo = nestedMessage.second.get()->_fieldInfo;
auto buffer = nestedMessage.second.get()->_buffer;
auto numElements = nestedMessage.second.get()->_numElements;
auto numClusters = nestedMessage.second.get()->_numElements;

auto metadata = fieldInfo._owner->FindMetadata(fieldInfo.embeddedMessageName);
const char* lv_ptr = (this->GetLVClusterHandleSharedPtr()) + fieldInfo.clusterOffset;
auto clusterSize = metadata->clusterSize;
auto alignment = metadata->alignmentRequirement;


// shrink the array to the correct size
auto arraySize = numElements * clusterSize;
NumericArrayResize(0x08, 1, reinterpret_cast<void*>(const_cast<char*>(lv_ptr)), arraySize);
// Allocate an array with the correct size and alignment for the cluster.
auto byteSize = numClusters * clusterSize;
auto alignedElementSize = byteSize / alignment;
if (byteSize % alignment != 0)
{
alignedElementSize++;
}
auto numDimensions = 1;
NumericArrayResize(GetTypeCodeForSize(alignment), numDimensions, reinterpret_cast<void*>(const_cast<char*>(lv_ptr)), alignedElementSize);
auto arrayHandle = *(LV1DArrayHandle*)lv_ptr;
(*arrayHandle)->cnt = numElements;
(*arrayHandle)->cnt = numClusters;

auto _vectorDataPtr = buffer.data();
auto _lvArrayDataPtr = (*arrayHandle)->bytes(0, alignment);
memcpy(_lvArrayDataPtr, _vectorDataPtr, arraySize);
memcpy(_lvArrayDataPtr, _vectorDataPtr, byteSize);
nestedMessage.second.get()->_validData = false;
}
}

}

Loading
Loading