Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release 1.12.0 redpanda rebase #272

Merged
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
e4e9eb0
Prefer make_shared
BenPope Aug 20, 2021
edd19c8
NodeSymbolic: Resolve should follow _actualNode
BenPope Aug 18, 2021
c7a2745
NodeSymbolic: Resolve should follow _actualNode
BenPope Aug 25, 2022
4571154
NodeEnum: Support default value
BenPope Aug 19, 2021
952bdbb
NodeEnum: Support compiling default value
BenPope Aug 19, 2021
e8bac21
NodeEnum: Support printJson for default value
BenPope Aug 24, 2021
5970178
NodePrimitive: Resolve AVRO_STRING <-> AVRO_BYTES
BenPope Aug 19, 2021
2da75f1
c++/Compiler: CustomAttributes should allow json
BenPope Nov 16, 2022
931dcd4
c++: NodePrimitive::resolve allow futherResolution for STRING and BYTES
BenPope Dec 21, 2022
014a915
build: add barebones cmake build
dotnwat Aug 12, 2023
3129565
build: move cmake module into separate directory
dotnwat Aug 12, 2023
2a56dbb
build: normalize include paths
dotnwat Aug 12, 2023
93fe7d9
fix -Winconsistent-missing-override warning
dotnwat Aug 16, 2023
1c6902a
Modify CMakeLists to fix build errors.
jcipar Jul 2, 2024
230fd84
Allow custom metadata in data files.
jcipar Jul 2, 2024
4326f23
Add `map` logical type for arrays.
jcipar Jul 2, 2024
cb32bf8
Add element-id support for Avro arrays.
jcipar Jul 2, 2024
fbc8385
allow for iceberg element-ids take 2
andrwng Jul 11, 2024
d95eee9
redpanda_build: add avrogencpp to RP build take 2
andrwng Jul 11, 2024
17a2373
avrogencpp: expose ValidSchema in generated code
andrwng Jul 15, 2024
6bd10af
C++: add elementId to programmatic API
andrwng Aug 2, 2024
af673ea
build: import fmt using find_package
pgellert Aug 12, 2024
a0dafd5
fix algorithm header include
pgellert Aug 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 3 additions & 11 deletions lang/c++/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -68,7 +68,7 @@ if (WIN32 AND NOT CYGWIN AND NOT MSYS)
endif()

if (CMAKE_COMPILER_IS_GNUCXX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wduplicated-cond -Wduplicated-branches -Wlogical-op -Wuseless-cast -Wconversion -pedantic -Werror")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wduplicated-cond -Wduplicated-branches -Wlogical-op -Wuseless-cast -Wconversion -pedantic -Werror -Wno-dangling-reference -Wno-maybe-uninitialized")
if (AVRO_ADD_PROTECTOR_FLAGS)
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fstack-protector-all -D_GLIBCXX_DEBUG")
# Unset _GLIBCXX_DEBUG for avrogencpp.cc because using Boost Program Options
@@ -82,15 +82,7 @@ endif ()
find_package (Boost 1.38 REQUIRED
COMPONENTS filesystem iostreams program_options regex system)

include(FetchContent)
FetchContent_Declare(
fmt
GIT_REPOSITORY https://github.com/fmtlib/fmt.git
GIT_TAG 10.2.1
GIT_PROGRESS TRUE
USES_TERMINAL_DOWNLOAD TRUE
)
FetchContent_MakeAvailable(fmt)
find_package(fmt 8.1.1 REQUIRED)

find_package(Snappy)
if (SNAPPY_FOUND)
@@ -239,7 +231,7 @@ install (TARGETS avrocpp avrocpp_s

install (TARGETS avrogencpp RUNTIME DESTINATION bin)

install (DIRECTORY include/avro DESTINATION include
install (DIRECTORY include/avro/ DESTINATION include/avro
FILES_MATCHING PATTERN *.hh)

if (NOT CMAKE_BUILD_TYPE)
File renamed without changes.
72 changes: 50 additions & 22 deletions lang/c++/impl/Compiler.cc
Original file line number Diff line number Diff line change
@@ -48,21 +48,21 @@ using SymbolTable = map<Name, NodePtr>;

static NodePtr makePrimitive(const string &t) {
if (t == "null") {
return NodePtr(new NodePrimitive(AVRO_NULL));
return std::make_shared<NodePrimitive>(AVRO_NULL);
} else if (t == "boolean") {
return NodePtr(new NodePrimitive(AVRO_BOOL));
return std::make_shared<NodePrimitive>(AVRO_BOOL);
} else if (t == "int") {
return NodePtr(new NodePrimitive(AVRO_INT));
return std::make_shared<NodePrimitive>(AVRO_INT);
} else if (t == "long") {
return NodePtr(new NodePrimitive(AVRO_LONG));
return std::make_shared<NodePrimitive>(AVRO_LONG);
} else if (t == "float") {
return NodePtr(new NodePrimitive(AVRO_FLOAT));
return std::make_shared<NodePrimitive>(AVRO_FLOAT);
} else if (t == "double") {
return NodePtr(new NodePrimitive(AVRO_DOUBLE));
return std::make_shared<NodePrimitive>(AVRO_DOUBLE);
} else if (t == "string") {
return NodePtr(new NodePrimitive(AVRO_STRING));
return std::make_shared<NodePrimitive>(AVRO_STRING);
} else if (t == "bytes") {
return NodePtr(new NodePrimitive(AVRO_BYTES));
return std::make_shared<NodePrimitive>(AVRO_BYTES);
} else {
return NodePtr();
}
@@ -94,7 +94,7 @@ static NodePtr makeNode(const string &t, SymbolTable &st, const string &ns) {

auto it = st.find(n);
if (it != st.end()) {
return NodePtr(new NodeSymbolic(asSingleAttribute(n), it->second));
return std::make_shared<NodeSymbolic>(asSingleAttribute(n), it->second);
}
throw Exception("Unknown type: {}", n);
}
@@ -277,7 +277,7 @@ static void getCustomAttributes(const Object &m, CustomAttributes &customAttribu
const std::unordered_set<std::string> &kKnownFields = getKnownFields();
for (const auto &entry : m) {
if (kKnownFields.find(entry.first) == kKnownFields.end()) {
customAttributes.addAttribute(entry.first, entry.second.stringValue());
customAttributes.addAttribute(entry.first, entry.second.toString());
}
}
}
@@ -324,15 +324,15 @@ static NodePtr makeRecordNode(const Entity &e, const Name &name,
customAttributes.add(f.customAttributes);
}

NodeRecord *node;
NodePtr node;
if (doc == nullptr) {
node = new NodeRecord(asSingleAttribute(name), fieldValues, fieldNames,
node = std::make_shared<NodeRecord>(asSingleAttribute(name), fieldValues, fieldNames,
fieldAliases, defaultValues, customAttributes);
} else {
node = new NodeRecord(asSingleAttribute(name), asSingleAttribute(*doc),
node = std::make_shared<NodeRecord>(asSingleAttribute(name), asSingleAttribute(*doc),
fieldValues, fieldNames, fieldAliases, defaultValues, customAttributes);
}
return NodePtr(node);
return node;
}

static LogicalType makeLogicalType(const Entity &e, const Object &m) {
@@ -373,6 +373,8 @@ static LogicalType makeLogicalType(const Entity &e, const Object &m) {
t = LogicalType::DURATION;
else if (typeField == "uuid")
t = LogicalType::UUID;
else if (typeField == "map")
t = LogicalType::MAP;
return LogicalType(t);
}

@@ -381,13 +383,35 @@ static NodePtr makeEnumNode(const Entity &e,
string symbolsName = "symbols";
const Array &v = getArrayField(e, m, symbolsName);
concepts::MultiAttribute<string> symbols;
GenericDatum defaultValue;
for (const auto &it : v) {
if (it.type() != json::EntityType::String) {
throw Exception("Enum symbol not a string: {}", it.toString());
}
symbols.add(it.stringValue());
}
NodePtr node = NodePtr(new NodeEnum(asSingleAttribute(name), symbols));

auto it = m.find("default");
if (it != m.end()) {
const json::Entity &e = it->second;
if (it->second.type() != json::EntityType::String) {
throw Exception("Enum default not a string: {}", e.toString());
}
string defStr = e.stringValue();
bool symbolsHasDefault = false;
for (size_t i = 0; i < symbols.size(); ++i) {
if (symbols.get(i) == defStr) {
symbolsHasDefault = true;
break;
}
}
if (!symbolsHasDefault) {
throw Exception("Enum default not in symbols: {}", defStr);
}
defaultValue = defStr;
}

NodePtr node = std::make_shared<NodeEnum>(asSingleAttribute(name), symbols, defaultValue);
if (containsField(m, "doc")) {
node->setDoc(getDocField(e, m));
}
@@ -401,7 +425,7 @@ static NodePtr makeFixedNode(const Entity &e,
throw Exception("Size for fixed is not positive: {}", e.toString());
}
NodePtr node =
NodePtr(new NodeFixed(asSingleAttribute(name), asSingleAttribute(static_cast<size_t>(v))));
std::make_shared<NodeFixed>(asSingleAttribute(name), asSingleAttribute(static_cast<size_t>(v)));
if (containsField(m, "doc")) {
node->setDoc(getDocField(e, m));
}
@@ -411,8 +435,12 @@ static NodePtr makeFixedNode(const Entity &e,
static NodePtr makeArrayNode(const Entity &e, const Object &m,
SymbolTable &st, const string &ns) {
auto it = findField(e, m, "items");
NodePtr node = NodePtr(new NodeArray(
asSingleAttribute(makeNode(it->second, st, ns))));
std::shared_ptr<NodeArray> nodePointer = std::make_shared<NodeArray>(
asSingleAttribute(makeNode(it->second, st, ns)));
NodePtr node = NodePtr(nodePointer);
if (containsField(m, "element-id")) {
nodePointer->elementId_ = getLongField(e, m, "element-id");
}
if (containsField(m, "doc")) {
node->setDoc(getDocField(e, m));
}
@@ -423,8 +451,8 @@ static NodePtr makeMapNode(const Entity &e, const Object &m,
SymbolTable &st, const string &ns) {
auto it = findField(e, m, "values");

NodePtr node = NodePtr(new NodeMap(
asSingleAttribute(makeNode(it->second, st, ns))));
NodePtr node = std::make_shared<NodeMap>(
asSingleAttribute(makeNode(it->second, st, ns)));
if (containsField(m, "doc")) {
node->setDoc(getDocField(e, m));
}
@@ -468,7 +496,7 @@ static NodePtr makeNode(const Entity &e, const Object &m,
if (type == "record" || type == "error" || type == "enum" || type == "fixed") {
Name nm = getName(e, m, ns);
if (type == "record" || type == "error") {
result = NodePtr(new NodeRecord());
result = std::make_shared<NodeRecord>();
st[nm] = result;
// Get field doc
if (containsField(m, "doc")) {
@@ -513,7 +541,7 @@ static NodePtr makeNode(const Entity &, const Array &m,
for (const auto &it : m) {
mm.add(makeNode(it, st, ns));
}
return NodePtr(new NodeUnion(mm));
return std::make_shared<NodeUnion>(mm);
}

static NodePtr makeNode(const json::Entity &e, SymbolTable &st, const string &ns) {
2 changes: 1 addition & 1 deletion lang/c++/impl/CustomAttributes.cc
Original file line number Diff line number Diff line change
@@ -48,6 +48,6 @@ void CustomAttributes::printJson(std::ostream &os,
if (attributes().find(name) == attributes().end()) {
throw Exception(name + " doesn't exist");
}
os << "\"" << name << "\": \"" << attributes().at(name) << "\"";
os << "\"" << name << "\": " << attributes().at(name);
}
} // namespace avro
23 changes: 18 additions & 5 deletions lang/c++/impl/DataFile.cc
Original file line number Diff line number Diff line change
@@ -20,6 +20,8 @@
#include "Compiler.hh"
#include "Exception.hh"

#include <map>
#include <optional>
#include <sstream>

#include <boost/crc.hpp> // for boost::crc_32_type
@@ -64,7 +66,7 @@ boost::iostreams::zlib_params get_zlib_params() {
} // namespace

DataFileWriterBase::DataFileWriterBase(const char *filename, const ValidSchema &schema, size_t syncInterval,
Codec codec) : filename_(filename),
Codec codec, const std::map<std::string, std::string> &customMetadata) : filename_(filename),
schema_(schema),
encoderPtr_(binaryEncoder()),
syncInterval_(syncInterval),
@@ -74,11 +76,11 @@ DataFileWriterBase::DataFileWriterBase(const char *filename, const ValidSchema &
sync_(makeSync()),
objectCount_(0),
lastSync_(0) {
init(schema, syncInterval, codec);
init(schema, syncInterval, codec, customMetadata);
}

DataFileWriterBase::DataFileWriterBase(std::unique_ptr<OutputStream> outputStream,
const ValidSchema &schema, size_t syncInterval, Codec codec) : filename_(),
const ValidSchema &schema, size_t syncInterval, Codec codec, const std::map<std::string, std::string> &customMetadata) : filename_(),
schema_(schema),
encoderPtr_(binaryEncoder()),
syncInterval_(syncInterval),
@@ -88,10 +90,10 @@ DataFileWriterBase::DataFileWriterBase(std::unique_ptr<OutputStream> outputStrea
sync_(makeSync()),
objectCount_(0),
lastSync_(0) {
init(schema, syncInterval, codec);
init(schema, syncInterval, codec, customMetadata);
}

void DataFileWriterBase::init(const ValidSchema &schema, size_t syncInterval, const Codec &codec) {
void DataFileWriterBase::init(const ValidSchema &schema, size_t syncInterval, const Codec &codec, const std::map<std::string, std::string> &customMetadata) {
if (syncInterval < minSyncInterval || syncInterval > maxSyncInterval) {
throw Exception(
"Invalid sync interval: {}. Should be between {} and {}",
@@ -111,6 +113,9 @@ void DataFileWriterBase::init(const ValidSchema &schema, size_t syncInterval, co
throw Exception("Unknown codec: {}", int(codec));
}
setMetadata(AVRO_SCHEMA_KEY, schema.toJson(false));
for (const auto &kv : customMetadata) {
setMetadata(kv.first, kv.second);
}

writeHeader();
encoderPtr_->init(*buffer_);
@@ -563,4 +568,12 @@ int64_t DataFileReaderBase::previousSync() const {
return blockStart_;
}

std::optional<std::string> DataFileReaderBase::getMetadata(const std::string& key) {
if (auto it = metadata_.find(key); it == metadata_.cend()) {
return std::nullopt;
} else {
return std::string(it->second.cbegin(), it->second.cend());
}
}

} // namespace avro
3 changes: 3 additions & 0 deletions lang/c++/impl/LogicalType.cc
Original file line number Diff line number Diff line change
@@ -77,6 +77,9 @@ void LogicalType::printJson(std::ostream &os) const {
case UUID:
os << R"("logicalType": "uuid")";
break;
case MAP:
os << R"("logicalType": "map")";
break;
}
}

7 changes: 7 additions & 0 deletions lang/c++/impl/Node.cc
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@
* limitations under the License.
*/

#include <algorithm>
#include <cmath>
#include <unordered_set>

@@ -201,6 +202,12 @@ void Node::setLogicalType(LogicalType logicalType) {
"STRING type");
}
break;
case LogicalType::MAP:
if (type_ != AVRO_ARRAY) {
throw Exception("MAP logical type can only annotate "
"ARRAY type");
}
break;
}

logicalType_ = logicalType;
41 changes: 38 additions & 3 deletions lang/c++/impl/NodeImpl.cc
Original file line number Diff line number Diff line change
@@ -129,6 +129,24 @@ NodePrimitive::resolve(const Node &reader) const {
return RESOLVE_PROMOTABLE_TO_DOUBLE;
}

break;

case AVRO_BYTES:

if (reader.type() == AVRO_STRING) {
return RESOLVE_MATCH;
}

break;

case AVRO_STRING:

if (reader.type() == AVRO_BYTES) {
return RESOLVE_MATCH;
}

break;

default: break;
}

@@ -209,7 +227,7 @@ NodeFixed::resolve(const Node &reader) const {

SchemaResolution
NodeSymbolic::resolve(const Node &reader) const {
const NodePtr &node = leafAt(0);
const NodePtr &node = getNode();
return node->resolve(reader);
}

@@ -540,7 +558,15 @@ void NodeEnum::printJson(std::ostream &os, size_t depth) const {
os << indent(depth) << '\"' << leafNameAttributes_.get(i) << '\"';
}
os << '\n';
os << indent(--depth) << "]\n";
os << indent(--depth) << "]";

if (defaultValue.type() != AVRO_NULL) {
os << ",\n"
<< indent(depth) << "\"default\": ";
NodePrimitive{defaultValue.type()}.printDefaultToJson(defaultValue, os, depth);
}

os << "\n";
os << indent(--depth) << '}';
}

@@ -551,6 +577,15 @@ void NodeArray::printJson(std::ostream &os, size_t depth) const {
os << indent(depth + 1) << R"("doc": ")"
<< escape(getDoc()) << "\",\n";
}
if (logicalType().type() != LogicalType::NONE) {
os << indent(depth);
logicalType().printJson(os);
os << ",\n";
}
if (elementId_.has_value()) {
os << indent(depth + 1) << "\"element-id\": "
<< *elementId_ << ",\n";
}
os << indent(depth + 1) << "\"items\": ";
leafAttributes_.get()->printJson(os, depth + 1);
os << '\n';
@@ -571,7 +606,7 @@ void NodeMap::printJson(std::ostream &os, size_t depth) const {
}

NodeMap::NodeMap() : NodeImplMap(AVRO_MAP) {
NodePtr key(new NodePrimitive(AVRO_STRING));
NodePtr key(std::make_shared<NodePrimitive>(AVRO_STRING));
doAddLeaf(key);
}

Loading