From c62d7b9bf86c0a2d24f86bd223a1cf5ca2ed908c Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Fri, 5 Jul 2024 15:25:11 +0800 Subject: [PATCH 1/5] extract interface Committed-by: xiaolei.zl from Dev container --- flex/engines/hqps_db/database/adj_list.h | 191 +++++++++++++++++ flex/engines/hqps_db/database/i_graph.h | 254 +++++++++++++++++++++++ 2 files changed, 445 insertions(+) create mode 100644 flex/engines/hqps_db/database/i_graph.h diff --git a/flex/engines/hqps_db/database/adj_list.h b/flex/engines/hqps_db/database/adj_list.h index 986bd30a0416..c6ee0e25948b 100644 --- a/flex/engines/hqps_db/database/adj_list.h +++ b/flex/engines/hqps_db/database/adj_list.h @@ -26,6 +26,197 @@ namespace gs { +namespace igraph { + +template +// Base interface for edge iterator +class EdgeIter { + public: + using label_id_t = LabelT; + + virtual void Next() const = 0; + virtual vid_t GetDstId() const = 0; + + virtual Any GetData() const = 0; + virtual bool IsValid() const = 0; + + virtual label_id_t GetDstLabel() const = 0; + + virtual label_id_t GetSrcLabel() const = 0; + + virtual Direction GetDirection() const = 0; + + virtual std::vector GetPropNames() const = 0; + + size_t Size() const = 0; +}; + +template +class SubGraph { + public: + using iterator = EdgeIter; + using label_id_t = LabelT; + + virtual iterator get_edges(VID_T vid) const = 0; + + // here the src, dst, refer the src, dst of the csr, not the direction. + virtual label_id_t GetSrcLabel() const = 0; + virtual label_id_t GetEdgeLabel() const = 0; + virtual label_id_t GetDstLabel() const = 0; + virtual Direction GetDirection() const = 0; + + virtual std::vector GetPropNames() const; +}; + +template +class SinglePropGetter { + public: + using value_type = T; + static constexpr size_t prop_num = 1; + + virtual value_type get_view(vid_t vid) const = 0; +}; + +template +class MultiPropGetter { + public: + using result_tuple_t = std::tuple; + static constexpr size_t prop_num = sizeof...(T); + + virtual result_tuple_t get_view(vid_t vid) const = 0; +}; + +class Nbr { + public: + Nbr() = default; + explicit Nbr(vid_t neighbor) : neighbor_(neighbor) {} + ~Nbr() = default; + + inline vid_t neighbor() const { return neighbor_; } + + private: + vid_t neighbor_; +}; + +class NbrList { + public: + NbrList(const Nbr* b, const Nbr* e) : begin_(b), end_(e) {} + NbrList() : begin_(nullptr), end_(nullptr) {} + ~NbrList() = default; + + const Nbr* begin() const { return begin_; } + const Nbr* end() const { return end_; } + inline size_t size() const { return end_ - begin_; } + + private: + const Nbr* begin_; + const Nbr* end_; +}; + +class NbrListArray { + public: + NbrListArray() {} + ~NbrListArray() = default; + + NbrList get(size_t index) const { + auto& list = nbr_lists_[index]; + return NbrList(list.data(), list.data() + list.size()); + } + + void put(std::vector&& list) { nbr_lists_.push_back(std::move(list)); } + + size_t size() const { return nbr_lists_.size(); } + + void resize(size_t size) { nbr_lists_.resize(size); } + + std::vector& get_vector(size_t index) { return nbr_lists_[index]; } + + private: + std::vector> nbr_lists_; +}; + +class NbrListArray { + public: + NbrListArray() {} + ~NbrListArray() = default; + + NbrList get(size_t index) const { + auto& list = nbr_lists_[index]; + return NbrList(list.data(), list.data() + list.size()); + } + + void put(std::vector&& list) { nbr_lists_.push_back(std::move(list)); } + + size_t size() const { return nbr_lists_.size(); } + + void resize(size_t size) { nbr_lists_.resize(size); } + + std::vector& get_vector(size_t index) { return nbr_lists_[index]; } + + private: + std::vector> nbr_lists_; +}; + +template +class Adj { + public: + Adj() = default; + ~Adj() = default; + + Adj(const Adj& other) : neighbor_(other.neighbor_), prop_(other.prop_) {} + + Adj(Adj&& other) + : neighbor_(other.neighbor_), prop_(std::move(other.prop_)) {} + + inline Adj& operator=(const Adj& from) { + this->neighbor_ = from.neighbor_; + this->prop_ = from.prop_; + return *this; + } + + vid_t neighbor() const { return neighbor_; } + const std::tuple& properties() const { return prop_; } + + vid_t neighbor_; + std::tuple prop_; +}; + +template +class AdjList { + // // Do we need a virtual Iterator? + // class Iterator { + // public: + // bool valid() const = 0; + // const Adj& operator*() const = 0; + // const Adj* operator->() const = 0; + + // vid_t neighbor() const = 0; + // const std::tuple& properties() const; + + // Iterator& operator++(); + // Iterator operator++(int); + + // bool operator==(const Iterator& rhs) const; + + // bool operator!=(const Iterator& rhs) const; + // }; + + public: + // virtual Iterator begin() const = 0; + // virtual Iterator end() const = 0; + size_t size() const = 0; +}; + +template +class AdjListArray { + public: + virtual size_t size() const = 0; + + virtual AdjList get(size_t i) const = 0; +}; + +}; // namespace igraph + namespace mutable_csr_graph_impl { template diff --git a/flex/engines/hqps_db/database/i_graph.h b/flex/engines/hqps_db/database/i_graph.h new file mode 100644 index 000000000000..f5b4133b1fd3 --- /dev/null +++ b/flex/engines/hqps_db/database/i_graph.h @@ -0,0 +1,254 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_ +#define ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_ + +#include + +#include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/graph_db/database/graph_db_session.h" +#include "flex/engines/hqps_db/core/null_record.h" +#include "flex/engines/hqps_db/core/params.h" + +#include "flex/engines/hqps_db/database/adj_list.h" +#include "grape/utils/bitset.h" + +#include "grape/util.h" + +namespace gs { + +template +bool exists_nullptr_in_tuple(const T& t) { + return std::apply([](auto&&... args) { return ((args == nullptr) || ...); }, + t); +} + +template +void get_tuple_from_column_tuple( + size_t index, std::tuple& t, + const std::tuple>...>& columns) { + using cur_ele_t = std::tuple_element_t>; + auto ptr = std::get(columns); + if (ptr) { + std::get(t) = ptr->get_view(index); + } else { + std::get(t) = NullRecordCreator::GetNull(); + } + + if constexpr (I + 1 < sizeof...(T)) { + get_tuple_from_column_tuple(index, t, columns); + } +} + +template +void get_tuple_from_column_tuple(size_t index, std::tuple& t, + const std::tuple& columns) { + auto ptr = std::get(columns); + using cur_ele_t = std::tuple_element_t>; + if (ptr) { + std::get(t) = ptr->get_view(index); + } else { + std::get(t) = NullRecordCreator::GetNull(); + } + + if constexpr (I + 1 < sizeof...(T)) { + get_tuple_from_column_tuple(index, t, columns); + } +} + +/** + * @brief MutableCSRInterface is the interface for the mutable CSR graph + * implementation. + * + */ +class MutableCSRInterface { + public: + using vertex_id_t = vid_t; + using gid_t = uint64_t; + using label_id_t = uint8_t; + + using nbr_list_array_t = mutable_csr_graph_impl::NbrListArray; + + template + using adj_list_array_t = mutable_csr_graph_impl::AdjListArray; + + template + using adj_list_t = mutable_csr_graph_impl::AdjList; + + template + using adj_t = mutable_csr_graph_impl::Adj; + + using nbr_t = mutable_csr_graph_impl::Nbr; + + using nbr_list_t = mutable_csr_graph_impl::NbrList; + + template + using single_prop_getter_t = mutable_csr_graph_impl::SinglePropGetter; + + template + using multi_prop_getter_t = mutable_csr_graph_impl::MultiPropGetter; + + using sub_graph_t = mutable_csr_graph_impl::SubGraph; + + /** + * @brief Get the Vertex Label id + * + * @param label + * @return label_id_t + */ + virtual label_id_t GetVertexLabelId(const std::string& label) const = 0; + + /** + * @brief Get the Edge Label id + * + * @param label + * @return label_id_t + */ + virtual label_id_t GetEdgeLabelId(const std::string& label) const = 0; + + /** + * @brief Scans all vertices with the given label and calls the + * given function on each vertex for filtering. + * @tparam FUNC_T + * @tparam SELECTOR + * @param label_id + * @param selectors The Property selectors. The selected properties will be + * fed to the filtering function + * @param func The lambda function for filtering. + */ + template + void ScanVertices(const label_id_t& label_id, + const std::tuple& selectors, + const FUNC_T& func) const; + + /** + * @brief ScanVertices scans all vertices with the given label with give + * original id. + * @param label_id The label id. + * @param oid The original id. + * @param vid The result internal id. + */ + template + bool ScanVerticesWithOid(const label_id_t& label_id, OID_T oid, + vertex_id_t& vid) const; + + /** + * @brief GetVertexProps gets the properties of single label vertices. + * @tparam T The property types. + * @param label + * @param vids + * @param prop_names + */ + template + std::vector> GetVertexPropsFromVid( + const label_id_t& label_id, const std::vector& vids, + const std::array>>& + prop_names) const { + // auto label_id = db_session_.schema().get_vertex_label_id(label); + CHECK(label_id < db_session_.schema().vertex_label_num()); + std::tuple>...> columns; + get_tuple_column_from_graph(label_id, prop_names, columns); + std::vector> props(vids.size()); + fetch_properties_in_column(vids, props, columns); + return props; + } + + /** + * @brief GetVertexProps gets the properties of vertices from multiple labels. + * @tparam T + * @param vids The vertex ids. + * @param label_ids The label ids. + * @param vid_inds The indices of the vertex ids in the input vids. + * @param prop_names The property names. + */ + template + std::vector> GetVertexPropsFromVid( + const std::vector& vids, + const std::vector& label_ids, + const std::vector>& vid_inds, + const std::array>>& + prop_names) const; + + /** + * @brief GetSubGraph gets the subgraph with the given label and edge label. + * @param src_label_id The source label id. + * @param dst_label_id The destination label id. + * @param edge_label_id The edge label id. + * @param direction_str The direction string. + * @param prop_names The property names. + */ + std::vector> + GetSubGraph(const label_id_t src_label_id, const label_id_t dst_label_id, + const label_id_t edge_label_id, const std::string& direction_str, + const std::vector& prop_names) const; + + /** + * @brief GetEdges gets the edges with the given label and edge label, and + * with the starting vertex internal ids. + * When the direction is "out", the edges are from the source label to the + * destination label, and vice versa when the direction is "in". When the + * direction is "both", the src and dst labels SHOULD be the same. + * @tparam T The property types. + * @param src_label_id The source label id. + * @param dst_label_id The destination label id. + * @param edge_label_id The edge label id. + * @param vids The starting vertex internal ids. + * @param direction_str The direction string. + * @param limit The limit of the edges. + * @param prop_names The property names. + */ + template + mutable_csr_graph_impl::AdjListArray GetEdges( + const label_id_t& src_label_id, const label_id_t& dst_label_id, + const label_id_t& edge_label_id, const std::vector& vids, + const std::string& direction_str, size_t limit, + const std::array>>& + prop_names) const; + + /** + * @brief Get vertices on the other side of edges, via the given edge label + * and the starting vertex internal ids. + * When the direction is "out", the vertices are on the destination label side + * of the edges, and vice versa when the direction is "in". When the direction + * is "both", the src and dst labels SHOULD be the same. + * @param src_label_id The source label id. + * @param dst_label_id The destination label id. + * @param edge_label_id The edge label id. + * @param vids The starting vertex internal ids. + * @param direction_str The direction string. + * @param limit The limit of the vertices. + */ + mutable_csr_graph_impl::NbrListArray GetOtherVertices( + const label_id_t& src_label_id, const label_id_t& dst_label_id, + const label_id_t& edge_label_id, const std::vector& vids, + const std::string& direction_str, size_t limit) const; + + template + mutable_csr_graph_impl::MultiPropGetter GetMultiPropGetter( + const label_id_t& label_id, + const std::array& prop_names) const; + + template + mutable_csr_graph_impl::SinglePropGetter GetSinglePropGetter( + const label_id_t& label_id, const std::string& prop_name) const { + using column_t = std::shared_ptr>; + column_t column = GetTypedRefColumn(label_id, prop_name); + return mutable_csr_graph_impl::SinglePropGetter(std::move(column)); + } +}; + +} // namespace gs + +#endif // ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_ From f2f496c9870a65cfe461976b630dfad3d3e0096b Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Wed, 10 Jul 2024 17:17:02 +0800 Subject: [PATCH 2/5] able to compile Committed-by: xiaolei.zl from Dev container --- flex/codegen/src/building_context.h | 6 +- .../hqps_db/core/operator/edge_expand.h | 132 ++--- flex/engines/hqps_db/core/operator/get_v.h | 10 +- .../hqps_db/core/operator/path_expand.h | 20 +- flex/engines/hqps_db/core/operator/project.h | 18 +- flex/engines/hqps_db/core/operator/scan.h | 33 +- flex/engines/hqps_db/core/operator/sink.h | 61 +-- flex/engines/hqps_db/core/sync_engine.h | 12 +- flex/engines/hqps_db/core/utils/graph_utils.h | 99 ++++ flex/engines/hqps_db/core/utils/keyed.h | 18 +- flex/engines/hqps_db/core/utils/props.h | 125 +++-- flex/engines/hqps_db/database/adj_list.h | 60 +-- flex/engines/hqps_db/database/adj_list_v2.h | 301 +++++++++++ flex/engines/hqps_db/database/i_graph.h | 254 ---------- .../hqps_db/database/mutable_csr_interface.h | 141 +++++- .../database/mutable_csr_interface_v2.h | 467 ++++++++++++++++++ flex/engines/hqps_db/database/nbr_list.h | 106 ++++ flex/engines/hqps_db/database/sub_graph.h | 189 +++++++ flex/engines/hqps_db/structures/collection.h | 4 +- .../multi_vertex_set/general_vertex_set.h | 67 ++- .../multi_vertex_set/row_vertex_set.h | 44 +- .../multi_vertex_set/two_label_vertex_set.h | 53 +- .../mutable_property_fragment.cc | 7 + .../mutable_property_fragment.h | 3 + flex/storages/rt_mutable_graph/schema.cc | 9 +- flex/storages/rt_mutable_graph/schema.h | 3 + flex/tests/hqps/context_test.cc | 17 +- flex/tests/hqps/dedup_test.cc | 2 +- flex/tests/hqps/path_set_test.cc | 1 - flex/tests/hqps/query_test.cc | 13 +- flex/utils/property/column.cc | 46 ++ flex/utils/property/column.h | 3 + 32 files changed, 1689 insertions(+), 635 deletions(-) create mode 100644 flex/engines/hqps_db/core/utils/graph_utils.h create mode 100644 flex/engines/hqps_db/database/adj_list_v2.h delete mode 100644 flex/engines/hqps_db/database/i_graph.h create mode 100644 flex/engines/hqps_db/database/mutable_csr_interface_v2.h create mode 100644 flex/engines/hqps_db/database/nbr_list.h create mode 100644 flex/engines/hqps_db/database/sub_graph.h diff --git a/flex/codegen/src/building_context.h b/flex/codegen/src/building_context.h index b4135943c86f..ec1c6f4487ad 100644 --- a/flex/codegen/src/building_context.h +++ b/flex/codegen/src/building_context.h @@ -27,7 +27,7 @@ static constexpr const char* time_stamp = "time_stamp"; static constexpr const char* graph_var = "graph"; static constexpr const char* GRAPE_INTERFACE_CLASS = "gs::MutableCSRInterface"; static constexpr const char* GRAPE_INTERFACE_HEADER = - "flex/engines/hqps_db/database/mutable_csr_interface.h"; + "flex/engines/hqps_db/database/mutable_csr_interface_v2.h"; static constexpr const char* EDGE_EXPAND_OPT_NAME = "edge_expand_opt"; static constexpr const char* SORT_OPT_NAME = "sort_opt"; static constexpr const char* GET_V_OPT_NAME = "get_v_opt"; @@ -126,9 +126,7 @@ struct TagIndMapping { tag_id_2_tag_inds_[tag_id] != -1; } - int32_t GetMaxTagId() const { - return tag_id_2_tag_inds_.size() - 1; - } + int32_t GetMaxTagId() const { return tag_id_2_tag_inds_.size() - 1; } // convert tag_ind (us) to tag ids std::vector tag_ind_2_tag_ids_; diff --git a/flex/engines/hqps_db/core/operator/edge_expand.h b/flex/engines/hqps_db/core/operator/edge_expand.h index 77aaa87c8133..eebd672d8c10 100644 --- a/flex/engines/hqps_db/core/operator/edge_expand.h +++ b/flex/engines/hqps_db/core/operator/edge_expand.h @@ -21,6 +21,7 @@ limitations under the License. #include "flex/engines/hqps_db/core/utils/hqps_utils.h" +#include "flex/engines/hqps_db/core/utils/graph_utils.h" #include "flex/engines/hqps_db/structures/multi_edge_set/adj_edge_set.h" #include "flex/engines/hqps_db/structures/multi_edge_set/flat_edge_set.h" #include "flex/engines/hqps_db/structures/multi_edge_set/general_edge_set.h" @@ -182,7 +183,7 @@ class EdgeExpand { if (state.direction_ == Direction::In || state.direction_ == Direction::Both) { auto tmp_nbr_list_array = state.graph_.GetOtherVertices( - src_label, dst_label, state.edge_label_, cur_vids, "In", + src_label, dst_label, state.edge_label_, cur_vids, Direction::In, state.limit_); CHECK(tmp_nbr_list_array.size() == active_inds.size()); @@ -196,7 +197,7 @@ class EdgeExpand { if (state.direction_ == Direction::Out || state.direction_ == Direction::Both) { auto tmp_nbr_list_array = state.graph_.GetOtherVertices( - src_label, dst_label, state.edge_label_, cur_vids, "Out", + src_label, dst_label, state.edge_label_, cur_vids, Direction::Out, state.limit_); CHECK(tmp_nbr_list_array.size() == active_inds.size()); @@ -284,8 +285,8 @@ class EdgeExpand { << ",dst: " << std::to_string(dst_label) << ",dire: " << state.direction_; auto tmp_nbr_list_array = state.graph_.GetOtherVertices( - src_label, dst_label, state.edge_label_, cur_vids, - gs::to_string(state.direction_), state.limit_); + src_label, dst_label, state.edge_label_, cur_vids, state.direction_, + state.limit_); // nbr_lists.emplace_back(std::move(nbr_list_array)); CHECK(tmp_nbr_list_array.size() == active_inds.size()); @@ -627,9 +628,9 @@ class EdgeExpand { << ", src label: " << gs::to_string(raw_src_label) << ", dst label: " << gs::to_string(raw_dst_label) << ", direction: " << gs::to_string(direction); - auto vid_and_offset1 = graph.GetOtherVerticesV2( - raw_src_label, raw_dst_label, edge_label, - cur_vertex_set.GetVertices(), gs::to_string(direction), INT_MAX); + auto vid_and_offset1 = get_other_vertices_in_batch( + graph, raw_src_label, raw_dst_label, edge_label, + cur_vertex_set.GetVertices(), direction); VLOG(10) << ", vid_and_offset1 size: " << vid_and_offset1.first.size() << ", vid_and_offset1 offset size: " << vid_and_offset1.second.size(); @@ -649,7 +650,7 @@ class EdgeExpand { auto [vids, active_inds] = cur_vertex_set.GetVertices(j); auto tmp_nbr_list_array = graph.GetOtherVertices(raw_src_label, raw_dst_label, edge_label, - vids, gs::to_string(direction), INT_MAX); + vids, direction, INT_MAX); for (size_t k = 0; k < active_inds.size(); ++k) { auto dst_ind = active_inds[k]; CHECK(nbr_list_array.get(dst_ind).size() == 0); @@ -887,12 +888,11 @@ class EdgeExpand { for (size_t i = 0; i < edge_labels.size(); ++i) { // Check whether the edge triplet match input vertices. // return a handler to get edges - auto sub_graph_vec = graph.GetSubGraph( - edge_labels[i][0], edge_labels[i][1], edge_labels[i][2], - gs::to_string(direction), prop_names_vec[i]); - for (auto sub_graph : sub_graph_vec) { - sub_graphs.emplace_back(sub_graph); - } + auto sub_graph = graph.GetSubGraph(edge_labels[i][0], edge_labels[i][1], + edge_labels[i][2], direction); + // for (auto sub_graph : sub_graph_vec) { + sub_graphs.emplace_back(sub_graph); + // } } std::vector> label_triplets; @@ -1003,12 +1003,9 @@ class EdgeExpand { for (size_t i = 0; i < edge_labels.size(); ++i) { // Check whether the edge triplet match input vertices. // return a handler to get edges - auto sub_graph_vec = graph.GetSubGraph( - edge_labels[i][0], edge_labels[i][1], edge_labels[i][2], - gs::to_string(direction), prop_names_vec[i]); - for (auto sub_graph : sub_graph_vec) { - sub_graphs.emplace_back(sub_graph); - } + auto sub_graph = graph.GetSubGraph(edge_labels[i][0], edge_labels[i][1], + edge_labels[i][2], direction); + sub_graphs.emplace_back(sub_graph); } std::vector> label_triplets; @@ -1156,27 +1153,6 @@ class EdgeExpand { return std::make_pair(std::move(new_set), std::move(old_offset)); } - // for input vertex set with only one label. - template ::type* = nullptr, - typename RES_T = std::pair, - std::vector>> - static RES_T EdgeExpandE( - const GRAPH_INTERFACE& graph, - RowVertexSet& cur_vertex_set, - Direction direction, label_id_t edge_label, label_id_t other_label, - EDGE_FILTER_T& edge_filter, PropNameArray& props, - size_t limit = INT_MAX) { - auto state = - EdgeExpandEState, - EDGE_FILTER_T>(graph, cur_vertex_set, direction, - edge_label, other_label, props, - edge_filter, limit); - - return EdgeExpandENoPropImpl(state); - } // EdgeExpandE when input vertex are single label. and get multiple props // Currently only support one edge property template vids; std::vector offset; offset.reserve(state.cur_vertex_set_.Size() + 1); @@ -1339,15 +1314,14 @@ class EdgeExpand { return pair; } - template + template static void fetch_adj_list_array_from_graph( const GRAPH_INTERFACE& graph, const VERTEX_SET_T& vertex_set, Direction direction, label_id_t edge_label, label_id_t other_label, - PropNameArray& prop_names, int32_t limit, - typename GRAPH_INTERFACE::template adj_list_array_t& + PropNameArray& prop_names, int32_t limit, + typename GRAPH_INTERFACE::template adj_list_array_t& res_adj_list_arrays) { auto num_labels = vertex_set.GetLabels().size(); - auto direction_str = gs::to_string(direction); label_id_t src_label, dst_label; for (size_t i = 0; i < num_labels; ++i) { if (direction == Direction::In) { @@ -1371,9 +1345,8 @@ class EdgeExpand { std::vector cur_vids; std::vector cur_active_inds; std::tie(cur_vids, cur_active_inds) = vertex_set.GetVertices(i); - auto tmp = graph.template GetEdges( - src_label, dst_label, edge_label, cur_vids, direction_str, limit, - prop_names); + auto tmp = graph.template GetEdges( + src_label, dst_label, edge_label, cur_vids, direction, limit); CHECK(tmp.size() == cur_active_inds.size()); if (i == 0) { // first time, update flag field. @@ -1576,8 +1549,7 @@ class EdgeExpand { state.direction_ == Direction::Both) { auto adj_list_array = state.graph_.template GetEdges( src_label, dst_label, state.edge_label_, - state.cur_vertex_set_.GetVertices(), gs::to_string(Direction::Out), - state.limit_, prop_names); + state.cur_vertex_set_.GetVertices(), Direction::Out, state.limit_); adj_list_array_vec.emplace_back( std::make_pair(std::move(adj_list_array), Direction::Out)); } @@ -1585,8 +1557,7 @@ class EdgeExpand { state.direction_ == Direction::Both) { auto adj_list_array = state.graph_.template GetEdges( src_label, dst_label, state.edge_label_, - state.cur_vertex_set_.GetVertices(), gs::to_string(Direction::In), - state.limit_, prop_names); + state.cur_vertex_set_.GetVertices(), Direction::In, state.limit_); adj_list_array_vec.emplace_back( std::make_pair(std::move(adj_list_array), Direction::In)); } @@ -1646,51 +1617,6 @@ class EdgeExpand { return filter(std::get(props)...); } - // EdgeExpandE for single label input vertex set. - template - static auto EdgeExpandENoPropImpl( - EdgeExpandEState, - EDGE_FILTER_T>& state) { - // no prop. - auto prop_names = state.prop_names_; - label_id_t src_label, dst_label; - if (state.direction_ == Direction::In) { - src_label = state.other_label_; - dst_label = state.cur_vertex_set_.GetLabel(); - } else { - src_label = state.cur_vertex_set_.GetLabel(); - dst_label = state.other_label_; - } - LOG(INFO) << "[EdgeExpandENoPropImpl] for single label vertex set. " - << (int) src_label << " " << (int) dst_label; - auto adj_list_array = state.graph_.template GetEdges<>( - src_label, dst_label, state.edge_label_, - state.cur_vertex_set_.GetVertices(), gs::to_string(state.direction_), - state.limit_, prop_names); - LOG(INFO) << "after get edges"; - std::vector offset; - offset.reserve(state.cur_vertex_set_.Size() + 1); - size_t size = 0; - size_t adj_list_ind = 0; - offset.emplace_back(size); - // Construct offset from adj_list. - for (auto iter : state.cur_vertex_set_) { - auto edges = adj_list_array.get(adj_list_ind); - size += edges.size(); // number of edges in this AdjList - offset.emplace_back(size); - adj_list_ind++; - } - LOG(INFO) << "total size of edges: " << size; - auto copied_vids(state.cur_vertex_set_.GetVertices()); - auto edge_set = - AdjEdgeSet( - std::move(copied_vids), std::move(adj_list_array), - state.edge_label_, state.cur_vertex_set_.GetLabel(), - state.other_label_, array_to_vec(prop_names), state.direction_); - return std::make_pair(std::move(edge_set), std::move(offset)); - } - // only support one property template @@ -1711,8 +1637,7 @@ class EdgeExpand { std::array prop_names = {selector.prop_name_}; auto adj_list_array = state.graph_.template GetEdges( src_label, dst_label, state.edge_label_, - state.cur_vertex_set_.GetVertices(), gs::to_string(state.direction_), - state.limit_, prop_names); + state.cur_vertex_set_.GetVertices(), state.direction_, state.limit_); return adj_list_array; } @@ -1726,9 +1651,8 @@ class EdgeExpand { CHECK(direction != Direction::Both); std::vector dst_vertices; std::vector tmp_offset; - std::tie(dst_vertices, tmp_offset) = - graph.GetOtherVerticesV2(src_label_id, dst_label_id, edge_label_id, - src_v, gs::to_string(direction), INT_MAX); + std::tie(dst_vertices, tmp_offset) = get_other_vertices_in_batch( + graph, src_label_id, dst_label_id, edge_label_id, src_v, direction); // put these vertices into tmp_nbr_vertices label_id_t label_id; if (direction == Direction::Out) { diff --git a/flex/engines/hqps_db/core/operator/get_v.h b/flex/engines/hqps_db/core/operator/get_v.h index e396a20e608b..f3d0063883eb 100644 --- a/flex/engines/hqps_db/core/operator/get_v.h +++ b/flex/engines/hqps_db/core/operator/get_v.h @@ -189,8 +189,8 @@ class GetVertex { for (size_t i = 0; i < num_labels; ++i) { auto& cur_set = multi_set.GetSet(i); VLOG(10) << "set: " << i << ", size: " << cur_set.Size(); - res_data_tuples[i] = graph.template GetVertexPropsFromVid( - cur_set.GetLabel(), cur_set.GetVertices(), props); + res_data_tuples[i] = get_vertex_props_from_vids( + graph, cur_set.GetLabel(), cur_set.GetVertices(), props); } VLOG(10) << "Finish get data tuples"; auto set_array = std::array{make_row_vertex_set( @@ -301,9 +301,9 @@ class GetVertex { const GRAPH_INTERFACE& graph, std::array& labels, Filter& filter, const RowVertexSet& set) { - // TODO: support for multiple selectors - auto property_getters_array = std::array{get_prop_getter_from_selectors( - graph, set.GetLabel(), filter.selectors_)}; + auto property_getters_array = + std::array{get_prop_getters_from_selectors_single_label( + graph, set.GetLabel(), filter.selectors_)}; return set.project_vertices(labels, filter.expr_, property_getters_array); } diff --git a/flex/engines/hqps_db/core/operator/path_expand.h b/flex/engines/hqps_db/core/operator/path_expand.h index 28e1acce79c6..38fae699e00a 100644 --- a/flex/engines/hqps_db/core/operator/path_expand.h +++ b/flex/engines/hqps_db/core/operator/path_expand.h @@ -19,6 +19,7 @@ limitations under the License. #include #include "flex/engines/hqps_db/core/params.h" +#include "flex/engines/hqps_db/core/utils/graph_utils.h" #include "flex/engines/hqps_db/core/utils/hqps_utils.h" #include "flex/engines/hqps_db/structures/multi_vertex_set/row_vertex_set.h" #include "flex/engines/hqps_db/structures/path.h" @@ -186,8 +187,8 @@ class PathExpand { graph, cur_label, vertex_set.GetVertices(), range, edge_expand_opt); auto& vids_vec = std::get<0>(tuple); - auto tuple_vec = graph.template GetVertexPropsFromVid( - cur_label, vids_vec, get_v_opt.props_); + auto tuple_vec = get_vertex_props_from_vids( + graph, cur_label, vids_vec, get_v_opt.props_); CHECK(tuple_vec.size() == vids_vec.size()); // prepend dist info. auto new_tuple_vec = @@ -488,9 +489,9 @@ class PathExpand { double visit_array_time = 0.0; for (size_t cur_hop = 1; cur_hop < range.limit_; ++cur_hop) { double t0 = -grape::GetCurrentTime(); - auto pair = graph.GetOtherVerticesV2( - real_src_label, dst_label, edge_expand_opt.edge_label_, - gids[cur_hop - 1], gs::to_string(edge_expand_opt.dir_), INT_MAX); + auto pair = get_other_vertices_in_batch( + graph, real_src_label, dst_label, edge_expand_opt.edge_label_, + gids[cur_hop - 1], edge_expand_opt.dir_); gids[cur_hop].swap(pair.first); CHECK(gids[cur_hop - 1].size() + 1 == pair.second.size()); @@ -705,9 +706,9 @@ class PathExpand { auto& prev_other_vertices = other_vertices[i - 1]; std::tie(cur_other_vertices, cur_other_offsets) = - graph.GetOtherVerticesV2(src_label, edge_opt.other_label_, - edge_opt.edge_label_, prev_other_vertices, - gs::to_string(edge_opt.dir_), INT_MAX); + get_other_vertices_in_batch(graph, src_label, edge_opt.other_label_, + edge_opt.edge_label_, prev_other_vertices, + edge_opt.dir_); VLOG(10) << "PathExpand at distance: " << i << ", got vertices: " << "size : " << cur_other_vertices.size(); } @@ -865,8 +866,7 @@ class PathExpand { } auto cur_nbr_list = graph.GetOtherVertices( real_src_label, real_dst_label, cur_edge_label, - other_vertices_for_cur_label, gs::to_string(direction), - INT_MAX); + other_vertices_for_cur_label, direction, INT_MAX); { size_t tmp_sum = 0; for (size_t i = 0; i < cur_nbr_list.size(); ++i) { diff --git a/flex/engines/hqps_db/core/operator/project.h b/flex/engines/hqps_db/core/operator/project.h index be7791dfc8fc..abe3b671a797 100644 --- a/flex/engines/hqps_db/core/operator/project.h +++ b/flex/engines/hqps_db/core/operator/project.h @@ -346,9 +346,8 @@ class ProjectOp { const GRAPH_INTERFACE& graph, NODE_T& node, const std::string& prop_name, const std::vector& repeat_array) { // Get property from storage. - auto prop_tuple_vec = graph.template GetVertexPropsFromVid( - node.GetLabel(), node.GetVertices(), {prop_name}); - // VLOG(10) << "Finish fetching properties"; + auto prop_tuple_vec = get_vertex_props_from_vids( + graph, node.GetLabel(), node.GetVertices(), {prop_name}); node.fillBuiltinProps(prop_tuple_vec, {prop_name}, repeat_array); std::vector res_prop_vec; for (size_t i = 0; i < repeat_array.size(); ++i) { @@ -356,9 +355,6 @@ class ProjectOp { res_prop_vec.push_back(std::get<0>(prop_tuple_vec[i])); } } - // check builtin properties. - // Found if there is any builtin properties need. - return Collection(std::move(res_prop_vec)); } @@ -370,8 +366,9 @@ class ProjectOp { const GRAPH_INTERFACE& graph, TwoLabelVertexSetImpl& node, const std::string& prop_name, const std::vector& repeat_array) { + auto selector_tuple = std::make_tuple(PropertySelector(prop_name)); auto tmp_prop_vec = - get_property_tuple_two_label(graph, node, {prop_name}); + get_property_tuple_two_label(graph, node, selector_tuple); // make_repeat; size_t sum = 0; @@ -410,10 +407,11 @@ class ProjectOp { static auto apply_single_project_impl( const GRAPH_INTERFACE& graph, GeneralVertexSet& node, - const std::string& prop_name_, const std::vector& repeat_array) { + const std::string& prop_name, const std::vector& repeat_array) { VLOG(10) << "start fetching properties"; - auto tmp_prop_vec = get_property_tuple_general( - graph, node, std::array{prop_name_}); + auto prop_selector_tuple = std::make_tuple(PropertySelector(prop_name)); + auto tmp_prop_vec = + get_property_tuple_general(graph, node, prop_selector_tuple); VLOG(10) << "Got properties for general vertex set: " << gs::to_string(tmp_prop_vec); std::vector res_prop_vec; diff --git a/flex/engines/hqps_db/core/operator/scan.h b/flex/engines/hqps_db/core/operator/scan.h index 3135a18b9308..3c20bc90ac93 100644 --- a/flex/engines/hqps_db/core/operator/scan.h +++ b/flex/engines/hqps_db/core/operator/scan.h @@ -45,7 +45,7 @@ class Scan { public: using label_id_t = typename GRAPH_INTERFACE::label_id_t; using vertex_id_t = typename GRAPH_INTERFACE::vertex_id_t; - using gid_t = typename GRAPH_INTERFACE::gid_t; + using gid_t = uint64_t; using vertex_set_t = DefaultRowVertexSet; using two_label_set_t = TwoLabelVertexSet; @@ -374,19 +374,32 @@ class Scan { const GRAPH_INTERFACE& graph, const label_id_t& v_label_id, const FUNC& func, const std::tuple& selectors) { std::vector gids; - auto filter = - [&](vertex_id_t v, - const std::tuple& real_props) { - if (apply_on_tuple(func, real_props)) { - gids.push_back(v); - } - }; // if FUNC has filter_null constexpr member, we can use it to filter // vertices if constexpr (FilterNull::value) { - graph.template ScanVertices(v_label_id, selectors, filter, true); + auto filter = + [&](vertex_id_t v, + const std::tuple& real_props) { + if (HasNull(real_props)) { + VLOG(10) << "When scanning for label " + << std::to_string(v_label_id) + << ", there is null column, using default NULL value"; + return; + } + if (apply_on_tuple(func, real_props)) { + gids.push_back(v); + } + }; + graph.template ScanVertices(v_label_id, selectors, filter); } else { - graph.template ScanVertices(v_label_id, selectors, filter, false); + auto filter = + [&](vertex_id_t v, + const std::tuple& real_props) { + if (apply_on_tuple(func, real_props)) { + gids.push_back(v); + } + }; + graph.template ScanVertices(v_label_id, selectors, filter); } return gids; } diff --git a/flex/engines/hqps_db/core/operator/sink.h b/flex/engines/hqps_db/core/operator/sink.h index 01ab8a782b6b..c9f41e8d5fad 100644 --- a/flex/engines/hqps_db/core/operator/sink.h +++ b/flex/engines/hqps_db/core/operator/sink.h @@ -23,7 +23,7 @@ #include "flex/engines/hqps_db/core/params.h" #include "flex/engines/hqps_db/core/utils/hqps_utils.h" #include "flex/engines/hqps_db/core/utils/props.h" -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" +#include "flex/engines/hqps_db/database/mutable_csr_interface_v2.h" #include "flex/engines/hqps_db/structures/multi_edge_set/flat_edge_set.h" #include "flex/engines/hqps_db/structures/multi_edge_set/general_edge_set.h" #include "flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h" @@ -114,9 +114,8 @@ void template_set_value(common::Value* value, T v) { } template ) && - (!std::is_same_v)>::type* = nullptr> + typename std::enable_if<(std::is_same_v) &&( + !std::is_same_v)>::type* = nullptr> void template_set_value(common::Value* value, T v) { value->set_i64(v); } @@ -432,14 +431,15 @@ class SinkOp { prop_names[0] = schema.get_vertex_property_names(labels[0]); prop_names[1] = schema.get_vertex_property_names(labels[1]); // get all properties - std::array>, 2> column_ptrs; + using prop_getter_t = typename GRAPH_INTERFACE::untyped_prop_getter_t; + std::array, 2> prop_getters; for (size_t i = 0; i < prop_names[0].size(); ++i) { - column_ptrs[0].emplace_back( - graph.GetRefColumnBase(labels[0], prop_names[0][i])); + prop_getters[0].emplace_back( + graph.GetUntypedVertexPropertyGetter(labels[0], prop_names[0][i])); } for (size_t i = 0; i < prop_names[1].size(); ++i) { - column_ptrs[1].emplace_back( - graph.GetRefColumnBase(labels[1], prop_names[1][i])); + prop_getters[1].emplace_back( + graph.GetUntypedVertexPropertyGetter(labels[1], prop_names[1][i])); } label_t label; @@ -460,14 +460,14 @@ class SinkOp { vertex->mutable_label()->set_id(label); vertex->set_id(encode_unique_vertex_id(label, vids[i])); // set properties. - auto columns = column_ptrs[label]; + auto columns = prop_getters[label]; for (size_t j = 0; j < columns.size(); ++j) { auto& column_ptr = columns[j]; // Only set non-none properties. - if (column_ptr) { + if (column_ptr.IsValid()) { auto new_prop = vertex->add_properties(); new_prop->mutable_key()->set_name(prop_names[label][j]); - set_any_to_common_value(column_ptr->get(vids[i]), + set_any_to_common_value(column_ptr.Get(vids[i]), new_prop->mutable_value()); } } @@ -500,14 +500,14 @@ class SinkOp { vertex->set_id(encode_unique_vertex_id(label, vids[i])); vertex->mutable_label()->set_id(label); // set properties. - auto columns = column_ptrs[label]; + auto columns = prop_getters[label]; for (size_t j = 0; j < columns.size(); ++j) { auto& column_ptr = columns[j]; // Only set non-none properties. - if (column_ptr) { + if (column_ptr.IsValid()) { auto new_prop = vertex->add_properties(); new_prop->mutable_key()->set_name(prop_names[label][j]); - set_any_to_common_value(column_ptr->get(vids[i]), + set_any_to_common_value(column_ptr.Get(vids[i]), new_prop->mutable_value()); } } @@ -528,9 +528,12 @@ class SinkOp { auto& schema = graph.schema(); auto prop_names = schema.get_vertex_property_names(label); // get all properties - std::vector> column_ptrs; + using untyped_prop_getter_t = + typename GRAPH_INTERFACE::untyped_prop_getter_t; + std::vector column_ptrs; for (size_t i = 0; i < prop_names.size(); ++i) { - column_ptrs.emplace_back(graph.GetRefColumnBase(label, prop_names[i])); + column_ptrs.emplace_back( + graph.GetUntypedVertexPropertyGetter(label, prop_names[i])); } VLOG(10) << "PropNames: " << prop_names.size(); if (repeat_offsets.empty()) { @@ -547,10 +550,10 @@ class SinkOp { for (size_t j = 0; j < column_ptrs.size(); ++j) { auto& column_ptr = column_ptrs[j]; // Only set non-none properties. - if (column_ptr) { + if (column_ptr.IsValid()) { auto new_prop = vertex->add_properties(); new_prop->mutable_key()->set_name(prop_names[j]); - set_any_to_common_value(column_ptr->get(vids[i]), + set_any_to_common_value(column_ptr.Get(vids[i]), new_prop->mutable_value()); } } @@ -579,10 +582,10 @@ class SinkOp { for (size_t j = 0; j < column_ptrs.size(); ++j) { auto& column_ptr = column_ptrs[j]; // Only set non-none properties. - if (column_ptr) { + if (column_ptr.IsValid()) { auto new_prop = vertex->add_properties(); new_prop->mutable_key()->set_name(prop_names[j]); - set_any_to_common_value(column_ptr->get(vids[i]), + set_any_to_common_value(column_ptr.Get(vids[i]), new_prop->mutable_value()); } } @@ -867,14 +870,14 @@ class SinkOp { for (size_t i = 0; i < labels_vec.size(); ++i) { prop_names.emplace_back(schema.get_vertex_property_names(labels_vec[i])); } + using prop_getter_t = typename GRAPH_INTERFACE::untyped_prop_getter_t; // get all properties - std::vector>> column_ptrs( - labels_vec.size()); + std::vector> column_ptrs(labels_vec.size()); if (labels_vec.size() > 0) { for (size_t i = 0; i < labels_vec.size(); ++i) { for (size_t j = 0; j < prop_names[i].size(); ++j) { - column_ptrs[i].emplace_back( - graph.GetRefColumnBase(labels_vec[i], prop_names[i][j])); + column_ptrs[i].emplace_back(graph.GetUntypedVertexPropertyGetter( + labels_vec[i], prop_names[i][j])); } } } @@ -916,10 +919,10 @@ class SinkOp { for (size_t j = 0; j < cur_column_ptrs.size(); ++j) { auto& column_ptr = cur_column_ptrs[j]; // Only set non-none properties. - if (column_ptr) { + if (column_ptr.IsValid()) { auto new_prop = mutable_vertex->add_properties(); new_prop->mutable_key()->set_name(prop_names[label_vec_ind][j]); - set_any_to_common_value(column_ptr->get(vertices_vec[i]), + set_any_to_common_value(column_ptr.Get(vertices_vec[i]), new_prop->mutable_value()); } } @@ -954,10 +957,10 @@ class SinkOp { for (size_t j = 0; j < cur_column_ptrs.size(); ++j) { auto& column_ptr = cur_column_ptrs[j]; // Only set non-none properties. - if (column_ptr) { + if (column_ptr.IsValid()) { auto new_prop = mutable_vertex->add_properties(); new_prop->mutable_key()->set_name(prop_names[label_vec_ind][j]); - set_any_to_common_value(column_ptr->get(vertices_vec[i]), + set_any_to_common_value(column_ptr.Get(vertices_vec[i]), new_prop->mutable_value()); } } diff --git a/flex/engines/hqps_db/core/sync_engine.h b/flex/engines/hqps_db/core/sync_engine.h index 5e17cf980440..96b7c58bcbb8 100644 --- a/flex/engines/hqps_db/core/sync_engine.h +++ b/flex/engines/hqps_db/core/sync_engine.h @@ -44,7 +44,6 @@ template class SyncEngine : public BaseEngine { using label_id_t = typename GRAPH_INTERFACE::label_id_t; using vertex_id_t = typename GRAPH_INTERFACE::vertex_id_t; - using gid_t = typename GRAPH_INTERFACE::gid_t; using default_vertex_set_t = DefaultRowVertexSet; using two_label_set_t = TwoLabelVertexSet; @@ -850,7 +849,8 @@ class SyncEngine : public BaseEngine { for (auto j = cur_begin; j < limit; ++j) { auto vid = vertices[j]; if (bitset.get_bit(j)) { - if (std::apply(expr, prop_getter_tuple[0].get_view(vid))) { + if (std::apply(expr, get_view_from_prop_getters(prop_getter_tuple[0], + vid))) { new_bitset.set_bit(cur); if (cur < j) { vertices[cur++] = vid; @@ -859,7 +859,8 @@ class SyncEngine : public BaseEngine { } } } else { - if (std::apply(expr, prop_getter_tuple[1].get_view(vid))) { + if (std::apply(expr, get_view_from_prop_getters(prop_getter_tuple[1], + vid))) { if (cur < j) { vertices[cur++] = vid; } else { @@ -898,9 +899,8 @@ class SyncEngine : public BaseEngine { auto& head = ctx.GetMutableHead(); auto label = head.GetLabel(); - auto prop_getter_tuple = - std::array{get_prop_getter_from_selectors(graph, label, selectors)}; - // TODO: implement + auto prop_getter_tuple = std::array{ + get_prop_getters_from_selectors_single_label(graph, label, selectors)}; SelectRowVertexSetImpl(ctx, head, prop_getter_tuple, expr, std::make_index_sequence()); diff --git a/flex/engines/hqps_db/core/utils/graph_utils.h b/flex/engines/hqps_db/core/utils/graph_utils.h new file mode 100644 index 000000000000..3399d4fd72fb --- /dev/null +++ b/flex/engines/hqps_db/core/utils/graph_utils.h @@ -0,0 +1,99 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLEX_ENGINES_HQPS_DB_CORE_UTILS_GRAPH_UTILS_H_ +#define FLEX_ENGINES_HQPS_DB_CORE_UTILS_GRAPH_UTILS_H_ + +#include + +#include "flex/engines/hqps_db/core/params.h" +#include "flex/engines/hqps_db/database/mutable_csr_interface_v2.h" + +namespace gs { + +// Helper functions for hqps engine + +template +std::pair, + std::vector> +get_other_vertices_in_batch( + const GRAPH_INTERFACE& graph, label_t src_label_id, label_t dst_label_id, + label_t edge_label, + const std::vector& vertices, + Direction direction) { + auto nbr_list = graph.GetOtherVertices(src_label_id, dst_label_id, edge_label, + vertices, direction); + CHECK(nbr_list.size() == vertices.size()); + std::vector other_vertices; + std::vector offsets; + offsets.push_back(0); + for (size_t i = 0; i < vertices.size(); ++i) { + auto list = nbr_list.get(i); + for (auto nbr : list) { + other_vertices.push_back(nbr.neighbor()); + } + offsets.push_back(other_vertices.size()); + } + return std::make_pair(other_vertices, offsets); +} + +template ::value>::type* = + nullptr> +inline auto get_view_from_prop_getters_impl(const PropGetterTuple& tuple, + vid_t vid, + std::index_sequence) { + static_assert(std::tuple_size_v == sizeof...(Is), + "The number of PropGetter should be equal to the number of Is"); + return std::make_tuple(std::get(tuple).get_view(vid)...); +} + +template +inline auto get_view_from_prop_getters(const std::tuple& tuple, + vid_t vid) { + return get_view_from_prop_getters_impl( + tuple, vid, std::make_index_sequence()); +} + +template +std::vector> get_vertex_props_from_vids_impl( + const GRAPH_INTERFACE& graph, label_t vertex_label, + const std::vector& vids, + const std::array>>& + prop_names, + std::index_sequence) { + std::vector> res; + auto prop_getters = std::make_tuple(graph.template GetVertexPropertyGetter( + vertex_label, std::get(prop_names))...); + for (auto vid : vids) { + res.push_back(get_view_from_prop_getters(prop_getters, vid)); + } + return res; +} + +template +std::vector> get_vertex_props_from_vids( + const GRAPH_INTERFACE& graph, label_t vertex_label, + const std::vector& vids, + const std::array>>& + prop_names) { + return get_vertex_props_from_vids_impl( + graph, vertex_label, vids, prop_names, + std::make_index_sequence()); +} + +} // namespace gs + +#endif // FLEX_ENGINES_HQPS_DB_CORE_UTILS_GRAPH_UTILS_H_ \ No newline at end of file diff --git a/flex/engines/hqps_db/core/utils/keyed.h b/flex/engines/hqps_db/core/utils/keyed.h index 98a447d6fdf4..375d6f41efd5 100644 --- a/flex/engines/hqps_db/core/utils/keyed.h +++ b/flex/engines/hqps_db/core/utils/keyed.h @@ -17,7 +17,7 @@ #define ENGINES_HQPS_ENGINE_KEYED_UTILS_H_ #include "flex/engines/hqps_db/core/utils/props.h" -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" +#include "flex/engines/hqps_db/database/mutable_csr_interface_v2.h" #include "flex/engines/hqps_db/structures/collection.h" #include "flex/engines/hqps_db/structures/multi_edge_set/adj_edge_set.h" #include "flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h" @@ -245,7 +245,7 @@ struct KeyedAggT, AggFunc::COUNT, using index_ele_t = typename RowVertexSet::index_ele_tuple_t; using prop_getter_t = RowVertexSetPropGetter< - tag_id, gs::mutable_csr_graph_impl::SinglePropGetter, index_ele_t>; + tag_id, gs::mutable_csr_graph_impl::PropertyGetter, index_ele_t>; // build a counter array. using aggregate_res_builder_t = PropCountBuilder; @@ -318,7 +318,7 @@ struct KeyedAggT, AggFunc::COUNT, typename TwoLabelVertexSet::index_ele_tuple_t; // build a counter array. using prop_getter_t = TwoLabelVertexSetImplPropGetter< - tag_id, gs::mutable_csr_graph_impl::SinglePropGetter, index_ele_t>; + tag_id, gs::mutable_csr_graph_impl::PropertyGetter, index_ele_t>; using aggregate_res_builder_t = PropCountBuilder; static aggregate_res_builder_t create_agg_builder( @@ -373,7 +373,7 @@ struct KeyedAggT, AggFunc::COUNT, using index_ele_t = typename GeneralVertexSet::index_ele_tuple_t; using prop_getter_t = GeneralVertexSetPropGetter< - tag_id, gs::mutable_csr_graph_impl::SinglePropGetter, index_ele_t>; + tag_id, gs::mutable_csr_graph_impl::PropertyGetter, index_ele_t>; // build a counter array. using aggregate_res_builder_t = PropCountBuilder; @@ -427,7 +427,7 @@ struct KeyedAggT, AggFunc::SUM, using index_ele_t = typename TwoLabelVertexSet::index_ele_tuple_t; using prop_getter_t = TwoLabelVertexSetImplPropGetter< - tag_id, gs::mutable_csr_graph_impl::SinglePropGetter, index_ele_t>; + tag_id, gs::mutable_csr_graph_impl::PropertyGetter, index_ele_t>; using aggregate_res_builder_t = SumBuilder; static aggregate_res_builder_t create_agg_builder( @@ -567,7 +567,7 @@ struct KeyedAggT, AggFunc::MIN, using index_ele_t = typename RowVertexSet::index_ele_tuple_t; using prop_getter_t = RowVertexSetPropGetter< - tag_id, gs::mutable_csr_graph_impl::SinglePropGetter, index_ele_t>; + tag_id, gs::mutable_csr_graph_impl::PropertyGetter, index_ele_t>; using aggregate_res_builder_t = MinBuilder; static aggregate_res_builder_t create_agg_builder( @@ -586,7 +586,7 @@ struct KeyedAggT, AggFunc::MIN, using index_ele_t = typename TwoLabelVertexSet::index_ele_tuple_t; using prop_getter_t = TwoLabelVertexSetImplPropGetter< - tag_id, gs::mutable_csr_graph_impl::SinglePropGetter, index_ele_t>; + tag_id, gs::mutable_csr_graph_impl::PropertyGetter, index_ele_t>; using aggregate_res_builder_t = MinBuilder; static aggregate_res_builder_t create_agg_builder( @@ -620,7 +620,7 @@ struct KeyedAggT, AggFunc::MAX, using index_ele_t = typename RowVertexSet::index_ele_tuple_t; using prop_getter_t = RowVertexSetPropGetter< - tag_id, gs::mutable_csr_graph_impl::SinglePropGetter, index_ele_t>; + tag_id, gs::mutable_csr_graph_impl::PropertyGetter, index_ele_t>; using aggregate_res_builder_t = MaxBuilder; static aggregate_res_builder_t create_agg_builder( @@ -639,7 +639,7 @@ struct KeyedAggT, AggFunc::MAX, using index_ele_t = typename TwoLabelVertexSet::index_ele_tuple_t; using prop_getter_t = TwoLabelVertexSetImplPropGetter< - tag_id, gs::mutable_csr_graph_impl::SinglePropGetter, index_ele_t>; + tag_id, gs::mutable_csr_graph_impl::PropertyGetter, index_ele_t>; using aggregate_res_builder_t = MaxBuilder; static aggregate_res_builder_t create_agg_builder( diff --git a/flex/engines/hqps_db/core/utils/props.h b/flex/engines/hqps_db/core/utils/props.h index 2ce201f621d8..d190ddf53bc2 100644 --- a/flex/engines/hqps_db/core/utils/props.h +++ b/flex/engines/hqps_db/core/utils/props.h @@ -58,96 +58,90 @@ class TwoLabelVertexSetImpl; template class GeneralVertexSet; -template -struct MultiPropGetterT; - -template -struct MultiPropGetterT> {}; - template class Collection; -template -static auto get_prop_getter_from_named_property( - const GRAPH_INTERFACE& graph, const LabelT& label, - const std::tuple...>& named_property) { - std::array prop_names; - int i = 0; - std::apply( - [&prop_names, &i](auto&... named_prop) { - ((prop_names[i++] = named_prop.name), ...); - }, - named_property); - return graph.template GetMultiPropGetter(label, prop_names); -} - -//// Get one property getter for one label -template -static auto get_prop_getter_from_selectors( - const GRAPH_INTERFACE& graph, const LabelT& label, - const std::tuple...>& selectors) { - std::array prop_names; - int i = 0; - std::apply( - [&prop_names, &i](auto&... named_prop) { - ((prop_names[i++] = named_prop.prop_name_), ...); - }, - selectors); - return graph.template GetMultiPropGetter(label, prop_names); -} /// get single property getter for one label template static auto get_single_prop_getter_from_selector( const GRAPH_INTERFACE& graph, const LabelT& label, const PropertySelector& selector) { auto prop_name = selector.prop_name_; - return graph.template GetSinglePropGetter(label, prop_name); + return graph.template GetVertexPropertyGetter(label, prop_name); +} + +//// Get one property getter for one label +template +static auto get_prop_getter_from_selectors_impl( + const GRAPH_INTERFACE& graph, const LabelT& label, + const std::tuple...>& selectors, + std::index_sequence) { + return std::make_tuple(get_single_prop_getter_from_selector( + graph, label, std::get(selectors))...); } // Get prop getters from Selector. template -static auto get_prop_getters_from_selectors_impl( +static auto get_prop_getters_from_selectors_impl_label_array( const GRAPH_INTERFACE& graph, const std::array& labels, std::tuple selectors, std::index_sequence) { - using prop_getter_t = typename GRAPH_INTERFACE::template multi_prop_getter_t< - typename SELECTOR::prop_t...>; - std::array prop_getter_array{ - get_prop_getter_from_selectors(graph, labels[Is], selectors)...}; + using prop_getter_tuple_t = + std::tuple...>; + std::array prop_getter_array{ + get_prop_getter_from_selectors_impl( + graph, labels[Is], selectors, + std::index_sequence_for{})...}; return prop_getter_array; } // Get prop getters from Selector, with label vector. -template +template static auto get_prop_getters_from_selectors_impl_label_vec( const GRAPH_INTERFACE& graph, const std::vector& labels, std::tuple selectors) { - using prop_getter_t = typename GRAPH_INTERFACE::template multi_prop_getter_t< - typename SELECTOR::prop_t...>; - std::vector prop_getter_array; + using prop_getter_tuple_t = + std::tuple...>; + std::vector prop_getter_array; + auto index_seq = std::index_sequence_for{}; for (size_t i = 0; i < labels.size(); ++i) { - prop_getter_array.emplace_back( - get_prop_getter_from_selectors(graph, labels[i], selectors)); + prop_getter_array.emplace_back(get_prop_getter_from_selectors_impl( + graph, labels[i], selectors, index_seq)); }; return prop_getter_array; } +///////////////////////// prop getter for vertex set////////////////////// + +// Get Property Getters from selectors, with label array. template static auto get_prop_getters_from_selectors( const GRAPH_INTERFACE& graph, const std::array& labels, - std::tuple named_property) { - return get_prop_getters_from_selectors_impl( - graph, labels, named_property, std::make_index_sequence{}); + std::tuple selectors) { + return get_prop_getters_from_selectors_impl_label_array( + graph, labels, selectors, std::make_index_sequence{}); } +// Get Property Getters from selectors, with label vector. template -static auto get_prop_getters_from_selectors( - const GRAPH_INTERFACE& graph, const std::vector& labels, - std::tuple named_property) { +static auto get_prop_getters_from_selectors(const GRAPH_INTERFACE& graph, + const std::vector& labels, + std::tuple selectors) { return get_prop_getters_from_selectors_impl_label_vec(graph, labels, - named_property); + selectors); +} + +// Get Property Getters from selectors, with only one label +template +static auto get_prop_getters_from_selectors_single_label( + const GRAPH_INTERFACE& graph, const LabelT& label, + const std::tuple...>& selectors) { + return get_prop_getter_from_selectors_impl( + graph, label, selectors, std::index_sequence_for{}); } ///////////////////////// prop getter for vertex set @@ -853,13 +847,13 @@ static auto create_prop_getter_impl( const RowVertexSetImpl& set, const GRAPH_INTERFACE& graph, const std::string& prop_name) { using prop_getter_t = - typename GRAPH_INTERFACE::template single_prop_getter_t; - // const std::array& labels = set.GetLabels(); + typename GRAPH_INTERFACE::template prop_getter_t; auto label = set.GetLabel(); VLOG(10) << "getting getter for " << prop_name << " for label " << gs::to_string(label); - auto getter = graph.template GetSinglePropGetter(label, prop_name); + auto getter = + graph.template GetVertexPropertyGetter(label, prop_name); return RowVertexSetPropGetter< tag_id, prop_getter_t, typename RowVertexSetImpl::index_ele_tuple_t>( @@ -901,14 +895,14 @@ static auto create_prop_getter_impl( const TwoLabelVertexSetImpl& set, const GRAPH_INTERFACE& graph, const std::string& prop_name) { using prop_getter_t = - typename GRAPH_INTERFACE::template single_prop_getter_t; + typename GRAPH_INTERFACE::template prop_getter_t; auto& labels = set.GetLabels(); std::array names{prop_name}; VLOG(10) << "Getting prop labels for " << prop_name << " for labels " << std::to_string(labels[0]) << ", " << std::to_string(labels[1]); std::array prop_getter{ - graph.template GetSinglePropGetter(labels[0], prop_name), - graph.template GetSinglePropGetter(labels[1], prop_name)}; + graph.template GetVertexPropertyGetter(labels[0], prop_name), + graph.template GetVertexPropertyGetter(labels[1], prop_name)}; return TwoLabelVertexSetImplPropGetter< tag_id, prop_getter_t, @@ -926,11 +920,11 @@ static auto create_prop_getter_impl( const KeyedRowVertexSetImpl& set, const GRAPH_INTERFACE& graph, const std::string& prop_name) { using prop_getter_t = - typename GRAPH_INTERFACE::template single_prop_getter_t; - // const std::array& labels = set.GetLabels(); + typename GRAPH_INTERFACE::template prop_getter_t; auto label = set.GetLabel(); - auto getter = graph.template GetSinglePropGetter(label, prop_name); + auto getter = + graph.template GetVertexPropertyGetter(label, prop_name); return KeyedRowVertexSetPropGetter< tag_id, prop_getter_t, typename KeyedRowVertexSetImpl& set, const GRAPH_INTERFACE& graph, const std::string& prop_name) { using prop_getter_t = - typename GRAPH_INTERFACE::template single_prop_getter_t; - // const std::array& labels = set.GetLabels(); + typename GRAPH_INTERFACE::template prop_getter_t; auto labels = set.GetLabels(); std::vector prop_getters; for (size_t i = 0; i < labels.size(); ++i) { prop_getters.emplace_back( - graph.template GetSinglePropGetter(labels[i], prop_name)); + graph.template GetVertexPropertyGetter(labels[i], prop_name)); } return GeneralVertexSetPropGetter< diff --git a/flex/engines/hqps_db/database/adj_list.h b/flex/engines/hqps_db/database/adj_list.h index c6ee0e25948b..00d81b691c0f 100644 --- a/flex/engines/hqps_db/database/adj_list.h +++ b/flex/engines/hqps_db/database/adj_list.h @@ -68,14 +68,14 @@ class SubGraph { virtual std::vector GetPropNames() const; }; -template -class SinglePropGetter { - public: - using value_type = T; - static constexpr size_t prop_num = 1; +// template +// class SinglePropGetter { +// public: +// using value_type = T; +// static constexpr size_t prop_num = 1; - virtual value_type get_view(vid_t vid) const = 0; -}; +// virtual value_type get_view(vid_t vid) const = 0; +// }; template class MultiPropGetter { @@ -317,29 +317,29 @@ class SubGraph { Direction dir_; }; -template -class SinglePropGetter { - public: - using value_type = T; - static constexpr size_t prop_num = 1; - SinglePropGetter() {} - SinglePropGetter(std::shared_ptr> c) : column(c) {} - - inline value_type get_view(vid_t vid) const { - if (vid == NONE || column == nullptr) { - return NullRecordCreator::GetNull(); - } - return column->get_view(vid); - } - - inline SinglePropGetter& operator=(const SinglePropGetter& d) { - column = d.column; - return *this; - } - - private: - std::shared_ptr> column; -}; +// template +// class SinglePropGetter { +// public: +// using value_type = T; +// static constexpr size_t prop_num = 1; +// SinglePropGetter() {} +// SinglePropGetter(std::shared_ptr> c) : column(c) {} + +// inline value_type get_view(vid_t vid) const { +// if (vid == NONE || column == nullptr) { +// return NullRecordCreator::GetNull(); +// } +// return column->get_view(vid); +// } + +// inline SinglePropGetter& operator=(const SinglePropGetter& d) { +// column = d.column; +// return *this; +// } + +// private: +// std::shared_ptr> column; +// }; // Property Getter hold the handle of the property column. diff --git a/flex/engines/hqps_db/database/adj_list_v2.h b/flex/engines/hqps_db/database/adj_list_v2.h new file mode 100644 index 000000000000..9b7fd41048a2 --- /dev/null +++ b/flex/engines/hqps_db/database/adj_list_v2.h @@ -0,0 +1,301 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef ENGINES_HQPS_DATABASE_ADJ_LIST_H_ +#define ENGINES_HQPS_DATABASE_ADJ_LIST_H_ + +#include +#include +#include +#include + +#include "flex/engines/hqps_db/core/null_record.h" +#include "flex/storages/rt_mutable_graph/csr/mutable_csr.h" +#include "flex/storages/rt_mutable_graph/types.h" +#include "flex/utils/property/types.h" + +namespace gs { + +namespace mutable_csr_graph_impl { + +/** + * @brief Adj is a simple struct to store a edge from a view of csr. Here we use + * a variadic template to store multiple properties of the edge. + * @tparam T The type of the property. + */ +template +class Adj { + public: + Adj() = default; + ~Adj() = default; + + Adj(const Adj& other) : neighbor_(other.neighbor_), prop_(other.prop_) {} + + Adj(Adj&& other) + : neighbor_(other.neighbor_), prop_(std::move(other.prop_)) {} + + inline Adj& operator=(const Adj& from) { + this->neighbor_ = from.neighbor_; + this->prop_ = from.prop_; + return *this; + } + + vid_t neighbor() const { return neighbor_; } + const std::tuple& properties() const { return prop_; } + + vid_t neighbor_; + std::tuple prop_; +}; + +///////////////////////////// AdjList ////////////////////////////// + +/** + * @brief AdjList is a simple class to store a list of edges from a view of csr. + * Here we use a variadic template to store multiple properties of the edge. + * @tparam T The type of the property. + */ +template +class AdjList { + using nbr_t = MutableNbr; + class Iterator { + public: + using edge_property_t = std::tuple; + Iterator() + : cur_(), + begin0_(nullptr), + begin1_(nullptr), + end0_(nullptr), + end1_(nullptr) {} + Iterator(const nbr_t* begin0, const nbr_t* end0, const nbr_t* begin1, + const nbr_t* end1) + : cur_(), begin0_(begin0), begin1_(begin1), end0_(end0), end1_(end1) { + probe_for_next(); + } + + void probe_for_next() { + if (begin0_ != NULL && begin0_ != end0_) { + cur_.neighbor_ = begin0_->neighbor; + std::get<0>(cur_.prop_) = begin0_->data; + return; + } + if (begin1_ != end1_ && begin1_ != NULL) { + cur_.neighbor_ = begin1_->neighbor; + std::get<0>(cur_.prop_) = begin1_->data; + return; + } + } + + bool valid() const { return begin0_ != end0_ || begin1_ != end1_; } + const Adj& operator*() const { return cur_; } + const Adj* operator->() const { return &cur_; } + + vid_t neighbor() const { return cur_.neighbor(); } + const std::tuple& properties() const { return cur_.properties(); } + + std::string to_string() const { + std::stringstream ss; + ss << "(neighbor: " << cur_.neighbor_ + << ", prop: " << std::get<0>(cur_.prop_) << ")"; + return ss.str(); + } + + Iterator& operator++() { + if (begin0_ < end0_) { + ++begin0_; + } else if (begin1_ < end1_) { + ++begin1_; + } else { + return *this; + } + probe_for_next(); + return *this; + } + Iterator operator++(int) { + Iterator tmp = *this; + ++(*this); + return tmp; + } + + bool operator==(const Iterator& rhs) const { + return begin0_ == rhs.begin0_ && begin1_ == rhs.begin1_; + } + + bool operator!=(const Iterator& rhs) const { + return begin1_ != rhs.begin1_ || begin0_ != rhs.begin0_; + } + inline Iterator& operator=(const Iterator& from) { + this->cur_ = from.cur_; + this->begin0_ = from.begin0_; + this->end0_ = from.end0_; + this->begin1_ = from.begin1_; + this->end1_ = from.end1_; + return *this; + } + + private: + Adj cur_; + const nbr_t *begin0_, *begin1_; + const nbr_t *end0_, *end1_; + }; + + public: + using slice_t = MutableNbrSlice; + using iterator = Iterator; + AdjList() = default; + // copy constructor + AdjList(const AdjList& adj_list) + : slice0_(adj_list.slice0_), slice1_(adj_list.slice1_) {} + // with single slice provided. + AdjList(const slice_t& slice0) : slice0_(slice0), slice1_() {} + AdjList(const slice_t& slice0, const slice_t& slice1) + : slice0_(slice0), slice1_(slice1) {} + + AdjList(AdjList&& adj_list) + : slice0_(std::move(adj_list.slice0_)), + slice1_(std::move(adj_list.slice1_)) {} + + AdjList(AdjList& adj_list) + : slice0_(adj_list.slice0_), slice1_(adj_list.slice1_) {} + + Iterator begin() const { + return Iterator(slice0_.begin(), slice0_.end(), slice1_.begin(), + slice1_.end()); + } + + Iterator end() const { + return Iterator(slice0_.end(), slice0_.end(), slice1_.end(), slice1_.end()); + } + size_t size() const { return slice0_.size() + slice1_.size(); } + + AdjList& operator=(const AdjList& other) { + slice0_ = other.slice0_; + slice1_ = other.slice1_; + return *this; + } + + const slice_t& slice0() const { return slice0_; } + const slice_t& slice1() const { return slice1_; } + + private: + slice_t slice0_, slice1_; +}; + +/** + * @brief Stores a list of AdjLists, each of which represents the edges of a + * vertex. + * @tparam T The type of the property. + */ +template +class AdjListArray { + public: + using slice_t = MutableNbrSlice; + AdjListArray(std::vector>&& slices) + : slices_(std::move(slices)) {} + + void resize(size_t new_size) { slices_.resize(new_size); } + + void set(size_t i, const AdjList& slice) { + slices_[i] = std::make_pair(slice.slice0(), slice.slice1()); + } + + AdjListArray(AdjListArray&& adj_list) + : slices_(std::move(adj_list.slices_)) {} + + size_t size() const { return slices_.size(); } + + AdjList get(size_t i) const { + return AdjList(slices_[i].first, slices_[i].second); + } + + void swap(AdjListArray& adj_list) { this->slices_.swap(adj_list.slices_); } + + private: + std::vector> slices_; +}; + +////////////////////////Util functions/////////////////////////// + +template +void iterate_and_set_slices( + std::vector, MutableNbrSlice>>& + slices, + const CsrBase* csr, const std::vector& vids) { + static_assert(I < 2, "I should be 0 or 1"); + // The CsrBase has multiple derived classes, and we need to cast it to the + // correct one. + if (csr) { + using typed_csr_base_t = MutableCsr; + using single_typed_csr_base_t = SingleMutableCsr; + const typed_csr_base_t* casted_csr = + dynamic_cast(csr); + if (casted_csr) { + for (size_t i = 0; i < vids.size(); ++i) { + std::get(slices[i]) = casted_csr->get_edges(vids[i]); + } + } else { + LOG(WARNING) << "cast to MutableCSR failed, try single csr"; + const single_typed_csr_base_t* casted_single_csr = + dynamic_cast(csr); + if (casted_single_csr) { + for (size_t i = 0; i < vids.size(); ++i) { + std::get(slices[i]) = casted_single_csr->get_edges(vids[i]); + } + } else { + LOG(WARNING) << "Cannot cast to MutableCSR or SingleMutableCSR"; + } + } + } else { + LOG(WARNING) << "No such edge, since csr is null"; + } +} + +// A helper function to create an AdjListArray<> from two MutableCsr. +// +template +void iterate_and_set_iterators( + std::vector, + std::shared_ptr>>& slices, + const CsrBase* csr, const std::vector& vids) { + static_assert(I < 2, "I should be 0 or 1"); + if (csr) { + for (size_t i = 0; i < vids.size(); ++i) { + std::get<0>(slices[i]) = csr->edge_iter(vids[i]); + } + } else { + LOG(WARNING) << "No such edge, since csr is null"; + } +} + +template +AdjListArray create_adj_list_array(const CsrBase* csr0, + const CsrBase* csr1, + const std::vector& vids) { + using slice_t = MutableNbrSlice; + std::vector> slices; + slices.resize(vids.size()); + iterate_and_set_slices<0, EDATA_T>(slices, csr0, vids); + iterate_and_set_slices<1, EDATA_T>(slices, csr1, vids); + return AdjListArray(std::move(slices)); +} + +template +AdjListArray create_adj_list_array(const CsrBase* csr, + const std::vector& vids) { + return create_adj_list_array(csr, nullptr, vids); +} + +} // namespace mutable_csr_graph_impl +} // namespace gs + +#endif // ENGINES_HQPS_DATABASE_ADJ_LIST_H_ \ No newline at end of file diff --git a/flex/engines/hqps_db/database/i_graph.h b/flex/engines/hqps_db/database/i_graph.h deleted file mode 100644 index f5b4133b1fd3..000000000000 --- a/flex/engines/hqps_db/database/i_graph.h +++ /dev/null @@ -1,254 +0,0 @@ -/** Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_ -#define ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_ - -#include - -#include "flex/engines/graph_db/database/graph_db.h" -#include "flex/engines/graph_db/database/graph_db_session.h" -#include "flex/engines/hqps_db/core/null_record.h" -#include "flex/engines/hqps_db/core/params.h" - -#include "flex/engines/hqps_db/database/adj_list.h" -#include "grape/utils/bitset.h" - -#include "grape/util.h" - -namespace gs { - -template -bool exists_nullptr_in_tuple(const T& t) { - return std::apply([](auto&&... args) { return ((args == nullptr) || ...); }, - t); -} - -template -void get_tuple_from_column_tuple( - size_t index, std::tuple& t, - const std::tuple>...>& columns) { - using cur_ele_t = std::tuple_element_t>; - auto ptr = std::get(columns); - if (ptr) { - std::get(t) = ptr->get_view(index); - } else { - std::get(t) = NullRecordCreator::GetNull(); - } - - if constexpr (I + 1 < sizeof...(T)) { - get_tuple_from_column_tuple(index, t, columns); - } -} - -template -void get_tuple_from_column_tuple(size_t index, std::tuple& t, - const std::tuple& columns) { - auto ptr = std::get(columns); - using cur_ele_t = std::tuple_element_t>; - if (ptr) { - std::get(t) = ptr->get_view(index); - } else { - std::get(t) = NullRecordCreator::GetNull(); - } - - if constexpr (I + 1 < sizeof...(T)) { - get_tuple_from_column_tuple(index, t, columns); - } -} - -/** - * @brief MutableCSRInterface is the interface for the mutable CSR graph - * implementation. - * - */ -class MutableCSRInterface { - public: - using vertex_id_t = vid_t; - using gid_t = uint64_t; - using label_id_t = uint8_t; - - using nbr_list_array_t = mutable_csr_graph_impl::NbrListArray; - - template - using adj_list_array_t = mutable_csr_graph_impl::AdjListArray; - - template - using adj_list_t = mutable_csr_graph_impl::AdjList; - - template - using adj_t = mutable_csr_graph_impl::Adj; - - using nbr_t = mutable_csr_graph_impl::Nbr; - - using nbr_list_t = mutable_csr_graph_impl::NbrList; - - template - using single_prop_getter_t = mutable_csr_graph_impl::SinglePropGetter; - - template - using multi_prop_getter_t = mutable_csr_graph_impl::MultiPropGetter; - - using sub_graph_t = mutable_csr_graph_impl::SubGraph; - - /** - * @brief Get the Vertex Label id - * - * @param label - * @return label_id_t - */ - virtual label_id_t GetVertexLabelId(const std::string& label) const = 0; - - /** - * @brief Get the Edge Label id - * - * @param label - * @return label_id_t - */ - virtual label_id_t GetEdgeLabelId(const std::string& label) const = 0; - - /** - * @brief Scans all vertices with the given label and calls the - * given function on each vertex for filtering. - * @tparam FUNC_T - * @tparam SELECTOR - * @param label_id - * @param selectors The Property selectors. The selected properties will be - * fed to the filtering function - * @param func The lambda function for filtering. - */ - template - void ScanVertices(const label_id_t& label_id, - const std::tuple& selectors, - const FUNC_T& func) const; - - /** - * @brief ScanVertices scans all vertices with the given label with give - * original id. - * @param label_id The label id. - * @param oid The original id. - * @param vid The result internal id. - */ - template - bool ScanVerticesWithOid(const label_id_t& label_id, OID_T oid, - vertex_id_t& vid) const; - - /** - * @brief GetVertexProps gets the properties of single label vertices. - * @tparam T The property types. - * @param label - * @param vids - * @param prop_names - */ - template - std::vector> GetVertexPropsFromVid( - const label_id_t& label_id, const std::vector& vids, - const std::array>>& - prop_names) const { - // auto label_id = db_session_.schema().get_vertex_label_id(label); - CHECK(label_id < db_session_.schema().vertex_label_num()); - std::tuple>...> columns; - get_tuple_column_from_graph(label_id, prop_names, columns); - std::vector> props(vids.size()); - fetch_properties_in_column(vids, props, columns); - return props; - } - - /** - * @brief GetVertexProps gets the properties of vertices from multiple labels. - * @tparam T - * @param vids The vertex ids. - * @param label_ids The label ids. - * @param vid_inds The indices of the vertex ids in the input vids. - * @param prop_names The property names. - */ - template - std::vector> GetVertexPropsFromVid( - const std::vector& vids, - const std::vector& label_ids, - const std::vector>& vid_inds, - const std::array>>& - prop_names) const; - - /** - * @brief GetSubGraph gets the subgraph with the given label and edge label. - * @param src_label_id The source label id. - * @param dst_label_id The destination label id. - * @param edge_label_id The edge label id. - * @param direction_str The direction string. - * @param prop_names The property names. - */ - std::vector> - GetSubGraph(const label_id_t src_label_id, const label_id_t dst_label_id, - const label_id_t edge_label_id, const std::string& direction_str, - const std::vector& prop_names) const; - - /** - * @brief GetEdges gets the edges with the given label and edge label, and - * with the starting vertex internal ids. - * When the direction is "out", the edges are from the source label to the - * destination label, and vice versa when the direction is "in". When the - * direction is "both", the src and dst labels SHOULD be the same. - * @tparam T The property types. - * @param src_label_id The source label id. - * @param dst_label_id The destination label id. - * @param edge_label_id The edge label id. - * @param vids The starting vertex internal ids. - * @param direction_str The direction string. - * @param limit The limit of the edges. - * @param prop_names The property names. - */ - template - mutable_csr_graph_impl::AdjListArray GetEdges( - const label_id_t& src_label_id, const label_id_t& dst_label_id, - const label_id_t& edge_label_id, const std::vector& vids, - const std::string& direction_str, size_t limit, - const std::array>>& - prop_names) const; - - /** - * @brief Get vertices on the other side of edges, via the given edge label - * and the starting vertex internal ids. - * When the direction is "out", the vertices are on the destination label side - * of the edges, and vice versa when the direction is "in". When the direction - * is "both", the src and dst labels SHOULD be the same. - * @param src_label_id The source label id. - * @param dst_label_id The destination label id. - * @param edge_label_id The edge label id. - * @param vids The starting vertex internal ids. - * @param direction_str The direction string. - * @param limit The limit of the vertices. - */ - mutable_csr_graph_impl::NbrListArray GetOtherVertices( - const label_id_t& src_label_id, const label_id_t& dst_label_id, - const label_id_t& edge_label_id, const std::vector& vids, - const std::string& direction_str, size_t limit) const; - - template - mutable_csr_graph_impl::MultiPropGetter GetMultiPropGetter( - const label_id_t& label_id, - const std::array& prop_names) const; - - template - mutable_csr_graph_impl::SinglePropGetter GetSinglePropGetter( - const label_id_t& label_id, const std::string& prop_name) const { - using column_t = std::shared_ptr>; - column_t column = GetTypedRefColumn(label_id, prop_name); - return mutable_csr_graph_impl::SinglePropGetter(std::move(column)); - } -}; - -} // namespace gs - -#endif // ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_ diff --git a/flex/engines/hqps_db/database/mutable_csr_interface.h b/flex/engines/hqps_db/database/mutable_csr_interface.h index 4d9547a0d3ff..e6ea6c4de274 100644 --- a/flex/engines/hqps_db/database/mutable_csr_interface.h +++ b/flex/engines/hqps_db/database/mutable_csr_interface.h @@ -75,8 +75,6 @@ void get_tuple_from_column_tuple(size_t index, std::tuple& t, */ class MutableCSRInterface { public: - const GraphDBSession& GetDBSession() const { return db_session_; } - using vertex_id_t = vid_t; using gid_t = uint64_t; using label_id_t = uint8_t; @@ -104,7 +102,7 @@ class MutableCSRInterface { using sub_graph_t = mutable_csr_graph_impl::SubGraph; - static constexpr bool is_grape = true; + const GraphDBSession& GetDBSession() const { return db_session_; } MutableCSRInterface(const MutableCSRInterface&) = delete; @@ -123,9 +121,6 @@ class MutableCSRInterface { * @return label_id_t */ label_id_t GetVertexLabelId(const std::string& label) const { - LOG(INFO) << "GetVertexLabelId: " << label; - LOG(INFO) << "label num: " << db_session_.schema().vertex_label_num(); - LOG(INFO) << "edge labelnum: " << db_session_.schema().edge_label_num(); return db_session_.schema().get_vertex_label_id(label); } @@ -861,6 +856,140 @@ class MutableCSRInterface { label, selectors, std::make_index_sequence()); } + /** + * @brief Scans all vertices with the given label and calls the + * given function on each vertex for filtering. + * @tparam FUNC_T + * @tparam SELECTOR + * @param label_id + * @param selectors The Property selectors. The selected properties will be + * fed to the filtering function + * @param func The lambda function for filtering. + * @param filter_null If true, then if null presents in the properties, will + * skip the vertex. + */ + template + void ScanVertices(const label_id_t& label_id, + const std::tuple& selectors, + const FUNC_T& func, bool filter_null = false) const; + + /** + * @brief ScanVertices scans all vertices with the given label with give + * original id. + * @param label_id The label id. + * @param oid The original id. + * @param vid The result internal id. + */ + template + bool ScanVerticesWithOid(const label_id_t& label_id, OID_T oid, + vertex_id_t& vid) const; + + /** + * @brief GetVertexProps gets the properties of single label vertices. + * @tparam T The property types. + * @param label + * @param vids + * @param prop_names + */ + template + std::vector> GetVertexPropsFromVid( + const label_id_t& label_id, const std::vector& vids, + const std::array>>& + prop_names) const; + + /** + * @brief GetVertexProps gets the properties of vertices from multiple labels. + * @tparam T + * @param vids The vertex ids. + * @param label_ids The label ids. + * @param vid_inds The indices of the vertex ids in the input vids. + * @param prop_names The property names. + */ + template + std::vector> GetVertexPropsFromVid( + const std::vector& vids, + const std::vector& label_ids, + const std::vector>& vid_inds, + const std::array>>& + prop_names) const; + + /** + * @brief GetSubGraph gets the subgraph with the given label and edge label. + * @param src_label_id The source label id. + * @param dst_label_id The destination label id. + * @param edge_label_id The edge label id. + * @param direction_str The direction string. + * @param prop_names The property names. + */ + std::vector> + GetSubGraph(const label_id_t src_label_id, const label_id_t dst_label_id, + const label_id_t edge_label_id, const std::string& direction_str, + const std::vector& prop_names) const; + + /** + * @brief GetEdges gets the edges with the given label and edge label, and + * with the starting vertex internal ids. + * When the direction is "out", the edges are from the source label to the + * destination label, and vice versa when the direction is "in". When the + * direction is "both", the src and dst labels SHOULD be the same. + * @tparam T The property types. + * @param src_label_id The source label id. + * @param dst_label_id The destination label id. + * @param edge_label_id The edge label id. + * @param vids The starting vertex internal ids. + * @param direction_str The direction string. + * @param limit The limit of the edges. + * @param prop_names The property names. + */ + template + mutable_csr_graph_impl::AdjListArray GetEdges( + const label_id_t& src_label_id, const label_id_t& dst_label_id, + const label_id_t& edge_label_id, const std::vector& vids, + const std::string& direction_str, size_t limit, + const std::array>>& + prop_names) const; + + /** + * @brief Get vertices on the other side of edges, via the given edge label + * and the starting vertex internal ids. + * When the direction is "out", the vertices are on the destination label side + * of the edges, and vice versa when the direction is "in". When the direction + * is "both", the src and dst labels SHOULD be the same. + * @param src_label_id The source label id. + * @param dst_label_id The destination label id. + * @param edge_label_id The edge label id. + * @param vids The starting vertex internal ids. + * @param direction_str The direction string. + * @param limit The limit of the vertices. + */ + mutable_csr_graph_impl::NbrListArray GetOtherVertices( + const label_id_t& src_label_id, const label_id_t& dst_label_id, + const label_id_t& edge_label_id, const std::vector& vids, + const std::string& direction_str, size_t limit) const; + + /** + * @brief Great a prop getter which can get multiple properties for a vertex + * within one call. + * @tparam T The property types. + * @param label_id The label id. + * @param prop_names The property names. + */ + template + mutable_csr_graph_impl::MultiPropGetter GetMultiPropGetter( + const label_id_t& label_id, + const std::array& prop_names) const; + + /** + * @brief Get a prop getter which can get a single property for a vertex + * within one call. + * @tparam T The property type. + * @param label_id The label id. + * @param prop_name The property name. + */ + template + mutable_csr_graph_impl::SinglePropGetter GetSinglePropGetter( + const label_id_t& label_id, const std::string& prop_name) const; + private: std::shared_ptr create_ref_column( std::shared_ptr column) const { diff --git a/flex/engines/hqps_db/database/mutable_csr_interface_v2.h b/flex/engines/hqps_db/database/mutable_csr_interface_v2.h new file mode 100644 index 000000000000..277faaf8d6b1 --- /dev/null +++ b/flex/engines/hqps_db/database/mutable_csr_interface_v2.h @@ -0,0 +1,467 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_V2 +#define ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_V2 + +#include + +#include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/graph_db/database/graph_db_session.h" +#include "flex/engines/hqps_db/core/null_record.h" +#include "flex/engines/hqps_db/core/params.h" + +#include "flex/engines/hqps_db/database/adj_list_v2.h" +#include "flex/engines/hqps_db/database/nbr_list.h" +#include "flex/engines/hqps_db/database/sub_graph.h" +#include "grape/utils/bitset.h" + +#include "grape/util.h" + +namespace gs { + +namespace mutable_csr_graph_impl { +template +void get_tuple_from_column_tuple( + size_t index, std::tuple& t, + const std::tuple>...>& columns) { + using cur_ele_t = std::tuple_element_t>; + auto ptr = std::get(columns); + if (ptr) { + std::get(t) = ptr->get_view(index); + } else { + std::get(t) = NullRecordCreator::GetNull(); + } + + if constexpr (I + 1 < sizeof...(T)) { + get_tuple_from_column_tuple(index, t, columns); + } +} + +// Here we use alias to make the code more readable, and make it easier to +// change the implementation in the future. +template +struct PropertyGetter { + using value_type = T; + std::shared_ptr> column; + bool is_valid; + PropertyGetter(std::shared_ptr> column) + : column(column), is_valid((column != nullptr)) {} + + bool IsValid() const { return is_valid; } + + inline T Get(size_t index) const { return column->get_view(index); } + + inline T get_view(size_t index) const { return column->get_view(index); } +}; + +struct UntypedPropertyGetter { + using value_type = Any; + std::shared_ptr column; + bool is_valid; + UntypedPropertyGetter(std::shared_ptr column) + : column(column), is_valid((column != nullptr)) {} + + inline bool IsValid() const { return is_valid; } + + inline Any Get(size_t index) const { return column->get(index); } + + inline Any get_view(size_t index) const { return column->get(index); } +}; + +} // namespace mutable_csr_graph_impl + +/** + * @brief The MutableCSRInterface class is the implementation of IGraphInterface + * on rt_mutable_graph store, providing a read-only view. + */ +class MutableCSRInterface { + public: + using vertex_id_t = vid_t; + using label_id_t = uint8_t; + + template + using adj_list_array_t = mutable_csr_graph_impl::AdjListArray; + + using nbr_list_array_t = mutable_csr_graph_impl::NbrListArray; + + using sub_graph_t = mutable_csr_graph_impl::SubGraph; + + template + using prop_getter_t = mutable_csr_graph_impl::PropertyGetter; + + using untyped_prop_getter_t = mutable_csr_graph_impl::UntypedPropertyGetter; + + /////////////////////////Constructors//////////////////////////// + const GraphDBSession& GetDBSession() const { return db_session_; } + + MutableCSRInterface(const MutableCSRInterface&) = delete; + + MutableCSRInterface(MutableCSRInterface&& other) + : db_session_(other.db_session_) {} + + explicit MutableCSRInterface(const GraphDBSession& session) + : db_session_(session) {} + + //////////////////////////////Schema Related///////////////////// + + const Schema& schema() const { return db_session_.schema(); } + + //////////////////////////////Graph Metadata Related//////////// + inline size_t VertexLabelNum() const { + return db_session_.schema().vertex_label_num(); + } + + inline size_t EdgeLabelNum() const { + return db_session_.schema().edge_label_num(); + } + + inline size_t VertexNum() const { return get_vertex_num_impl(); } + + inline size_t VertexNum(const label_t& label) const { + return db_session_.graph().vertex_num(label); + } + + inline size_t EdgeNum() const { + size_t ret = 0; + for (label_t src_label = 0; src_label < VertexLabelNum(); ++src_label) { + for (label_t dst_label = 0; dst_label < VertexLabelNum(); ++dst_label) { + for (label_t edge_label = 0; edge_label < EdgeLabelNum(); + ++edge_label) { + if (ExitEdgeTriplet(src_label, dst_label, edge_label)) { + ret += get_edge_num_impl(src_label, dst_label, edge_label); + } + } + } + } + return ret; + } + + inline size_t EdgeNum(label_t src_label, label_t dst_label, + label_t edge_label) const { + return get_edge_num_impl(src_label, dst_label, edge_label); + } + + label_id_t GetVertexLabelId(const std::string& label) const { + return db_session_.schema().get_vertex_label_id(label); + } + + label_id_t GetEdgeLabelId(const std::string& label) const { + return db_session_.schema().get_edge_label_id(label); + } + + std::string GetVertexLabelName(label_t index) const { + return db_session_.schema().get_vertex_label_name(index); + } + + std::string GetEdgeLabelName(label_t index) const { + return db_session_.schema().get_edge_label_name(index); + } + + bool ExitVertexLabel(const std::string& label) const { + return db_session_.schema().contains_vertex_label(label); + } + + bool ExitEdgeLabel(const std::string& edge_label) const { + return db_session_.schema().contains_edge_label(edge_label); + } + + bool ExitEdgeTriplet(const label_t& src_label, const label_t& dst_label, + const label_t& edge_label) const { + return db_session_.schema().has_edge_label(src_label, dst_label, + edge_label); + } + + std::vector> GetEdgeTripletProperties( + const label_t& src_label, const label_t& dst_label, + const label_t& label) const { + const std::vector& props = + db_session_.schema().get_edge_properties(src_label, dst_label, label); + const std::vector& prop_names = + db_session_.schema().get_edge_property_names(src_label, dst_label, + label); + std::vector> res; + for (size_t i = 0; i < props.size(); ++i) { + res.emplace_back(prop_names[i], props[i]); + } + return res; + } + + std::vector> GetVertexProperties( + label_t label) const { + const std::vector& props = + db_session_.schema().get_vertex_properties(label); + const std::vector& prop_names = + db_session_.schema().get_vertex_property_names(label); + std::vector> res; + for (size_t i = 0; i < props.size(); ++i) { + res.emplace_back(prop_names[i], props[i]); + } + return res; + } + + //////////////////////////////Vertex-related Interface//////////// + + /** + * @brief + Scan all points with label label_id, for each point, get the properties + specified by selectors, and input them into func. The function signature of + func_t should be: void func(vertex_id_t v, const std::tuple& props) + Users implement their own logic in the function. This function has no return + value. In the example below, we scan all person points, find all points with + age , and save them to a vector. std::vector vids; + graph.ScanVertices(person_label_id, + gs::PropertySelector("age"), + [&vids](vertex_id_t vid, const std::tuple& props){ + if (std::get<0>(props) == 18){ + vids.emplace_back(vid); + } + }); + It is important to note that the properties specified by selectors will be + input into the lambda function in a tuple manner. + * @tparam FUNC_T + * @tparam SELECTOR + * @param label_id + * @param selectors The Property selectors. The selected properties will be + * fed to the function + * @param func The lambda function for filtering. + */ + // TODO(zhanglei): fix filter_null in scan.h + template + void ScanVertices(const label_id_t& label_id, + const std::tuple...>& selectors, + const FUNC_T& func) const { + auto vnum = db_session_.graph().vertex_num(label_id); + std::tuple t; + if constexpr (sizeof...(T) == 0) { + for (size_t v = 0; v != vnum; ++v) { + func(v, t); + } + } else { + auto columns = get_vertex_property_columns(label_id, selectors); + for (size_t v = 0; v != vnum; ++v) { + mutable_csr_graph_impl::get_tuple_from_column_tuple(v, t, columns); + func(v, t); + } + } + } + + /** + * @brief ScanVertices scans all vertices with the given label with give + * original id. + * @param label_id The label id. + * @param oid The original id. + * @param vid The result internal id. + */ + bool ScanVerticesWithOid(const label_id_t& label_id, Any oid, + vertex_id_t& vid) const { + return db_session_.graph().get_lid(label_id, oid, vid); + } + + /** + * @brief GetVertexPropertyGetter gets the property getter for the given + * vertex label and property name. + * @tparam T The property type. + * @param label_id The vertex label id. + * @param prop_name The property name. + * @return The property getter. + */ + template + mutable_csr_graph_impl::PropertyGetter GetVertexPropertyGetter( + const label_id_t& label_id, const std::string& prop_name) const { + auto column = get_vertex_property_column(label_id, prop_name); + return mutable_csr_graph_impl::PropertyGetter(column); + } + + mutable_csr_graph_impl::UntypedPropertyGetter GetUntypedVertexPropertyGetter( + const label_t& label_id, const std::string& prop_name) const { + auto column = get_vertex_property_column(label_id, prop_name); + return mutable_csr_graph_impl::UntypedPropertyGetter(column); + } + + //////////////////////////////Edge-related Interface//////////// + + /** + * @brief GetEdges gets the edges with the given label and edge label, and + * with the starting vertex internal ids. + * When the direction is "out", the edges are from the source label to the + * destination label, and vice versa when the direction is "in". When the + * direction is "both", the src and dst labels SHOULD be the same. + */ + template + mutable_csr_graph_impl::AdjListArray GetEdges( + const label_id_t& src_label_id, const label_id_t& dst_label_id, + const label_id_t& edge_label_id, const std::vector& vids, + const Direction& direction, size_t limit = INT_MAX) const { + if (direction == Direction::Out) { + auto csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, + edge_label_id); + // return mutable_csr_graph_impl::AdjListArray(csr, vids); + return mutable_csr_graph_impl::create_adj_list_array(csr, vids); + } else if (direction == Direction::In) { + auto csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id, + edge_label_id); + // return mutable_csr_graph_impl::AdjListArray(csr, vids); + return mutable_csr_graph_impl::create_adj_list_array(csr, vids); + } else if (direction == Direction::Both) { + auto csr0 = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, + edge_label_id); + auto csr1 = db_session_.graph().get_ie_csr(dst_label_id, src_label_id, + edge_label_id); + return mutable_csr_graph_impl::create_adj_list_array(csr0, csr1, vids); + } else { + throw std::runtime_error("Not implemented - " + direction); + } + } + + /** + * @brief Get vertices on the other side of edges, via the given edge label + * and the starting vertex internal ids. + * When the direction is "out", the vertices are on the destination label side + * of the edges, and vice versa when the direction is "in". When the direction + * is "both", the src and dst labels SHOULD be the same. + */ + mutable_csr_graph_impl::NbrListArray GetOtherVertices( + const label_id_t& src_label_id, const label_id_t& dst_label_id, + const label_id_t& edge_label_id, const std::vector& vids, + const Direction& direction, size_t limit = INT_MAX) const { + if (direction == Direction::Out) { + auto csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, + edge_label_id); + return mutable_csr_graph_impl::create_nbr_list_array(csr, vids); + } else if (direction == Direction::In) { + auto csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id, + edge_label_id); + return mutable_csr_graph_impl::create_nbr_list_array(csr, vids); + } else if (direction == Direction::Both) { + auto csr0 = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, + edge_label_id); + auto csr1 = db_session_.graph().get_ie_csr(dst_label_id, src_label_id, + edge_label_id); + return mutable_csr_graph_impl::create_nbr_list_array(csr0, csr1, vids); + } else { + throw std::runtime_error("Not implemented - " + gs::to_string(direction)); + } + } + + //////////////////////////////Subgraph-related Interface//////////// + mutable_csr_graph_impl::SubGraph GetSubGraph( + const label_id_t src_label_id, const label_id_t dst_label_id, + const label_id_t edge_label_id, const Direction& direction) const { + const CsrBase *csr = nullptr, *other_csr = nullptr; + if (direction == Direction::Out) { + csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, + edge_label_id); + return mutable_csr_graph_impl::SubGraph{ + csr, {src_label_id, dst_label_id, edge_label_id}, Direction::Out}; + } else if (direction == Direction::In) { + csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id, + edge_label_id); + return mutable_csr_graph_impl::SubGraph{ + csr, {dst_label_id, src_label_id, edge_label_id}, Direction::In}; + } else if (direction == Direction::Both) { + csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, + edge_label_id); + other_csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id, + edge_label_id); + return mutable_csr_graph_impl::SubGraph{ + csr, + other_csr, + {src_label_id, dst_label_id, edge_label_id}, + Direction::Both}; + } else { + throw std::runtime_error("Not implemented - " + gs::to_string(direction)); + } + } + + //////////////////////////////Private Functions//////////////////// + private: + template + inline auto get_vertex_property_columns( + label_t label, + const std::tuple...>& selectors) const { + return get_vertex_property_columns_impl( + label, selectors, std::make_index_sequence()); + } + + template + auto get_vertex_property_columns_impl( + label_t label, const std::tuple...>& selectors, + std::index_sequence) const { + return std::make_tuple(get_vertex_property_column( + label, std::get(selectors).prop_name_)...); + } + + // get the vertex property + template + std::shared_ptr> get_vertex_property_column( + const label_t& label_id, const std::string& prop_name) const { + using column_t = std::shared_ptr>; + column_t column; + if constexpr (std::is_same_v) { + return std::make_shared>(label_id); + } + if constexpr (std::is_same_v) { + return std::make_shared>(label_id); + } + if (prop_name == "id" || prop_name == "ID" || prop_name == "Id") { + column = std::dynamic_pointer_cast>( + db_session_.get_vertex_id_column(label_id)); + } else { + auto ptr = db_session_.get_vertex_property_column(label_id, prop_name); + if (ptr) { + column = std::dynamic_pointer_cast>( + create_ref_column(ptr)); + } else { + return nullptr; + } + } + return column; + } + + std::shared_ptr get_vertex_property_column( + const label_t& label_id, const std::string& prop_name) const { + if (prop_name == "id" || prop_name == "ID" || prop_name == "Id") { + return db_session_.get_vertex_id_column(label_id); + } else { + auto col = db_session_.get_vertex_property_column(label_id, prop_name); + if (col) { + return create_ref_column(col); + } else { + return nullptr; + } + } + } + + size_t get_vertex_num_impl() const { + size_t ret = 0; + for (label_t label_id = 0; + label_id < db_session_.schema().vertex_label_num(); ++label_id) { + ret += db_session_.graph().vertex_num(label_id); + } + return ret; + } + + size_t get_edge_num_impl(const label_t& src_label_id, + const label_t& dst_label_id, + const label_t& edge_label_id) const { + return db_session_.graph().edge_num(src_label_id, dst_label_id, + edge_label_id); + } + + const GraphDBSession& db_session_; +}; +} // namespace gs + +#endif // ENGINES_HQPS_DATABASE_MUTABLE_CSR_INTERFACE_H_V2 \ No newline at end of file diff --git a/flex/engines/hqps_db/database/nbr_list.h b/flex/engines/hqps_db/database/nbr_list.h new file mode 100644 index 000000000000..7654d93a7260 --- /dev/null +++ b/flex/engines/hqps_db/database/nbr_list.h @@ -0,0 +1,106 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENGINES_HQPS_DATABASE_NBR_LIST_H_ +#define ENGINES_HQPS_DATABASE_NBR_LIST_H_ + +#include +#include "flex/utils/property/types.h" + +namespace gs { +namespace mutable_csr_graph_impl { +class Nbr { + public: + Nbr() = default; + explicit Nbr(vid_t neighbor) : neighbor_(neighbor) {} + ~Nbr() = default; + + inline vid_t neighbor() const { return neighbor_; } + + private: + vid_t neighbor_; +}; + +class NbrList { + public: + NbrList(const Nbr* b, const Nbr* e) : begin_(b), end_(e) {} + NbrList() : begin_(nullptr), end_(nullptr) {} + ~NbrList() = default; + + const Nbr* begin() const { return begin_; } + const Nbr* end() const { return end_; } + inline size_t size() const { return end_ - begin_; } + + private: + const Nbr* begin_; + const Nbr* end_; +}; + +class NbrListArray { + public: + NbrListArray() {} + ~NbrListArray() = default; + + NbrList get(size_t index) const { + auto& list = nbr_lists_[index]; + return NbrList(list.data(), list.data() + list.size()); + } + + void put(std::vector&& list) { nbr_lists_.push_back(std::move(list)); } + + size_t size() const { return nbr_lists_.size(); } + + void resize(size_t size) { nbr_lists_.resize(size); } + + std::vector& get_vector(size_t index) { return nbr_lists_[index]; } + + private: + std::vector> nbr_lists_; +}; + +NbrListArray create_nbr_list_array(const CsrBase* csr0, const CsrBase* csr1, + const std::vector& vids) { + NbrListArray ret; + ret.resize(vids.size()); + for (size_t i = 0; i < vids.size(); ++i) { + auto v = vids[i]; + auto& vec = ret.get_vector(i); + if (csr0) { + auto iter = csr0->edge_iter(v); + while (iter->is_valid()) { + vec.emplace_back(Nbr(iter->get_neighbor())); + iter->next(); + } + } + if (csr1) { + auto iter = csr1->edge_iter(v); + while (iter->is_valid()) { + vec.emplace_back(Nbr(iter->get_neighbor())); + iter->next(); + } + } + } + return ret; +} + +NbrListArray create_nbr_list_array(const CsrBase* csr, + const std::vector& vids) { + return create_nbr_list_array(csr, nullptr, vids); +} +} // namespace mutable_csr_graph_impl + +} // namespace gs + +#endif // ENGINES_HQPS_DATABASE_NBR_LIST_H_ \ No newline at end of file diff --git a/flex/engines/hqps_db/database/sub_graph.h b/flex/engines/hqps_db/database/sub_graph.h new file mode 100644 index 000000000000..1779152443f5 --- /dev/null +++ b/flex/engines/hqps_db/database/sub_graph.h @@ -0,0 +1,189 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FLEX_ENGINES_HQPS_DATABASE_SUB_GRAPH_H_ +#define FLEX_ENGINES_HQPS_DATABASE_SUB_GRAPH_H_ + +#include +#include +#include +#include + +#include "flex/engines/hqps_db/core/null_record.h" +#include "flex/storages/rt_mutable_graph/csr/mutable_csr.h" +#include "flex/utils/property/types.h" + +namespace gs { + +namespace mutable_csr_graph_impl { + +// Base interface for edge iterator +class EdgeIter { + public: + using label_id_t = label_t; + EdgeIter() + : ptr1_(nullptr), + ptr2_(nullptr), + dir_(Direction::Out), + label_triplet_({0, 0, 0}) {} + + EdgeIter(std::shared_ptr ptr1, + std::shared_ptr ptr2, const Direction& dir, + const std::array& label_triplet) + : ptr1_(ptr1), ptr2_(ptr2), dir_(dir), label_triplet_(label_triplet) {} + + EdgeIter(std::shared_ptr ptr1, const Direction& dir, + const std::array& label_triplet) + : ptr1_(ptr1), ptr2_(nullptr), dir_(dir), label_triplet_(label_triplet) {} + + EdgeIter(const EdgeIter& other) + : ptr1_(other.ptr1_), + ptr2_(other.ptr2_), + dir_(other.dir_), + label_triplet_(other.label_triplet_) {} + + inline void Next() const { + if (ptr1_ && ptr1_->is_valid()) { + ptr1_->next(); + } + if (ptr2_ && ptr2_->is_valid()) { + ptr2_->next(); + } + } + + inline vid_t GetDstId() const { + if (ptr1_ && ptr1_->is_valid()) { + return ptr1_->get_neighbor(); + } else if (ptr2_ && ptr2_->is_valid()) { + return ptr2_->get_neighbor(); + } else { + throw std::runtime_error("Invalid edge iterator"); + } + } + + inline vid_t GetSrcId() const { + // TODO(fixme): + LOG(FATAL) << "Not implemented yet"; + } + + inline label_id_t GetDstLabel() const { return label_triplet_[1]; } + + inline label_id_t GetSrcLabel() const { return label_triplet_[0]; } + + inline Direction GetDirection() const { return dir_; } + + inline Any GetData() const { + if (ptr1_ && ptr1_->is_valid()) { + return ptr1_->get_data(); + } else if (ptr2_ && ptr2_->is_valid()) { + return ptr2_->get_data(); + } else { + throw std::runtime_error("Invalid edge iterator"); + } + } + inline bool IsValid() const { + if (ptr1_ && ptr1_->is_valid()) { + return true; + } else if (ptr2_ && ptr2_->is_valid()) { + return true; + } else { + return false; + } + } + + EdgeIter& operator=(const EdgeIter& rhs) { + this->ptr1_ = rhs.ptr1_; + this->ptr2_ = rhs.ptr2_; + this->dir_ = rhs.dir_; + this->label_triplet_ = rhs.label_triplet_; + return *this; + } + + size_t Size() const { + size_t ret = 0; + if (ptr1_) { + ret += ptr1_->size(); + } + if (ptr2_) { + ret += ptr2_->size(); + } + return ret; + } + + private: + std::shared_ptr ptr1_, ptr2_; + Direction dir_; + std::array label_triplet_; +}; + +class SubGraph { + public: + using label_id_t = label_t; + using iterator = EdgeIter; + SubGraph(const CsrBase* first, const CsrBase* second, + const std::array& label_triplet, Direction dir) + : first_csr_(first), + second_csr_(second), + label_triplet_(label_triplet), + dir_(dir) { + if (dir == Direction::Both) { + CHECK_NOTNULL(first_csr_); + CHECK_NOTNULL(second_csr_); + } else { + CHECK_NOTNULL(first_csr_); + } + } + + SubGraph(const CsrBase* first, const std::array& label_triplet, + Direction dir) + : first_csr_(first), + second_csr_(nullptr), + label_triplet_(label_triplet), + dir_(dir) { + CHECK_NOTNULL(first_csr_); + CHECK(dir != Direction::Both); + } + + inline iterator get_edges(vid_t vid) const { + // TODO(zhanglei): Fixme + if (dir_ == Direction::Out) { + return iterator(first_csr_->edge_iter(vid), dir_, label_triplet_); + } else if (dir_ == Direction::In) { + return iterator(first_csr_->edge_iter(vid), dir_, label_triplet_); + } else { + return iterator(first_csr_->edge_iter(vid), second_csr_->edge_iter(vid), + dir_, label_triplet_); + } + } + + // here the src, dst, refer the sub graph, not the csr. + label_id_t GetSrcLabel() const { return label_triplet_[0]; } + label_id_t GetEdgeLabel() const { return label_triplet_[2]; } + label_id_t GetDstLabel() const { return label_triplet_[1]; } + Direction GetDirection() const { return dir_; } + + private: + // dir_ = Direction::Out: first_csr_ = out_csr, second_csr_ = nullptr + // dir_ = Direction::In: first_csr_ = in_csr, second_csr_ = nullptr + // dir_ = Direction::Both: first_csr_ = out_csr, second_csr_ = in_csr + const CsrBase *first_csr_, *second_csr_; + std::array label_triplet_; + Direction dir_; +}; + +} // namespace mutable_csr_graph_impl + +} // namespace gs + +#endif // FLEX_ENGINES_HQPS_DATABASE_SUB_GRAPH_H_ \ No newline at end of file diff --git a/flex/engines/hqps_db/structures/collection.h b/flex/engines/hqps_db/structures/collection.h index 0c9addabc95e..3820d2845959 100644 --- a/flex/engines/hqps_db/structures/collection.h +++ b/flex/engines/hqps_db/structures/collection.h @@ -434,9 +434,7 @@ class CountBuilder { return true; } - Collection Build() { - return Collection(std::move(vec_)); - } + Collection Build() { return Collection(std::move(vec_)); } private: std::vector vec_; diff --git a/flex/engines/hqps_db/structures/multi_vertex_set/general_vertex_set.h b/flex/engines/hqps_db/structures/multi_vertex_set/general_vertex_set.h index 8c39c9ba9e39..683099c8564d 100644 --- a/flex/engines/hqps_db/structures/multi_vertex_set/general_vertex_set.h +++ b/flex/engines/hqps_db/structures/multi_vertex_set/general_vertex_set.h @@ -25,20 +25,22 @@ #include "grape/utils/bitset.h" #include "flex/engines/hqps_db/core/null_record.h" +#include "flex/engines/hqps_db/core/utils/graph_utils.h" #include "flex/engines/hqps_db/core/utils/hqps_type.h" #include "flex/engines/hqps_db/core/utils/hqps_utils.h" namespace gs { // return the old labels, that are active in filter. -template +template auto general_project_vertices_impl( const std::vector& old_vec, const std::vector& old_bit_sets, const std::vector& old_labels, const std::array& filter_labels, - const EXPR& expr, const std::vector& prop_getters) { + const EXPR& expr, + const std::vector>& prop_getters) { std::vector res_vec; CHECK(old_bit_sets.size() == old_labels.size()); CHECK(prop_getters.size() == old_labels.size()); @@ -79,14 +81,15 @@ auto general_project_vertices_impl( ++j; } CHECK(j < old_num_labels) << "no label is active at ind: " << i; - if constexpr (PROP_GETTER::prop_num == 0) { + if constexpr (sizeof...(PropGetterT) == 0) { if (expr()) { res_bitsets[j].set_bit(res_vec.size()); res_vec.push_back(old_vec[i]); } } else { if (label_ind_set.find(j) != label_ind_set.end()) { - if (std::apply(expr, prop_getters[j].get_view(old_vec[i]))) { + if (std::apply(expr, get_view_from_prop_getters(prop_getters[j], + old_vec[i]))) { res_bitsets[j].set_bit(res_vec.size()); res_vec.push_back(old_vec[i]); } @@ -121,14 +124,15 @@ auto general_project_vertices_impl( } template + typename... PropGetterT, size_t filter_num_labels> auto general_project_vertices_impl( const std::vector& old_vec, const std::vector& old_data_vec, const std::vector& old_bit_sets, const std::vector& old_labels, const std::array& filter_labels, - const EXPR& expr, const std::vector& prop_getters) { + const EXPR& expr, + const std::vector>& prop_getters) { std::vector res_vec; std::vector res_data_vec; CHECK(old_bit_sets.size() == old_labels.size()); @@ -163,8 +167,7 @@ auto general_project_vertices_impl( for (size_t i = 0; i < old_vec.size(); ++i) { for (auto label_id : select_label_id) { if (old_bit_sets[label_id].get_bit(i)) { - auto eles = prop_getters[label_id].get_view(old_vec[i]); - if constexpr (PROP_GETTER::prop_num == 0) { + if constexpr (sizeof...(PropGetterT) == 0) { if (expr()) { res_bitsets[label_id].set_bit(res_vec.size()); res_vec.push_back(old_vec[i]); @@ -172,6 +175,8 @@ auto general_project_vertices_impl( break; } } else { + auto eles = + get_view_from_prop_getters(prop_getters[label_id], old_vec[i]); if (std::apply(expr, eles)) { res_bitsets[label_id].set_bit(res_vec.size()); res_vec.push_back(old_vec[i]); @@ -1689,20 +1694,34 @@ std::vector> bitsets_to_vids_inds( } template -static auto get_property_tuple_general( + typename... SET_T, size_t... Is> +static auto get_property_tuple_general_impl( const GRAPH_INTERFACE& graph, const GeneralVertexSet& general_set, - const std::array& prop_names) { + const std::tuple...>& selectors, + std::index_sequence) { auto label_vec = general_set.GetLabels(); - VLOG(10) << "general set label vec: " << gs::to_string(label_vec) - << ", vid size: " << general_set.Size(); - auto vids_inds = bitsets_to_vids_inds(general_set.GetBitsets()); - - auto data_tuples = graph.template GetVertexPropsFromVid( - general_set.GetVertices(), label_vec, vids_inds, prop_names); - + const auto& vids = general_set.GetVertices(); + const auto& bitset = general_set.GetBitsets(); + using prop_getter_tuple = + std::tuple...>; + std::vector prop_getters; + for (size_t i = 0; i < label_vec.size(); ++i) { + prop_getters.emplace_back(get_prop_getters_from_selectors_single_label( + graph, label_vec[i], selectors)); + } + std::vector> data_tuples; + data_tuples.reserve(vids.size()); + for (size_t i = 0; i < vids.size(); ++i) { + for (size_t j = 0; j < label_vec.size(); ++j) { + if (bitset[j].get_bit(i)) { + data_tuples.emplace_back( + get_view_from_prop_getters(prop_getters[j], vids[i])); + break; + } + } + } return data_tuples; } @@ -1712,13 +1731,9 @@ static auto get_property_tuple_general( const GRAPH_INTERFACE& graph, const GeneralVertexSet& general_set, - const std::tuple...>& named_prop) { - std::array prop_names; - int ind = 0; - std::apply([&prop_names, - &ind](auto&&... args) { ((prop_names[ind++] = args.name), ...); }, - named_prop); - return get_property_tuple_general(graph, general_set, prop_names); + const std::tuple...>& selectors) { + return get_property_tuple_general_impl( + graph, general_set, selectors, std::make_index_sequence()); } } // namespace gs diff --git a/flex/engines/hqps_db/structures/multi_vertex_set/row_vertex_set.h b/flex/engines/hqps_db/structures/multi_vertex_set/row_vertex_set.h index 6b181d002387..6449657f470d 100644 --- a/flex/engines/hqps_db/structures/multi_vertex_set/row_vertex_set.h +++ b/flex/engines/hqps_db/structures/multi_vertex_set/row_vertex_set.h @@ -451,16 +451,14 @@ auto rowSetFlatImpl( // if num_labels == 0, we deem it as take all labels. template , std::vector>> -RES_T row_project_vertices_impl(const std::vector& lids, - LabelT cur_label, - std::array& labels, - EXPRESSION& expr, - std::array& prop_getters) { +RES_T row_project_vertices_impl( + const std::vector& lids, LabelT cur_label, + std::array& labels, EXPRESSION& expr, + std::array& prop_getters) { std::vector offsets; std::vector new_lids; - size_t cnt = 0; offsets.reserve(lids.size() + 1); int label_ind = -1; @@ -479,22 +477,26 @@ RES_T row_project_vertices_impl(const std::vector& lids, // for current set, we don't need. auto size = lids.size(); for (size_t i = 0; i < size; ++i) { - offsets.emplace_back(cnt); + offsets.emplace_back(new_lids.size()); } } else { // VLOG(10) << "Found label in query params"; - auto& cur_prop_getter = prop_getters[0]; - for (size_t i = 0; i < lids.size(); ++i) { - offsets.emplace_back(cnt); - auto cur_lid = lids[i]; - auto prop = cur_prop_getter.get_view(cur_lid); - if (std::apply(expr, prop)) { - // if (expr(eles[i])) { - new_lids.emplace_back(cur_lid); - cnt += 1; + if constexpr (std::tuple_size_v == 0) { + for (size_t i = 0; i < lids.size(); ++i) { + offsets.emplace_back(new_lids.size()); + new_lids.emplace_back(lids[i]); + } + } else { + auto& cur_prop_getter = prop_getters[0]; + for (size_t i = 0; i < lids.size(); ++i) { + offsets.emplace_back(new_lids.size()); + auto prop = cur_prop_getter.get_view(lids[i]); + if (std::apply(expr, prop)) { + new_lids.emplace_back(lids[i]); + } } } - offsets.emplace_back(cnt); + offsets.emplace_back(new_lids.size()); } VLOG(10) << "Project vertices, new lids" << new_lids.size() << ", offset size: " << offsets.size(); @@ -1400,7 +1402,8 @@ class RowVertexSetImpl { RES_T project_vertices(std::array& labels, EXPRESSION& exprs, std::array& prop_getter) const { - // TODO: vector-based cols should be able to be selected with certain rows. + // TODO: vector-based cols should be able to be selected with certain + // rows. auto new_lids_and_offsets = row_project_vertices_impl(vids_, v_label_, labels, exprs, prop_getter); @@ -1414,7 +1417,8 @@ class RowVertexSetImpl { template >> RES_T project_vertices(std::array& labels) const { - // TODO: vector-based cols should be able to be selected with certain rows. + // TODO: vector-based cols should be able to be selected with certain + // rows. auto new_lids_datas_and_offset = select_labels(vids_, v_label_, labels); self_type_t res_set(std::move(new_lids_datas_and_offset.first), v_label_); diff --git a/flex/engines/hqps_db/structures/multi_vertex_set/two_label_vertex_set.h b/flex/engines/hqps_db/structures/multi_vertex_set/two_label_vertex_set.h index c1998b5c4670..95789ffd8547 100644 --- a/flex/engines/hqps_db/structures/multi_vertex_set/two_label_vertex_set.h +++ b/flex/engines/hqps_db/structures/multi_vertex_set/two_label_vertex_set.h @@ -1446,38 +1446,41 @@ two_label_bitset_to_vids_indsV2(const grape::Bitset& bitset, return std::make_pair(std::move(res_vids), std::move(res)); } -template -static auto get_property_tuple_two_label( +template +static auto get_property_tuple_two_label_impl( const GRAPH_INTERFACE& graph, - const TwoLabelVertexSet& general_set, - const std::array& prop_names) { - auto& label_array = general_set.GetLabels(); - - // Get data for multilabel vertices, mixed - // double t1 = -grape::GetCurrentTime(); - auto data_tuples = graph.template GetVertexPropsFromVidV2( - general_set.GetVertices(), label_array, general_set.GetBitset(), - prop_names); - - return data_tuples; + const TwoLabelVertexSetImpl& set, + const std::tuple...>& selectors) { + auto labels = set.GetLabels(); + const auto& bitset = set.GetBitset(); + const auto& vids = set.GetVertices(); + CHECK(labels.size() == 2) << "Two label set should have two labels"; + auto prop_getters0 = + get_prop_getters_from_selectors_single_label(graph, labels[0], selectors); + auto prop_getters1 = + get_prop_getters_from_selectors_single_label(graph, labels[1], selectors); + std::vector> res; + res.reserve(set.Size()); + for (size_t i = 0; i < vids.size(); ++i) { + if (bitset.get_bit(i)) { + res.emplace_back(get_view_from_prop_getters(prop_getters0, vids[i])); + } else { + res.emplace_back(get_view_from_prop_getters(prop_getters1, vids[i])); + } + } + return res; } template + typename... VERTEX_SET_PROP_T, typename... T> static auto get_property_tuple_two_label( const GRAPH_INTERFACE& graph, const TwoLabelVertexSetImpl& general_set, - const std::tuple& named_prop) { - std::array prop_names; - size_t ind = 0; - std::apply([&prop_names, - &ind](auto&... args) { ((prop_names[ind++] = args.name), ...); }, - named_prop); - return get_property_tuple_two_label( - graph, general_set, prop_names); + VERTEX_SET_PROP_T...>& set, + const std::tuple...>& selector) { + return get_property_tuple_two_label_impl(graph, set, selector); } } // namespace gs diff --git a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc index 360d16ad0b8f..fb03543d34f2 100644 --- a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc +++ b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc @@ -394,6 +394,13 @@ vid_t MutablePropertyFragment::vertex_num(label_t vertex_label) const { return static_cast(lf_indexers_[vertex_label].size()); } +size_t MutablePropertyFragment::edge_num(label_t src_label, label_t edge_label, + label_t dst_label) const { + size_t index = src_label * vertex_label_num_ * edge_label_num_ + + dst_label * edge_label_num_ + edge_label; + return oe_[index]->size(); +} + bool MutablePropertyFragment::get_lid(label_t label, const Any& oid, vid_t& lid) const { return lf_indexers_[label].get_index(oid, lid); diff --git a/flex/storages/rt_mutable_graph/mutable_property_fragment.h b/flex/storages/rt_mutable_graph/mutable_property_fragment.h index f2bc198c9576..d8bccbe55c85 100644 --- a/flex/storages/rt_mutable_graph/mutable_property_fragment.h +++ b/flex/storages/rt_mutable_graph/mutable_property_fragment.h @@ -70,6 +70,9 @@ class MutablePropertyFragment { vid_t vertex_num(label_t vertex_label) const; + size_t edge_num(label_t src_label, label_t edge_label, + label_t dst_label) const; + bool get_lid(label_t label, const Any& oid, vid_t& lid) const; Any get_oid(label_t label, vid_t lid) const; diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index e0052a6c38fa..b861494e5af6 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -1335,13 +1335,18 @@ bool Schema::has_edge_label(const std::string& src_label, const std::string& dst_label, const std::string& label) const { label_t edge_label_id; + // TODO: check if src_label and dst_label exists auto src_label_id = get_vertex_label_id(src_label); auto dst_label_id = get_vertex_label_id(dst_label); if (!elabel_indexer_.get_index(label, edge_label_id)) { return false; } - auto e_label_id = - generate_edge_label(src_label_id, dst_label_id, edge_label_id); + return has_edge_label(src_label_id, dst_label_id, edge_label_id); +} + +bool Schema::has_edge_label(label_t src_label, label_t dst_label, + label_t edge_label) const { + label_t e_label_id = generate_edge_label(src_label, dst_label, edge_label); return eprop_names_.find(e_label_id) != eprop_names_.end(); } diff --git a/flex/storages/rt_mutable_graph/schema.h b/flex/storages/rt_mutable_graph/schema.h index 19aa17dee140..e84b11ba7901 100644 --- a/flex/storages/rt_mutable_graph/schema.h +++ b/flex/storages/rt_mutable_graph/schema.h @@ -159,6 +159,9 @@ class Schema { const std::string& dst_label, const std::string& edge_label) const; + bool has_edge_label(label_t src_label, label_t dst_label, + label_t edge_label) const; + bool valid_edge_property(const std::string& src_label, const std::string& dst_label, const std::string& label) const; diff --git a/flex/tests/hqps/context_test.cc b/flex/tests/hqps/context_test.cc index c08c8f5fede2..4a075bfa598b 100644 --- a/flex/tests/hqps/context_test.cc +++ b/flex/tests/hqps/context_test.cc @@ -25,7 +25,8 @@ #include "flex/engines/hqps_db/core/base_engine.h" #include "flex/engines/hqps_db/core/operator/project.h" -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" +#include "flex/engines/hqps_db/database/mutable_csr_interface_v2.h" +#include "flex/engines/hqps_db/database/nbr_list.h" #include "flex/storages/rt_mutable_graph/types.h" #include "flex/utils/property/column.h" #include "glog/logging.h" @@ -176,7 +177,7 @@ int main() { { auto ref_col = std::make_shared>(*col1); - gs::mutable_csr_graph_impl::SinglePropGetter getter(ref_col); + gs::mutable_csr_graph_impl::PropertyGetter getter(ref_col); double t1 = -grape::GetCurrentTime(); int32_t res = 0; for (auto ind : indices) { @@ -231,8 +232,8 @@ int main() { { auto ref_col1 = std::make_shared>(*col1); auto ref_col2 = std::make_shared>(*col2); - gs::mutable_csr_graph_impl::SinglePropGetter getter1(ref_col1); - gs::mutable_csr_graph_impl::SinglePropGetter getter2(ref_col2); + gs::mutable_csr_graph_impl::PropertyGetter getter1(ref_col1); + gs::mutable_csr_graph_impl::PropertyGetter getter2(ref_col2); double t1 = -grape::GetCurrentTime(); int32_t res = 0; @@ -306,10 +307,10 @@ int main() { // test two label vertex set prop getter. auto ref_col1 = std::make_shared>(*col1); auto ref_col2 = std::make_shared>(*col2); - gs::mutable_csr_graph_impl::SinglePropGetter getter1(ref_col1); - gs::mutable_csr_graph_impl::SinglePropGetter getter2(ref_col2); - std::array, 2> - array{getter1, getter2}; + gs::mutable_csr_graph_impl::PropertyGetter getter1(ref_col1); + gs::mutable_csr_graph_impl::PropertyGetter getter2(ref_col2); + std::array, 2> array{ + getter1, getter2}; // generate index ele std::vector> index_eles; diff --git a/flex/tests/hqps/dedup_test.cc b/flex/tests/hqps/dedup_test.cc index 7479860ec67e..1c536592897a 100644 --- a/flex/tests/hqps/dedup_test.cc +++ b/flex/tests/hqps/dedup_test.cc @@ -23,7 +23,7 @@ #include "flex/engines/hqps_db/structures/multi_vertex_set/multi_label_vertex_set.h" #include "flex/engines/hqps_db/structures/multi_vertex_set/row_vertex_set.h" -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" +#include "flex/engines/hqps_db/database/mutable_csr_interface_v2.h" #include "grape/types.h" diff --git a/flex/tests/hqps/path_set_test.cc b/flex/tests/hqps/path_set_test.cc index fba936bfc07f..a4cb59b2e753 100644 --- a/flex/tests/hqps/path_set_test.cc +++ b/flex/tests/hqps/path_set_test.cc @@ -19,7 +19,6 @@ #include "flex/engines/hqps_db/structures/multi_vertex_set/row_vertex_set.h" #include "flex/storages/rt_mutable_graph/types.h" -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" #include "flex/engines/hqps_db/structures/path.h" #include "grape/types.h" diff --git a/flex/tests/hqps/query_test.cc b/flex/tests/hqps/query_test.cc index 1b1c79a6f392..f4600623ad7c 100644 --- a/flex/tests/hqps/query_test.cc +++ b/flex/tests/hqps/query_test.cc @@ -12,8 +12,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "flex/engines/hqps_db/core/params.h" #include "flex/engines/hqps_db/core/sync_engine.h" -#include "flex/engines/hqps_db/database/mutable_csr_interface.h" +#include "flex/engines/hqps_db/database/mutable_csr_interface_v2.h" #include "flex/tests/hqps/match_query.h" #include "flex/tests/hqps/sample_query.h" @@ -49,7 +50,7 @@ int main(int argc, char** argv) { gs::MutableCSRInterface interface(sess); std::array prop_names{"creationDate"}; auto edges = - interface.GetEdges(1, 1, 8, vids, "Both", INT_MAX, prop_names); + interface.GetEdges(1, 1, 8, vids, gs::Direction::Both); double t = -grape::GetCurrentTime(); size_t cnt = 0; for (size_t i = 0; i < vids.size(); ++i) { @@ -63,11 +64,11 @@ int main(int argc, char** argv) { LOG(INFO) << "visiting edges: cost: " << t << ", num edges: " << cnt; // visiting vertices properties - auto vertex_prop = - interface.GetVertexPropsFromVid(1, vids, {"id"}); + auto vertex_prop_getter = + interface.GetVertexPropertyGetter(1, "id"); for (size_t i = 0; i < 10; ++i) { - VLOG(10) << "vid: " << vids[i] - << ", prop: " << gs::to_string(vertex_prop[i]); + VLOG(10) << "vid: " << vids[i] << ", prop: " + << gs::to_string(vertex_prop_getter.get_view(vids[i])); } } diff --git a/flex/utils/property/column.cc b/flex/utils/property/column.cc index 8f2ec478266e..fa710c65c291 100644 --- a/flex/utils/property/column.cc +++ b/flex/utils/property/column.cc @@ -174,4 +174,50 @@ std::shared_ptr CreateColumn(PropertyType type, } } +std::shared_ptr create_ref_column( + std::shared_ptr column) { + auto type = column->type(); + if (type == PropertyType::kBool) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kDay) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kDate) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kUInt8) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kUInt16) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kInt32) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kInt64) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kUInt32) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kUInt64) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kStringView || type.IsVarchar()) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kFloat) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else if (type == PropertyType::kDouble) { + return std::make_shared>( + *std::dynamic_pointer_cast>(column)); + } else { + LOG(FATAL) << "unexpected type to create column, " + << static_cast(type.type_enum); + return nullptr; + } +} + } // namespace gs diff --git a/flex/utils/property/column.h b/flex/utils/property/column.h index 10c58043aad0..e04c576ad056 100644 --- a/flex/utils/property/column.h +++ b/flex/utils/property/column.h @@ -749,6 +749,9 @@ class TypedRefColumn : public RefColumnBase { label_t label_key_; }; +std::shared_ptr create_ref_column( + std::shared_ptr column); + } // namespace gs #endif // GRAPHSCOPE_PROPERTY_COLUMN_H_ From dacb1786ff0b2c6c75004c8d861669f83b5aa8f9 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Thu, 11 Jul 2024 15:03:20 +0800 Subject: [PATCH 3/5] fix sub_graph Committed-by: xiaolei.zl from Dev container --- flex/bin/test.sh | 8 ++ .../hqps_db/core/operator/edge_expand.h | 28 ++-- .../database/mutable_csr_interface_v2.h | 24 ++-- flex/engines/hqps_db/database/sub_graph.h | 134 +++++++----------- .../multi_edge_set/untyped_edge_set.h | 46 +----- flex/storages/rt_mutable_graph/schema.cc | 2 +- flex/tests/hqps/engine_config_test.yaml | 2 +- 7 files changed, 97 insertions(+), 147 deletions(-) create mode 100644 flex/bin/test.sh diff --git a/flex/bin/test.sh b/flex/bin/test.sh new file mode 100644 index 000000000000..268b00138051 --- /dev/null +++ b/flex/bin/test.sh @@ -0,0 +1,8 @@ + for i in 1 2 3 4 5 6 7 8 9 10 11 12; + do + cmd="./load_plan_and_gen.sh -e=hqps -i=../resources/queries/ic/adhoc/ic${i}_adhoc.cypher -w=/tmp/codegen/" + cmd=${cmd}" -o=/tmp/plugin --ir_conf=../tests/hqps/engine_config_test.yaml " + cmd=${cmd}" --graph_schema_path=${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml" + echo $cmd + eval ${cmd} || exit 1 + done diff --git a/flex/engines/hqps_db/core/operator/edge_expand.h b/flex/engines/hqps_db/core/operator/edge_expand.h index eebd672d8c10..a454f65226a5 100644 --- a/flex/engines/hqps_db/core/operator/edge_expand.h +++ b/flex/engines/hqps_db/core/operator/edge_expand.h @@ -888,11 +888,16 @@ class EdgeExpand { for (size_t i = 0; i < edge_labels.size(); ++i) { // Check whether the edge triplet match input vertices. // return a handler to get edges - auto sub_graph = graph.GetSubGraph(edge_labels[i][0], edge_labels[i][1], - edge_labels[i][2], direction); - // for (auto sub_graph : sub_graph_vec) { - sub_graphs.emplace_back(sub_graph); - // } + if (direction == Direction::Both || direction == Direction::Out) { + sub_graphs.emplace_back( + graph.GetSubGraph(edge_labels[i][0], edge_labels[i][1], + edge_labels[i][2], Direction::Out)); + } + if (direction == Direction::Both || direction == Direction::In) { + sub_graphs.emplace_back( + graph.GetSubGraph(edge_labels[i][0], edge_labels[i][1], + edge_labels[i][2], Direction::In)); + } } std::vector> label_triplets; @@ -1003,9 +1008,16 @@ class EdgeExpand { for (size_t i = 0; i < edge_labels.size(); ++i) { // Check whether the edge triplet match input vertices. // return a handler to get edges - auto sub_graph = graph.GetSubGraph(edge_labels[i][0], edge_labels[i][1], - edge_labels[i][2], direction); - sub_graphs.emplace_back(sub_graph); + if (direction == Direction::Both || direction == Direction::Out) { + sub_graphs.emplace_back( + graph.GetSubGraph(edge_labels[i][0], edge_labels[i][1], + edge_labels[i][2], Direction::Out)); + } + if (direction == Direction::Both || direction == Direction::In) { + sub_graphs.emplace_back( + graph.GetSubGraph(edge_labels[i][0], edge_labels[i][1], + edge_labels[i][2], Direction::In)); + } } std::vector> label_triplets; diff --git a/flex/engines/hqps_db/database/mutable_csr_interface_v2.h b/flex/engines/hqps_db/database/mutable_csr_interface_v2.h index 277faaf8d6b1..0950bbb2f637 100644 --- a/flex/engines/hqps_db/database/mutable_csr_interface_v2.h +++ b/flex/engines/hqps_db/database/mutable_csr_interface_v2.h @@ -359,7 +359,7 @@ class MutableCSRInterface { mutable_csr_graph_impl::SubGraph GetSubGraph( const label_id_t src_label_id, const label_id_t dst_label_id, const label_id_t edge_label_id, const Direction& direction) const { - const CsrBase *csr = nullptr, *other_csr = nullptr; + const CsrBase *csr = nullptr; if (direction == Direction::Out) { csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, edge_label_id); @@ -370,16 +370,18 @@ class MutableCSRInterface { edge_label_id); return mutable_csr_graph_impl::SubGraph{ csr, {dst_label_id, src_label_id, edge_label_id}, Direction::In}; - } else if (direction == Direction::Both) { - csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, - edge_label_id); - other_csr = db_session_.graph().get_ie_csr(dst_label_id, src_label_id, - edge_label_id); - return mutable_csr_graph_impl::SubGraph{ - csr, - other_csr, - {src_label_id, dst_label_id, edge_label_id}, - Direction::Both}; + // } else if (direction == Direction::Both) { + // csr = db_session_.graph().get_oe_csr(src_label_id, dst_label_id, + // edge_label_id); + // other_csr = db_session_.graph().get_ie_csr(dst_label_id, + // src_label_id, + // edge_label_id); + // return mutable_csr_graph_impl::SubGraph{ + // csr, + // other_csr, + // {src_label_id, dst_label_id, edge_label_id}, + // Direction::Both}; + // } } else { throw std::runtime_error("Not implemented - " + gs::to_string(direction)); } diff --git a/flex/engines/hqps_db/database/sub_graph.h b/flex/engines/hqps_db/database/sub_graph.h index 1779152443f5..ae8509fab6b0 100644 --- a/flex/engines/hqps_db/database/sub_graph.h +++ b/flex/engines/hqps_db/database/sub_graph.h @@ -33,23 +33,15 @@ class EdgeIter { public: using label_id_t = label_t; EdgeIter() - : ptr1_(nullptr), - ptr2_(nullptr), - dir_(Direction::Out), - label_triplet_({0, 0, 0}) {} + : ptr1_(nullptr), dir_(Direction::Out), label_triplet_({0, 0, 0}) {} - EdgeIter(std::shared_ptr ptr1, - std::shared_ptr ptr2, const Direction& dir, - const std::array& label_triplet) - : ptr1_(ptr1), ptr2_(ptr2), dir_(dir), label_triplet_(label_triplet) {} - - EdgeIter(std::shared_ptr ptr1, const Direction& dir, - const std::array& label_triplet) - : ptr1_(ptr1), ptr2_(nullptr), dir_(dir), label_triplet_(label_triplet) {} + EdgeIter(vid_t vid, std::shared_ptr ptr1, + const Direction& dir, const std::array& label_triplet) + : src_vid_(vid), ptr1_(ptr1), dir_(dir), label_triplet_(label_triplet) {} EdgeIter(const EdgeIter& other) - : ptr1_(other.ptr1_), - ptr2_(other.ptr2_), + : src_vid_(other.src_vid_), + ptr1_(other.ptr1_), dir_(other.dir_), label_triplet_(other.label_triplet_) {} @@ -57,127 +49,99 @@ class EdgeIter { if (ptr1_ && ptr1_->is_valid()) { ptr1_->next(); } - if (ptr2_ && ptr2_->is_valid()) { - ptr2_->next(); - } } inline vid_t GetDstId() const { - if (ptr1_ && ptr1_->is_valid()) { + if (dir_ == Direction::Out) { return ptr1_->get_neighbor(); - } else if (ptr2_ && ptr2_->is_valid()) { - return ptr2_->get_neighbor(); } else { - throw std::runtime_error("Invalid edge iterator"); + return src_vid_; } } inline vid_t GetSrcId() const { - // TODO(fixme): - LOG(FATAL) << "Not implemented yet"; + if (dir_ == Direction::Out) { + return src_vid_; + } else { + return ptr1_->get_neighbor(); + } } + inline vid_t GetOtherId() const { return ptr1_->get_neighbor(); } - inline label_id_t GetDstLabel() const { return label_triplet_[1]; } - - inline label_id_t GetSrcLabel() const { return label_triplet_[0]; } - - inline Direction GetDirection() const { return dir_; } - - inline Any GetData() const { - if (ptr1_ && ptr1_->is_valid()) { - return ptr1_->get_data(); - } else if (ptr2_ && ptr2_->is_valid()) { - return ptr2_->get_data(); + inline label_id_t GetDstLabel() const { + if (dir_ == Direction::Out) { + return label_triplet_[1]; } else { - throw std::runtime_error("Invalid edge iterator"); + return label_triplet_[0]; } } - inline bool IsValid() const { - if (ptr1_ && ptr1_->is_valid()) { - return true; - } else if (ptr2_ && ptr2_->is_valid()) { - return true; + + inline label_id_t GetSrcLabel() const { + if (dir_ == Direction::Out) { + return label_triplet_[0]; } else { - return false; + return label_triplet_[1]; } } + inline label_id_t GetOtherLabel() const { return label_triplet_[1]; } + + inline Direction GetDirection() const { return dir_; } + + inline Any GetData() const { return ptr1_->get_data(); } + inline bool IsValid() const { return ptr1_ && ptr1_->is_valid(); } + EdgeIter& operator=(const EdgeIter& rhs) { + this->src_vid_ = rhs.src_vid_; this->ptr1_ = rhs.ptr1_; - this->ptr2_ = rhs.ptr2_; this->dir_ = rhs.dir_; this->label_triplet_ = rhs.label_triplet_; return *this; } - size_t Size() const { - size_t ret = 0; - if (ptr1_) { - ret += ptr1_->size(); - } - if (ptr2_) { - ret += ptr2_->size(); - } - return ret; - } + size_t Size() const { return ptr1_ ? ptr1_->size() : 0; } private: - std::shared_ptr ptr1_, ptr2_; + vid_t src_vid_; // the src vid of adj list, not the src of the directed edge. + std::shared_ptr ptr1_; Direction dir_; std::array label_triplet_; }; +/** + * @brief SubGraph is a wrapper of CsrBase, which provides a way to iterate + * over the sub graph defined via an edge triplet. + * When the direction is set to both, We actually need to iterate over both the + * in edges and out edges. + */ class SubGraph { public: using label_id_t = label_t; using iterator = EdgeIter; - SubGraph(const CsrBase* first, const CsrBase* second, - const std::array& label_triplet, Direction dir) - : first_csr_(first), - second_csr_(second), - label_triplet_(label_triplet), - dir_(dir) { - if (dir == Direction::Both) { - CHECK_NOTNULL(first_csr_); - CHECK_NOTNULL(second_csr_); - } else { - CHECK_NOTNULL(first_csr_); - } - } SubGraph(const CsrBase* first, const std::array& label_triplet, Direction dir) - : first_csr_(first), - second_csr_(nullptr), - label_triplet_(label_triplet), - dir_(dir) { - CHECK_NOTNULL(first_csr_); - CHECK(dir != Direction::Both); + : first_(first), label_triplet_(label_triplet), dir_(dir) { + if (dir_ == Direction::Both) { + throw std::runtime_error("SubGraph does not support both direction"); + } } inline iterator get_edges(vid_t vid) const { - // TODO(zhanglei): Fixme - if (dir_ == Direction::Out) { - return iterator(first_csr_->edge_iter(vid), dir_, label_triplet_); - } else if (dir_ == Direction::In) { - return iterator(first_csr_->edge_iter(vid), dir_, label_triplet_); - } else { - return iterator(first_csr_->edge_iter(vid), second_csr_->edge_iter(vid), - dir_, label_triplet_); + if (first_) { + return iterator(vid, first_->edge_iter(vid), dir_, label_triplet_); } + return iterator(vid, nullptr, dir_, label_triplet_); } // here the src, dst, refer the sub graph, not the csr. label_id_t GetSrcLabel() const { return label_triplet_[0]; } - label_id_t GetEdgeLabel() const { return label_triplet_[2]; } label_id_t GetDstLabel() const { return label_triplet_[1]; } + label_id_t GetEdgeLabel() const { return label_triplet_[2]; } Direction GetDirection() const { return dir_; } private: - // dir_ = Direction::Out: first_csr_ = out_csr, second_csr_ = nullptr - // dir_ = Direction::In: first_csr_ = in_csr, second_csr_ = nullptr - // dir_ = Direction::Both: first_csr_ = out_csr, second_csr_ = in_csr - const CsrBase *first_csr_, *second_csr_; + const CsrBase* first_; std::array label_triplet_; Direction dir_; }; diff --git a/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h b/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h index f0befaace3ef..83a9fde6479f 100644 --- a/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h +++ b/flex/engines/hqps_db/structures/multi_edge_set/untyped_edge_set.h @@ -74,51 +74,15 @@ class UnTypedEdgeSetIter { inline LabelT GetEdgeLabel() const { return cur_iter_.GetEdgeLabel(); } - inline VID_T GetSrc() const { - auto dir = get_cur_direction(); - if (dir == Direction::Out) { - return src_vertices_[vid_ind_]; - } else if (dir == Direction::In) { - return cur_iter_.GetDstId(); - } else { - LOG(FATAL) << "Not supported direction " << gs::to_string(dir); - } - } + inline VID_T GetSrc() const { return cur_iter_.GetSrcId(); } - inline VID_T GetDst() const { - auto dir = get_cur_direction(); - if (dir == Direction::Out) { - return cur_iter_.GetDstId(); - } else if (dir == Direction::In) { - return src_vertices_[vid_ind_]; - } else { - LOG(FATAL) << "Not supported direction " << gs::to_string(dir); - } - } + inline VID_T GetDst() const { return cur_iter_.GetDstId(); } - inline label_t GetDstLabel() const { - auto dir = get_cur_direction(); - if (dir == Direction::Out) { - return cur_iter_.GetDstLabel(); - } else if (dir == Direction::In) { - return cur_iter_.GetSrcLabel(); - } else { - LOG(FATAL) << "Not supported direction " << gs::to_string(dir); - } - } + inline label_t GetDstLabel() const { return cur_iter_.GetDstLabel(); } - inline label_t GetSrcLabel() const { - auto dir = get_cur_direction(); - if (dir == Direction::Out) { - return cur_iter_.GetSrcLabel(); - } else if (dir == Direction::In) { - return cur_iter_.GetDstLabel(); - } else { - LOG(FATAL) << "Not supported direction " << gs::to_string(dir); - } - } + inline label_t GetSrcLabel() const { return cur_iter_.GetSrcLabel(); } - inline label_t GetOtherLabel() const { return cur_iter_.GetDstLabel(); } + inline label_t GetOtherLabel() const { return cur_iter_.GetOtherLabel(); } inline VID_T GetOther() const { return cur_iter_.GetDstId(); } diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index b861494e5af6..7af1f5ad9199 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -1346,7 +1346,7 @@ bool Schema::has_edge_label(const std::string& src_label, bool Schema::has_edge_label(label_t src_label, label_t dst_label, label_t edge_label) const { - label_t e_label_id = generate_edge_label(src_label, dst_label, edge_label); + uint32_t e_label_id = generate_edge_label(src_label, dst_label, edge_label); return eprop_names_.find(e_label_id) != eprop_names_.end(); } diff --git a/flex/tests/hqps/engine_config_test.yaml b/flex/tests/hqps/engine_config_test.yaml index d4f1ab2f5864..a11bc928c525 100644 --- a/flex/tests/hqps/engine_config_test.yaml +++ b/flex/tests/hqps/engine_config_test.yaml @@ -5,7 +5,7 @@ directories: logs: logs conf: conf log_level: INFO -default_graph: modern_graph +default_graph: ldbc compute_engine: type: hiactor workers: From 3bb20d4dc4cb7560067aab7043b9bd86de260fd8 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 16 Jul 2024 20:52:18 +0800 Subject: [PATCH 4/5] minor Committed-by: xiaolei.zl from Dev container --- flex/codegen/src/hqps_generator.h | 1 - 1 file changed, 1 deletion(-) diff --git a/flex/codegen/src/hqps_generator.h b/flex/codegen/src/hqps_generator.h index 71d682a6a001..dc9b16dfdcd2 100644 --- a/flex/codegen/src/hqps_generator.h +++ b/flex/codegen/src/hqps_generator.h @@ -58,7 +58,6 @@ static constexpr const char* QUERY_TEMPLATE_STR = " using Engine = SyncEngine<%4%>;\n" " using label_id_t = typename %4%::label_id_t;\n" " using vertex_id_t = typename %4%::vertex_id_t;\n" - " using gid_t = typename %4%::gid_t;\n" " // constructor\n" " %3%() {}\n" "// Query function for query class\n" From 7f3583946532f35ea68651e0064c236f21dba8b0 Mon Sep 17 00:00:00 2001 From: "xiaolei.zl" Date: Tue, 23 Jul 2024 15:26:40 +0800 Subject: [PATCH 5/5] add unit test example Committed-by: xiaolei.zl from Dev container --- flex/tests/CMakeLists.txt | 1 + flex/tests/interface/CMakeLists.txt | 2 + .../tests/interface/storage_interface_test.cc | 388 ++++++++++++++++++ 3 files changed, 391 insertions(+) create mode 100644 flex/tests/interface/CMakeLists.txt create mode 100644 flex/tests/interface/storage_interface_test.cc diff --git a/flex/tests/CMakeLists.txt b/flex/tests/CMakeLists.txt index e2bb952d7eb9..771c1ab46b49 100644 --- a/flex/tests/CMakeLists.txt +++ b/flex/tests/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(hqps) add_subdirectory(rt_mutable_graph) +add_subdirectory(interface) diff --git a/flex/tests/interface/CMakeLists.txt b/flex/tests/interface/CMakeLists.txt new file mode 100644 index 000000000000..f05ee7b0eac8 --- /dev/null +++ b/flex/tests/interface/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(storage_interface_test storage_interface_test.cc) +target_link_libraries(storage_interface_test hqps_plan_proto flex_rt_mutable_graph flex_graph_db flex_utils) \ No newline at end of file diff --git a/flex/tests/interface/storage_interface_test.cc b/flex/tests/interface/storage_interface_test.cc new file mode 100644 index 000000000000..0451f17a0765 --- /dev/null +++ b/flex/tests/interface/storage_interface_test.cc @@ -0,0 +1,388 @@ + +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/app/app_base.h" +#include "flex/storages/rt_mutable_graph/types.h" +#include "flex/utils/property/types.h" + +#include "flex/engines/hqps_db/database/mutable_csr_interface_v2.h" +#include "flex/proto_generated_gie/results.pb.h" + +// A unit test case which shows how to customize a graph interface, and write a +// stored procedure and run. + +struct MyColumnBase { + virtual gs::Any get(size_t index) = 0; +}; + +template +struct MyColumn : public MyColumnBase { + T get_view(size_t index) const { return T(); } + gs::Any get(size_t index) const { return gs::Any(T()); } +}; + +template +struct PropertyGetter { + using value_type = T; + std::shared_ptr> column; + bool is_valid; + PropertyGetter(std::shared_ptr> column) + : column(column), is_valid((column != nullptr)) {} + + bool IsValid() const { return is_valid; } + + inline T Get(size_t index) const { return column->get_view(index); } + + inline T get_view(size_t index) const { return column->get_view(index); } +}; + +class NbrList { + public: + class Iterator {}; + Iterator begin() const; + Iterator end() const; + inline size_t size() const; +}; + +class NbrListArray { + public: + NbrListArray() {} + ~NbrListArray() = default; + + NbrList get(size_t index) const; + + size_t size() const; + + void resize(size_t size); +}; + +template +class AdjList { + class Iterator {}; + + public: + Iterator begin() const; + Iterator end() const; + size_t size() const; +}; + +//提供自己的实现 +class SubGraph { + public: + using vid_t = uint64_t; + using label_id_t = uint8_t; + class Iterator { + inline void Next() const; + + inline vid_t GetDstId() const; + + inline vid_t GetSrcId() const; + + inline vid_t GetOtherId() const; + + inline label_id_t GetDstLabel() const; + + inline label_id_t GetSrcLabel() const; + + inline label_id_t GetOtherLabel() const; + + inline gs::Direction GetDirection(); + + inline gs::Any GetData() const; + inline bool IsValid() const; + }; + inline Iterator get_edges(vid_t vid) const; + + // here the src, dst, refer the sub graph, not the csr. + label_id_t GetSrcLabel() const; + label_id_t GetDstLabel() const; + label_id_t GetEdgeLabel() const; + gs::Direction GetDirection() const; +}; + +/** + * @brief Stores a list of AdjLists, each of which represents the edges of a + * vertex. + * @tparam T The type of the property. + */ +template +class AdjListArray { + public: + size_t size() const; + + AdjList get(size_t i) const; +}; + +// Real implementation of the storage +class ActualStorage {}; + +//基于实际的存储实现，封装出访问接口 +class TestGraph { + public: + using vertex_id_t = uint64_t; + using label_id_t = uint8_t; + + TestGraph(const ActualStorage& storage) : storage_(storage) {} + + template + using adj_list_array_t = AdjListArray; + + using nbr_list_array_t = NbrListArray; + + using sub_graph_t = SubGraph; + + template + using prop_getter_t = gs::mutable_csr_graph_impl::PropertyGetter; + + using untyped_prop_getter_t = + gs::mutable_csr_graph_impl::UntypedPropertyGetter; + + //////////////////////////////Graph Metadata Related//////////// + inline size_t VertexLabelNum() const { + throw std::runtime_error("Not implemented"); + } + + inline size_t EdgeLabelNum() const { + throw std::runtime_error("Not implemented"); + } + + inline size_t VertexNum() const { + throw std::runtime_error("Not implemented"); + } + + inline size_t VertexNum(const label_id_t& label) const { + throw std::runtime_error("Not implemented"); + } + + inline size_t EdgeNum() const { throw std::runtime_error("Not implemented"); } + + inline size_t EdgeNum(label_id_t src_label, label_id_t dst_label, + label_id_t edge_label) const { + throw std::runtime_error("Not implemented"); + } + + label_id_t GetVertexLabelId(const std::string& label) const { + throw std::runtime_error("Not implemented"); + } + + label_id_t GetEdgeLabelId(const std::string& label) const { + throw std::runtime_error("Not implemented"); + } + + std::string GetVertexLabelName(label_id_t index) const { + throw std::runtime_error("Not implemented"); + } + + std::string GetEdgeLabelName(label_id_t index) const { + throw std::runtime_error("Not implemented"); + } + + bool ExitVertexLabel(const std::string& label) const { + throw std::runtime_error("Not implemented"); + } + + bool ExitEdgeLabel(const std::string& edge_label) const { + throw std::runtime_error("Not implemented"); + } + + bool ExitEdgeTriplet(const label_id_t& src_label, const label_id_t& dst_label, + const label_id_t& edge_label) const { + throw std::runtime_error("Not implemented"); + } + + std::vector> + GetEdgeTripletPropertyMeta(const label_id_t& src_label, + const label_id_t& dst_label, + const label_id_t& label) const { + throw std::runtime_error("Not implemented"); + } + + std::vector> GetVertexPropertyMeta( + label_id_t label) const { + throw std::runtime_error("Not implemented"); + } + + //////////////////////////////Vertex-related Interface//////////// + + /** + * @brief + Scan all points with label label_id, for each point, get the properties + specified by selectors, and input them into func. The function signature of + func_t should be: void func(vertex_id_t v, const std::tuple& props) + Users implement their own logic in the function. This function has no return + value. In the example below, we scan all person points, find all points with + age , and save them to a vector. std::vector vids; + graph.ScanVertices(person_label_id, + gs::PropertySelector("age"), + [&vids](vertex_id_t vid, const std::tuple& props){ + if (std::get<0>(props) == 18){ + vids.emplace_back(vid); + } + }); + It is important to note that the properties specified by selectors will be + input into the lambda function in a tuple manner. + * @tparam FUNC_T + * @tparam SELECTOR + * @param label_id + * @param selectors The Property selectors. The selected properties will be + * fed to the function + * @param func The lambda function for filtering. + */ + // TODO(zhanglei): fix filter_null in scan.h + template + void ScanVertices(const label_id_t& label_id, + const std::tuple...>& selectors, + const FUNC_T& func) const { + throw std::runtime_error("Not implemented"); + } + + /** + * @brief ScanVertices scans all vertices with the given label with give + * original id. + * @param label_id The label id. + * @param oid The original id. + * @param vid The result internal id. + */ + bool ScanVerticesWithOid(const label_id_t& label_id, gs::Any oid, + vertex_id_t& vid) const { + throw std::runtime_error("Not implemented"); + } + + /** + * @brief GetVertexPropertyGetter gets the property getter for the given + * vertex label and property name. + * @tparam T The property type. + * @param label_id The vertex label id. + * @param prop_name The property name. + * @return The property getter. + */ + template + gs::mutable_csr_graph_impl::PropertyGetter GetVertexPropertyGetter( + const label_id_t& label_id, const std::string& prop_name) const { + throw std::runtime_error("Not implemented"); + } + + gs::mutable_csr_graph_impl::UntypedPropertyGetter + GetUntypedVertexPropertyGetter(const label_id_t& label_id, + const std::string& prop_name) const { + throw std::runtime_error("Not implemented"); + } + + //////////////////////////////Edge-related Interface//////////// + + /** + * @brief GetEdges gets the edges with the given label and edge label, and + * with the starting vertex internal ids. + * When the direction is "out", the edges are from the source label to the + * destination label, and vice versa when the direction is "in". When the + * direction is "both", the src and dst labels SHOULD be the same. + */ + template + AdjListArray GetEdges(const label_id_t& src_label_id, + const label_id_t& dst_label_id, + const label_id_t& edge_label_id, + const std::vector& vids, + const gs::Direction& direction, + size_t limit = INT_MAX) const { + throw std::runtime_error("Not implemented"); + } + + /** + * @brief Get vertices on the other side of edges, via the given edge label + * and the starting vertex internal ids. + * When the direction is "out", the vertices are on the destination label side + * of the edges, and vice versa when the direction is "in". When the direction + * is "both", the src and dst labels SHOULD be the same. + */ + NbrListArray GetOtherVertices(const label_id_t& src_label_id, + const label_id_t& dst_label_id, + const label_id_t& edge_label_id, + const std::vector& vids, + const gs::Direction& direction, + size_t limit = INT_MAX) const { + throw std::runtime_error("Not implemented"); + } + + //////////////////////////////Subgraph-related Interface//////////// + gs::mutable_csr_graph_impl::SubGraph GetSubGraph( + const label_id_t src_label_id, const label_id_t dst_label_id, + const label_id_t edge_label_id, const gs::Direction& direction) const { + throw std::runtime_error("Not implemented"); + } + + private: + const ActualStorage& storage_; +}; + +//查询通过procedure的方式去实现。这块虚基类的接口还未确定好，但是确定的是就是一个query函数。 +class ReadExample { + public: + using vertex_id_t = TestGraph::vertex_id_t; + using label_id_t = TestGraph::label_id_t; + + ReadExample() {} + // Query function for query class + results::CollectiveResults Query(TestGraph& graph) const { + // Query the graph + // Get the vertex label id + label_id_t person_label_id = graph.GetVertexLabelId("person"); + // Get the property getter for the vertex label + auto prop_getter = + graph.GetVertexPropertyGetter(person_label_id, "age"); + // Get the property getter for the vertex label + auto prop_getter2 = + graph.GetVertexPropertyGetter(person_label_id, "name"); + + results::CollectiveResults results; + // find the person with id 1 + vertex_id_t vid; + if (graph.ScanVerticesWithOid(person_label_id, 1, vid)) { + // Get the age of the person + int32_t age = prop_getter.Get(vid); + // Get the name of the person + std::string name = prop_getter2.Get(vid); + // Print the age and name + std::cout << "Person with id 1 has age: " << age << " and name: " << name + << std::endl; + auto record = results.add_results()->mutable_record(); + { + auto col = record->add_columns(); + col->mutable_name_or_id()->set_name("age"); + col->mutable_entry()->mutable_element()->mutable_object()->set_i32(age); + } + { + auto col = record->add_columns(); + col->mutable_name_or_id()->set_name("name"); + col->mutable_entry()->mutable_element()->mutable_object()->set_str( + name); + } + + } else { + std::cout << "Person with id 1 not found" << std::endl; + } + return results; + } +}; + +int main(int argc, char** argv) { + // + + ActualStorage storage; + TestGraph graph(storage); + ReadExample app; + auto results = app.Query(graph); + return 0; +} \ No newline at end of file