From f5da33ccc216f317ddc637db088ec064de3e0493 Mon Sep 17 00:00:00 2001 From: yangyx <360508847@qq.com> Date: Wed, 10 Jul 2024 13:55:55 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B7=AF=E5=BE=84=E8=BD=AC=E6=8D=A2hook?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...onversion hook\344\275\277\347\224\250.md" | 41 +++++++++++++++++++ ...r.java => PathConvertMetastoreFilter.java} | 6 +-- .../PathConversionConfiguration.java | 2 +- .../model/PrefixingMetastoreFilter.java | 2 +- 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 "PathConversion hook\344\275\277\347\224\250.md" rename waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/{PathConverMetastoreFilter.java => PathConvertMetastoreFilter.java} (94%) diff --git "a/PathConversion hook\344\275\277\347\224\250.md" "b/PathConversion hook\344\275\277\347\224\250.md" new file mode 100644 index 00000000..21afbc12 --- /dev/null +++ "b/PathConversion hook\344\275\277\347\224\250.md" @@ -0,0 +1,41 @@ +## Overview +PathConvertMetastoreFilter是一个实现了hive 3.X MetaStoreFilterHook接口的过滤器钩子(filter hook)，用于对表的hdfs路径进行相关规则的转换，而无需更改底层元数据。 +支持在waggle-dance-federation.yml中，为单个HMS指定是否启用该hook。 + +## 路径转换范围 +仅对table及partition的路径进行转换。 +| Hook Type | Enabled? | +|-----------------|----------| +| Table(s) | true | +| Partition(s) | true | +| Database(s) | false | +| Index(es) | false | +| Table Names | false | +| Partition Specs | false | +| Index Names | false | + +## 配置 +| Property | Description | Default | +|--------------------------------------------------------------------|-----------------------------------------------------------------------------------------|-----------------------------| +| hive-metastore-filter-hook | 指定filter hook类，如路径转换钩子的类名为：com.hotels.bdp.waggledance.client.hooks.PathConvertMetastoreFilter | | +| configuration-properties.waggledance.hook.path.replacement.enabled | true or false,是否启用路径转换钩子，默认为false。 | false | + +### 配置路径转换规则：可配置多组 +| Property | Description | Default | +|-----------------------------------------------|----------------------------------------------------------------------------------------|-----------------------------| +| configuration-properties.waggledance.hook.path.replacement.regex. | Defined regex patterns to check for replacement. Requires matching value. | [] | +| configuration-properties.waggledance.hook.path.replacement.value. | Defined value patterns to check for replacement. Requires matching regex. | [] | +| configuration-properties.waggledance.hook.path.replacement.capturegroups. | (Optional) Comma delimited list of capture group indexes to use for regex replacement. | [1] | + +## 举例： +primary-meta-store: + name: dp6 + database-prefix: '' + remote-meta-store-uris: thrift://host-hms:9083 + access-control-type: READ_AND_WRITE_AND_CREATE + hive-metastore-filter-hook: com.hotels.bdp.waggledance.client.hooks.PathConvertMetastoreFilter + configuration-properties: + waggledance.hook.path.replacement.enabled: false + waggledance.hook.path.replacement.regex.rbf: ^(hdfs://sharecluster/)(?:.*) + waggledance.hook.path.replacement.capturegroups.rbf: 1 + waggledance.hook.path.replacement.value.rbf: hdfs://dp6/ \ No newline at end of file diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/PathConverMetastoreFilter.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/PathConvertMetastoreFilter.java similarity index 94% rename from waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/PathConverMetastoreFilter.java rename to waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/PathConvertMetastoreFilter.java index 684392f1..e9c8ef93 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/PathConverMetastoreFilter.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/PathConvertMetastoreFilter.java @@ -9,11 +9,11 @@ import java.util.List; -public class PathConverMetastoreFilter implements MetaStoreFilterHook { - private final static Logger log = LoggerFactory.getLogger(PathConverMetastoreFilter.class); +public class PathConvertMetastoreFilter implements MetaStoreFilterHook { + private final static Logger log = LoggerFactory.getLogger(PathConvertMetastoreFilter.class); private final PathConverter converter; - public PathConverMetastoreFilter(HiveConf conf) { + public PathConvertMetastoreFilter(HiveConf conf) { converter = new PathConverter(conf); } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/convert/pathconvert/PathConversionConfiguration.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/convert/pathconvert/PathConversionConfiguration.java index dac4105d..31436f62 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/convert/pathconvert/PathConversionConfiguration.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/hooks/convert/pathconvert/PathConversionConfiguration.java @@ -15,7 +15,7 @@ import static java.lang.String.format; @Slf4j public class PathConversionConfiguration { - private static final String PATH_REPLACEMENT_PREFIX = "hive.metastore.hooks.path.replacement"; + private static final String PATH_REPLACEMENT_PREFIX = "waggledance.hook.path.replacement"; public static final String PATH_REPLACEMENT_ENABLED = format("%s.enabled", PATH_REPLACEMENT_PREFIX); public static final String PATH_REPLACEMENT_REGEX = format("%s.regex", PATH_REPLACEMENT_PREFIX); public static final String PATH_REPLACEMENT_VALUES = format("%s.value", PATH_REPLACEMENT_PREFIX); diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java index 82ee3e16..c2afe03d 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java @@ -33,7 +33,7 @@ * */ public class PrefixingMetastoreFilter implements MetaStoreFilterHook { - public static final String PREFIX_KEY = "hive.metastore.hooks.prefix"; + public static final String PREFIX_KEY = "waggledance.hook.prefix"; public static final String PREFIX_DEFAULT = "prefix-"; private final String prefix;