diff --git a/src/main/java/com/jd/jdbc/monitor/HealthCheckCollector.java b/src/main/java/com/jd/jdbc/monitor/HealthCheckCollector.java index 97eea79..cdeccb9 100644 --- a/src/main/java/com/jd/jdbc/monitor/HealthCheckCollector.java +++ b/src/main/java/com/jd/jdbc/monitor/HealthCheckCollector.java @@ -70,15 +70,14 @@ public List collect() { } GaugeMetricFamily labeledGauge = new GaugeMetricFamily(COLLECT_NAME, COLLECT_HELP, DefaultConfig.HEALTH_CHECK_LABEL_NAMES); - int notServing = -1; for (Map.Entry entry : healthByAlias.entrySet()) { TabletHealthCheck tabletHealthCheck = entry.getValue(); - buildGaugeMetric(labeledGauge, notServing, tabletHealthCheck); + buildGaugeMetric(labeledGauge, tabletHealthCheck); } return Collections.singletonList(labeledGauge); } - public static void buildGaugeMetric(GaugeMetricFamily labeledGauge, int notServing, TabletHealthCheck tabletHealthCheck) { + public static void buildGaugeMetric(GaugeMetricFamily labeledGauge, TabletHealthCheck tabletHealthCheck) { Topodata.Tablet tablet = tabletHealthCheck.getTablet(); Query.Target target = tabletHealthCheck.getTarget(); List labelValues = Lists.newArrayList(tablet.getAlias().getCell(), @@ -90,6 +89,7 @@ public static void buildGaugeMetric(GaugeMetricFamily labeledGauge, int notServi TopoProto.getPoolName(tablet), tablet.getMysqlHostname()); - labeledGauge.addMetric(labelValues, tabletHealthCheck.getServing().get() ? 1 : notServing--); + long uid = tablet.getAlias().getUid(); + labeledGauge.addMetric(labelValues, tabletHealthCheck.getServing().get() ? uid : -uid); } } diff --git a/src/main/java/com/jd/jdbc/monitor/HealthyCollector.java b/src/main/java/com/jd/jdbc/monitor/HealthyCollector.java index 81dcca4..78fce24 100644 --- a/src/main/java/com/jd/jdbc/monitor/HealthyCollector.java +++ b/src/main/java/com/jd/jdbc/monitor/HealthyCollector.java @@ -56,12 +56,11 @@ public List collect() { } GaugeMetricFamily labeledGauge = new GaugeMetricFamily(COLLECT_NAME, COLLECT_HELP, DefaultConfig.HEALTH_CHECK_LABEL_NAMES); - int notServing = -1; for (Map.Entry> entry : healthyCopy.entrySet()) { List tabletHealthCheckList = entry.getValue(); for (TabletHealthCheck tabletHealthCheck : tabletHealthCheckList) { - HealthCheckCollector.buildGaugeMetric(labeledGauge, notServing, tabletHealthCheck); + HealthCheckCollector.buildGaugeMetric(labeledGauge, tabletHealthCheck); } } diff --git a/src/main/java/com/jd/jdbc/monitor/SplitTableUtilCollector.java b/src/main/java/com/jd/jdbc/monitor/SplitTableUtilCollector.java new file mode 100644 index 0000000..611b9ed --- /dev/null +++ b/src/main/java/com/jd/jdbc/monitor/SplitTableUtilCollector.java @@ -0,0 +1,31 @@ +/* +Copyright 2021 JD Project Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package com.jd.jdbc.monitor; + +import io.prometheus.client.Gauge; + +public class SplitTableUtilCollector { + private static final Gauge SPLIT_TABLE_GAUGE = Gauge.build() + .name("split_table_flag") + .labelNames("Keyspace") + .help("split table flag") + .register(MonitorServer.getCollectorRegistry()); + + public static Gauge getSplitTableGauge() { + return SPLIT_TABLE_GAUGE; + } +} diff --git a/src/main/java/com/jd/jdbc/monitor/SqlErrorCollector.java b/src/main/java/com/jd/jdbc/monitor/SqlErrorCollector.java index b3a3d1e..eb41847 100644 --- a/src/main/java/com/jd/jdbc/monitor/SqlErrorCollector.java +++ b/src/main/java/com/jd/jdbc/monitor/SqlErrorCollector.java @@ -19,6 +19,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; +import com.jd.jdbc.common.util.CollectionUtils; import com.jd.jdbc.sqlparser.SQLUtils; import com.jd.jdbc.sqlparser.SqlParser; import com.jd.jdbc.sqlparser.ast.SQLStatement; @@ -50,16 +51,16 @@ public final class SqlErrorCollector extends Collector { private static final Integer MINUTES_TO_KEEP = 10; - private static final List LABEL_NAMES = Lists.newArrayList("Keyspace", "sqlStatement", "bindVariableMap", "errorClassName", "errorMessage", "errorTime", "sql"); + private static final List LABEL_NAMES = Lists.newArrayList("Keyspace", "sqlStatement", "errorClassName", "errorMessage", "errorTime", "sql"); private static final String COLLECT_NAME = "error_sql"; private static final String COLLECT_HELP = "error sql info."; private static final Cache SQL_ERROR_RECORDER_CACHE = CacheBuilder.newBuilder() - .maximumSize(DEFAULT_CAPACITY) - .expireAfterWrite(MINUTES_TO_KEEP, TimeUnit.MINUTES) - .build(); + .maximumSize(DEFAULT_CAPACITY) + .expireAfterWrite(MINUTES_TO_KEEP, TimeUnit.MINUTES) + .build(); private static final SqlErrorCollector INSTANCE = new SqlErrorCollector(); @@ -78,12 +79,11 @@ public List collect() { for (Map.Entry entry : copyMap.entrySet()) { SqlErrorRecorder recorder = entry.getValue(); List labelValues = Lists.newArrayList(recorder.getKeyspace(), - recorder.getSqlStatement().toString(), - recorder.getBindVariableMap().isEmpty() ? "" : recorder.getBindVariableMap().toString(), - recorder.getErrorClassName(), - recorder.getErrorMessage(), - DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(recorder.getErrorTime()), - recorder.getSql()); + recorder.getSqlStatement().toString(), + recorder.getErrorClassName(), + recorder.getErrorMessage(), + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(recorder.getErrorTime()), + recorder.getSql()); labeledGauge.addMetric(labelValues, index--); } return Collections.singletonList(labeledGauge); @@ -93,12 +93,15 @@ public void add(final String keyspace, final String userSQL, final Map MAX_BINDVAR_SIZE) { + if (LOG.isDebugEnabled()) { + LOG.debug("Error sql size too long, this record has been ignored: userSQL=" + userSQL + ",BindVarMap=" + userBindVarMap); + } + return; + } SQLStatement stmt = SQLUtils.parseSingleMysqlStatement(userSQL); Map bdMap = userBindVarMap; - if (bdMap == null) { - bdMap = new HashMap<>(); - } String sql = userSQL; try { SqlParser.PrepareAstResult prepareAstResult = SqlParser.prepareAst(stmt, userBindVarMap, charEncoding); @@ -112,20 +115,18 @@ public void add(final String keyspace, final String userSQL, final Map MAX_KEY_SIZE || bdMap.size() > MAX_BINDVAR_SIZE) { + if (key.length() > MAX_KEY_SIZE || (CollectionUtils.isNotEmpty(bdMap) && bdMap.size() > MAX_BINDVAR_SIZE)) { if (LOG.isDebugEnabled()) { - LOG.debug("Error sql size too long, this record has been ignored: key=" + key + "bdMap=" + bdMap); + LOG.debug("Error sql size too long, this record has been ignored: key=" + key + ",BindVarMap=" + bdMap); } return; } LocalDateTime date = LocalDateTime.now(); - SqlErrorRecorder sqlErrorRecorder = new SqlErrorRecorder(keyspace, e.getMessage() == null ? "" : e.getMessage(), - e.getClass().getSimpleName(), stmt, date, bdMap, sql); + SqlErrorRecorder sqlErrorRecorder = new SqlErrorRecorder(keyspace, e.getMessage() == null ? "" : e.getMessage(), e.getClass().getSimpleName(), stmt, date, sql); synchronized (this) { SqlErrorRecorder recorder = SQL_ERROR_RECORDER_CACHE.getIfPresent(key); if (recorder != null && recorder.equals(sqlErrorRecorder)) { - recorder.setBindVariableMap(bdMap); recorder.setErrorTime(date); recorder.setSql(sql); } else { @@ -137,7 +138,7 @@ public void add(final String keyspace, final String userSQL, final Map bindVariableMap; - + @Setter private String sql; @Override diff --git a/src/main/java/com/jd/jdbc/monitor/SrvKeyspaceCollector.java b/src/main/java/com/jd/jdbc/monitor/SrvKeyspaceCollector.java index a0eea8f..f234e02 100644 --- a/src/main/java/com/jd/jdbc/monitor/SrvKeyspaceCollector.java +++ b/src/main/java/com/jd/jdbc/monitor/SrvKeyspaceCollector.java @@ -53,7 +53,7 @@ public final class SrvKeyspaceCollector extends Collector { private static final Counter SRV_KEYSPACE_TASK_ERROR_COUNTER = Counter.build() .name("SrvKeyspaceTask_error_counter_total") .labelNames("Keyspace", "Cell") - .help("SrvKeyspaceTaskerror counter info") + .help("SrvKeyspaceTask error counter info") .register(MonitorServer.getCollectorRegistry()); private static final SrvKeyspaceCollector srvKeyspaceCollector = new SrvKeyspaceCollector(); diff --git a/src/main/java/com/jd/jdbc/tindexes/SplitTableUtil.java b/src/main/java/com/jd/jdbc/tindexes/SplitTableUtil.java index 6f9a658..d01f9f8 100644 --- a/src/main/java/com/jd/jdbc/tindexes/SplitTableUtil.java +++ b/src/main/java/com/jd/jdbc/tindexes/SplitTableUtil.java @@ -17,6 +17,8 @@ package com.jd.jdbc.tindexes; import com.jd.jdbc.common.Constant; +import com.jd.jdbc.common.util.CollectionUtils; +import com.jd.jdbc.monitor.SplitTableUtilCollector; import com.jd.jdbc.sqlparser.support.logging.Log; import com.jd.jdbc.sqlparser.support.logging.LogFactory; import com.jd.jdbc.sqlparser.utils.StringUtils; @@ -49,6 +51,8 @@ public final class SplitTableUtil { private static String preConfigPath; + private static boolean springFlag = false; + private SplitTableUtil() { } @@ -123,16 +127,20 @@ public static synchronized Map> getTableIndexesM throw new RuntimeException("configPath should not empty"); } if (Objects.equals(configPath, preConfigPath)) { - return SplitTableUtil.tableIndexesMap; + return tableIndexesMap; + } + if (springFlag && CollectionUtils.isNotEmpty(tableIndexesMap)) { + return tableIndexesMap; } - SplitTableUtil.tableIndexesMap = initTableIndexesMapFromYaml(configPath); + tableIndexesMap = initTableIndexesMapFromYaml(configPath); preConfigPath = configPath; return tableIndexesMap; } //call by vtdriver-spring-boot-starter public static void setSplitIndexesMapFromSpring(SplitTableConfig splitTableConfig) { - SplitTableUtil.tableIndexesMap = buildTableIndexesMap(splitTableConfig); + tableIndexesMap = buildTableIndexesMap(splitTableConfig); + springFlag = true; } public static Map> initTableIndexesMapFromYaml(final String configPath) { @@ -167,6 +175,7 @@ public static Map> buildTableIndexesMap(final Sp } } map.put(schema.getSchema().toLowerCase(), logicTableMap); + SplitTableUtilCollector.getSplitTableGauge().labels(schema.getSchema().toLowerCase()).set(1.0D); } } catch (InstantiationException | IllegalAccessException e) { if (logger.isDebugEnabled()) { diff --git a/src/main/java/com/jd/jdbc/vitess/VitessDataSource.java b/src/main/java/com/jd/jdbc/vitess/VitessDataSource.java index be8232b..01767f5 100755 --- a/src/main/java/com/jd/jdbc/vitess/VitessDataSource.java +++ b/src/main/java/com/jd/jdbc/vitess/VitessDataSource.java @@ -35,9 +35,11 @@ public class VitessDataSource extends VitessWrapper implements javax.sql.DataSou private static final Histogram HISTOGRAM = ConnectionCollector.getConnectionHistogram(); - private static volatile Map> tableIndexesMap; + private static Map> tableIndexesMap; - private static volatile boolean tableIndexesMapLoaded = false; + static { + tableIndexesMap = SplitTableUtil.getTableIndexesMap(); + } protected final VitessDriver driver = new VitessDriver(); @@ -67,14 +69,6 @@ public VitessDataSource(String url) throws Exception { } public static LogicTable getLogicTable(final String keyspace, final String logicTable) { - if (tableIndexesMap == null && !tableIndexesMapLoaded) { - synchronized (VitessDataSource.class) { - if (tableIndexesMap == null) { - VitessDataSource.tableIndexesMap = SplitTableUtil.getTableIndexesMap(); - tableIndexesMapLoaded = true; - } - } - } if (tableIndexesMap == null || tableIndexesMap.isEmpty() || StringUtils.isEmpty(keyspace) || StringUtils.isEmpty(logicTable)) { return null; } diff --git a/src/main/java/com/jd/jdbc/vitess/VitessDriver.java b/src/main/java/com/jd/jdbc/vitess/VitessDriver.java index 836f439..b9e7fb9 100755 --- a/src/main/java/com/jd/jdbc/vitess/VitessDriver.java +++ b/src/main/java/com/jd/jdbc/vitess/VitessDriver.java @@ -40,11 +40,13 @@ import com.jd.jdbc.srvtopo.SrvTopo; import com.jd.jdbc.srvtopo.TabletGateway; import com.jd.jdbc.srvtopo.TxConn; +import com.jd.jdbc.tindexes.SplitTableUtil; import com.jd.jdbc.topo.Topo; import com.jd.jdbc.topo.TopoServer; import com.jd.jdbc.util.threadpool.impl.VtDaemonExecutorService; import com.jd.jdbc.util.threadpool.impl.VtHealthCheckExecutorService; import com.jd.jdbc.util.threadpool.impl.VtQueryExecutorService; +import com.jd.jdbc.vindexes.hash.BinaryHash; import io.prometheus.client.Histogram; import io.vitess.proto.Topodata; import io.vitess.proto.Vtgate; @@ -75,6 +77,11 @@ public class VitessDriver implements java.sql.Driver { } } + static { + SplitTableUtil.getTableIndexesMap(); + new BinaryHash(); + } + private final ReentrantLock lock = new ReentrantLock(); private volatile boolean inited = false; diff --git a/src/test/java/com/jd/jdbc/table/SplitTableUtilTest.java b/src/test/java/com/jd/jdbc/table/SplitTableUtilTest.java index 9514d63..3ec2ab1 100644 --- a/src/test/java/com/jd/jdbc/table/SplitTableUtilTest.java +++ b/src/test/java/com/jd/jdbc/table/SplitTableUtilTest.java @@ -16,14 +16,24 @@ package com.jd.jdbc.table; +import com.google.common.collect.Lists; import com.jd.jdbc.sqlparser.utils.StringUtils; import com.jd.jdbc.tindexes.SplitTableUtil; +import com.jd.jdbc.tindexes.config.LogicTableConfig; +import com.jd.jdbc.tindexes.config.SchemaConfig; +import com.jd.jdbc.tindexes.config.SplitTableConfig; import org.apache.commons.lang3.RandomUtils; +import org.junit.After; import org.junit.Assert; import org.junit.Test; public class SplitTableUtilTest { + @After + public void clean() throws Exception { + TableTestUtil.setDefaultTableConfig(); + } + @Test public void getActualTableNames() { int start = RandomUtils.nextInt(); @@ -62,4 +72,40 @@ public void getShardingColumnName4() { String shardingColumnName = SplitTableUtil.getShardingColumnName("commerce3", "table_engine_test3"); Assert.assertNull(shardingColumnName); } + + @Test + public void testSpring() { + initConfigBySpring(); + String actualTableName = SplitTableUtil.getActualTableName("customer", "t_users", RandomUtils.nextInt()); + Assert.assertTrue("actualTableName should not empty", StringUtils.isNotEmpty(actualTableName)); + String shardingColumnName = SplitTableUtil.getShardingColumnName("customer", "t_users"); + Assert.assertTrue("getShardingColumnName error", "id".equalsIgnoreCase(shardingColumnName)); + shardingColumnName = SplitTableUtil.getShardingColumnName("customer", "t_user"); + Assert.assertNull(shardingColumnName); + } + + private void initConfigBySpring() { + /* + - { actualTableExprs: 't_users_${1..8}', + logicTable: t_users, + shardingAlgorithms: ShardTableByLong, + shardingColumnName: id, + shardingColumnType: INT32 } + */ + LogicTableConfig logicTableConfig = new LogicTableConfig(); + logicTableConfig.setLogicTable("t_users"); + logicTableConfig.setActualTableExprs("t_users_${1..8}"); + logicTableConfig.setShardingColumnName("id"); + logicTableConfig.setShardingColumnType("INT32"); + logicTableConfig.setShardingAlgorithms("TableRuleMod"); + + SchemaConfig schemaConfig = new SchemaConfig(); + schemaConfig.setSchema("customer"); + schemaConfig.setLogicTables(Lists.newArrayList(logicTableConfig)); + + SplitTableConfig config = new SplitTableConfig(); + config.setSchemas(Lists.newArrayList(schemaConfig)); + + SplitTableUtil.setSplitIndexesMapFromSpring(config); + } } \ No newline at end of file