From bf5bd7c1d2ad9a27877df9ae135a4962c0568ec4 Mon Sep 17 00:00:00 2001 From: jinan159 Date: Mon, 16 Jan 2023 00:46:12 +0900 Subject: [PATCH 01/19] Add agent connection port to docker README.md --- docker/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/README.md b/docker/README.md index 4ac56934f..035c2336a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -42,6 +42,8 @@ Port information: * __80__: Default controller web UI port. +* __16001__: Controller port for agent connection. + * __9010-9019__: agents connect to the controller cluster through these ports. * __12000-12029__: controllers allocate stress tests through these ports. From bbd984a624a46c892913a15a03b8815bf13bb41b Mon Sep 17 00:00:00 2001 From: junoyoon Date: Fri, 21 Jul 2023 16:38:26 +0900 Subject: [PATCH 02/19] Use controller.host property instead of old controller.ip property --- .../java/org/ngrinder/common/constant/ControllerConstants.java | 2 +- .../src/main/java/org/ngrinder/infra/config/Config.java | 2 +- .../src/main/resources/controller-properties.map | 2 +- ngrinder-core/src/main/java/net/grinder/util/NetworkUtils.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/common/constant/ControllerConstants.java b/ngrinder-controller/src/main/java/org/ngrinder/common/constant/ControllerConstants.java index 8449f2249..d3e5cb88c 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/common/constant/ControllerConstants.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/common/constant/ControllerConstants.java @@ -53,7 +53,7 @@ public interface ControllerConstants { String PROP_CONTROLLER_FRONT_PAGE_QNA_RSS = "controller.front_page_qna_rss"; String PROP_CONTROLLER_FRONT_PAGE_RESOURCES_MORE_URL = "controller.front_page_resources_more_url"; String PROP_CONTROLLER_HELP_URL = "controller.help_url"; - String PROP_CONTROLLER_IP = "controller.ip"; + String PROP_CONTROLLER_HOST = "controller.host"; String PROP_CONTROLLER_MAX_AGENT_PER_TEST = "controller.max_agent_per_test"; String PROP_CONTROLLER_MAX_CONCURRENT_TEST = "controller.max_concurrent_test"; String PROP_CONTROLLER_MAX_RUN_COUNT = "controller.max_run_count"; diff --git a/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java b/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java index 379dbee50..ecdb62b8c 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java @@ -703,7 +703,7 @@ public String getCurrentIP() { if (cluster) { return StringUtils.trimToEmpty(getClusterProperties().getProperty(PROP_CLUSTER_HOST)); } else { - return StringUtils.trimToEmpty(getControllerProperties().getProperty(PROP_CONTROLLER_IP)); + return StringUtils.trimToEmpty(getControllerProperties().getProperty(PROP_CONTROLLER_HOST)); } } diff --git a/ngrinder-controller/src/main/resources/controller-properties.map b/ngrinder-controller/src/main/resources/controller-properties.map index 830aa67ee..26e9bdce9 100644 --- a/ngrinder-controller/src/main/resources/controller-properties.map +++ b/ngrinder-controller/src/main/resources/controller-properties.map @@ -17,7 +17,7 @@ controller.monitor_port,13243,monitor.listen.port controller.url,,ngrinder.http.url,http.url controller.console_port_base,12000,ngrinder.console.portbase controller.controller_port,16001,ngrinder.agent.control.port -controller.ip,,ngrinder.controller.ipaddress,ngrinder.controller.ip +controller.host,,controller.ip,ngrinder.controller.ipaddress,ngrinder.controller.ip controller.validation_timeout,100,ngrinder.validation.timeout controller.enable_agent_auto_approval,true, controller.enable_script_console,false, diff --git a/ngrinder-core/src/main/java/net/grinder/util/NetworkUtils.java b/ngrinder-core/src/main/java/net/grinder/util/NetworkUtils.java index 0f828bb0e..c91fdb650 100644 --- a/ngrinder-core/src/main/java/net/grinder/util/NetworkUtils.java +++ b/ngrinder-core/src/main/java/net/grinder/util/NetworkUtils.java @@ -226,7 +226,7 @@ public static List getAvailablePorts(String ip, int size, int from, int if (!inetAddress.isReachable(MAX_REACHABLE_TIMEOUT)) { processException( "Can not check available ports because given local IP address '" + ip + "' is unreachable. " + - "Please check the `/etc/hosts` file or manually specify the local IP address in `${NGRINDER_HOME}/system.conf`." + "Specify controller.host property in `${NGRINDER_HOME}/system.conf`." ); } } catch (SecurityException | IOException e) { From e67db935c35a5fd24c101167deb8a72ba2b50963 Mon Sep 17 00:00:00 2001 From: leedonggyu Date: Thu, 21 Sep 2023 11:44:56 +0900 Subject: [PATCH 03/19] Add fallback to cleanup perftest resources --- .../org/ngrinder/perftest/service/PerfTestRunnable.java | 7 +++++-- ngrinder-core/src/main/java/net/grinder/SingleConsole.java | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/perftest/service/PerfTestRunnable.java b/ngrinder-controller/src/main/java/org/ngrinder/perftest/service/PerfTestRunnable.java index d826b181f..fa1e7123d 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/perftest/service/PerfTestRunnable.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/perftest/service/PerfTestRunnable.java @@ -54,7 +54,8 @@ import java.util.List; import java.util.Set; -import static java.time.Instant.*; +import static java.time.Instant.now; +import static java.time.Instant.ofEpochMilli; import static java.time.temporal.ChronoUnit.MINUTES; import static java.util.Arrays.asList; import static java.util.Objects.requireNonNull; @@ -229,6 +230,8 @@ public void doTest(final PerfTest perfTest) { LOG.debug("Stack Trace is : ", ex); doTerminate(perfTest, singleConsole, ex.getMessage()); notifyFinish(perfTest, StopReason.ERROR_WHILE_PREPARE); + } finally { + cleanUp(perfTest); } } @@ -254,7 +257,7 @@ private void deleteCachedDistFiles(File distDir, } /** - * Extract non cached distribution files for send to each agents. + * Extract non cached distribution files for send to each agent. * * @param distFilesDigest Required file's digest for currently running test. * @param agentCachedDistFilesDigestList Digest of files in each agent cache directory. diff --git a/ngrinder-core/src/main/java/net/grinder/SingleConsole.java b/ngrinder-core/src/main/java/net/grinder/SingleConsole.java index e3c5b1d04..d3d20d119 100644 --- a/ngrinder-core/src/main/java/net/grinder/SingleConsole.java +++ b/ngrinder-core/src/main/java/net/grinder/SingleConsole.java @@ -65,7 +65,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ForkJoinPool; - import static java.util.Arrays.stream; import static java.util.Collections.synchronizedMap; import static org.ngrinder.common.util.CollectionUtils.*; @@ -461,7 +460,7 @@ private void checkSafetyWithCacheState(final FileDistribution fileDistribution, /** * Wait until the given size of agents are all connected and receive digest from there cached files. - * It wait until 10 sec. + * It waits until 10 sec. * * @param size size of agent. */ From 9959953571bda95bdf4e2f363ea8e105057f821c Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Mon, 8 Jan 2024 19:29:18 +0900 Subject: [PATCH 04/19] Clean up util classes (#994) * Move StringUtils to ngrinder-core * Change to lazy search localhost address * Clean up EncodingUtils * Add profile for testing * Temporarily disable failed tests --- .../java/org/ngrinder/common/model/Home.java | 26 ++----- ...{EncodingUtils.java => PropertyUtils.java} | 69 ++++++----------- .../org/ngrinder/infra/config/Config.java | 7 +- .../config/UserDefinedMessageSource.java | 24 +++--- .../infra/hazelcast/task/RegionInfoTask.java | 4 +- .../perftest/service/ConsoleManager.java | 14 ++-- .../region/service/RegionService.java | 3 +- .../repository/FileEntryRepository.java | 6 +- ...gUtilsTest.java => PropertyUtilsTest.java} | 28 ++----- .../org/ngrinder/infra/config/MockConfig.java | 17 +++-- .../service/PerfTestRunnableTest.java | 3 + .../ngrinder/common/util/EncodingUtils.java | 23 ++++++ .../org/ngrinder/common/util/StringUtils.java | 76 +++++++++++-------- .../common/util/EncodingUtilsTest.java | 19 +++++ .../ngrinder/common/util/StringUtilsTest.java | 53 +++++++------ 15 files changed, 194 insertions(+), 178 deletions(-) rename ngrinder-controller/src/main/java/org/ngrinder/common/util/{EncodingUtils.java => PropertyUtils.java} (56%) rename ngrinder-controller/src/test/java/org/ngrinder/common/util/{EncodingUtilsTest.java => PropertyUtilsTest.java} (56%) rename {ngrinder-controller => ngrinder-core}/src/main/java/org/ngrinder/common/util/StringUtils.java (81%) create mode 100644 ngrinder-core/src/test/java/org/ngrinder/common/util/EncodingUtilsTest.java rename {ngrinder-controller => ngrinder-core}/src/test/java/org/ngrinder/common/util/StringUtilsTest.java (65%) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/common/model/Home.java b/ngrinder-controller/src/main/java/org/ngrinder/common/model/Home.java index a6afd1a33..df7fb4a34 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/common/model/Home.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/common/model/Home.java @@ -16,8 +16,8 @@ import org.apache.commons.io.FileUtils; import org.ngrinder.common.constants.GrinderConstants; import org.ngrinder.common.exception.ConfigurationException; -import org.ngrinder.common.util.EncodingUtils; import org.ngrinder.common.util.NoOp; +import org.ngrinder.common.util.PropertyUtils; import org.ngrinder.model.PerfTest; import org.ngrinder.model.User; import org.slf4j.Logger; @@ -26,7 +26,6 @@ import java.io.File; import java.io.IOException; -import java.io.StringReader; import java.util.Properties; import static java.util.Objects.requireNonNull; @@ -55,8 +54,8 @@ public class Home { private static final String PATH_DIST = "dist"; private static final String PATH_STAT = "stat"; private final static Logger LOGGER = LoggerFactory.getLogger(Home.class); - private final File directory; private static final String REPORT_CSV = "output.csv"; + private final File directory; /** * Constructor. @@ -81,8 +80,7 @@ public Home(File directory, boolean create) { } } if (directory.exists() && !directory.canWrite()) { - throw new ConfigurationException(String.format(" ngrinder home directory %s is not writable.", directory), - null); + throw new ConfigurationException(String.format(" ngrinder home directory %s is not writable.", directory), null); } this.directory = directory; } @@ -140,22 +138,8 @@ public void makeSubPath(String subPathName) { * @return loaded {@link Properties} */ public Properties getProperties(String confFileName) { - try { - File configFile = getSubFile(confFileName); - if (configFile.exists()) { - byte[] propByte = FileUtils.readFileToByteArray(configFile); - String propString = EncodingUtils.getAutoDecodedString(propByte, "UTF-8"); - Properties prop = new Properties(); - prop.load(new StringReader(propString)); - return prop; - } else { - // default empty properties. - return new Properties(); - } - - } catch (IOException e) { - throw processException("Fail to load property file " + confFileName, e); - } + File configFile = getSubFile(confFileName); + return PropertyUtils.loadProperties(configFile); } /** diff --git a/ngrinder-controller/src/main/java/org/ngrinder/common/util/EncodingUtils.java b/ngrinder-controller/src/main/java/org/ngrinder/common/util/PropertyUtils.java similarity index 56% rename from ngrinder-controller/src/main/java/org/ngrinder/common/util/EncodingUtils.java rename to ngrinder-controller/src/main/java/org/ngrinder/common/util/PropertyUtils.java index 3e21405c6..d6728b0b4 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/common/util/EncodingUtils.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/common/util/PropertyUtils.java @@ -15,14 +15,14 @@ import com.ibm.icu.text.CharsetDetector; import com.ibm.icu.text.CharsetMatch; +import org.apache.commons.io.FileUtils; +import java.io.File; import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.net.URLEncoder; +import java.io.StringReader; import java.nio.charset.Charset; +import java.util.Properties; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.ngrinder.common.util.ExceptionUtils.processException; /** @@ -30,20 +30,17 @@ * * @since 3.0 */ -public abstract class EncodingUtils { +public abstract class PropertyUtils { private static final int MINIMAL_CONFIDENCE_LEVEL = 70; /** * Decode the byte array with auto encoding detection feature. * - * @param data - * byte array - * @param defaultEncoding - * the default encoding if no encoding is sure. + * @param data byte array + * @param defaultEncoding the default encoding if no encoding is sure. * @return decoded string - * @throws IOException - * occurs when the decoding is failed. + * @throws IOException occurs when the decoding is failed. */ public static String getAutoDecodedString(byte[] data, String defaultEncoding) throws IOException { return new String(data, detectEncoding(data, defaultEncoding)); @@ -52,10 +49,8 @@ public static String getAutoDecodedString(byte[] data, String defaultEncoding) t /** * Detect encoding of given data. * - * @param data - * byte array - * @param defaultEncoding - * the default encoding if no encoding is sure. + * @param data byte array + * @param defaultEncoding the default encoding if no encoding is sure. * @return encoding name detected encoding name */ public static String detectEncoding(byte[] data, String defaultEncoding) { @@ -67,40 +62,20 @@ public static String detectEncoding(byte[] data, String defaultEncoding) { return isReliable ? estimatedEncoding : defaultEncoding; } - /** - * Encode the given path with UTF-8. - * - * "/" is not encoded. - * @param path path - * @return encoded path - */ - public static String encodePathWithUTF8(String path) { + public static Properties loadProperties(File file) { try { - StringBuilder result = new StringBuilder(); - for (char each : path.toCharArray()) { - if (each == '/') { - result.append("/"); - } else { - result.append(URLEncoder.encode(String.valueOf(each), "UTF-8")); - } + if (file.exists()) { + byte[] propByte = FileUtils.readFileToByteArray(file); + String propString = PropertyUtils.getAutoDecodedString(propByte, "UTF-8"); + Properties prop = new Properties(); + prop.load(new StringReader(propString)); + return prop; + } else { + // default empty properties. + return new Properties(); } - return result.toString(); - } catch (UnsupportedEncodingException e) { - throw processException(e); - } - } - - /** - * Decode the given path with UTF-8. - * - * @param path path - * @return decoded path - */ - public static String decodePathWithUTF8(String path) { - try { - return URLDecoder.decode(path, UTF_8.name()); - } catch (UnsupportedEncodingException e) { - throw processException(e); + } catch (IOException e) { + throw processException("Fail to load property file " + file.getAbsolutePath(), e); } } } diff --git a/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java b/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java index ecdb62b8c..bf841f8ee 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java @@ -1,4 +1,4 @@ - /* +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Profile; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; @@ -53,7 +54,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static net.grinder.util.NoOp.noOp; -import static org.apache.commons.io.FileUtils.*; +import static org.apache.commons.io.FileUtils.readFileToString; import static org.apache.commons.lang.StringUtils.defaultIfEmpty; import static org.apache.commons.lang.StringUtils.isEmpty; import static org.ngrinder.common.constant.CacheConstants.REGION_ATTR_KEY; @@ -71,7 +72,7 @@ * * @since 3.0 */ - +@Profile("production") @Component public class Config extends AbstractConfig implements ControllerConstants, ClusterConstants { public static final String NONE_REGION = "NONE"; diff --git a/ngrinder-controller/src/main/java/org/ngrinder/infra/config/UserDefinedMessageSource.java b/ngrinder-controller/src/main/java/org/ngrinder/infra/config/UserDefinedMessageSource.java index 5181b7e2e..482505875 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/infra/config/UserDefinedMessageSource.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/infra/config/UserDefinedMessageSource.java @@ -11,9 +11,8 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang.builder.EqualsBuilder; -import org.ngrinder.common.util.EncodingUtils; +import org.ngrinder.common.util.PropertyUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.support.AbstractMessageSource; @@ -81,19 +80,16 @@ private Map getLangMessageMap() { if (messagesDirectory.exists()) { for (String each : Locale.getISOLanguages()) { File file = new File(messagesDirectory, "messages_" + each + ".properties"); - if (file.exists()) { - try { - byte[] propByte = FileUtils.readFileToByteArray(file); - String propString = EncodingUtils.getAutoDecodedString(propByte, "UTF-8"); - Properties prop = new Properties(); - prop.load(new StringReader(propString)); - for (Map.Entry eachEntry : prop.entrySet()) { - map.put(new LocaleAndCode(each, (String) eachEntry.getKey()), new MessageFormat( - (String) eachEntry.getValue())); - } - } catch (Exception e) { - LOGGER.error("Error while loading {}", file.getAbsolutePath(), e); + try { + Properties prop = PropertyUtils.loadProperties(file); + for (Map.Entry eachEntry : prop.entrySet()) { + map.put( + new LocaleAndCode(each, (String) eachEntry.getKey()), + new MessageFormat((String) eachEntry.getValue()) + ); } + } catch (Exception e) { + LOGGER.error("Error while loading {}", file.getAbsolutePath(), e); } } } diff --git a/ngrinder-controller/src/main/java/org/ngrinder/infra/hazelcast/task/RegionInfoTask.java b/ngrinder-controller/src/main/java/org/ngrinder/infra/hazelcast/task/RegionInfoTask.java index a367613a6..8178b0995 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/infra/hazelcast/task/RegionInfoTask.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/infra/hazelcast/task/RegionInfoTask.java @@ -11,10 +11,10 @@ import java.util.Set; import java.util.concurrent.Callable; -import static org.apache.commons.lang.StringUtils.defaultIfBlank; import static org.ngrinder.common.constant.CacheConstants.REGION_ATTR_KEY; import static org.ngrinder.common.constant.CacheConstants.SUBREGION_ATTR_KEY; import static org.ngrinder.common.util.RegionUtils.convertSubregionsStringToSet; +import static org.ngrinder.common.util.StringUtils.defaultIfBlank; /** * Task for getting region info from clustered controller. @@ -28,7 +28,7 @@ public class RegionInfoTask implements Callable, Serializable { private transient Config config; public RegionInfo call() { - final String regionIP = defaultIfBlank(config.getCurrentIP(), NetworkUtils.getLocalHostAddress()); + final String regionIP = defaultIfBlank(config.getCurrentIP(), NetworkUtils::getLocalHostAddress); Map regionWithSubregion = config.getRegionWithSubregion(); Set subregion = convertSubregionsStringToSet(regionWithSubregion.get(SUBREGION_ATTR_KEY)); return new RegionInfo(regionWithSubregion.get(REGION_ATTR_KEY), subregion, regionIP, config.getControllerPort()); diff --git a/ngrinder-controller/src/main/java/org/ngrinder/perftest/service/ConsoleManager.java b/ngrinder-controller/src/main/java/org/ngrinder/perftest/service/ConsoleManager.java index 12cca5507..22f74549d 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/perftest/service/ConsoleManager.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/perftest/service/ConsoleManager.java @@ -24,6 +24,7 @@ import org.ngrinder.perftest.model.NullSingleConsole; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -36,10 +37,10 @@ import java.util.concurrent.TimeUnit; import static net.grinder.util.NetworkUtils.getAvailablePorts; -import static org.apache.commons.lang.StringUtils.defaultIfEmpty; import static org.ngrinder.common.constant.ControllerConstants.*; import static org.ngrinder.common.util.ExceptionUtils.processException; import static org.ngrinder.common.util.NoOp.noOp; +import static org.ngrinder.common.util.StringUtils.defaultIfBlank; /** * Console manager is responsible for console instance management. @@ -51,6 +52,7 @@ * * @since 3.0 */ +@Profile("production") @Component @RequiredArgsConstructor public class ConsoleManager { @@ -72,7 +74,7 @@ public class ConsoleManager { public void init() { int consoleSize = getConsoleSize(); consoleQueue = new ArrayBlockingQueue<>(consoleSize); - final String currentIP = defaultIfEmpty(config.getCurrentIP(), NetworkUtils.getLocalHostAddress()); + final String currentIP = defaultIfBlank(config.getCurrentIP(), NetworkUtils::getLocalHostAddress); for (int port : getAvailablePorts(currentIP, consoleSize, getConsolePortBase(), MAX_PORT_NUMBER)) { final ConsoleEntry consoleEntry = new ConsoleEntry(currentIP, port); try { @@ -150,7 +152,7 @@ public SingleConsole getAvailableConsole(ConsoleProperties baseConsoleProperties consoleCommunicationSetting.setInactiveClientTimeOut(config.getInactiveClientTimeOut()); } SingleConsole singleConsole = new SingleConsole(config.getCurrentIP(), consoleEntry.getPort(), - consoleCommunicationSetting, baseConsoleProperties); + consoleCommunicationSetting, baseConsoleProperties); getConsoleInUse().add(singleConsole); singleConsole.setCsvSeparator(config.getCsvSeparator()); return singleConsole; @@ -180,7 +182,7 @@ public void returnBackConsole(String testIdentifier, SingleConsole console) { console.sendStopMessageToAgents(); } catch (Exception e) { LOG.error("Exception occurred during console return back for test {}.", - testIdentifier, e); + testIdentifier, e); // But the port is getting back. } finally { // This is very careful implementation.. @@ -189,7 +191,7 @@ public void returnBackConsole(String testIdentifier, SingleConsole console) { console.waitUntilAllAgentDisconnected(); } catch (Exception e) { LOG.error("Exception occurred during console return back for test {}.", - testIdentifier, e); + testIdentifier, e); // If it's not disconnected still, stop them by force. agentManager.stopAgent(console.getConsolePort()); } @@ -197,7 +199,7 @@ public void returnBackConsole(String testIdentifier, SingleConsole console) { console.shutdown(); } catch (Exception e) { LOG.error("Exception occurred during console return back for test {}.", - testIdentifier, e); + testIdentifier, e); } int consolePort; String consoleIP; diff --git a/ngrinder-controller/src/main/java/org/ngrinder/region/service/RegionService.java b/ngrinder-controller/src/main/java/org/ngrinder/region/service/RegionService.java index 820d06441..bf7cda8cd 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/region/service/RegionService.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/region/service/RegionService.java @@ -40,6 +40,7 @@ import static org.ngrinder.common.constant.CacheConstants.*; import static org.ngrinder.common.util.ExceptionUtils.processException; import static org.ngrinder.common.util.RegionUtils.convertSubregionsStringToSet; +import static org.ngrinder.common.util.StringUtils.defaultIfBlank; /** * Region service class. This class responsible to keep the status of available regions. @@ -66,7 +67,7 @@ public Map get() { regions.put(regionInfo.getRegionName(), regionInfo); } } else { - final String regionIP = StringUtils.defaultIfBlank(config.getCurrentIP(), NetworkUtils.getLocalHostAddress()); + final String regionIP = defaultIfBlank(config.getCurrentIP(), NetworkUtils::getLocalHostAddress); regions.put(config.getRegion(), new RegionInfo(config.getRegion(), emptySet(), regionIP, config.getControllerPort())); } return regions; diff --git a/ngrinder-controller/src/main/java/org/ngrinder/script/repository/FileEntryRepository.java b/ngrinder-controller/src/main/java/org/ngrinder/script/repository/FileEntryRepository.java index 06d36526c..abe07238a 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/script/repository/FileEntryRepository.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/script/repository/FileEntryRepository.java @@ -20,7 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.ngrinder.common.exception.NGrinderRuntimeException; import org.ngrinder.common.model.Home; -import org.ngrinder.common.util.EncodingUtils; +import org.ngrinder.common.util.PropertyUtils; import org.ngrinder.infra.config.Config; import org.ngrinder.model.User; import org.ngrinder.script.model.FileCategory; @@ -239,8 +239,8 @@ public FileEntry findOne(User user, String path, SVNRevision revision) { } script.setFileType(FileType.getFileTypeByExtension(FilenameUtils.getExtension(script.getFileName()))); if (script.getFileType().isEditable()) { - String autoDetectedEncoding = EncodingUtils.detectEncoding(byteArray, "UTF-8"); - script.setContent((new String(byteArray, autoDetectedEncoding)).replaceAll(""","\"")); + String autoDetectedEncoding = PropertyUtils.detectEncoding(byteArray, "UTF-8"); + script.setContent((new String(byteArray, autoDetectedEncoding)).replaceAll(""", "\"")); script.setEncoding(autoDetectedEncoding); } script.setContentBytes(byteArray); diff --git a/ngrinder-controller/src/test/java/org/ngrinder/common/util/EncodingUtilsTest.java b/ngrinder-controller/src/test/java/org/ngrinder/common/util/PropertyUtilsTest.java similarity index 56% rename from ngrinder-controller/src/test/java/org/ngrinder/common/util/EncodingUtilsTest.java rename to ngrinder-controller/src/test/java/org/ngrinder/common/util/PropertyUtilsTest.java index 6683d094f..97255a85e 100644 --- a/ngrinder-controller/src/test/java/org/ngrinder/common/util/EncodingUtilsTest.java +++ b/ngrinder-controller/src/test/java/org/ngrinder/common/util/PropertyUtilsTest.java @@ -13,51 +13,39 @@ */ package org.ngrinder.common.util; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; +import org.junit.Test; import java.io.IOException; -import org.junit.Test; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; -public class EncodingUtilsTest { +public class PropertyUtilsTest { /** - * Test method for {@link EncodingUtils#getAutoDecodedString(byte[], java.lang.String)}. + * Test method for {@link PropertyUtils#getAutoDecodedString(byte[], java.lang.String)}. * * @throws IOException */ @Test public void testGetAutoDecodedString() throws IOException { String testStr = "12345678ikbsdfghjklsdfghjklzxcvbnm,.:LGF)(&^%^RYVG"; - String rtnEncode = EncodingUtils.detectEncoding(testStr.getBytes("UTF-8"), "UTF-8"); + String rtnEncode = PropertyUtils.detectEncoding(testStr.getBytes("UTF-8"), "UTF-8"); assertThat(rtnEncode, is("UTF-8")); } @Test public void testGetAutoDecodedStringChinese() throws IOException { String testStr = "12345678ikbsdfghjklsd你好lzxcvbnm,.:LGF)(&^%^RYVG"; - String rtnEncode = EncodingUtils.detectEncoding(testStr.getBytes("EUC-KR"), "EUC-KR"); + String rtnEncode = PropertyUtils.detectEncoding(testStr.getBytes("EUC-KR"), "EUC-KR"); assertThat(rtnEncode, is("EUC-KR")); } @Test public void testDetectEncoding() throws IOException { String testStr = "12345678ikbsdfghjklsd你好lzxcvbnm,.:LGF)(&^%^RYVG"; - String rtnStr = EncodingUtils.getAutoDecodedString(testStr.getBytes("UTF-8"), "UTF-8"); + String rtnStr = PropertyUtils.getAutoDecodedString(testStr.getBytes("UTF-8"), "UTF-8"); assertThat(rtnStr, is(testStr)); } - @Test - public void testPathEncoding() { - assertThat(EncodingUtils.encodePathWithUTF8("hello"), is("hello")); - assertThat(EncodingUtils.encodePathWithUTF8("한국"), is("%ED%95%9C%EA%B5%AD")); - assertThat(EncodingUtils.encodePathWithUTF8("hello/한국"), is("hello/%ED%95%9C%EA%B5%AD")); - assertThat(EncodingUtils.encodePathWithUTF8("hello/한국/와우"), is("hello/%ED%95%9C%EA%B5%AD/%EC%99%80%EC%9A%B0")); - assertThat(EncodingUtils.encodePathWithUTF8("--hello/한국/와우/"), is("--hello/%ED%95%9C%EA%B5%AD/%EC%99%80%EC%9A%B0/")); - assertThat(EncodingUtils.encodePathWithUTF8("/hello/한국/와우/"), is("/hello/%ED%95%9C%EA%B5%AD/%EC%99%80%EC%9A%B0/")); - assertThat(EncodingUtils.encodePathWithUTF8("/hello/한국.-/와우/"), is("/hello/%ED%95%9C%EA%B5%AD.-/%EC%99%80%EC%9A%B0/")); - - } - } diff --git a/ngrinder-controller/src/test/java/org/ngrinder/infra/config/MockConfig.java b/ngrinder-controller/src/test/java/org/ngrinder/infra/config/MockConfig.java index 5f9748506..0e71da807 100644 --- a/ngrinder-controller/src/test/java/org/ngrinder/infra/config/MockConfig.java +++ b/ngrinder-controller/src/test/java/org/ngrinder/infra/config/MockConfig.java @@ -1,4 +1,4 @@ -/* +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -9,7 +9,7 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ package org.ngrinder.infra.config; @@ -27,15 +27,15 @@ public class MockConfig extends Config { public boolean cluster; public boolean doRealOnRegion = false; - public void setControllerProperties(PropertiesWrapper wrapper) { - this.wrapper = wrapper; - } - @Override public PropertiesWrapper getControllerProperties() { return wrapper; } + public void setControllerProperties(PropertiesWrapper wrapper) { + this.wrapper = wrapper; + } + @Override public void loadProperties() { super.loadProperties(); @@ -52,4 +52,9 @@ public String getRegion() { return isClustered() ? (doRealOnRegion == true ? super.getRegion() : "TestRegion") : NONE_REGION; } + @Override + public String getCurrentIP() { + return "127.0.0.1"; + } + } diff --git a/ngrinder-controller/src/test/java/org/ngrinder/perftest/service/PerfTestRunnableTest.java b/ngrinder-controller/src/test/java/org/ngrinder/perftest/service/PerfTestRunnableTest.java index 182db58a5..0502846d0 100644 --- a/ngrinder-controller/src/test/java/org/ngrinder/perftest/service/PerfTestRunnableTest.java +++ b/ngrinder-controller/src/test/java/org/ngrinder/perftest/service/PerfTestRunnableTest.java @@ -22,7 +22,9 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; +import org.junit.jupiter.api.Disabled; import org.ngrinder.agent.service.AgentService; import org.ngrinder.agent.store.AgentInfoStore; import org.ngrinder.common.constant.ControllerConstants; @@ -119,6 +121,7 @@ public void before() throws IOException { } @Test + @Ignore public void testDoTest() throws IOException { assertThat(agentService.getAllAttachedFreeApprovedAgents().size(), is(1)); perfTestRunnable.doStart(); diff --git a/ngrinder-core/src/main/java/org/ngrinder/common/util/EncodingUtils.java b/ngrinder-core/src/main/java/org/ngrinder/common/util/EncodingUtils.java index d108733a2..e2b44e3b2 100644 --- a/ngrinder-core/src/main/java/org/ngrinder/common/util/EncodingUtils.java +++ b/ngrinder-core/src/main/java/org/ngrinder/common/util/EncodingUtils.java @@ -15,6 +15,7 @@ import java.io.UnsupportedEncodingException; import java.net.URLDecoder; +import java.net.URLEncoder; import static java.nio.charset.StandardCharsets.UTF_8; import static org.ngrinder.common.util.ExceptionUtils.processException; @@ -25,6 +26,28 @@ * @since 3.5.0 */ public abstract class EncodingUtils { + /** + * Encode the given path with UTF-8. + * "/" is not encoded. + * + * @param path path + * @return encoded path + */ + public static String encodePathWithUTF8(String path) { + try { + StringBuilder result = new StringBuilder(); + for (char each : path.toCharArray()) { + if (each == '/') { + result.append("/"); + } else { + result.append(URLEncoder.encode(String.valueOf(each), "UTF-8")); + } + } + return result.toString(); + } catch (UnsupportedEncodingException e) { + throw processException(e); + } + } /** * Decode the given path with UTF-8. diff --git a/ngrinder-controller/src/main/java/org/ngrinder/common/util/StringUtils.java b/ngrinder-core/src/main/java/org/ngrinder/common/util/StringUtils.java similarity index 81% rename from ngrinder-controller/src/main/java/org/ngrinder/common/util/StringUtils.java rename to ngrinder-core/src/main/java/org/ngrinder/common/util/StringUtils.java index 833398d61..bb8ee316c 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/common/util/StringUtils.java +++ b/ngrinder-core/src/main/java/org/ngrinder/common/util/StringUtils.java @@ -1,33 +1,43 @@ -/* - * Copyright (c) 2012-present NAVER Corp. - * - * This file is part of The nGrinder software distribution. Refer to - * the file LICENSE which is part of The nGrinder distribution for - * licensing details. The nGrinder distribution is available on the - * Internet at https://naver.github.io/ngrinder - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.ngrinder.common.util; - -/** - * String utility. - * - * @since 3.5.1-p1 - */ -public class StringUtils { - - public static String replaceLast(String str, String regex, String replacement) { - return str.replaceFirst("(?s)(.*)" + regex, "$1" + replacement); - } -} +/* + * Copyright (c) 2012-present NAVER Corp. + * + * This file is part of The nGrinder software distribution. Refer to + * the file LICENSE which is part of The nGrinder distribution for + * licensing details. The nGrinder distribution is available on the + * Internet at https://naver.github.io/ngrinder + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.ngrinder.common.util; + +import java.util.function.Supplier; + +/** + * String utility. + * + * @since 3.5.1-p1 + */ +public class StringUtils { + + public static String replaceLast(String str, String regex, String replacement) { + return str.replaceFirst("(?s)(.*)" + regex, "$1" + replacement); + } + + public static String defaultIfBlank(String str, Supplier defaultStringSupplier) { + if (str != null && !str.trim().isEmpty()) { + return str; + } else { + return defaultStringSupplier.get(); + } + } +} diff --git a/ngrinder-core/src/test/java/org/ngrinder/common/util/EncodingUtilsTest.java b/ngrinder-core/src/test/java/org/ngrinder/common/util/EncodingUtilsTest.java new file mode 100644 index 000000000..d988f0d6c --- /dev/null +++ b/ngrinder-core/src/test/java/org/ngrinder/common/util/EncodingUtilsTest.java @@ -0,0 +1,19 @@ +package org.ngrinder.common.util; + +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +public class EncodingUtilsTest { + @Test + public void testPathEncoding() { + assertThat(EncodingUtils.encodePathWithUTF8("hello"), is("hello")); + assertThat(EncodingUtils.encodePathWithUTF8("한국"), is("%ED%95%9C%EA%B5%AD")); + assertThat(EncodingUtils.encodePathWithUTF8("hello/한국"), is("hello/%ED%95%9C%EA%B5%AD")); + assertThat(EncodingUtils.encodePathWithUTF8("hello/한국/와우"), is("hello/%ED%95%9C%EA%B5%AD/%EC%99%80%EC%9A%B0")); + assertThat(EncodingUtils.encodePathWithUTF8("--hello/한국/와우/"), is("--hello/%ED%95%9C%EA%B5%AD/%EC%99%80%EC%9A%B0/")); + assertThat(EncodingUtils.encodePathWithUTF8("/hello/한국/와우/"), is("/hello/%ED%95%9C%EA%B5%AD/%EC%99%80%EC%9A%B0/")); + assertThat(EncodingUtils.encodePathWithUTF8("/hello/한국.-/와우/"), is("/hello/%ED%95%9C%EA%B5%AD.-/%EC%99%80%EC%9A%B0/")); + } +} diff --git a/ngrinder-controller/src/test/java/org/ngrinder/common/util/StringUtilsTest.java b/ngrinder-core/src/test/java/org/ngrinder/common/util/StringUtilsTest.java similarity index 65% rename from ngrinder-controller/src/test/java/org/ngrinder/common/util/StringUtilsTest.java rename to ngrinder-core/src/test/java/org/ngrinder/common/util/StringUtilsTest.java index 88a52e2c3..609671c30 100644 --- a/ngrinder-controller/src/test/java/org/ngrinder/common/util/StringUtilsTest.java +++ b/ngrinder-core/src/test/java/org/ngrinder/common/util/StringUtilsTest.java @@ -1,22 +1,31 @@ -package org.ngrinder.common.util; - -import org.junit.Test; - -import static org.hamcrest.Matchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.ngrinder.common.util.StringUtils.replaceLast; - -public class StringUtilsTest { - - @Test - public void testReplaceLast() { - String str1 = "ngrinder-core-3.5.1-p1.jar"; - assertThat(replaceLast(str1, "-p[1-9]", ""), is("ngrinder-core-3.5.1.jar")); - - String str2 = "ngrin-p2der-c-p3ore-3.5.1-p1.jar"; - assertThat(replaceLast(str2, "-p[1-9]", ""), is("ngrin-p2der-c-p3ore-3.5.1.jar")); - - String str3 = "ngr-pginder-core-3.5.1-p0.jar"; - assertThat(replaceLast(str3, "-p[1-9]", ""), is("ngr-pginder-core-3.5.1-p0.jar")); - } -} +package org.ngrinder.common.util; + +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.ngrinder.common.util.StringUtils.defaultIfBlank; +import static org.ngrinder.common.util.StringUtils.replaceLast; + +public class StringUtilsTest { + + @Test + public void testReplaceLast() { + String str1 = "ngrinder-core-3.5.1-p1.jar"; + assertThat(replaceLast(str1, "-p[1-9]", ""), is("ngrinder-core-3.5.1.jar")); + + String str2 = "ngrin-p2der-c-p3ore-3.5.1-p1.jar"; + assertThat(replaceLast(str2, "-p[1-9]", ""), is("ngrin-p2der-c-p3ore-3.5.1.jar")); + + String str3 = "ngr-pginder-core-3.5.1-p0.jar"; + assertThat(replaceLast(str3, "-p[1-9]", ""), is("ngr-pginder-core-3.5.1-p0.jar")); + } + + @Test + public void testDefaultIfBlank() { + assertThat(defaultIfBlank("helloWorld", () -> "foo"), is("helloWorld")); + assertThat(defaultIfBlank("\t\n", () -> "foo"), is("foo")); + assertThat(defaultIfBlank(" ", () -> "foo"), is("foo")); + assertThat(defaultIfBlank("", () -> "foo"), is("foo")); + } +} From 033814601c7ab9e267c23265e5cc4591629fa31d Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Mon, 8 Jan 2024 19:29:28 +0900 Subject: [PATCH 05/19] Add access control for webhook (#993) * Add access control for webhook * Set webhook creatorId as server authenticated userId --- .../infra/webhook/controller/WebhookApiController.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/infra/webhook/controller/WebhookApiController.java b/ngrinder-controller/src/main/java/org/ngrinder/infra/webhook/controller/WebhookApiController.java index 358227504..b4b7c428b 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/infra/webhook/controller/WebhookApiController.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/infra/webhook/controller/WebhookApiController.java @@ -47,7 +47,8 @@ public class WebhookApiController { private final WebhookActivationService webhookActivationService; @PostMapping - public void save(@RequestBody WebhookConfig webhookConfig) { + public void save(User user, @RequestBody WebhookConfig webhookConfig) { + webhookConfig.setCreatorId(user.getUserId()); webhookConfigService.save(webhookConfig); } @@ -62,8 +63,7 @@ public WebhookConfig getOne(User user) { } @GetMapping("/activation") - public List getActivations(@RequestParam String creatorId, - @PageableDefault Pageable pageable) { - return webhookActivationService.findAll(creatorId, pageable); + public List getActivations(User user, @PageableDefault Pageable pageable) { + return webhookActivationService.findAll(user.getUserId(), pageable); } } From f6d067271ad10716fe4f75ba77216a9fc7143306 Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Tue, 9 Jan 2024 11:04:32 +0900 Subject: [PATCH 06/19] Remove unused delayed health check API (#992) * Remove unused delayed health check API * Fix test --- .../home/controller/HealthCheckApiController.java | 15 --------------- .../home/controller/HomeControllerTest.java | 3 +-- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/home/controller/HealthCheckApiController.java b/ngrinder-controller/src/main/java/org/ngrinder/home/controller/HealthCheckApiController.java index 989eec355..f53caa42f 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/home/controller/HealthCheckApiController.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/home/controller/HealthCheckApiController.java @@ -64,19 +64,4 @@ public Map healthCheck(HttpServletResponse response) { map.put("regions", regionService.getAll()); return map; } - - /** - * Return health check message with 1 sec delay. If there is shutdown lock, - * it returns 503. Otherwise, it returns region lists. - * - * @param sleep in milliseconds. - * @param response response - * @return region list - */ - @GetMapping("/check/healthcheck_slow") - public Map healthCheckSlowly(@RequestParam(value = "delay", defaultValue = "1000") int sleep, - HttpServletResponse response) { - ThreadUtils.sleep(sleep); - return healthCheck(response); - } } diff --git a/ngrinder-controller/src/test/java/org/ngrinder/home/controller/HomeControllerTest.java b/ngrinder-controller/src/test/java/org/ngrinder/home/controller/HomeControllerTest.java index 8897dd6e2..091cd5ec2 100644 --- a/ngrinder-controller/src/test/java/org/ngrinder/home/controller/HomeControllerTest.java +++ b/ngrinder-controller/src/test/java/org/ngrinder/home/controller/HomeControllerTest.java @@ -65,8 +65,7 @@ public void testHome() { @Test public void testHealthCheck() { MockHttpServletResponse res = new MockHttpServletResponse(); - healthCheckApiController.healthCheck(res); - Map message = healthCheckApiController.healthCheckSlowly(500, res); + Map message = healthCheckApiController.healthCheck(res); assertEquals(message.get("current"), regionService.getCurrent()); } From 1a287412597ee1aecbf41a245a58eb2de4e7f99e Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Tue, 9 Jan 2024 17:26:17 +0900 Subject: [PATCH 07/19] Change yaml parser to yamlbeans (#995) * Change yaml parser * Add a unit test --- ngrinder-controller/build.gradle | 2 +- .../service/GitHubFileEntryService.java | 33 +++---- .../service/GitHubFileEntryServiceTest.java | 88 +++++++++++++++++++ 3 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java diff --git a/ngrinder-controller/build.gradle b/ngrinder-controller/build.gradle index a0dace928..1faa1c1f0 100644 --- a/ngrinder-controller/build.gradle +++ b/ngrinder-controller/build.gradle @@ -59,7 +59,7 @@ dependencies { implementation (group: "jaxen", name: "jaxen", version: "1.1.4") implementation (group: "com.beust", name: "jcommander", version: "1.32") implementation (group: "org.pf4j", name: "pf4j", version: "3.0.1") - implementation (group: "org.yaml", name: "snakeyaml", version: "1.25") + implementation (group: "com.esotericsoftware.yamlbeans", name: "yamlbeans", version: "1.17") implementation (group: "commons-collections", name: "commons-collections", version: "3.2.1") implementation (group: "org.reflections", name: "reflections", version: "0.9.9") implementation (group: "com.hazelcast", name: "hazelcast", version: hazelcast_version) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java b/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java index f9529cd7f..c509b474f 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java @@ -1,5 +1,6 @@ package org.ngrinder.script.service; +import com.esotericsoftware.yamlbeans.YamlReader; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; @@ -26,7 +27,6 @@ import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.auth.BasicAuthenticationManager; import org.tmatesoft.svn.core.wc.*; -import org.yaml.snakeyaml.Yaml; import java.io.File; import java.io.FileNotFoundException; @@ -280,23 +280,24 @@ public Set getAllGitHubConfig(User user) throws FileNotFoundExcept private Set getAllGithubConfig(FileEntry gitConfigYaml) { Set gitHubConfig = new HashSet<>(); - // Yaml is not thread safe. so create it every time. - Yaml yaml = new Yaml(); - Iterable> gitConfigs = cast(yaml.loadAll(gitConfigYaml.getContent())); - for (Map configMap : gitConfigs) { - if (configMap == null) { - continue; - } - configMap.put("revision", gitConfigYaml.getRevision()); - GitHubConfig config = objectMapper.convertValue(configMap, GitHubConfig.class); + try (YamlReader reader = new YamlReader(gitConfigYaml.getContent())) { + Map gitConfigMap = cast(reader.read()); + while (gitConfigMap != null) { + gitConfigMap.put("revision", gitConfigYaml.getRevision()); + GitHubConfig config = objectMapper.convertValue(gitConfigMap, GitHubConfig.class); + + if (gitHubConfig.contains(config)) { + throw new InvalidGitHubConfigurationException("GitHub configuration '" + + config.getName() + "' is duplicated.\nPlease check your .gitconfig.yml"); + } - if (gitHubConfig.contains(config)) { - throw new InvalidGitHubConfigurationException("GitHub configuration '" - + config.getName() + "' is duplicated.\nPlease check your .gitconfig.yml"); - } + gitHubConfig.add(config); - gitHubConfig.add(config); - } + gitConfigMap = cast(reader.read()); + } + } catch (IOException e) { + throw new InvalidGitHubConfigurationException("Fail to read GitHub configuration.", e); + } return gitHubConfig; } diff --git a/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java b/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java new file mode 100644 index 000000000..6bd0a6e11 --- /dev/null +++ b/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java @@ -0,0 +1,88 @@ +package org.ngrinder.script.service; + +import org.apache.commons.io.FileUtils; +import org.junit.Before; +import org.junit.Test; +import org.ngrinder.AbstractNGrinderTransactionalTest; +import org.ngrinder.common.util.CompressionUtils; +import org.ngrinder.model.User; +import org.ngrinder.script.model.FileEntry; +import org.ngrinder.script.model.GitHubConfig; +import org.ngrinder.script.repository.MockFileEntityRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; + +import java.io.File; +import java.io.IOException; +import java.util.Set; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.is; + +public class GitHubFileEntryServiceTest extends AbstractNGrinderTransactionalTest { + + private static final String GITHUB_CONFIG_NAME = ".gitconfig.yml"; + + @Autowired + private GitHubFileEntryService gitHubFileEntryService; + + @Autowired + private MockFileEntityRepository mockFileEntityRepository; + + @Before + public void before() throws IOException { + File file = new File(System.getProperty("java.io.tmpdir"), "repo"); + FileUtils.deleteQuietly(file); + CompressionUtils.unzip(new ClassPathResource("TEST_USER.zip").getFile(), file); + mockFileEntityRepository.setUserRepository(new File(file, getTestUser().getUserId())); + } + + @Test + public void getAllGitHubConfig() throws Exception { + User testUser = getTestUser(); + + FileEntry fileEntry = new FileEntry(); + fileEntry.setContent("name: My Github Config\n" + + "owner: naver\n" + + "repo: ngrinder\n" + + "access-token: e1a47e652762b60a...3ddc0713b07g13k\n" + + "---\n" + + "name: Another Config\n" + + "owner: naver\n" + + "repo: pinpoint\n" + + "access-token: t9a47e6ff262b60a...3dac0713b07e82a\n" + + "branch: feature/add-ngrinder-scripts\n" + + "base-url: https://api.mygithub.com\n" + + "script-root: ngrinder-scripts\n" + ); + fileEntry.setEncoding("UTF-8"); + fileEntry.setPath(GITHUB_CONFIG_NAME); + fileEntry.setDescription("for unit test"); + mockFileEntityRepository.save(testUser, fileEntry, fileEntry.getEncoding()); + + Set gitHubConfigs = gitHubFileEntryService.getAllGitHubConfig(testUser); + + assertThat(gitHubConfigs.size(), is(2)); + assertThat( + gitHubConfigs, + hasItems( + GitHubConfig.builder() + .name("My Github Config") + .owner("naver") + .repo("ngrinder") + .accessToken("e1a47e652762b60a...3ddc0713b07g13k") + .build(), + GitHubConfig.builder() + .name("Another Config") + .owner("naver") + .repo("pinpoint") + .accessToken("t9a47e6ff262b60a...3dac0713b07e82a") + .branch("feature/add-ngrinder-scripts") + .baseUrl("https://api.mygithub.com") + .scriptRoot("ngrinder-scripts") + .build() + ) + ); + } +} From 4efcbe9dd2972582bc63c2cdc056b53918611009 Mon Sep 17 00:00:00 2001 From: leedonggyu Date: Wed, 21 Feb 2024 12:17:55 +0900 Subject: [PATCH 08/19] Add controller properties for monitoring enable or not --- .../agent/controller/MonitorManagerApiController.java | 8 ++++++++ .../org/ngrinder/common/constant/ControllerConstants.java | 1 + .../src/main/java/org/ngrinder/infra/config/Config.java | 4 ++++ .../src/main/resources/controller-properties.map | 1 + .../src/main/resources/ngrinder_home_template/system.conf | 8 +++++--- .../js/components/perftest/modal/TargetHostInfoModal.vue | 4 ++-- 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/agent/controller/MonitorManagerApiController.java b/ngrinder-controller/src/main/java/org/ngrinder/agent/controller/MonitorManagerApiController.java index 02fc923d4..e60fa8205 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/agent/controller/MonitorManagerApiController.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/agent/controller/MonitorManagerApiController.java @@ -13,6 +13,7 @@ */ package org.ngrinder.agent.controller; +import org.ngrinder.common.exception.NGrinderRuntimeException; import org.ngrinder.common.util.AopUtils; import org.ngrinder.infra.config.Config; import org.ngrinder.monitor.controller.model.SystemDataModel; @@ -53,6 +54,13 @@ public class MonitorManagerApiController { */ @GetMapping("/state") public SystemDataModel getRealTimeMonitorData(@RequestParam final String ip) throws InterruptedException, ExecutionException, TimeoutException { + if (!config.isMonitorEnabled()) { + throw new NGrinderRuntimeException( + "Monitoring is disabled. If you want to enable monitoring feature," + + " modify system settings. but there is a risk of attack from the monitoring server." + ); + } + int port = config.getMonitorPort(); Future systemInfoFuture = AopUtils.proxy(this).getAsyncSystemInfo(ip, port); SystemInfo systemInfo = checkNotNull(systemInfoFuture.get(2, TimeUnit.SECONDS), "Monitoring data is not available."); diff --git a/ngrinder-controller/src/main/java/org/ngrinder/common/constant/ControllerConstants.java b/ngrinder-controller/src/main/java/org/ngrinder/common/constant/ControllerConstants.java index d3e5cb88c..7fd1e8421 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/common/constant/ControllerConstants.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/common/constant/ControllerConstants.java @@ -60,6 +60,7 @@ public interface ControllerConstants { String PROP_CONTROLLER_MAX_RUN_HOUR = "controller.max_run_hour"; String PROP_CONTROLLER_MAX_VUSER_PER_AGENT = "controller.max_vuser_per_agent"; String PROP_CONTROLLER_MONITOR_PORT = "controller.monitor_port"; + String PROP_CONTROLLER_ENABLE_MONITOR = "controller.enable_monitor"; String PROP_CONTROLLER_PLUGIN_SUPPORT = "controller.plugin_support"; String PROP_CONTROLLER_SAFE_DIST = "controller.safe_dist"; String PROP_CONTROLLER_SAFE_DIST_THRESHOLD = "controller.safe_dist_threshold"; diff --git a/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java b/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java index bf841f8ee..8720019d0 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/infra/config/Config.java @@ -238,6 +238,10 @@ public int getMonitorPort() { return getControllerProperties().getPropertyInt(PROP_CONTROLLER_MONITOR_PORT); } + public boolean isMonitorEnabled() { + return getControllerProperties().getPropertyBoolean(PROP_CONTROLLER_ENABLE_MONITOR); + } + /** * Check if the periodic usage report is enabled. * diff --git a/ngrinder-controller/src/main/resources/controller-properties.map b/ngrinder-controller/src/main/resources/controller-properties.map index 26e9bdce9..cb9af4428 100644 --- a/ngrinder-controller/src/main/resources/controller-properties.map +++ b/ngrinder-controller/src/main/resources/controller-properties.map @@ -14,6 +14,7 @@ controller.max_run_count,10000,agent.max.runcount controller.max_run_hour,8,agent.max.runhour controller.max_concurrent_test,10,ngrinder.max.concurrenttest controller.monitor_port,13243,monitor.listen.port +controller.enable_monitor,false, controller.url,,ngrinder.http.url,http.url controller.console_port_base,12000,ngrinder.console.portbase controller.controller_port,16001,ngrinder.agent.control.port diff --git a/ngrinder-controller/src/main/resources/ngrinder_home_template/system.conf b/ngrinder-controller/src/main/resources/ngrinder_home_template/system.conf index b3ba075fe..c72f4533f 100644 --- a/ngrinder-controller/src/main/resources/ngrinder_home_template/system.conf +++ b/ngrinder-controller/src/main/resources/ngrinder_home_template/system.conf @@ -1,4 +1,4 @@ -# Set verbose to print the detailed log +# Set verbose to print the detailed log #controller.verbose=true # If dev_mode is true, the log goes to stdout and @@ -32,7 +32,6 @@ # true if you want to controller uses high level password encoding(sha256). #controller.user_password_sha256=false - # The maximum number of agents which can be attached per one test. #controller.max_agent_per_test=10 @@ -52,6 +51,9 @@ # The monitor connecting port. The default value is 13243. #controller.monitor_port=13243 +# monitoring feature enable or not. +#controller.enable_monitor=false + # The base URL of the controller. If not set, the controller URL is automatically selected. #controller.url= @@ -135,7 +137,7 @@ # clustering configuration. # This is not the option applied on the fly. You need to reboot to apply this. ###################################################################################### -# These should be very carefully set. +# These should be very carefully set. # You can refer http://www.cubrid.org/wiki_ngrinder/entry/controller-clustering-guide # if you want to enable controller clustering. please enable below. diff --git a/ngrinder-frontend/src/js/components/perftest/modal/TargetHostInfoModal.vue b/ngrinder-frontend/src/js/components/perftest/modal/TargetHostInfoModal.vue index 73781608f..13ae3b80c 100644 --- a/ngrinder-frontend/src/js/components/perftest/modal/TargetHostInfoModal.vue +++ b/ngrinder-frontend/src/js/components/perftest/modal/TargetHostInfoModal.vue @@ -90,13 +90,13 @@ this.memory.queue.enQueue(res.data.totalMemory - res.data.freeMemory); this.cpu.chart.load({ json: { 'cpu-usage': this.cpu.queue.getArray() } }); this.memory.chart.load({ json: { 'memory-usage': this.memory.queue.getArray() } }); - }); + }).catch(() => {}); // ignore error. } closeMonitorConnection() { this.$http.get('/monitor/api/close', { params: { - ip: this.ip + ip: this.ip, }, }); } From 1c6ba7ca5d9d2976cca7829f69c7d2b3e8ab3d51 Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Wed, 21 Feb 2024 13:52:30 +0900 Subject: [PATCH 09/19] Fix unit test (#996) * Fix unit test * fix test --- .../service/PerfTestRunnableTest.java | 4 --- .../java/org/ngrinder/infra/AgentConfig.java | 13 +++++++++- .../java/net/grinder/AgentControllerTest.java | 25 +++++++++---------- .../AbstractMultiGrinderTestBase.java | 4 +-- .../org/ngrinder/infra/AgentConfigTest.java | 6 ++--- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/ngrinder-controller/src/test/java/org/ngrinder/perftest/service/PerfTestRunnableTest.java b/ngrinder-controller/src/test/java/org/ngrinder/perftest/service/PerfTestRunnableTest.java index 0502846d0..742f4a7ec 100644 --- a/ngrinder-controller/src/test/java/org/ngrinder/perftest/service/PerfTestRunnableTest.java +++ b/ngrinder-controller/src/test/java/org/ngrinder/perftest/service/PerfTestRunnableTest.java @@ -121,7 +121,6 @@ public void before() throws IOException { } @Test - @Ignore public void testDoTest() throws IOException { assertThat(agentService.getAllAttachedFreeApprovedAgents().size(), is(1)); perfTestRunnable.doStart(); @@ -135,8 +134,6 @@ public void testDoTest() throws IOException { assertThat(consoleManager.getConsoleInUse().size(), is(0)); } - private boolean ended = false; - @Test public void testStartConsole() throws IOException { // Get perf test @@ -168,7 +165,6 @@ public void onSamplingStarted() { @Override public void onSamplingEnded() { - ended = true; } @Override diff --git a/ngrinder-core/src/main/java/org/ngrinder/infra/AgentConfig.java b/ngrinder-core/src/main/java/org/ngrinder/infra/AgentConfig.java index 38b528d24..f74c51741 100644 --- a/ngrinder-core/src/main/java/org/ngrinder/infra/AgentConfig.java +++ b/ngrinder-core/src/main/java/org/ngrinder/infra/AgentConfig.java @@ -41,6 +41,7 @@ import static org.ngrinder.common.util.ExceptionUtils.processException; import static org.ngrinder.common.util.NoOp.noOp; import static org.ngrinder.common.util.Preconditions.checkNotNull; +import static org.ngrinder.common.util.StringUtils.defaultIfBlank; /** * Spring component which is responsible to get the nGrinder config which is stored @@ -328,7 +329,7 @@ public void setControllerIP(String ip) { } public String getControllerIP() { - return defaultIfEmpty(controllerIP, NetworkUtils.getLocalHostAddress()); + return defaultIfBlank(controllerIP, NetworkUtils::getLocalHostAddress); } public int getControllerPort() { @@ -420,5 +421,15 @@ protected AgentHome resolveHome() { } return resolveHome; } + + @Override + public String getControllerHost() { + return "127.0.0.1"; + } + + @Override + public String getControllerIP() { + return "127.0.0.1"; + } } } diff --git a/ngrinder-core/src/test/java/net/grinder/AgentControllerTest.java b/ngrinder-core/src/test/java/net/grinder/AgentControllerTest.java index 56760c3ec..bed864b0c 100644 --- a/ngrinder-core/src/test/java/net/grinder/AgentControllerTest.java +++ b/ngrinder-core/src/test/java/net/grinder/AgentControllerTest.java @@ -48,13 +48,15 @@ public void before() { final int freePort = getFreePort(); agentControllerServerDaemon = new AgentControllerServerDaemon(freePort); agentControllerServerDaemon.start(); + agentConfig1.setControllerPort(freePort); agentControllerDaemon = new AgentControllerDaemon(agentConfig1); - agentControllerDaemon.run(); + agentConfig2.setControllerPort(freePort); agentControllerDaemon2 = new AgentControllerDaemon(agentConfig2); agentControllerDaemon2.run(); + sleep(2000); // Validate if all agents are well-attached. @@ -172,7 +174,7 @@ public void testConsoleCommunicationSettingTimeout() throws GrinderProperties.Pe FileUtils.copyFileToDirectory( new File(this.getClass().getResource("/long-time-prepare-test.py").getFile()), - new File("./tmp/agent-home/tmp_1/file-store/_default/incoming") + new File("./tmp/agent-home/tmp_0/file-store/_default/incoming") ); URL scriptUrl = this.getClass().getResource("/long-time-prepare-test.properties"); @@ -180,17 +182,14 @@ public void testConsoleCommunicationSettingTimeout() throws GrinderProperties.Pe GrinderProperties properties = new GrinderProperties(scriptFile); properties.setAssociatedFile(new File("long-time-prepare-test.properties")); final MutableBoolean timeouted = new MutableBoolean(false); - console1.addListener(new SingleConsole.ConsoleShutdownListener() { - @Override - public void readyToStop(StopReason stopReason) { - // Notice: it couldn't distinguish between a script error or - // timed out of the keepalive connection. - System.out.println("The stop signal is received " + stopReason); - if (stopReason.equals(SCRIPT_ERROR)) { - timeouted.setValue(true); - } - } - }); + console1.addListener(stopReason -> { + // Notice: it couldn't distinguish between a script error or + // timed out of the keepalive connection. + System.out.println("The stop signal is received " + stopReason); + if (stopReason.equals(SCRIPT_ERROR)) { + timeouted.setValue(true); + } + }); console1.startTest(properties); for (int i = 0; i < 20; i++) { diff --git a/ngrinder-core/src/test/java/org/ngrinder/AbstractMultiGrinderTestBase.java b/ngrinder-core/src/test/java/org/ngrinder/AbstractMultiGrinderTestBase.java index da535f72a..d5c1f7351 100644 --- a/ngrinder-core/src/test/java/org/ngrinder/AbstractMultiGrinderTestBase.java +++ b/ngrinder-core/src/test/java/org/ngrinder/AbstractMultiGrinderTestBase.java @@ -43,13 +43,13 @@ abstract public class AbstractMultiGrinderTestBase { @Before public void agentInit() { - agentConfig1 = new AgentConfig.NullAgentConfig(1); + agentConfig1 = new AgentConfig.NullAgentConfig(0); agentConfig1.init(); agentConfig2 = new AgentConfig.NullAgentConfig(1); agentConfig2.init(); - agentConfig3 = new AgentConfig.NullAgentConfig(1); + agentConfig3 = new AgentConfig.NullAgentConfig(2); agentConfig3.init(); } diff --git a/ngrinder-core/src/test/java/org/ngrinder/infra/AgentConfigTest.java b/ngrinder-core/src/test/java/org/ngrinder/infra/AgentConfigTest.java index 3999ec07e..6dbe946b3 100644 --- a/ngrinder-core/src/test/java/org/ngrinder/infra/AgentConfigTest.java +++ b/ngrinder-core/src/test/java/org/ngrinder/infra/AgentConfigTest.java @@ -1,4 +1,4 @@ -/* +/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -9,13 +9,12 @@ * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License. + * limitations under the License. */ package org.ngrinder.infra; import org.apache.commons.io.FileUtils; -import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -36,7 +35,6 @@ public class AgentConfigTest { * @throws IOException */ @Test - @Ignore public void testAgentConfigInitialization() throws IOException { // Given AgentConfig config = new AgentConfig(); From 85efa4a075354e077a700262ef78e2e9119881bf Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Fri, 23 Feb 2024 14:40:50 +0900 Subject: [PATCH 10/19] Add object input filter on Connector (#1000) * Add object input filter on Connector * Fix ambiguous import * Change setup jdk to 11 --- .../workflows/run-build-with-test-on-pr.yml | 4 ++-- build.gradle | 4 ++-- .../net/grinder/communication/Connector.java | 20 +++++++++++++++++++ .../org/ngrinder/dns/LocalManagedDnsImpl.java | 9 ++++++++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-build-with-test-on-pr.yml b/.github/workflows/run-build-with-test-on-pr.yml index cb44a4afb..e6b06a339 100644 --- a/.github/workflows/run-build-with-test-on-pr.yml +++ b/.github/workflows/run-build-with-test-on-pr.yml @@ -21,11 +21,11 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Setup JDK 8 + - name: Setup JDK 11 uses: actions/setup-java@v3 with: distribution: 'zulu' - java-version: '8.0.345' + java-version: '11.0.22' architecture: x64 cache: 'gradle' diff --git a/build.gradle b/build.gradle index 61130f8ad..183c2bf16 100644 --- a/build.gradle +++ b/build.gradle @@ -28,8 +28,8 @@ subprojects { compileJava.options.encoding = "UTF-8" compileTestJava.options.encoding = "UTF-8" - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + sourceCompatibility = 11 + targetCompatibility = 11 ext { profile = project.hasProperty("profile") ? profile : "production" diff --git a/ngrinder-core/src/main/java/net/grinder/communication/Connector.java b/ngrinder-core/src/main/java/net/grinder/communication/Connector.java index 3a1f2a5d9..8fa883526 100644 --- a/ngrinder-core/src/main/java/net/grinder/communication/Connector.java +++ b/ngrinder-core/src/main/java/net/grinder/communication/Connector.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.ObjectInputFilter; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; @@ -31,6 +32,8 @@ import java.net.UnknownHostException; import net.grinder.common.UncheckedInterruptedException; +import net.grinder.common.processidentity.ProcessIdentity; +import net.grinder.engine.controller.AbstractAgentControllerIdentityImplementation; /** @@ -197,6 +200,23 @@ static ConnectDetails read(InputStream in) throws CommunicationException { try { final ObjectInputStream objectInputStream = new ObjectInputStream(in); + objectInputStream.setObjectInputFilter(filterInfo -> { + if (filterInfo.serialClass() == null) { + return ObjectInputFilter.Status.UNDECIDED; + } + + if (Address.class.isAssignableFrom(filterInfo.serialClass()) + || ConnectionType.class.isAssignableFrom(filterInfo.serialClass()) + || Enum.class.isAssignableFrom(filterInfo.serialClass()) + || filterInfo.serialClass().getCanonicalName().startsWith("net.grinder.engine") + || filterInfo.serialClass().getCanonicalName().startsWith("net.grinder.common") + ) { + return ObjectInputFilter.Status.ALLOWED; + } + + return ObjectInputFilter.Status.REJECTED; + }); + final ConnectionType type = (ConnectionType) objectInputStream.readObject(); final Address address = (Address) objectInputStream.readObject(); diff --git a/ngrinder-runtime/src/main/java/org/ngrinder/dns/LocalManagedDnsImpl.java b/ngrinder-runtime/src/main/java/org/ngrinder/dns/LocalManagedDnsImpl.java index fb3c701da..96dbe13b0 100644 --- a/ngrinder-runtime/src/main/java/org/ngrinder/dns/LocalManagedDnsImpl.java +++ b/ngrinder-runtime/src/main/java/org/ngrinder/dns/LocalManagedDnsImpl.java @@ -13,7 +13,14 @@ */ package org.ngrinder.dns; -import org.xbill.DNS.*; +import org.xbill.DNS.ARecord; +import org.xbill.DNS.Cache; +import org.xbill.DNS.DClass; +import org.xbill.DNS.Lookup; +import org.xbill.DNS.PTRRecord; +import org.xbill.DNS.Record; +import org.xbill.DNS.TextParseException; +import org.xbill.DNS.Type; import java.net.InetAddress; import java.net.UnknownHostException; From add38d87fe31f560d30702ab541fef24df5b53a2 Mon Sep 17 00:00:00 2001 From: Yonghyun Kim <62998666+facewise@users.noreply.github.com> Date: Fri, 23 Feb 2024 17:47:33 +0900 Subject: [PATCH 11/19] Fix failing to call mvn or gradle at MAVEN_HOME, GRADLE_HOME respectively(#975) (#998) --- .../org/ngrinder/starter/InstallationChecker.java | 14 +++++++++----- .../ngrinder/starter/InstallationCheckerTest.java | 11 +++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 ngrinder-controller/src/test/java/org/ngrinder/starter/InstallationCheckerTest.java diff --git a/ngrinder-controller/src/main/java/org/ngrinder/starter/InstallationChecker.java b/ngrinder-controller/src/main/java/org/ngrinder/starter/InstallationChecker.java index f9088ff74..8b1c5fa81 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/starter/InstallationChecker.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/starter/InstallationChecker.java @@ -20,6 +20,9 @@ */ package org.ngrinder.starter; +import com.sun.jna.Platform; + +import java.io.File; import java.util.List; import static java.lang.System.*; @@ -38,20 +41,19 @@ * */ public enum InstallationChecker { - MAVEN(MAVEN_HOME_ENV_NAME, "mvn -version", + MAVEN(MAVEN_HOME_ENV_NAME, Platform.isWindows() ? "mvn.cmd -version" : "mvn -version", "Maven isn't installed, You can't run Maven groovy scripts. Please install Maven and set MAVEN_HOME. "), - GRADLE(GRADLE_HOME_ENV_NAME, "gradle -version", + GRADLE(GRADLE_HOME_ENV_NAME, Platform.isWindows() ? "gradle.bat -version" : "gradle -version", "Gradle isn't installed, You can't run Gradle groovy scripts. Please install Gradle and set GRADLE_HOME."); - private final String homePath; + private final String homeEnvName; private final String installationCheckingCommand; private final String warningMessage; InstallationChecker(String homeEnvName, String installationCheckingCommand, String warningMessage) { + this.homeEnvName = homeEnvName; this.warningMessage = warningMessage; this.installationCheckingCommand = installationCheckingCommand; - - homePath = getenv(homeEnvName) == null ? "" : getenv(homeEnvName) + "/bin/"; } public static void checkAll() { @@ -63,6 +65,8 @@ public static void checkAll() { } private boolean isInstalled() { + String env = getenv(homeEnvName); + String homePath = env == null ? "" : env + File.separator + "bin" + File.separator; String checkCommand = homePath + installationCheckingCommand; List result = runNative(checkCommand.split(" "), null); return !result.isEmpty(); diff --git a/ngrinder-controller/src/test/java/org/ngrinder/starter/InstallationCheckerTest.java b/ngrinder-controller/src/test/java/org/ngrinder/starter/InstallationCheckerTest.java new file mode 100644 index 000000000..045624e2c --- /dev/null +++ b/ngrinder-controller/src/test/java/org/ngrinder/starter/InstallationCheckerTest.java @@ -0,0 +1,11 @@ +package org.ngrinder.starter; + +import org.junit.Test; + +public class InstallationCheckerTest { + + @Test + public void checkInstalled() { + InstallationChecker.checkAll(); + } +} From 2d9ae336a8aca69096a25b2b39c05d21e92885ad Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Mon, 26 Feb 2024 11:12:32 +0900 Subject: [PATCH 12/19] Release/3.5.9 (#1002) * Fix test * Update version * Update docker base image to jre 11 --- NOTICE | 12 ++++++++++++ README.md | 2 +- RELEASE-NOTE.md | 6 ++++++ build.gradle | 2 +- docker/README.md | 10 +++++----- docker/agent/Dockerfile | 2 +- docker/controller/Dockerfile | 4 ++-- .../src/main/resources/application.yml | 2 +- .../script_template/groovy_gradle/build.gradle | 2 +- .../java/org/ngrinder/infra/config/MockConfig.java | 4 ++++ ngrinder-frontend/package-lock.json | 4 ++-- ngrinder-frontend/package.json | 2 +- ngrinder-frontend/webpack.config.js | 2 +- 13 files changed, 38 insertions(+), 16 deletions(-) diff --git a/NOTICE b/NOTICE index 478f00e6f..93ddfd9dc 100644 --- a/NOTICE +++ b/NOTICE @@ -581,3 +581,15 @@ See the License for the specific language governing permissions and limitations under the License. ===== + +Copyright (c) 2008, SnakeYAML + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License +is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +or implied. See the License for the specific language governing permissions and limitations under +the License. diff --git a/README.md b/README.md index c768a1c12..c172f740b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ * Please post questions in [Discussions](https://github.com/naver/ngrinder/discussions) not Issues. -* nGrinder 3.5.8 has been released. See https://github.com/naver/ngrinder/releases +* nGrinder 3.5.9 has been released. See https://github.com/naver/ngrinder/releases * Currently, we are working on the internal project not ngrinder. We will resume ngrinder dev from July, 2022. nGrinder diff --git a/RELEASE-NOTE.md b/RELEASE-NOTE.md index 210f4360b..a85d165ad 100644 --- a/RELEASE-NOTE.md +++ b/RELEASE-NOTE.md @@ -1,3 +1,9 @@ +3.5.9 (2023.02.28) +================= +- Changes + - Fix security vulnerabilities + - Bump base JDK version up to 11 + 3.5.8 (2022.12.30) ================= - Changes diff --git a/build.gradle b/build.gradle index 183c2bf16..bcf5ab760 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,7 @@ allprojects { apply plugin: "idea" group = "org.ngrinder" - version = "3.5.8" + version = "3.5.9" idea { module { diff --git a/docker/README.md b/docker/README.md index 035c2336a..5ee7b78d6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -16,7 +16,7 @@ nGrinder consists of two major components. Version --------- -Current Version: 3.4 +Current Version: 3.5.9 How to run nGrinder with dockers =========================== @@ -27,13 +27,13 @@ Install docker 1.5.0 or above on your host. Pull the ngrinder/controller image. ``` -$ docker pull ngrinder/controller:3.5.8 +$ docker pull ngrinder/controller:3.5.9 ``` Start controller. ``` -docker run -d -v ~/ngrinder-controller:/opt/ngrinder-controller -p 80:80 -p 16001:16001 -p 12000-12009:12000-12009 ngrinder/controller:3.5.8 +docker run -d -v ~/ngrinder-controller:/opt/ngrinder-controller -p 80:80 -p 16001:16001 -p 12000-12009:12000-12009 ngrinder/controller:3.5.9 ``` The controller creates a data folder under /opt/ngrinder-controller to maintain test history and configuration data. In order to keep the data persistently, you should map the folder /opt/ngrinder-controller on the container to a folder on your host . @@ -56,11 +56,11 @@ Install docker 1.5.0 or above on your another host. You should run your agent on Pull the ngrinder/agent image. ``` -$ docker pull ngrinder/agent:3.5.8 +$ docker pull ngrinder/agent:3.5.9 ``` Start agent. ``` -docker run -v ~/ngrinder-agent:/opt/ngrinder-agent -d ngrinder/agent:3.5.8 controller_ip:controller_web_port +docker run -v ~/ngrinder-agent:/opt/ngrinder-agent -d ngrinder/agent:3.5.9 controller_ip:controller_web_port ``` diff --git a/docker/agent/Dockerfile b/docker/agent/Dockerfile index 6967eca4f..6ce2c84f4 100644 --- a/docker/agent/Dockerfile +++ b/docker/agent/Dockerfile @@ -1,4 +1,4 @@ -FROM jeanblanchard/java:serverjre-8 +FROM azul/zulu-openjdk-alpine:11.0.22-jre MAINTAINER JunHo Yoon "junoyoon@gmail.com" RUN apk update; apk add curl bash udev diff --git a/docker/controller/Dockerfile b/docker/controller/Dockerfile index b5aecce49..9446a455b 100644 --- a/docker/controller/Dockerfile +++ b/docker/controller/Dockerfile @@ -1,9 +1,9 @@ -FROM jeanblanchard/java:serverjre-8 +FROM azul/zulu-openjdk-alpine:11.0.22-jre MAINTAINER JunHo Yoon "junoyoon@gmail.com" RUN apk update; apk add curl bash tar -ARG MAVEN_VERSION=3.6.3 +ARG MAVEN_VERSION=3.9.6 ARG MAVEN_DOWNLOAD_BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries ARG GRADLE_VERSION=6.7.1 diff --git a/ngrinder-controller/src/main/resources/application.yml b/ngrinder-controller/src/main/resources/application.yml index 2561e2ba5..c7c4927f5 100644 --- a/ngrinder-controller/src/main/resources/application.yml +++ b/ngrinder-controller/src/main/resources/application.yml @@ -1,5 +1,5 @@ ngrinder: - version: 3.5.8 + version: 3.5.9 server: default-encoding: UTF-8 diff --git a/ngrinder-controller/src/main/resources/script_template/groovy_gradle/build.gradle b/ngrinder-controller/src/main/resources/script_template/groovy_gradle/build.gradle index 5d936d11e..219b73286 100644 --- a/ngrinder-controller/src/main/resources/script_template/groovy_gradle/build.gradle +++ b/ngrinder-controller/src/main/resources/script_template/groovy_gradle/build.gradle @@ -15,7 +15,7 @@ repositories { } ext { - ngrinder_version = "3.5.8" + ngrinder_version = "3.5.9" } dependencies { diff --git a/ngrinder-controller/src/test/java/org/ngrinder/infra/config/MockConfig.java b/ngrinder-controller/src/test/java/org/ngrinder/infra/config/MockConfig.java index 0e71da807..5fd4999d9 100644 --- a/ngrinder-controller/src/test/java/org/ngrinder/infra/config/MockConfig.java +++ b/ngrinder-controller/src/test/java/org/ngrinder/infra/config/MockConfig.java @@ -57,4 +57,8 @@ public String getCurrentIP() { return "127.0.0.1"; } + @Override + public boolean isMonitorEnabled() { + return true; + } } diff --git a/ngrinder-frontend/package-lock.json b/ngrinder-frontend/package-lock.json index 6c9a0242c..bf399aec9 100644 --- a/ngrinder-frontend/package-lock.json +++ b/ngrinder-frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "ngrinder", - "version": "3.5.8", + "version": "3.5.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "ngrinder", - "version": "3.5.8", + "version": "3.5.9", "license": "ISC", "dependencies": { "axios": "^0.21.1", diff --git a/ngrinder-frontend/package.json b/ngrinder-frontend/package.json index 999b828d9..e44159ec5 100644 --- a/ngrinder-frontend/package.json +++ b/ngrinder-frontend/package.json @@ -1,6 +1,6 @@ { "name": "ngrinder", - "version": "3.5.8", + "version": "3.5.9", "description": "ngrinder GUI", "private": true, "repository": { diff --git a/ngrinder-frontend/webpack.config.js b/ngrinder-frontend/webpack.config.js index dd318b50b..f69c7cc76 100644 --- a/ngrinder-frontend/webpack.config.js +++ b/ngrinder-frontend/webpack.config.js @@ -35,7 +35,7 @@ var lessLoader = { }; module.exports = function (env) { - var ngrinderVersion = '3.5.8'; + var ngrinderVersion = '3.5.9'; if (env !== undefined && env.ngrinderVersion !== undefined) { ngrinderVersion = env.ngrinderVersion; } From 5c702fff279e0695f692302751fea112dba96e19 Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Tue, 27 Feb 2024 13:45:27 +0900 Subject: [PATCH 13/19] Fix oshi (#1004) --- ngrinder-core/build.gradle | 4 ++-- .../main/java/net/grinder/engine/agent/PropertyBuilder.java | 3 +-- .../main/java/org/ngrinder/common/util/SystemInfoUtils.java | 5 ----- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/ngrinder-core/build.gradle b/ngrinder-core/build.gradle index 7602dfbb2..0bc1a89fb 100644 --- a/ngrinder-core/build.gradle +++ b/ngrinder-core/build.gradle @@ -3,7 +3,7 @@ description = "ngrinder-core" dependencies { api (project(":ngrinder-groovy")) api (group: "org.apache.commons", name: "commons-compress", version: "1.4.1") - api (group: "com.github.oshi", name: "oshi-core", version: "6.1.6") + api (group: "com.github.oshi", name: "oshi-core-java11", version: "6.4.13") implementation files("${rootDir}/lib/grinder-3.9.1-patch.jar") implementation (group: "org.pf4j", name: "pf4j", version: "3.0.1") @@ -11,7 +11,7 @@ dependencies { implementation (group: "commons-collections", name: "commons-collections", version: "3.2.1") implementation (group: "com.fasterxml.jackson.core", name: "jackson-annotations", version: jackson_version) implementation (group: "com.fasterxml.jackson.core", name: "jackson-databind", version: jackson_version) - implementation (group: "net.java.dev.jna", name: "jna", version: "5.6.0") + implementation (group: "net.java.dev.jna", name: "jna", version: "5.14.0") implementation (group: "commons-codec", name: "commons-codec", version: "1.14") implementation (group: "com.beust", name: "jcommander", version: "1.32") implementation (group: "ch.qos.logback", name: "logback-classic", version: "1.0.0") diff --git a/ngrinder-core/src/main/java/net/grinder/engine/agent/PropertyBuilder.java b/ngrinder-core/src/main/java/net/grinder/engine/agent/PropertyBuilder.java index 7271b6f5e..5e6bc2e46 100644 --- a/ngrinder-core/src/main/java/net/grinder/engine/agent/PropertyBuilder.java +++ b/ngrinder-core/src/main/java/net/grinder/engine/agent/PropertyBuilder.java @@ -40,7 +40,6 @@ import static org.ngrinder.common.util.Preconditions.checkNotEmpty; import static org.ngrinder.common.util.Preconditions.checkNotNull; import static org.ngrinder.common.util.SystemInfoUtils.getAvailableMemory; -import static org.ngrinder.common.util.SystemInfoUtils.getJDKVersion; /** * Class which is responsible to build custom jvm arguments. @@ -205,7 +204,7 @@ public String buildJVMArgumentWithoutMemory() { addAdditionalJavaOpt(jvmArguments); } - String jdkVersion = getJDKVersion(); + String jdkVersion = System.getProperty("java.specification.version"); if (jdkVersion != null && !jdkVersion.startsWith("1.")) { jvmArguments.append(" --add-opens java.base/java.net=ALL-UNNAMED "); } diff --git a/ngrinder-core/src/main/java/org/ngrinder/common/util/SystemInfoUtils.java b/ngrinder-core/src/main/java/org/ngrinder/common/util/SystemInfoUtils.java index 272c8244b..27ffa714e 100644 --- a/ngrinder-core/src/main/java/org/ngrinder/common/util/SystemInfoUtils.java +++ b/ngrinder-core/src/main/java/org/ngrinder/common/util/SystemInfoUtils.java @@ -33,7 +33,6 @@ import static com.sun.jna.Platform.isWindows; import static java.lang.System.currentTimeMillis; -import static java.lang.System.getProperty; import static oshi.util.ExecutingCommand.runNative; /** @@ -104,10 +103,6 @@ public static BandWidth getNetworkUsage() { return getBandWidth(rx, tx); } - public static String getJDKVersion() { - return getProperty("java.specification.version"); - } - private static BandWidth getBandWidth(long rx, long tx) { BandWidth bandWidth = new BandWidth(currentTimeMillis()); bandWidth.setReceived(bandWidth.getReceived() + rx); From 0ab78f2fe8867ac879633ed7dcd5ebbdc10dcb7a Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Tue, 27 Feb 2024 14:51:10 +0900 Subject: [PATCH 14/19] Update release note (#1005) --- RELEASE-NOTE.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/RELEASE-NOTE.md b/RELEASE-NOTE.md index a85d165ad..cfb480dc0 100644 --- a/RELEASE-NOTE.md +++ b/RELEASE-NOTE.md @@ -1,8 +1,11 @@ 3.5.9 (2023.02.28) ================= - Changes - - Fix security vulnerabilities - - Bump base JDK version up to 11 + - Fix security vulnerabilities + - Bump base JDK version up to 11 +- Bug fix + - #998 Fix failing to call mvn and gradle command in Windows + - #1004 Fix script validation error in docker env 3.5.8 (2022.12.30) ================= From 98892b1afba9a79e333b1a5d915d1da9c2a9521a Mon Sep 17 00:00:00 2001 From: Johnny Lim Date: Tue, 26 Mar 2024 19:24:30 +0900 Subject: [PATCH 15/19] Fix typo --- .../src/main/resources/script_template/basic_template_py.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ngrinder-controller/src/main/resources/script_template/basic_template_py.ftl b/ngrinder-controller/src/main/resources/script_template/basic_template_py.ftl index dfe823504..0fdc25a73 100644 --- a/ngrinder-controller/src/main/resources/script_template/basic_template_py.ftl +++ b/ngrinder-controller/src/main/resources/script_template/basic_template_py.ftl @@ -66,7 +66,7 @@ cookies.append(Cookie("${cookie["name"]?j_string}", "${cookie["value"]?j_string} class TestRunner: - # initlialize a thread + # initialize a thread def __init__(self): test1.record(TestRunner.__call__) grinder.statistics.delayReports=True From cffd50b767799a08ea37a1ba9cb24dc150d03252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20St=C3=B6ckli?= Date: Thu, 11 Apr 2024 15:51:19 +0200 Subject: [PATCH 16/19] add CodeQL workflow --- .github/workflows/codeql.yml | 83 ++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..fc959a9a4 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,83 @@ +name: "CodeQL" + +on: + push: + branches: [ 'develop', 'master' ] + pull_request: + branches: [ 'develop', 'master' ] + schedule: + - cron: '19 6 * * 2' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: 'java-kotlin' + build-mode: none + - language: 'python' + - language: 'javascript' + # CodeQL supports the following values keywords for 'language': 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + queries: security-extended + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + run: | + echo 'If you are using a "manual" build mode for one or more of the' \ + 'languages you are analyzing, replace this with the commands to build' \ + 'your code, for example:' + echo ' make bootstrap' + echo ' make release' + exit 1 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From 17dbe9f2646e081e81ea524edbbea5726f4a2c72 Mon Sep 17 00:00:00 2001 From: imb Date: Wed, 15 May 2024 02:12:14 +0900 Subject: [PATCH 17/19] Revert "Change yaml parser to yamlbeans (#995)" This reverts commit 1a287412597ee1aecbf41a245a58eb2de4e7f99e. --- ngrinder-controller/build.gradle | 2 +- .../service/GitHubFileEntryService.java | 33 ++++--- .../service/GitHubFileEntryServiceTest.java | 88 ------------------- 3 files changed, 17 insertions(+), 106 deletions(-) delete mode 100644 ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java diff --git a/ngrinder-controller/build.gradle b/ngrinder-controller/build.gradle index 1faa1c1f0..a0dace928 100644 --- a/ngrinder-controller/build.gradle +++ b/ngrinder-controller/build.gradle @@ -59,7 +59,7 @@ dependencies { implementation (group: "jaxen", name: "jaxen", version: "1.1.4") implementation (group: "com.beust", name: "jcommander", version: "1.32") implementation (group: "org.pf4j", name: "pf4j", version: "3.0.1") - implementation (group: "com.esotericsoftware.yamlbeans", name: "yamlbeans", version: "1.17") + implementation (group: "org.yaml", name: "snakeyaml", version: "1.25") implementation (group: "commons-collections", name: "commons-collections", version: "3.2.1") implementation (group: "org.reflections", name: "reflections", version: "0.9.9") implementation (group: "com.hazelcast", name: "hazelcast", version: hazelcast_version) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java b/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java index c509b474f..f9529cd7f 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java @@ -1,6 +1,5 @@ package org.ngrinder.script.service; -import com.esotericsoftware.yamlbeans.YamlReader; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; @@ -27,6 +26,7 @@ import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.auth.BasicAuthenticationManager; import org.tmatesoft.svn.core.wc.*; +import org.yaml.snakeyaml.Yaml; import java.io.File; import java.io.FileNotFoundException; @@ -280,24 +280,23 @@ public Set getAllGitHubConfig(User user) throws FileNotFoundExcept private Set getAllGithubConfig(FileEntry gitConfigYaml) { Set gitHubConfig = new HashSet<>(); - try (YamlReader reader = new YamlReader(gitConfigYaml.getContent())) { - Map gitConfigMap = cast(reader.read()); - while (gitConfigMap != null) { - gitConfigMap.put("revision", gitConfigYaml.getRevision()); - GitHubConfig config = objectMapper.convertValue(gitConfigMap, GitHubConfig.class); - - if (gitHubConfig.contains(config)) { - throw new InvalidGitHubConfigurationException("GitHub configuration '" - + config.getName() + "' is duplicated.\nPlease check your .gitconfig.yml"); - } - - gitHubConfig.add(config); + // Yaml is not thread safe. so create it every time. + Yaml yaml = new Yaml(); + Iterable> gitConfigs = cast(yaml.loadAll(gitConfigYaml.getContent())); + for (Map configMap : gitConfigs) { + if (configMap == null) { + continue; + } + configMap.put("revision", gitConfigYaml.getRevision()); + GitHubConfig config = objectMapper.convertValue(configMap, GitHubConfig.class); - gitConfigMap = cast(reader.read()); + if (gitHubConfig.contains(config)) { + throw new InvalidGitHubConfigurationException("GitHub configuration '" + + config.getName() + "' is duplicated.\nPlease check your .gitconfig.yml"); } - } catch (IOException e) { - throw new InvalidGitHubConfigurationException("Fail to read GitHub configuration.", e); - } + + gitHubConfig.add(config); + } return gitHubConfig; } diff --git a/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java b/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java deleted file mode 100644 index 6bd0a6e11..000000000 --- a/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.ngrinder.script.service; - -import org.apache.commons.io.FileUtils; -import org.junit.Before; -import org.junit.Test; -import org.ngrinder.AbstractNGrinderTransactionalTest; -import org.ngrinder.common.util.CompressionUtils; -import org.ngrinder.model.User; -import org.ngrinder.script.model.FileEntry; -import org.ngrinder.script.model.GitHubConfig; -import org.ngrinder.script.repository.MockFileEntityRepository; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ClassPathResource; - -import java.io.File; -import java.io.IOException; -import java.util.Set; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasItems; -import static org.hamcrest.Matchers.is; - -public class GitHubFileEntryServiceTest extends AbstractNGrinderTransactionalTest { - - private static final String GITHUB_CONFIG_NAME = ".gitconfig.yml"; - - @Autowired - private GitHubFileEntryService gitHubFileEntryService; - - @Autowired - private MockFileEntityRepository mockFileEntityRepository; - - @Before - public void before() throws IOException { - File file = new File(System.getProperty("java.io.tmpdir"), "repo"); - FileUtils.deleteQuietly(file); - CompressionUtils.unzip(new ClassPathResource("TEST_USER.zip").getFile(), file); - mockFileEntityRepository.setUserRepository(new File(file, getTestUser().getUserId())); - } - - @Test - public void getAllGitHubConfig() throws Exception { - User testUser = getTestUser(); - - FileEntry fileEntry = new FileEntry(); - fileEntry.setContent("name: My Github Config\n" + - "owner: naver\n" + - "repo: ngrinder\n" + - "access-token: e1a47e652762b60a...3ddc0713b07g13k\n" + - "---\n" + - "name: Another Config\n" + - "owner: naver\n" + - "repo: pinpoint\n" + - "access-token: t9a47e6ff262b60a...3dac0713b07e82a\n" + - "branch: feature/add-ngrinder-scripts\n" + - "base-url: https://api.mygithub.com\n" + - "script-root: ngrinder-scripts\n" - ); - fileEntry.setEncoding("UTF-8"); - fileEntry.setPath(GITHUB_CONFIG_NAME); - fileEntry.setDescription("for unit test"); - mockFileEntityRepository.save(testUser, fileEntry, fileEntry.getEncoding()); - - Set gitHubConfigs = gitHubFileEntryService.getAllGitHubConfig(testUser); - - assertThat(gitHubConfigs.size(), is(2)); - assertThat( - gitHubConfigs, - hasItems( - GitHubConfig.builder() - .name("My Github Config") - .owner("naver") - .repo("ngrinder") - .accessToken("e1a47e652762b60a...3ddc0713b07g13k") - .build(), - GitHubConfig.builder() - .name("Another Config") - .owner("naver") - .repo("pinpoint") - .accessToken("t9a47e6ff262b60a...3dac0713b07e82a") - .branch("feature/add-ngrinder-scripts") - .baseUrl("https://api.mygithub.com") - .scriptRoot("ngrinder-scripts") - .build() - ) - ); - } -} From 6df6b325590a215a0ca7b6bb663a118c320f68d7 Mon Sep 17 00:00:00 2001 From: imb Date: Wed, 15 May 2024 03:03:16 +0900 Subject: [PATCH 18/19] Change to use SafeConstructor --- .../service/GitHubFileEntryService.java | 9 +-- .../service/GitHubFileEntryServiceTest.java | 69 +++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java diff --git a/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java b/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java index f9529cd7f..18966fd81 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/script/service/GitHubFileEntryService.java @@ -27,6 +27,7 @@ import org.tmatesoft.svn.core.auth.BasicAuthenticationManager; import org.tmatesoft.svn.core.wc.*; import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; import java.io.File; import java.io.FileNotFoundException; @@ -281,14 +282,14 @@ public Set getAllGitHubConfig(User user) throws FileNotFoundExcept private Set getAllGithubConfig(FileEntry gitConfigYaml) { Set gitHubConfig = new HashSet<>(); // Yaml is not thread safe. so create it every time. - Yaml yaml = new Yaml(); - Iterable> gitConfigs = cast(yaml.loadAll(gitConfigYaml.getContent())); - for (Map configMap : gitConfigs) { + Yaml yaml = new Yaml(new SafeConstructor()); + Iterable> gitConfigs = cast(yaml.loadAll(gitConfigYaml.getContent())); + for (Map configMap : gitConfigs) { if (configMap == null) { continue; } - configMap.put("revision", gitConfigYaml.getRevision()); GitHubConfig config = objectMapper.convertValue(configMap, GitHubConfig.class); + config.setRevision(String.valueOf(gitConfigYaml.getRevision())); if (gitHubConfig.contains(config)) { throw new InvalidGitHubConfigurationException("GitHub configuration '" diff --git a/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java b/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java new file mode 100644 index 000000000..17f3579dd --- /dev/null +++ b/ngrinder-controller/src/test/java/org/ngrinder/script/service/GitHubFileEntryServiceTest.java @@ -0,0 +1,69 @@ +package org.ngrinder.script.service; + +import org.junit.Test; +import org.ngrinder.AbstractNGrinderTransactionalTest; +import org.ngrinder.common.exception.NGrinderRuntimeException; +import org.ngrinder.script.model.FileEntry; +import org.springframework.beans.factory.annotation.Autowired; +import org.yaml.snakeyaml.constructor.ConstructorException; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class GitHubFileEntryServiceTest extends AbstractNGrinderTransactionalTest { + @Autowired + private GitHubFileEntryService gitHubFileEntryService; + + @Test + public void testValidateInvalidConfigNameLength() { + FileEntry fileEntry = new FileEntry(); + fileEntry.setContent( + "name: My Long Long Long Long Long Long Github Config Name\n" + + "owner: naver\n" + + "repo: ngrinder\n" + + "access-token: e1a47e652762b60a...3ddc0713b07g13k\n" + ); + fileEntry.setRevision(-1L); + + try { + gitHubFileEntryService.validate(fileEntry); + } catch (Exception e) { + assertTrue(e instanceof NGrinderRuntimeException); + assertTrue(e.getMessage().contains("Configuration name must be shorter than")); + } + } + + @Test + public void testValidateInvalidYamlValue() { + FileEntry fileEntry = new FileEntry(); + fileEntry.setContent( + "!!com.sun.rowset.JdbcRowSetImpl\n " + + "dataSourceName: rmi://127.0.0.1:13243/jmxrmi\n " + + "autoCommit: true" + ); + fileEntry.setRevision(-1L); + + try { + gitHubFileEntryService.validate(fileEntry); + } catch (Exception e) { + assertTrue(e instanceof ConstructorException); + assertTrue(e.getMessage().contains("could not determine a constructor for the tag")); + } + } + + @Test + public void testValidateInvalidYamlValue2() { + FileEntry fileEntry = new FileEntry(); + fileEntry.setContent( + "some_var: !!javax.script.ScriptEngineManager " + + "[!!java.net.URLClassLoader [[!!java.net.URL [\"http://localhost:8080\"]]]]" + ); + fileEntry.setRevision(-1L); + + try { + gitHubFileEntryService.validate(fileEntry); + } catch (Exception e) { + assertTrue(e instanceof ConstructorException); + assertTrue(e.getMessage().contains("could not determine a constructor for the tag")); + } + } +} From 2e44e04f405d8f2902f04b99a64d1049cb63c258 Mon Sep 17 00:00:00 2001 From: Imbyungjun Date: Mon, 20 May 2024 15:39:52 +0900 Subject: [PATCH 19/19] Add authorization settings for API (#1017) * Add authorization for file entry APIs * Changed not to expose other users' perftest running information * Add authorization for perftest APIs --- .../perftest/controller/PerfTestApiController.java | 8 +++++++- .../ngrinder/perftest/controller/PerfTestController.java | 6 ++++++ .../script/controller/FileEntryApiController.java | 2 ++ .../src/js/components/perftest/list/List.vue | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/ngrinder-controller/src/main/java/org/ngrinder/perftest/controller/PerfTestApiController.java b/ngrinder-controller/src/main/java/org/ngrinder/perftest/controller/PerfTestApiController.java index 14e28d097..b3aa9709f 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/perftest/controller/PerfTestApiController.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/perftest/controller/PerfTestApiController.java @@ -44,6 +44,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; import org.springframework.data.web.PageableDefault; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import java.net.URL; @@ -74,6 +75,7 @@ @RestController @RequestMapping("/perftest/api") @RequiredArgsConstructor +@PreAuthorize("hasAnyRole('A', 'S', 'U')") public class PerfTestApiController { private final PerfTestService perfTestService; @@ -337,6 +339,7 @@ public Map refreshTestRunning(User user, @PathVariable long id) } @GetMapping("/{id}/detail_report") + @PreAuthorize("permitAll") public Map getReport(@PathVariable long id) { Map model = newHashMap(); model.put("test", perfTestService.getOne(id)); @@ -531,7 +534,7 @@ public Map getStatuses(User user, @RequestParam(defaultValue = " .collect(toList()); return buildMap( - "perfTestInfo", perfTestService.getCurrentPerfTestStatistics(), + "runningTestsCount", perfTestService.getCurrentPerfTestStatistics().size(), "status", statuses ); } @@ -601,6 +604,7 @@ public List getLogs(User user, @PathVariable long id) { * @return perf test result list. */ @GetMapping({"/{id}/perf", "/{id}/graph"}) + @PreAuthorize("permitAll") public Map getPerfGraph(@PathVariable long id, @RequestParam(defaultValue = "") String dataType, @RequestParam(defaultValue = "false") boolean onlyTotal, @@ -630,6 +634,7 @@ private Map getPerfGraphData(Long id, String[] dataTypes, boolea * @return json message */ @GetMapping("/{id}/monitor") + @PreAuthorize("permitAll") public Map getMonitorGraph(@PathVariable long id, @RequestParam String targetIP, @RequestParam int imgWidth) { int interval = perfTestService.getMonitorGraphInterval(id, targetIP, imgWidth); @@ -649,6 +654,7 @@ public Map getMonitorGraph(@PathVariable long id, * @return json message */ @GetMapping("/{id}/plugin/{plugin}") + @PreAuthorize("permitAll") public Map getPluginGraph(@PathVariable long id, @PathVariable String plugin, @RequestParam String kind, diff --git a/ngrinder-controller/src/main/java/org/ngrinder/perftest/controller/PerfTestController.java b/ngrinder-controller/src/main/java/org/ngrinder/perftest/controller/PerfTestController.java index 4afb2f707..db414e2a1 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/perftest/controller/PerfTestController.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/perftest/controller/PerfTestController.java @@ -24,6 +24,7 @@ import org.ngrinder.infra.spring.RemainedPath; import org.ngrinder.model.*; import org.ngrinder.perftest.service.PerfTestService; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; @@ -44,6 +45,7 @@ @RequestMapping("/perftest") @GlobalControllerModel @RequiredArgsConstructor +@PreAuthorize("hasAnyRole('A', 'S', 'U')") public class PerfTestController { private final PerfTestService perfTestService; @@ -150,6 +152,7 @@ public void showLog(User user, @PathVariable("id") long id, @RemainedPath String * @return perftest/detail_report */ @GetMapping({"/{id}/detail_report", /* for backward compatibility */"/{id}/report"}) + @PreAuthorize("permitAll") public String getReport(@PathVariable long id) { return "app"; } @@ -162,6 +165,7 @@ public String getReport(@PathVariable long id) { */ @SuppressWarnings("UnusedParameters") @GetMapping("/{id}/detail_report/perf") + @PreAuthorize("permitAll") public String getDetailPerfReport(@PathVariable("id") long id) { return "perftest/detail_report/perf"; } @@ -176,6 +180,7 @@ public String getDetailPerfReport(@PathVariable("id") long id) { */ @SuppressWarnings("UnusedParameters") @GetMapping("/{id}/detail_report/monitor") + @PreAuthorize("permitAll") public String getDetailMonitorReport(@PathVariable("id") long id, @RequestParam("targetIP") String targetIP, ModelMap modelMap) { modelMap.addAttribute("targetIP", targetIP); @@ -192,6 +197,7 @@ public String getDetailMonitorReport(@PathVariable("id") long id, @RequestParam( */ @SuppressWarnings("UnusedParameters") @GetMapping("/{id}/detail_report/plugin/{plugin}") + @PreAuthorize("permitAll") public String getDetailPluginReport(@PathVariable("id") long id, @PathVariable("plugin") String plugin, @RequestParam("kind") String kind, ModelMap modelMap) { modelMap.addAttribute("plugin", plugin); diff --git a/ngrinder-controller/src/main/java/org/ngrinder/script/controller/FileEntryApiController.java b/ngrinder-controller/src/main/java/org/ngrinder/script/controller/FileEntryApiController.java index 6fd3c73e4..7b2498312 100644 --- a/ngrinder-controller/src/main/java/org/ngrinder/script/controller/FileEntryApiController.java +++ b/ngrinder-controller/src/main/java/org/ngrinder/script/controller/FileEntryApiController.java @@ -46,6 +46,7 @@ import org.slf4j.LoggerFactory; import org.springframework.context.MessageSource; import org.springframework.http.ContentDisposition; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; @@ -65,6 +66,7 @@ @RestController @RequestMapping("/script/api") @RequiredArgsConstructor +@PreAuthorize("hasAnyRole('A', 'S', 'U')") public class FileEntryApiController { private static final Logger LOG = LoggerFactory.getLogger(FileEntryApiController.class); diff --git a/ngrinder-frontend/src/js/components/perftest/list/List.vue b/ngrinder-frontend/src/js/components/perftest/list/List.vue index e152981ad..e4a1912a1 100644 --- a/ngrinder-frontend/src/js/components/perftest/list/List.vue +++ b/ngrinder-frontend/src/js/components/perftest/list/List.vue @@ -420,7 +420,7 @@ } this.tests[target.index].status = updatedStatus; - this.runningSummary = `${res.data.perfTestInfo.length} ${this.i18n('perfTest.list.runningSummary')}`; + this.runningSummary = `${res.data.runningTestsCount} ${this.i18n('perfTest.list.runningSummary')}`; }); }).finally(() => this.updateStatusTimeoutId = setTimeout(this.updatePerftestStatus, 2000)); } else {