diff --git a/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/Constants.java b/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/Constants.java index 2cb08aee6eef..a176ae14ce40 100644 --- a/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/Constants.java +++ b/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/Constants.java @@ -28,4 +28,13 @@ public class Constants { public static final String ANONYMOUS_VALUE = "anonymous"; public static final String UNKNOWN_VALUE = "UNKNOWN"; public static final int UNKNOWN_INT_VALUE = -1; + public static final String IPV4_PROP_TYPE = "IPV4"; + public static final String IPV6_PROP_TYPE = "IPV6"; + public static final String EMAIL_PROP_TYPE = "EMAIL"; + public static final String USERNAME_PROP_TYPE = "USERNAME"; + + public static final String IPV4_MASK_VALUE = "***"; + public static final String IPV6_MASK_VALUE = "**"; + public static final String EMAIL_MASK_VALUE = "*****"; + public static final String USERNAME_MASK_VALUE = "*****"; } diff --git a/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/AnalyticsDataProvider.java b/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/AnalyticsDataProvider.java index 5f502f75beb6..0e71407a5cd9 100644 --- a/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/AnalyticsDataProvider.java +++ b/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/AnalyticsDataProvider.java @@ -73,4 +73,6 @@ public interface AnalyticsDataProvider { default Map getProperties() { return Collections.EMPTY_MAP; } + + Map getMaskProperties(); } diff --git a/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/impl/SuccessRequestDataCollector.java b/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/impl/SuccessRequestDataCollector.java index e541ac6c30cf..f8c9ecc6ecf8 100644 --- a/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/impl/SuccessRequestDataCollector.java +++ b/components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/impl/SuccessRequestDataCollector.java @@ -32,6 +32,19 @@ import org.wso2.carbon.apimgt.common.analytics.publishers.dto.Target; import org.wso2.carbon.apimgt.common.analytics.publishers.impl.SuccessRequestDataPublisher; +import java.util.Iterator; +import java.util.Map; + +import static org.wso2.carbon.apimgt.common.analytics.Constants.EMAIL_MASK_VALUE; +import static org.wso2.carbon.apimgt.common.analytics.Constants.EMAIL_PROP_TYPE; +import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV4_MASK_VALUE; +import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV4_PROP_TYPE; +import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV6_MASK_VALUE; +import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV6_PROP_TYPE; +import static org.wso2.carbon.apimgt.common.analytics.Constants.USERNAME_MASK_VALUE; +import static org.wso2.carbon.apimgt.common.analytics.Constants.USERNAME_PROP_TYPE; + + /** * Success request data collector. */ @@ -58,6 +71,22 @@ public void collectData() throws AnalyticsException { Event event = new Event(); event.setProperties(provider.getProperties()); + + // Masking the configured data + Map maskData = provider.getMaskProperties(); + Iterator> iterator = maskData.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + Map props = event.getProperties(); + if (props != null) { + Object value = props.get(entry.getKey()); + if (value != null) { + String maskStr = maskAnalyticsData(entry.getValue(), value); + props.replace(entry.getKey(), maskStr); + } + } + } + API api = provider.getApi(); Operation operation = provider.getOperation(); Target target = provider.getTarget(); @@ -72,12 +101,32 @@ public void collectData() throws AnalyticsException { MetaInfo metaInfo = provider.getMetaInfo(); String userAgent = provider.getUserAgentHeader(); String userName = provider.getUserName(); + + // Mask UserName if configured + if (userName != null) { + if (maskData.containsKey("api.ut.userName")) { + userName = maskAnalyticsData(maskData.get("api.ut.userName"), userName); + } else if (maskData.containsKey("api.ut.userId")) { + userName = maskAnalyticsData(maskData.get("api.ut.userId"), userName); + } + } + String userIp = provider.getEndUserIP(); if (userIp == null) { userIp = Constants.UNKNOWN_VALUE; + } else { + // Mask User IP if configured + if (maskData.containsKey("api.analytics.user.ip")) { + userIp = maskAnalyticsData(maskData.get("api.analytics.user.ip"), userIp); + } } + if (userAgent == null) { userAgent = Constants.UNKNOWN_VALUE; + } else { + if (maskData.containsKey("api.analytics.user.agent")) { + userAgent = maskAnalyticsData(maskData.get("api.analytics.user.agent"), userAgent); + } } event.setApi(api); @@ -95,4 +144,41 @@ public void collectData() throws AnalyticsException { this.processor.publish(event); } + private String maskAnalyticsData(String type, Object value) { + if (value instanceof String) { + switch (type) { + case IPV4_PROP_TYPE: + String[] octets = value.toString().split("\\."); + octets[2] = IPV4_MASK_VALUE; + + // Sample output: 192.168.***.98 + return octets[0] + "." + octets[1] + "." + octets[2] + "." + octets[3]; + case IPV6_PROP_TYPE: + octets = value.toString().split(":"); + octets[3] = IPV6_MASK_VALUE; + octets[4] = IPV6_MASK_VALUE; + octets[5] = IPV6_MASK_VALUE; + octets[6] = IPV6_MASK_VALUE; + + // Sample output: 2001:0db8:85a3:****:****:****:****:7334 + return octets[0] + ":" + octets[1] + ":" + octets[2] + ":" + octets[3] + ":" + octets[4] + ":" + + octets[5] + ":" + octets[6] + ":" + octets[7]; + case EMAIL_PROP_TYPE: + String[] email = value.toString().split("@"); + email[0] = EMAIL_MASK_VALUE; + + // Sample output: *****@gmail.com + return email[0] + "@" + email[1]; + case USERNAME_PROP_TYPE: + String username = USERNAME_MASK_VALUE; + return username; + default: + String str = USERNAME_MASK_VALUE; + + // Sample output: ******** + return str; + } + } + return null; + } } diff --git a/components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/analytics/SynapseAnalyticsDataProvider.java b/components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/analytics/SynapseAnalyticsDataProvider.java index 41dbae64eb99..32a04a53a98c 100644 --- a/components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/analytics/SynapseAnalyticsDataProvider.java +++ b/components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/analytics/SynapseAnalyticsDataProvider.java @@ -251,6 +251,13 @@ public MetaInfo getMetaInfo() { return metaInfo; } + @Override + public Map getMaskProperties() { + Map maskProperties = ServiceReferenceHolder.getInstance().getApiManagerConfigurationService() + .getAPIAnalyticsConfiguration().getMaskDataProperties(); + return maskProperties; + } + @Override public int getProxyResponseCode() { diff --git a/components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/streaming/websocket/WebSocketAnalyticsDataProvider.java b/components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/streaming/websocket/WebSocketAnalyticsDataProvider.java index d7ddfa98b982..795d0d69aa4c 100644 --- a/components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/streaming/websocket/WebSocketAnalyticsDataProvider.java +++ b/components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/streaming/websocket/WebSocketAnalyticsDataProvider.java @@ -254,6 +254,13 @@ public MetaInfo getMetaInfo() { return metaInfo; } + @Override + public Map getMaskProperties() { + Map maskProperties = ServiceReferenceHolder.getInstance().getApiManagerConfigurationService() + .getAPIAnalyticsConfiguration().getMaskDataProperties(); + return maskProperties; + } + @Override public int getProxyResponseCode() { if (isSuccessRequest()) { diff --git a/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerAnalyticsConfiguration.java b/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerAnalyticsConfiguration.java index e133df813de6..a3398de51305 100644 --- a/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerAnalyticsConfiguration.java +++ b/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerAnalyticsConfiguration.java @@ -57,6 +57,7 @@ public class APIManagerAnalyticsConfiguration { private String responseSchemaName; private String faultSchemaName; private Map reporterProperties; + private Map maskDataProperties; private APIManagerAnalyticsConfiguration() { } @@ -77,6 +78,7 @@ public void setAPIManagerConfiguration(APIManagerConfiguration config){ this.responseSchemaName = config.getFirstProperty(APIConstants.API_ANALYTICS_RESPONSE_SCHEMA_NAME); this.faultSchemaName = config.getFirstProperty(APIConstants.API_ANALYTICS_FAULT_SCHEMA_NAME); this.reporterProperties = config.getAnalyticsProperties(); + this.maskDataProperties = config.getAnalyticsMaskProperties(); } } @@ -247,4 +249,8 @@ public String getResponseSchemaName() { public String getFaultSchemaName() { return faultSchemaName; } + + public Map getMaskDataProperties() { + return maskDataProperties; + } } diff --git a/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java b/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java index 4c43df271c36..e47ed44a414a 100644 --- a/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java +++ b/components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java @@ -127,6 +127,7 @@ public class APIManagerConfiguration { private RedisConfig redisConfig = new RedisConfig(); private Map> restApiJWTAuthAudiences = new HashMap<>(); private JSONObject subscriberAttributes = new JSONObject(); + private static Map analyticsMaskProps; public Map> getRestApiJWTAuthAudiences() { return restApiJWTAuthAudiences; @@ -357,6 +358,19 @@ private void readChildElements(OMElement serverConfig, analyticsProps.put(name, value); } } + + // Load all the mask properties + OMElement maskProperties = element.getFirstChildWithName(new QName("MaskProperties")); + Iterator maskPropertiesIterator = maskProperties.getChildrenWithLocalName("Property"); + Map maskProps = new HashMap<>(); + while (maskPropertiesIterator.hasNext()) { + OMElement propertyElem = (OMElement) maskPropertiesIterator.next(); + String name = propertyElem.getAttributeValue(new QName("name")); + String value = propertyElem.getText(); + maskProps.put(name, value.toUpperCase()); + } + analyticsMaskProps = maskProps; + OMElement authTokenElement = element.getFirstChildWithName(new QName("AuthToken")); String resolvedAuthToken = MiscellaneousUtil.resolve(authTokenElement, secretResolver); analyticsProps.put("auth.api.token", resolvedAuthToken); @@ -2203,6 +2217,8 @@ public GatewayCleanupSkipList getGatewayCleanupSkipList() { public static Map getAnalyticsProperties() { return analyticsProperties; } + + public static Map getPersistenceProperties() { return persistenceProperties; @@ -2269,4 +2285,8 @@ public HttpClientConfigurationDTO getHttpClientConfiguration() { public void setHttpClientConfiguration(HttpClientConfigurationDTO httpClientConfiguration) { this.httpClientConfiguration = httpClientConfiguration; } + + public static Map getAnalyticsMaskProperties() { + return analyticsMaskProps; + } } diff --git a/features/apimgt/org.wso2.carbon.apimgt.core.feature/src/main/resources/conf_templates/templates/repository/conf/api-manager.xml.j2 b/features/apimgt/org.wso2.carbon.apimgt.core.feature/src/main/resources/conf_templates/templates/repository/conf/api-manager.xml.j2 index edc725d2c1cd..14282d5ad04f 100644 --- a/features/apimgt/org.wso2.carbon.apimgt.core.feature/src/main/resources/conf_templates/templates/repository/conf/api-manager.xml.j2 +++ b/features/apimgt/org.wso2.carbon.apimgt.core.feature/src/main/resources/conf_templates/templates/repository/conf/api-manager.xml.j2 @@ -304,6 +304,11 @@ {% endfor %} + + {% for key,value in apim.analytics.mask.items() %} + {{value}} + {% endfor %} +