Skip to content

Commit d586e2f

Browse files
raghavsethiraghavsethi
committed
Add built-in file based session property manager
1 parent a2df42d commit d586e2f

13 files changed

+748
-0
lines changed

‎pom.xml

+7
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@
112112
<module>presto-plugin-toolkit</module>
113113
<module>presto-resource-group-managers</module>
114114
<module>presto-password-authenticators</module>
115+
<module>presto-session-property-managers</module>
115116
<module>presto-benchto-benchmarks</module>
116117
<module>presto-thrift-connector-api</module>
117118
<module>presto-thrift-testing-server</module>
@@ -156,6 +157,12 @@
156157
<version>${project.version}</version>
157158
</dependency>
158159

160+
<dependency>
161+
<groupId>com.facebook.presto</groupId>
162+
<artifactId>presto-session-property-managers</artifactId>
163+
<version>${project.version}</version>
164+
</dependency>
165+
159166
<dependency>
160167
<groupId>com.facebook.presto</groupId>
161168
<artifactId>presto-array</artifactId>

‎presto-docs/src/main/sphinx/admin.rst

+1
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ Administration
1010
admin/properties
1111
admin/spill
1212
admin/resource-groups
13+
admin/session-property-managers
1314
admin/dist-sort

‎presto-docs/src/main/sphinx/admin/session-property-managers.rst

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
=========================
2+
Session Property Managers
3+
=========================
4+
5+
Administrators can add session properties to control the behavior for subsets of their workload.
6+
These properties are defaults and can be overridden by users (if authorized to do so). Session
7+
properties can be used to control resource usage, enable or disable features, and change query
8+
characteristics. Session property managers are pluggable.
9+
10+
Add an ``etc/session-property-config.properties`` file with the following contents to enable
11+
the built-in manager that reads a JSON config file:
12+
13+
.. code-block:: none
14+
15+
session-property-config.configuration-manager=file
16+
session-property-manager.config-file=etc/session-property-config.json
17+
18+
Change the value of ``session-property-manager.config-file`` to point to a JSON config file,
19+
which can be an absolute path, or a path relative to the Presto data directory.
20+
21+
This configuration file consists of a list of match rules, each of which specify a list of
22+
conditions that the query must meet, and a list of session properties that should be applied
23+
by default. All matching rules contribute to constructing a list of session properties. Rules
24+
are applied in the order they are specified. Rules specified later in the file override values
25+
for properties that have been previously encountered.
26+
27+
Match Rules
28+
-----------
29+
30+
* ``user`` (optional): regex to match against user name.
31+
32+
* ``source`` (optional): regex to match against source string.
33+
34+
* ``queryType`` (optional): string to match against the type of the query submitted:
35+
* ``DATA_DEFINITION``: Queries that alter/create/drop the metadata of schemas/tables/views, and that manage
36+
prepared statements, privileges, sessions, and transactions.
37+
* ``DELETE``: ``DELETE`` queries.
38+
* ``DESCRIBE``: ``DESCRIBE``, ``DESCRIBE INPUT``, ``DESCRIBE OUTPUT``, and ``SHOW`` queries.
39+
* ``EXPLAIN``: ``EXPLAIN`` queries.
40+
* ``INSERT``: ``INSERT`` and ``CREATE TABLE AS`` queries.
41+
* ``SELECT``: ``SELECT`` queries.
42+
43+
* ``clientTags`` (optional): list of tags. To match, every tag in this list must be in the list of
44+
client-provided tags associated with the query.
45+
46+
* ``group`` (optional): regex to match against the fully qualified name of the resource group the query is
47+
routed to.
48+
49+
* ``sessionProperties``: map with string keys and values. Each entry is a system or catalog property name and
50+
corresponding value. Values must be specified as strings, no matter the actual data type.
51+
52+
Example
53+
-------
54+
55+
Consider the following set of requirements:
56+
57+
* All queries running under the ``global`` resource group must have an execution time limit of 8 hours.
58+
59+
* All interactive queries are routed to subgroups under the ``global.interactive`` group, and have an execution time
60+
limit of 1 hour (tighter than the constraint on ``global``).
61+
62+
* All ETL queries (tagged with 'etl') are routed to subgroups under the ``global.pipeline`` group, and must be
63+
configured with certain properties to control writer behavior.
64+
65+
These requirements can be expressed with the following rules:
66+
67+
.. code-block:: json
68+
69+
[
70+
{
71+
"resourceGroup": "global.*",
72+
"sessionProperties": {
73+
"query_max_execution_time": "8h",
74+
}
75+
},
76+
{
77+
"resourceGroup": "global.interactive.*",
78+
"sessionProperties": {
79+
"query_max_execution_time": "1h"
80+
}
81+
},
82+
{
83+
"resourceGroup": "global.pipeline.*",
84+
"clientTags": ["etl"],
85+
"sessionProperties": {
86+
"scale_writers": "true",
87+
"writer_min_size": "1GB"
88+
}
89+
}
90+
]

‎presto-server/src/main/provisio/presto.xml

+6
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@
3838
</artifact>
3939
</artifactSet>
4040

41+
<artifactSet to="plugin/session-property-managers">
42+
<artifact id="${project.groupId}:presto-session-property-managers:zip:${project.version}">
43+
<unpack />
44+
</artifact>
45+
</artifactSet>
46+
4147
<artifactSet to="plugin/jmx">
4248
<artifact id="${project.groupId}:presto-jmx:zip:${project.version}">
4349
<unpack />

‎presto-session-property-managers/pom.xml

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
<?xml version="1.0"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
5+
<parent>
6+
<groupId>com.facebook.presto</groupId>
7+
<artifactId>presto-root</artifactId>
8+
<version>0.211-SNAPSHOT</version>
9+
</parent>
10+
11+
<artifactId>presto-session-property-managers</artifactId>
12+
<description>Presto - Session Property Managers</description>
13+
<packaging>presto-plugin</packaging>
14+
15+
<properties>
16+
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
17+
</properties>
18+
19+
<dependencies>
20+
<dependency>
21+
<groupId>io.airlift</groupId>
22+
<artifactId>bootstrap</artifactId>
23+
</dependency>
24+
25+
<dependency>
26+
<groupId>io.airlift</groupId>
27+
<artifactId>json</artifactId>
28+
</dependency>
29+
30+
<dependency>
31+
<groupId>io.airlift</groupId>
32+
<artifactId>configuration</artifactId>
33+
</dependency>
34+
35+
<dependency>
36+
<groupId>com.google.guava</groupId>
37+
<artifactId>guava</artifactId>
38+
</dependency>
39+
40+
<dependency>
41+
<groupId>com.google.inject</groupId>
42+
<artifactId>guice</artifactId>
43+
</dependency>
44+
45+
<dependency>
46+
<groupId>javax.inject</groupId>
47+
<artifactId>javax.inject</artifactId>
48+
</dependency>
49+
50+
<dependency>
51+
<groupId>javax.validation</groupId>
52+
<artifactId>validation-api</artifactId>
53+
</dependency>
54+
55+
<dependency>
56+
<groupId>com.fasterxml.jackson.core</groupId>
57+
<artifactId>jackson-databind</artifactId>
58+
</dependency>
59+
60+
<dependency>
61+
<groupId>com.fasterxml.jackson.core</groupId>
62+
<artifactId>jackson-core</artifactId>
63+
</dependency>
64+
65+
<!-- Presto SPI -->
66+
<dependency>
67+
<groupId>com.facebook.presto</groupId>
68+
<artifactId>presto-spi</artifactId>
69+
<scope>provided</scope>
70+
</dependency>
71+
72+
<dependency>
73+
<groupId>io.airlift</groupId>
74+
<artifactId>units</artifactId>
75+
<scope>provided</scope>
76+
</dependency>
77+
78+
<dependency>
79+
<groupId>com.fasterxml.jackson.core</groupId>
80+
<artifactId>jackson-annotations</artifactId>
81+
<scope>provided</scope>
82+
</dependency>
83+
84+
<!-- for testing -->
85+
<dependency>
86+
<groupId>org.testng</groupId>
87+
<artifactId>testng</artifactId>
88+
<scope>test</scope>
89+
</dependency>
90+
91+
<dependency>
92+
<groupId>io.airlift</groupId>
93+
<artifactId>testing</artifactId>
94+
<scope>test</scope>
95+
</dependency>
96+
</dependencies>
97+
</project>

‎presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManager.java

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.session;
15+
16+
import com.facebook.presto.spi.session.SessionConfigurationContext;
17+
import com.facebook.presto.spi.session.SessionPropertyConfigurationManager;
18+
import com.fasterxml.jackson.databind.JsonMappingException;
19+
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
20+
import com.google.common.collect.ImmutableMap;
21+
import io.airlift.json.JsonCodec;
22+
import io.airlift.json.JsonCodecFactory;
23+
import io.airlift.json.ObjectMapperProvider;
24+
25+
import javax.inject.Inject;
26+
27+
import java.io.IOException;
28+
import java.io.UncheckedIOException;
29+
import java.nio.file.Files;
30+
import java.nio.file.Path;
31+
import java.util.HashMap;
32+
import java.util.List;
33+
import java.util.Map;
34+
35+
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
36+
import static java.lang.String.format;
37+
import static java.util.Objects.requireNonNull;
38+
39+
public class FileSessionPropertyManager
40+
implements SessionPropertyConfigurationManager
41+
{
42+
public static final JsonCodec<List<SessionMatchSpec>> CODEC = new JsonCodecFactory(
43+
() -> new ObjectMapperProvider().get().enable(FAIL_ON_UNKNOWN_PROPERTIES))
44+
.listJsonCodec(SessionMatchSpec.class);
45+
46+
private final List<SessionMatchSpec> sessionMatchSpecs;
47+
48+
@Inject
49+
public FileSessionPropertyManager(FileSessionPropertyManagerConfig config)
50+
{
51+
requireNonNull(config, "config is null");
52+
53+
Path configurationFile = config.getConfigFile().toPath();
54+
try {
55+
sessionMatchSpecs = CODEC.fromJson(Files.readAllBytes(configurationFile));
56+
}
57+
catch (IOException e) {
58+
throw new UncheckedIOException(e);
59+
}
60+
catch (IllegalArgumentException e) {
61+
Throwable cause = e.getCause();
62+
if (cause instanceof UnrecognizedPropertyException) {
63+
UnrecognizedPropertyException ex = (UnrecognizedPropertyException) cause;
64+
String message = format("Unknown property at line %s:%s: %s",
65+
ex.getLocation().getLineNr(),
66+
ex.getLocation().getColumnNr(),
67+
ex.getPropertyName());
68+
throw new IllegalArgumentException(message, e);
69+
}
70+
if (cause instanceof JsonMappingException) {
71+
// remove the extra "through reference chain" message
72+
if (cause.getCause() != null) {
73+
cause = cause.getCause();
74+
}
75+
throw new IllegalArgumentException(cause.getMessage(), e);
76+
}
77+
throw e;
78+
}
79+
}
80+
81+
@Override
82+
public Map<String, String> getSystemSessionProperties(SessionConfigurationContext context)
83+
{
84+
// later properties override earlier properties
85+
Map<String, String> combinedProperties = new HashMap<>();
86+
for (SessionMatchSpec sessionMatchSpec : sessionMatchSpecs) {
87+
combinedProperties.putAll(sessionMatchSpec.match(context));
88+
}
89+
90+
return ImmutableMap.copyOf(combinedProperties);
91+
}
92+
93+
@Override
94+
public Map<String, Map<String, String>> getCatalogSessionProperties(SessionConfigurationContext context)
95+
{
96+
return ImmutableMap.of();
97+
}
98+
}

‎presto-session-property-managers/src/main/java/com/facebook/presto/session/FileSessionPropertyManagerConfig.java

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package com.facebook.presto.session;
15+
16+
import io.airlift.configuration.Config;
17+
18+
import javax.validation.constraints.NotNull;
19+
20+
import java.io.File;
21+
22+
public class FileSessionPropertyManagerConfig
23+
{
24+
private File configFile;
25+
26+
@NotNull
27+
public File getConfigFile()
28+
{
29+
return configFile;
30+
}
31+
32+
@Config("session-property-manager.config-file")
33+
public FileSessionPropertyManagerConfig setConfigFile(File configFile)
34+
{
35+
this.configFile = configFile;
36+
return this;
37+
}
38+
}

0 commit comments

Comments
 (0)