Merge pull request #38 from Ontotext-AD/10.0

Changes needed for GraphDB 10.0
Ontotext-AD · Jun 30, 2022 · 266ed2c · 266ed2c
2 parents 833e9d2 + 2811b23
commit 266ed2c
Show file tree

Hide file tree

Showing 52 changed files with 1,024 additions and 2,675 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,17 @@
 # GraphDB Helm chart release notes
+## Version 10.0
+
+### Breaking
+New major release that isn't compatible with the old chart, due to major breaking changes in Graphdb 10.
+Migration steps can be found [here](README.md#cluster-migration-from-graphdb-9x-to-100).
+
+### New
+- Changed to work with the new GraphDB 10.
+- Removed Kong.
+- Moved from multiple stateful sets with 1 replica to statefulsets with multiple replicas.
+- Configurable liveness, readiness, startup probes.
+- Can use standalone without license by default. Don't forget to set your license for a working cluster and connectors!
+- New overridable configmaps for users, settings and logback.
 
 ## Version 9.9.0
 

diff --git a/Chart.yaml b/Chart.yaml
@@ -1,12 +1,12 @@
 #
-# Helm chart for GraphDB Free/SE/EE
+# Helm chart for GraphDB
 #
 apiVersion: v2
 name: graphdb
-description: Helm chart for GraphDB Free/SE/EE
+description: Helm chart for GraphDB
 type: application
-version: 9.11.0
-appVersion: 9.11.0
+version: 10.0.0
+appVersion: 10.0.0
 home: https://graphdb.ontotext.com/
 icon: https://graphdb.ontotext.com/home/images/visual_Logo_GraphDB_02_12_2015.png
 maintainers:

diff --git a/README.md b/README.md
diff --git a/files/config/cluster-config.json b/files/config/cluster-config.json
@@ -0,0 +1,13 @@
+{
+  "electionMinTimeout": {{ $.Values.graphdb.clusterConfig.electionMinTimeout }},
+  "electionRangeTimeout": {{ $.Values.graphdb.clusterConfig.electionRangeTimeout }},
+  "heartbeatInterval": {{ $.Values.graphdb.clusterConfig.heartbeatInterval }},
+  "messageSize": {{ $.Values.graphdb.clusterConfig.messageSize }},
+  "verificationTimeout": {{ $.Values.graphdb.clusterConfig.verificationTimeout }},
+  "nodes": [
+    {{- range $i, $node_index := until ( (int $.Values.graphdb.clusterConfig.nodesCount) )}}
+    "graphdb-node-{{ $node_index }}.graphdb-node.{{ $.Release.Namespace }}.svc.cluster.local:7300"{{- if gt (sub (int $.Values.graphdb.clusterConfig.nodesCount) 1 ) $node_index }},
+    {{- end }}
+    {{- end }}
+  ]
+}
diff --git a/files/config/graphdb.properties b/files/config/graphdb.properties
@@ -42,6 +42,12 @@
 # graphdb.home.work =
 #
 #
+# GraphDB external plugins directory. This can be used to add
+# additional external plugins outside the distribution's lib/plugins directory.
+#
+# graphdb.extra.plugins =
+#
+#
 # Graphdb global page cache memory parameter. By default the page cache will
 # allocate 50% of the maximum Java heap memory (-Xmx) for the JVM process.
 # Setting this value too high will cause OME during the execution of memory intensive queries.
@@ -146,6 +152,12 @@
 # OpenID issuer URL, used to derive keys, endpoints and token validation. No default value.
 # graphdb.auth.openid.issuer = https://accounts.example.com
 #
+# OpenID well-known config URL, used to fetch the OpenID configuration of the OpenID provider.
+# The default value will be derived by appending /.well-known/openid-configuration to the issuer URL.
+# Use this setting if your OpenID does not conform to the specification and the well-known config is
+# served on a different URL than the default standard-mandated value.
+# graphdb.auth.openid.well_known_config_url = https://openid.example.com/custom/.well-known/openid-configuration
+#
 # OpenID client ID, used to authenticate and validate tokens. No default value.
 # graphdb.auth.openid.client_id = my-client-id
 #
@@ -166,12 +178,22 @@
 # OpenID expected audience in tokens, used to validate tokens. The default value is the same as the client ID.
 # graphdb.auth.openid.token_audience = my-audience
 #
+# OpenID extra scopes to request. Multiple scopes can be specified by separating them with a space.
+# By default GraphDB requests only the 'openid' scope and, if supported, the 'offline_access' scope.
+# Scopes are used to request sets of claims, e.g. you might need to set this to a provider-specific value
+# in order to obtain the username_name or the roles_claim (if using OAuth as well). The default value is empty.
+# graphdb.auth.openid.extra_scopes = profile email
+#
 # OpenID extra parameters for the authorize endpoint. Some OpenID providers require additional parameters sent
 # to the authorize endpoint (e.g. resource=xxx). This is a URL encoded string where each parameter-value pair
 # is delimited by &. The string will be appended to the rest of the authorize URL parameters.
 # The default value is the empty string.
 # graphdb.auth.openid.authorize_parameters = param1=value%201&param2=value%202
 #
+# OpenID Oracle identity domain. Oracle Access Manager has a non-standard OpenID implementation that requires
+# an additional parameter: the Oracle identity domain name. If you use OAM set this to your identity domain name.
+# graphdb.auth.openid.oracle_domain = my-oracle-domain
+#
 # OpenID use GraphDB as proxy for the JWKS URL and token endpoints. This can be used to bypass an OpenID provider
 # without a proper CORS configuration. The value is a boolean true/false. False by default.
 # graphdb.auth.openid.proxy = false
@@ -187,6 +209,10 @@
 # Enable OAuth authorization. The default value is 'local' corresponding to local users.
 # graphdb.auth.database = oauth
 #
+# Set case-insensitive validation for user accounts so that users can log in regardless of the case used at login time.
+# The default value is false.
+# graphdb.auth.database.case_insensitive = false
+#
 # OpenID issuer URL, used to derive keys, endpoints and token validation. No default value.
 # graphdb.auth.openid.issuer = https://accounts.example.com
 #
@@ -227,6 +253,10 @@
 # The prefix will be stripped when the roles are mapped. The default value is the empty string.
 # graphdb.auth.oauth.roles_prefix = GDB_
 #
+# OAuth roles suffix to strip. The roles claim may provide the GraphDB roles with some suffix, e.g. ROLE_USER_GDB.
+# The suffix will be stripped when the roles are mapped. The default value is the empty string.
+# graphdb.auth.oauth.roles_suffix = ROLE_USER_GDB
+#
 # OAuth default roles to assign. It may be convenient to always assign certain roles without listing them in the roles
 # claim. The value is a comma-delimited list of GraphDB roles. The default value is the empty list.
 #
@@ -241,7 +271,7 @@
 # graphdb.auth.database = ldap
 # graphdb.auth.ldap.url = ldap://localhost:10389/dc=example,dc=org
 
-# Permit access for all users that are part of the “people” unit of the fictional “example.org” organisation.
+# Permit access for all users that are part of the \u201Cpeople\u201D unit of the fictional \u201Cexample.org\u201D organisation.
 #
 # graphdb.auth.ldap.user.search.base = ou=people
 # graphdb.auth.ldap.user.search.filter = (cn={0})
@@ -329,3 +359,15 @@
 # the request it will be generated randomly in UUID type 5 format.
 #
 # graphdb.append.request.id.headers = true
+
+###### HEAP DUMPS ######
+#
+# GraphDB can dump the heap on out of memory errors in order to provide insight to the cause
+# for excessive memory usage.
+#
+# Enable or disable the heap dump (enabled by default)
+# graphdb.heapdump.enable = true
+#
+# File to write the heap dump to, by default this is the file heapdump.hprof in the configured logs directory.
+# See also the properties graphdb.home and graphdb.home.logs.
+# graphdb.heapdump.path =
diff --git a/files/config/kong.dbless.yaml b/files/config/kong.dbless.yaml
diff --git a/files/config/logback.xml b/files/config/logback.xml
@@ -80,33 +80,6 @@
         </filter>
     </appender>
 
-    <!-- Enterprise log. Contains the cluster-related logs -->
-    <appender name="EnterpriseLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-            <fileNamePattern>${logDestinationDirectory}/enterprise-log-%d{yyyy-MM-dd}.log</fileNamePattern>
-            <maxHistory>${keepLogDays}</maxHistory>
-            <totalSizeCap>${logMaxSize}</totalSizeCap>
-            <cleanHistoryOnStart>true</cleanHistoryOnStart>
-        </rollingPolicy>
-        <encoder>
-            <pattern>${defaultPattern}</pattern>
-            <charset>${encoding}</charset>
-        </encoder>
-    </appender>
-
-    <appender name="HttpLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
-            <fileNamePattern>${logDestinationDirectory}/http-log-%d{yyyy-MM-dd}.log</fileNamePattern>
-            <maxHistory>${keepLogDays}</maxHistory>
-            <totalSizeCap>${logMaxSize}</totalSizeCap>
-            <cleanHistoryOnStart>true</cleanHistoryOnStart>
-        </rollingPolicy>
-        <encoder>
-            <pattern>${defaultPattern}</pattern>
-            <charset>${encoding}</charset>
-        </encoder>
-    </appender>
-
     <appender name="QueryLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <fileNamePattern>${logDestinationDirectory}/query-log-%d{yyyy-MM-dd}.log</fileNamePattern>
@@ -151,45 +124,26 @@
         <appender-ref ref="AuditLog"/>
     </logger>
 
-    <!-- Log update operations on workers. Set to "INFO" level for logging all updates in workers' QueryLog.
+    <!-- Log update operations on workers. Set to "INFO" level by default for logging all updates in workers' QueryLog.
      Will log exceptions on "ERROR" level. Additivity is set to "false" and called first to prevent the messages
      from cluttering the other logs.-->
     <logger name="com.ontotext.trree.monitorRepository.MonitorRepositoryConnection" level="INFO" additivity="false">
         <appender-ref ref="QueryLog"/>
     </logger>
 
-    <!-- Log operations on the master or/and the workers. Set to "INFO" level for logging all operations.
-     Will log exceptions on "ERROR" level. Additivity is set to "false" and called first to prevent the messages
-     from cluttering the other logs.-->
-    <logger name="com.ontotext.trree.replicationcluster.api.ClusterOperation" level="INFO" additivity="false">
-        <appender-ref ref="QueryLog"/>
-    </logger>
+    <!-- Log query operations on the repository. Set to "DEBUG" level for logging all querries. Will log exceptions on "ERROR"
+          level. Additivity is set to "false" to prevent the messages from cluttering the other logs. -->
     <logger name="com.ontotext.trree.query.LoggingClosableIteration" level="INFO" additivity="false">
         <appender-ref ref="QueryLog"/>
     </logger>
 
     <!-- Log slow queries on "INFO" level. Queries are deemed "slow" if they take more than "SlowOpThresholdMs" from the
-          master JMX bean. Set the level to "OFF" to stop this log. Additivity is set to "false" and called first to prevent the messages
+          RepositorySettings property. Set the level to "OFF" to stop this log. Additivity is set to "false" to prevent the messages
           from cluttering the other logs. -->
     <logger name="slow-queries" level="INFO" additivity="false">
         <appender-ref ref="SlowQueryLog"/>
     </logger>
 
-    <!-- Enterprise loggers. Additivity is set to "false" so that the messages do not appear in the main log as well -->
-    <logger name="com.ontotext.trree.replicationcluster" level="INFO" additivity="false">
-        <appender-ref ref="EnterpriseLog"/>
-    </logger>
-
-    <logger name="com.ontotext.repository.http" level="INFO" additivity="false">
-        <appender-ref ref="EnterpriseLog"/>
-    </logger>
-
-
-    <!-- HTTP logger -->
-    <logger name="com.ontotext.trree.util.Http" level="INFO" additivity="false">
-        <appender-ref ref="HttpLog"/>
-    </logger>
-
     <root>
         <level value="${graphdb.logger.root.level:-INFO}"/>
         <appender-ref ref="MainLog"/>
@@ -207,45 +161,6 @@
     <logger name="com.github.ziplet" level="WARN"/>
     <logger name="springfox.documentation" level="WARN"/>
 
-    <!-- OpenRefine loggers be less verbose -->
-    <logger name="CsvExporter" level="WARN"/>
-    <logger name="FileProjectManager" level="WARN"/>
-    <logger name="HistoryEntry" level="WARN"/>
-    <logger name="ImportingParserBase" level="WARN"/>
-    <logger name="JsonParser" level="WARN"/>
-    <logger name="ProjectManager" level="WARN"/>
-    <logger name="RecordModel" level="WARN"/>
-    <logger name="TreeImportUtilities" level="WARN"/>
-    <logger name="velocity" level="WARN"/>
-    <logger name="Velocity.*" level="WARN"/>
-    <logger name="XmlImportUtilities" level="WARN"/>
-    <logger name="binning_clusterer" level="WARN"/>
-    <logger name="butterfly" level="WARN"/>
-    <logger name="butterfly.*" level="WARN"/>
-    <logger name="command" level="WARN"/>
-    <logger name="compute-clusters_command" level="WARN"/>
-    <logger name="create-import-job_command" level="WARN"/>
-    <logger name="create-project_command" level="WARN"/>
-    <logger name="get-scatterplot_command" level="WARN"/>
-    <logger name="import-project_command" level="WARN"/>
-    <logger name="importing" level="WARN"/>
-    <logger name="importing-controller_command" level="WARN"/>
-    <logger name="importing-utilities" level="WARN"/>
-    <logger name="javascript" level="WARN"/>
-    <logger name="kNN_clusterer" level="WARN"/>
-    <logger name="open" level="WARN"/>
-    <logger name="office" level="WARN"/>
-    <logger name="project" level="WARN"/>
-    <logger name="project_metadata" level="WARN"/>
-    <logger name="project_metadata_utilities" level="WARN"/>
-    <logger name="project_utilities" level="WARN"/>
-    <logger name="recon-config" level="WARN"/>
-    <logger name="recon-operation" level="WARN"/>
-    <logger name="refine" level="WARN"/>
-    <logger name="refine-standard-recon" level="WARN"/>
-    <logger name="refine_clientSideResourceManager" level="WARN"/>
-    <logger name="scatterplot_facet" level="WARN"/>
-
     <!-- GeoSPAQRL related deps be less verbose -->
     <logger name="hsqldb.*" level="WARN"/>
     <logger name="org.geotoolkit.*" level="WARN"/>

diff --git a/files/config/master.default.ttl b/files/config/master.default.ttl