From 6b66f75e313b7536882a0608ee69cf700ceb4a97 Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Mon, 4 Sep 2023 13:00:03 +0000 Subject: [PATCH 01/20] Added in new user guide section --- .../user-guide/gaffer-basics/gaffer-basics.md | 0 .../gaffer-basics/what-is-a-graph.md | 0 .../gaffer-basics/what-is-gaffer.md | 24 +++ docs/user-guide/gaffer-basics/what-is-json.md | 0 .../gaffer-basics/what-is-python.md | 0 docs/user-guide/getting-started/api.md | 94 ++++++++ .../getting-started/getting-started.md | 0 docs/user-guide/getting-started/schema.md | 94 ++++++++ .../query/api-querying/api-querying.md | 0 .../user-guide/query/api-querying/examples.md | 0 .../query/api-querying/import-export-data.md | 201 ++++++++++++++++++ docs/user-guide/query/gremlin/examples.md | 0 docs/user-guide/query/gremlin/gremlin.md | 0 .../query/python-extension/examples.md | 0 .../python-extension/python-extension.md | 0 docs/user-guide/query/query.md | 0 mkdocs.yml | 20 ++ 17 files changed, 433 insertions(+) create mode 100644 docs/user-guide/gaffer-basics/gaffer-basics.md create mode 100644 docs/user-guide/gaffer-basics/what-is-a-graph.md create mode 100644 docs/user-guide/gaffer-basics/what-is-gaffer.md create mode 100644 docs/user-guide/gaffer-basics/what-is-json.md create mode 100644 docs/user-guide/gaffer-basics/what-is-python.md create mode 100644 docs/user-guide/getting-started/api.md create mode 100644 docs/user-guide/getting-started/getting-started.md create mode 100644 docs/user-guide/getting-started/schema.md create mode 100644 docs/user-guide/query/api-querying/api-querying.md create mode 100644 docs/user-guide/query/api-querying/examples.md create mode 100644 docs/user-guide/query/api-querying/import-export-data.md create mode 100644 docs/user-guide/query/gremlin/examples.md create mode 100644 docs/user-guide/query/gremlin/gremlin.md create mode 100644 docs/user-guide/query/python-extension/examples.md create mode 100644 docs/user-guide/query/python-extension/python-extension.md create mode 100644 docs/user-guide/query/query.md diff --git a/docs/user-guide/gaffer-basics/gaffer-basics.md b/docs/user-guide/gaffer-basics/gaffer-basics.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/gaffer-basics/what-is-a-graph.md b/docs/user-guide/gaffer-basics/what-is-a-graph.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/gaffer-basics/what-is-gaffer.md b/docs/user-guide/gaffer-basics/what-is-gaffer.md new file mode 100644 index 0000000000..921ba59127 --- /dev/null +++ b/docs/user-guide/gaffer-basics/what-is-gaffer.md @@ -0,0 +1,24 @@ +# What is Gaffer? + +Gaffer is a graph database framework, it acts similarly to an interface providing a graph data +structure on top of a chosen storage technology to enable storage of large graphs and traversal of +it's nodes and edges. In a nutshell Gaffer allows you to take data, convert it into a graph, store it +in a database and then run queries and analytics on it. + +The high level interactions of loading data and querying are demonstrated in the diagrams below. + +```mermaid +flowchart TD + subgraph Graph Query + E((User)) --> F + G{{Schema}} --> F + F([Query]) ---> H(Gaffer) + J(key-value store) <--> H + H --> K([Result]) + end + subgraph Data Input + A(Data)-->B{{Schema}} + B --> C(Gaffer) + C --> D(key-value store) + end +``` \ No newline at end of file diff --git a/docs/user-guide/gaffer-basics/what-is-json.md b/docs/user-guide/gaffer-basics/what-is-json.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/gaffer-basics/what-is-python.md b/docs/user-guide/gaffer-basics/what-is-python.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/getting-started/api.md b/docs/user-guide/getting-started/api.md new file mode 100644 index 0000000000..e0e605c40a --- /dev/null +++ b/docs/user-guide/getting-started/api.md @@ -0,0 +1,94 @@ +# The API + +This section covers the currently available APIs that can be used to interact with a Gaffer graph. + +## Swagger Rest API + +Most of the interaction with a deployed Gaffer graph covered in this documentation will be through +the rest API. When deployed, the rest API will be available at a configurable address, accessing +this address via a browser brings up a [Swagger UI](https://swagger.io/) with various GET and POST +requests predefined for you to start using. + +Most of the GET requests simply retrieve information about the graph which can be useful to ensure +your config files have been loaded correctly and the graph is operating normally. The POST requests +allow you to interact with the graph by loading data and running queries. + +The main POST request end point you will use is `/graph/operations/execute`, this will ingest raw +JSON to carry out operations on the Gaffer graph. Gaffer provides many pre built operations that are +available to use and can be chained together for more complex use cases. However be aware, that +operation chains are usually highly specific to the data and results you wish to extract from the +graph so please refer to the reference guide on [Gaffer +operations](../reference/operations-guide/operations.md) for more detail on this. + +!!! example "Example operation chain using rest API" + + The following operation chain gets all the elements in the graph then will count them and + return the total. + + ```json + { + "class": "OperationChain", + "operations": [ + { + "class": "GetAllElements" + }, + { + "class": "Count" + } + ] + } + ``` + +## Python API + +Along side the rest API there also exists a Python API. Commonly referred to as `gafferpy` this API +enables similar querying capabilities as the rest API but from Python code. Fundamentally it wraps +the rest API to use the same JSON under the hood this means you should be able to access any +features or end points available in the main rest API. + +To get started with `gafferpy` simply import the module and connect to an existing graph. + +```python +from gafferpy import gaffer +from gafferpy import gaffer_connector +g_connector = gaffer_connector.GafferConnector("http://localhost:8080/rest/latest") +``` + +Then once connected you can access and run the same endpoints and operations as you would via the +usual rest API. + +!!! example "Example operation chain using `gafferpy`" + + The following operation chain gets all the elements in the graph then will count them and + store the total in the variable `count`. + + ```python + count = g_connector.execute_operation_chain( + operation_chain = gaffer.OperationChain( + operations=[ + gaffer.GetAllElements(), + gaffer.Count() + ] + ) + ) + ``` + +## Java API + +As Gaffer is written in Java there is native support to allow use of all its public classes. Using +Gaffer via the Java interface does differ from the rest API and `gafferpy` but is fully featured +with extensive [Javadocs](https://gchq.github.io/Gaffer/overview-summary.html). + +!!! example "Example operation chain using Java" + + The following operation chain gets all the elements in the graph then will count them and + store the result in a `Long`. + + ```java + OperationChain countAllElements = new OperationChain.Builder() + .first(new GetAllElements()) + .then(new Count<>()) + .build(); + + Long result = graph.execute(countAllElements, user); + ``` diff --git a/docs/user-guide/getting-started/getting-started.md b/docs/user-guide/getting-started/getting-started.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/getting-started/schema.md b/docs/user-guide/getting-started/schema.md new file mode 100644 index 0000000000..430e9f1284 --- /dev/null +++ b/docs/user-guide/getting-started/schema.md @@ -0,0 +1,94 @@ +# Gaffer Schemas + +One of the main differences between Gaffer and other graph database tools are its schemas. In Gaffer +JSON based schemas need to be written upfront for it to understand how to load and treat the data in +the graph. These schemas define all aspects of the nodes and edges in the graph, and can even be +used to automatically do basic analysis or aggregation on queries and ingested data. + +You can kind of think of the schema as sort of a filter or validator for the incoming data. A +given bit of data must conform with part of the schema or it will simply be ignored as it doesn't +fit the structure of the graph. + +## Elements Schema + +All distinct bits of data in a Gaffer graph (e.g. nodes and edges) are referred to as 'elements'. +The structure and properties of these graph elements are defined in the elements schema. The general +format of an element schema are two lists; one of the `"edges"` and the other of the `"entities"` +like the following: + +!!! example "Basic elements syntax" + + ```json + { + "edges": { + "Edge": { + "source": "type", + "destination": "type", + "directed": "true", + "properties": { + "property": "type" + } + } + }, + "entities": { + "Node": { + "description": "A Node", + "vertex": "type", + "properties": { + "property": "type" + } + } + } + } + ``` + +As you can see there are a few fields for both the example `"Edge"` and `"Node"`, many of these +require a type as their value (discussed in the [next section](#types-schema)) which are +essentially handlers or object types for the value associated with the field. + +For an `edge` the following fields are required: + +- `source` - A user defined type for the source node the edge originated from. + +- `directed` - Boolean true or false to define if the edge is directed or not. When an Edge is + undirected in Gaffer, it is treated as if the relationship was bidirectional and the vertices of + the edge do not have an authoritative source and destination. + + !!! note "" + The type here, `"true"` or `"false"` needs to be defined in the types schema using a class + that evaluates to it. This is demonstrated in the [example + deployment](./example-deployment/writing-the-schema.md) document. + +- `destination` - A user defined type for the destination node the edge goes to. + +For an `entity` only one field is required: + +- `vertex` - A user defined type for the node/vertex. + +!!! note "" + The example includes some of the common optional fields too such as a `"properties"` list and + `"description"`. + +## Types Schema + +Following on from the elements schema, the other necessary schema needed for a Gaffer deployment is +the types schema. The types schema allows user defined types for all elements in the graph. It can +also demonstrate the power of Gaffer as it allows for custom functions classes to be used on the +types; however, this can make it quite complex to write a full schema for a graph. + +!!! example "Example types syntax" + + ```json + { + "types": { + "type.string": { + "description": "A basic type to hold the string value of an element", + "class": "java.lang.String" + }, + "type.int": { + "description": "A basic type to hold the int value of an element", + "class": "java.lang.Integer" + } + } + } + ``` \ No newline at end of file diff --git a/docs/user-guide/query/api-querying/api-querying.md b/docs/user-guide/query/api-querying/api-querying.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/query/api-querying/examples.md b/docs/user-guide/query/api-querying/examples.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/query/api-querying/import-export-data.md b/docs/user-guide/query/api-querying/import-export-data.md new file mode 100644 index 0000000000..df29b75f8d --- /dev/null +++ b/docs/user-guide/query/api-querying/import-export-data.md @@ -0,0 +1,201 @@ +# CSV Import and Export + +!!! info "Work in Progress" + + This page is under construction. + + Proposed content: *Guide to cover the available import and export options for CSV*. + +Gaffer supports both importing from and exporting to csv. + +If you configure your Gaffer graph to support the `ImportFromLocalFile` and `ExportToLocalFile` +operations, then it can do this from/to a local file. + +??? tip "Enabling these operations on your Gaffer graph" + + To enable these operations on your Gaffer graph, you would need to add the following to your + `operationsDeclarations.json`. + + ```json + { + "operations": [ + { + "operation": "uk.gov.gchq.gaffer.operation.impl.export.localfile.ImportFromLocalFile", + "handler": { + "class": "uk.gov.gchq.gaffer.store.operation.handler.export.localfile.ImportFromLocalFileHandler" + } + }, + { + "operation": "uk.gov.gchq.gaffer.operation.impl.export.localfile.ExportToLocalFile", + "handler": { + "class": "uk.gov.gchq.gaffer.store.operation.handler.export.localfile.ExportToLocalFileHandler" + } + } + ] + } + ``` + +## CSV Import + +Importing is done with an OperationChain in multiple parts. + +```json +{ + "class": "OperationChain", + "operations": [ + { + "class": "ImportFromLocalFile", //(1)! + "filePath": "mydata.csv" + }, + { + "class": "GenerateElements", //(2)! + "elementGenerator": { + "class": "Neo4jCsvElementGenerator" + } + }, + { + "class": "AddElements" //(3)! + } + ] +} +``` + +1. The `ImportFromLocalFile` operation reads each line from the file `mydata.csv` and will stream + each string into the next parts of the chain. +2. The `GenerateElements` operation will transform each line of the file into a Gaffer Element. You + will need to provide an element generator that is suitable for the file you have provided. The + two CsvElementGenerators provided in core Gaffer are [`Neo4jElementGenerator`](#neo4j-format) and + [`NeptuneCsvElementGenerator`](#neptune-format). +3. Finally, the stream of Gaffer Elements are added with an `AddElements` operation. + +## CSV Export + +Exporting to csv is done with a similar OperationChain. + +```json +{ + "class": "OperationChain", + "operations": [ + { + "class": "GetAllElements" //(1)! + }, + { + "class": "ToCsv", //(2)! + "csvGenerator": "Neo4jCsvGenerator" + }, + { + "class": "ExportToLocalFile", //(3)! + "filePath": "output.csv" + } + ] +} +``` + +1. Firstly, you need to get the Elements which you want to export, in this example we simply + `GetAllElements`. +2. The `ToCsv` operation is then used to turn each Element into a csv formatted string. You must + supply a `CsvGenerator` to do this. You can build a custom [`CsvGenerator`](#custom-formats), or + use a supplied one. The two `CsvGenerators` provided in core Gaffer are + [`Neo4jCsvGenerator`](#neo4j-format) and [`NeptuneCsvGenerator`](#neptune-format). +3. Then the `ExportToLocalFile` operation is used to save this string output into a local file. + +## Formats + +### Custom formats + +You can customise CsvGenerator to create a custom export format in a ToCsv operation. +For example, the following operation. + +```json +{ + "class": "ToCsv", + "csvGenerator": { + "class": "CsvGenerator", + "fields": ["prop1", "SOURCE", "DESTINATION", "prop2", "GROUP"], + "constants": ["constant1", "constant2"] + } +} +``` + +Would produce csv rows that look like: + +=== "Table" + | prop1Value | sourceValue | destinationValue | prop2 | groupValue | constant1 | constant2 | + | ---------- | ----------- | ---------------- | ----- | ---------- | --------- | --------- | + +=== "CSV" + ```csv + prop1Value,sourceValue,destinationValue,prop2,groupValue,constant1,constant2 + ``` + +Currently, custom import formats are not supported. Instead you should use one of the two +[OpenCypher formats](#opencypher-formats). + +### OpenCypher Formats + +Core Gaffer has some generators provided that can import from and export to OpenCypher csvs. These +will work with other graph databases like Neo4j and Neptune. + +Please note that when using these, Gaffer might change your property name headers. All instances of +`-` are replaced with `_`, and invalid characters are stripped as outlined in +[PropertiesUtil](https://github.com/gchq/Gaffer/blob/f16de7c3eccfe7a800cad1d7eea5fbae4cf01d44/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/PropertiesUtil.java#L26). + +As shown [later](#neo4j-format) in the examples, OpenCypher formats let you dictate property types +in the header, like `propertyName:type`. Below is a table that shows which Gaffer transform function +is used to deserialise each [OpenCypher data +type](https://docs.aws.amazon.com/neptune/latest/userguide/bulk-load-tutorial-format-opencypher.html#bulk-load-tutorial-format-opencypher-data-types) +during import. + +| Gaffer Transform Function | OpenCypher Data Types | +| ------------------------- | --------------------------------------------------------------------- | +| `ToString` | `String` `Char` `Duration` `Point` `Date` `LocalDate` `LocalDateTime` | +| `ToBoolean` | `Bool` `Boolean` | +| `ToInteger` | `Int` `Short` `Byte` | +| `ToLong` | `Long` | +| `ToFloat` | `Float` | +| `ToDouble` | `Double` | +| `ParseTime` | `DateTime` | + +## Neo4j Generators + +You can import CSV from Neo4j using the `Neo4jCsvElementGenerator` and export using the +`Neo4jCsvGenerator`. The format used is defined +[here](https://neo4j.com/labs/apoc/4.4/export/csv/#export-database-csv). + +???+ note "Example" + === "Table" + | _id | name | age | lang | _labels | _start | _end | _type | weight | + |-----|-------|-----|------|----------|--------|------|---------|--------| + | v1 | marko | 29 | | Person | | | | | + | v2 | lop | | java | Software | | | | | + | e1 | | | | | v1 | v2 | Created | 0.4 | + + === "CSV" + ```csv + _id,name:String,age:Int,lang:String,_labels,_start,_end,_type,weight:Float + v1,marko,29,,Person,,,, + v2,lop,,java,Software,,,, + e1,,,,,v1,v2,Created,0.4 + ``` + +## Neptune Generators + +You can import csv from Neptune using the `NeptuneCsvElementGenerator` and export using the +`NeptuneCsvGenerator`. The format used is defined +[here](https://docs.aws.amazon.com/neptune/latest/userguide/bulk-load-tutorial-format-opencypher.html). + +???+ note "Example" + === "Table" + | :ID | name | age | lang | :LABEL | :START_ID | :END_ID | :TYPE | weight | + |-----|-------|-----|------|----------|-----------|---------|---------|--------| + | v1 | marko | 29 | | Person | | | | | + | v2 | lop | | java | Software | | | | | + | e1 | | | | | v1 | v2 | Created | 0.4 | + + === "CSV" + ```csv + :ID,name:String,age:Int,lang:String,:LABEL,:START_ID,:END_ID,:TYPE,weight:Double + v1,marko,29,,person,,,, + v2,lop,,java,software,,,, + e1,,,,,v1,v2,created,0.4 + ``` diff --git a/docs/user-guide/query/gremlin/examples.md b/docs/user-guide/query/gremlin/examples.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/query/gremlin/gremlin.md b/docs/user-guide/query/gremlin/gremlin.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/query/python-extension/examples.md b/docs/user-guide/query/python-extension/examples.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/query/python-extension/python-extension.md b/docs/user-guide/query/python-extension/python-extension.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/query/query.md b/docs/user-guide/query/query.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mkdocs.yml b/mkdocs.yml index 43c9c92fa3..d9f3e9fcfb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -98,6 +98,26 @@ nav: - dev/components/libraries/sketches.md - dev/components/libraries/spark.md - dev/components/libraries/time.md + - User Guide: + - 'Gaffer Basics': + - 'What is Gaffer?': 'user-guide/gaffer-basics/what-is-gaffer.md' + - 'What is Graph?': 'user-guide/gaffer-basics/what-is-graph.md' + - 'What is JSON?': 'user-guide/gaffer-basics/what-is-json.md' + - 'What is Python?': 'user-guide/gaffer-basics/what-is-python.md' + - 'Getting Started': + - 'API': 'user-guide/getting-started/api.md' + - 'Schema': 'user-guide/getting-started/schema.md' + - 'Query': + - 'Python Extension': + - 'Python Extension Guide': 'user-guide/query/python-extension/python-extension.md' + - 'Examples': 'user-guide/query/python-extension/examples.md' + - 'API Querying': + - 'API Querying Guide': 'user-guide/query/api-querying/api-querying.md' + - 'Import/Export Data': 'user-guide/query/api-querying/import-export-data.md' + - 'Examples': 'user-guide/query/api-querying/examples.md' + - 'Gremlin': + - 'Gremlin Guide': 'user-guide/query/gremlin/gremlin.md' + - 'Examples': 'user-guide/query/gremlin/examples.md' - 'Reference': - 'Reference Guide': 'reference/intro.md' - Stores: From 50e9955ba7fd0ddb4f371e02199868205d5097e8 Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:01:22 +0000 Subject: [PATCH 02/20] Added new development guide structure --- docs/development-guide/extending-gaffer.md | 0 docs/development-guide/getting-started.md | 0 .../components/accumulo-store.md | 213 + .../project-structure/components/cache.md | 24 + .../components/components.md | 99 + .../project-structure/components/core-rest.md | 72 + .../project-structure/components/data.md | 28 + .../project-structure/components/graph.md | 107 + .../components/integration-test.md | 6 + .../components/libraries/bitmap.md | 20 + .../components/libraries/flink.md | 19 + .../components/libraries/sketches.md | 14 + .../components/libraries/spark.md | 23 + .../components/libraries/time.md | 16 + .../project-structure/components/operation.md | 82 + .../components/serialisation.md | 7 + .../components/spring-rest.md | 47 + .../project-structure/components/store.md | 15 + .../maven-dependencies/core.svg | 307 + .../maven-dependencies/gaffer-complete.svg | 5772 +++++++++++++++++ .../maven-dependencies/map-store.svg | 1104 ++++ .../maven-dependencies/spring.md | 105 + .../remote-coding-environments.md | 32 + docs/development-guide/ways-of-working.md | 91 + mkdocs.yml | 26 + 25 files changed, 8229 insertions(+) create mode 100644 docs/development-guide/extending-gaffer.md create mode 100644 docs/development-guide/getting-started.md create mode 100644 docs/development-guide/project-structure/components/accumulo-store.md create mode 100644 docs/development-guide/project-structure/components/cache.md create mode 100644 docs/development-guide/project-structure/components/components.md create mode 100644 docs/development-guide/project-structure/components/core-rest.md create mode 100644 docs/development-guide/project-structure/components/data.md create mode 100644 docs/development-guide/project-structure/components/graph.md create mode 100644 docs/development-guide/project-structure/components/integration-test.md create mode 100644 docs/development-guide/project-structure/components/libraries/bitmap.md create mode 100644 docs/development-guide/project-structure/components/libraries/flink.md create mode 100644 docs/development-guide/project-structure/components/libraries/sketches.md create mode 100644 docs/development-guide/project-structure/components/libraries/spark.md create mode 100644 docs/development-guide/project-structure/components/libraries/time.md create mode 100644 docs/development-guide/project-structure/components/operation.md create mode 100644 docs/development-guide/project-structure/components/serialisation.md create mode 100644 docs/development-guide/project-structure/components/spring-rest.md create mode 100644 docs/development-guide/project-structure/components/store.md create mode 100644 docs/development-guide/project-structure/maven-dependencies/core.svg create mode 100644 docs/development-guide/project-structure/maven-dependencies/gaffer-complete.svg create mode 100644 docs/development-guide/project-structure/maven-dependencies/map-store.svg create mode 100644 docs/development-guide/project-structure/maven-dependencies/spring.md create mode 100644 docs/development-guide/remote-coding-environments.md create mode 100644 docs/development-guide/ways-of-working.md diff --git a/docs/development-guide/extending-gaffer.md b/docs/development-guide/extending-gaffer.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/development-guide/getting-started.md b/docs/development-guide/getting-started.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/development-guide/project-structure/components/accumulo-store.md b/docs/development-guide/project-structure/components/accumulo-store.md new file mode 100644 index 0000000000..4fb633a477 --- /dev/null +++ b/docs/development-guide/project-structure/components/accumulo-store.md @@ -0,0 +1,213 @@ +# Accumulo Store Implementation + +The [accumulo-store module](https://github.com/gchq/Gaffer/tree/master/core/store) is an implementation of the Store API which uses Apache Accumulo. + +This page contains brief details on the internal implementation of the `AccumuloStore`. For information on configuring and using this store, see [the Accumulo Store reference page](../../reference/stores-guide/accumulo.md). + +## Introduction + +It is assumed that the reader has some familiarity with the design of Accumulo (see the [Design page in Accumulo's Docs](https://accumulo.apache.org/docs/2.x/getting-started/design)). + +The important features for Gaffer are: + +- Accumulo stores data in key-value pairs. A key has multiple parts, namely a row ID, a column family, a column qualifier, a column visibility, and a timestamp. Each of these is simply a byte array, with the exception of the timestamp which is a long. A value is simply a byte array. +- Data in Accumulo is stored ordered by key. Keys are stored sorted by increasing row ID, then column family, then column qualifier, then column visibility, then by decreasing timestamp. +- Accumulo allows locality groups to be set which group together column families. This means that scans that only need to read certain column families can skip families they do not need to read. +- Accumulo allows data to be tagged with a visibility which restricts which users can view it. +- Accumulo allows the user to configure iterators that run at scan time, at compaction time or both. Gaffer adds iterators to scans to filter data. It uses compaction time iterators to persistently aggregate the properties of elements together, and to continually validate data. +- Accumulo provides an `InputFormat` that allows data to be retrieved via MapReduce jobs. + +The core of the functionality is implemented in the key-packages, the iterators and the retrievers. Each of these is described in some detail below. + +## Key-packages + +As noted in the [Key-packages section of the Accumulo Store reference](../../reference/stores-guide/accumulo.md#key-packages), key-packages are responsible for converting `Element`s to and from key-value pairs, for creating ranges of keys containing all data relevant to a particular query, and for configuring the Iterators. Gaffer provides two key-packages: `ByteEntityKeyPackage` and `ClassicKeyPackage`. Advanced users are able to create their own key-packages if they wish - see [options for future key-packages](#options-for-future-key-packages) for some ideas. + +Before these key-packages are described, we review the main design goals: + +- To be able to retrieve all `Edge`s for a vertex by seeking to a single point in the table and scanning forwards. +- To be able to retrieve all `Entity`s for a vertex by seeking to a single point in the table, and reading only relevant key-value pairs, i.e. not reading any of the `Edge`s associated to the vertex. +- A vertex should be uniquely identified by its serialised value. It should not be necessary to consult an external source to find the value that identifies a vertex. In particular unlike most graph databases we do not use longs to identify vertices. +- To ensure that there are no "fat" rows, i.e. that there are not very large numbers of key-value pairs with the same row-key. +- To allow efficient aggregation of properties. + +Both key-packages convert an `Entity` into a single Accumulo key-value pair and an `Edge` into two key-value pairs. The row ID (also known as the row-key) of the key-value formed from the `Entity` is the vertex serialised to a byte array, followed by a flag to indicate that this is an `Entity`. This allows the `Entity`s associated to a vertex to be quickly retrieved. It is necessary to store each `Edge` as two key-values so that it can found from both the source vertex and the destination vertex: one key-value has a row ID consisting of the source vertex serialised to a byte array, followed by a delimiter, followed by the destination vertex serialised to a byte array; the other key-value has the opposite, with the destination vertex followed by the source vertex. A flag is also stored to indicate which of these two versions the key is so that the original `Edge` can be recreated. + +An important feature of the row IDs created by both key-packages is that it is possible to create ranges of keys that either only contain the `Entity`s or only contain the `Edge`s or contain both. This means that if, for example, a user states that they only want to retrieve the `Entity`s for a particular vertex then only relevant key-value pairs need to be read. In the case of a high-degree vertex, this means that queries for just the `Entity`s will still be very quick. + +The two key-packages differ in subtle details of how the row ID is created. In the following descriptions the notation "(serialised_vertex)" refers to the vertex serialised to a byte array with any occurrences of the zero byte removed. This is necessary so that the zero byte delimiter can be used to separate different parts of the row-key. The zero bytes are removed in such a way that the original byte array can be recreated, and so that ordering is preserved. + +### `ClassicKeyPackage` details + +The `ClassicKeyPackage` constructs the following Accumulo key-value pair for an `Entity`: + +| Row ID | Column Family | Column Qualifier | Visibility | Timestamp | Value | +| ------------------- | ------------- | ------------------- | ------------------- | --------- | -------------------- | +| (serialised_vertex) | group | group by properties | visibility property | timestamp | all other properties | + +The following Accumulo key-value pairs are created for an `Edge`: + +| Row ID | Column Family | Column Qualifier | Visibility | Timestamp | Value | +| ------------------------------------------------------------ | ------------- | ------------------- | ------------------- | --------- | -------------------- | +| (serialised_source_vertex)0(serialised_destination_vertex)0x | group | group by properties | visibility property | timestamp | all other properties | +| (serialised_destination_vertex)0(serialised_source_vertex)0y | group | group by properties | visibility property | timestamp | all other properties | + +If the `Edge` is undirected then `x` and `y` are both 1 for both key-values. If the `Edge` is directed then `x` is 2 and `y` is 3. + +This is very similar to the design of the key-value pairs in version 1 of Gaffer, with the exception that version 1 did not store a delimiter or flag at the end of the row-key for an `Entity`. This necessitated a scan of the row-key counting the number of delimiters to determine whether it was an `Entity` or `Edge`. If it is an `Entity` the vertex could be created directly from the row-key. For the `ClassicKeyPackage`, this scan is not needed but an array copy of the row-key minus the delimiter and flag is needed. In practice, the difference in performance between the two is likely to be negligible. + +### `ByteEntityKeyPackage` details + +The ByteEntity key-package constructs the following Accumulo key-value pair for an `Entity`: + +| Row ID | Column Family | Column Qualifier | Visibility | Timestamp | Value | +| ------------------- | ------------- | ------------------- | ------------------- | --------- | -------------------- | +| (serialised_vertex)01 | group | group by properties | visibility property | timestamp | all other properties | + +In the row ID the 0 is a delimiter to split the serialised vertex from the 1. The 1 indicates that this is an `Entity`. By having this flag at the end of the row id it is easy to determine if the key relates to an `Entity` or an `Edge`. + +The following Accumulo key-value pairs are created for an `Edge`: + +| Row ID | Column Family | Column Qualifier | Visibility | Timestamp | Value | +| -------------------------------------------------------------- | ------------- | ------------------- | ------------------- | --------- | -------------------- | +| (serialised_source_vertex)0x0(serialised_destination_vertex)0x | group | group by properties | visibility property | timestamp | all other properties | +| (serialised_destination_vertex)0y0(serialised_source_vertex)0y | group | group by properties | visibility property | timestamp | all other properties | + +If the `Edge` is undirected then both `x` and `y` are 4. If the `Edge` is directed then `x` is 2 and `y` is 3. + +The flag is repeated twice to allow filters that need to know whether the key corresponds to a `Entity` or an `Edge` to avoid having to fully deserialise the row ID. For a query such as find all out-going edges from this vertex, the flag that is directly after the source vertex can be used to restrict the range of row IDs queried for. + +Note that in a range query filtering to restrict the results to say only out-going edges happens in an iterator. + +### Options for future key-packages + +Numerous variations on the above key-packages could be implemented. These would generally improve the performance for some types of query, at the expense of decreasing the performance for other types of query. Some examples are: + +- The row-keys could be sharded. The current design is optimised for retrieving all `Edge`s for a given vertex, when there are relatively few such `Edge`s. If there are a million edges for a vertex then all of these have to be read by a small number of tablet servers (typically one, unless the range spans multiple tablets). This limits the query performance. An alternative approach is to introduce a shard key at the start of the row-key to cause different edges for the same vertex to be spread uniformly across the table. This would increase the parallelism for queries which would lead to better performance when large numbers of edges need to be retrieved for a vertex. The trade-off is that all queries would need to query all shards which would reduce the performance when a vertex has only a small number of edges. +- If there are a very large number of `Edge`s with the same source, destination and group-by properties then this could cause unbalanced tablets. A sharding scheme similar to the above would deal with this. +- Remove the flag at the end of the row-key that indicates whether it corresponds to an `Entity` or an `Edge`. This is used to quickly determine whether it is an `Entity` or an `Edge`. This is actually superfluous information as the group is stored in the column family and that indicates whether the key-value is an `Entity` or an `Edge`. Storing the flag there creates the need for an array copy when an `Entity` is created from the key-value. Instead of storing the group string in the column family, two bytes could be stored. The first would indicate whether this is an `Entity` or an `Edge`, and if an `Edge` whether it needs reversing or not; the second would indicate what group it is. +- Store each group in a separate table. This should slightly improve the performance of queries that only require a subset of the groups, especially if the query scans lots of data (as Accumulo's locality groups are set in the above key-packages the performance improvement will probably be minor). It would worsen the query performance when multiple groups are being retrieved. +- If the vertices serialise to a fixed length, or if a maximum length is known, then the row-keys could be of fixed length. This would eliminate the need for the use of delimiters which forces the escaping of the zero byte inside the serialised value. This would potentially provide a small increase in ingest and query speed. + +## Iterators + +Gaffer makes substantial use of Accumulo's iterator functionality to perform permanent aggregation and validation of data at compaction time, and filtering and aggregation at query time. See the [Iterators](https://accumulo.apache.org/docs/2.x/development/iterators) section of Accumulo's Docs for more information on iterators. + +The following subsections describes the iterators that are used in Gaffer. They are listed in decreasing order of priority, i.e. the first iterator runs first. The text in brackets after the name of the iterator gives the scopes that the iterator is applied in. Some iterators that are only used for very specific operations are not listed here. + +### `AggregatorIterator` (compaction, scan) + +This iterator aggregates together all properties that are not group-by properties for `Element`s that are otherwise identical. As the non-group-by properties are stored in the `Value` this means that all `Value`s for identical keys are merged together. + +### `ValidatorFilter` (compaction, scan) + +The `ValidatorFilter` iterator validates every `Element` using the validation logic defined in the schema. When this is run during a compaction it causes invalid data to be deleted. This is typically used to delete data that is older than a certain date. + +### `ClassicEdgeDirectedUndirectedFilterIterator` (scan) + +!!! note + This is only used in the `ClassicKeyPackage`. + +This is used to filter out edges that are not required because the user has specified filters relating to edge direction (outgoing or incoming) and edge "directedness" (directed or undirected) in their query. Note that it is possible to ask for various combinations of these, e.g.: + +- Directed edges only: if the seed is A then directed edges A->B and B->A would be returned, but an undirected edge A-B wouldn't be. +- Directed outgoing edges only: if the seed is A then a directed edge A->B would be returned, but a directed edge B->A wouldn't be, nor would an undirected edge A-B. +- Directed incoming edges only: if the seed is A then a directed edge B->A would be returned, but a directed edge A->B wouldn't be, nor would an undirected edge A-B. +- Undirected edges only: if the seed is A then an undirected edge A-B would be returned, but directed edges A->B and B->A wouldn't be. +- Undirected outgoing edges only: if the seed is A then an undirected edge A-B would be returned, but directed edges A->B and B->A wouldn't be. +- Undirected incoming edges only: if the seed is A then an undirected edge A-B would be returned, but directed edges A->B and B->A wouldn't be. + +In the latter two examples, note that an undirected edge A-B is defined to be both outgoing from, and incoming to, both A and B. + +### `ElementPreAggregationFilter` (scan) + +This iterator filters out `Element`s that are not valid according to the `View`. This filtering happens before the aggregation. + +### `CoreKeyGroupByAggregatorIterator` (scan) + +This iterator aggregates together all properties according to the group-by in the view. + +### `ElementPostAggregationFilter` (scan) + +This iterator filters out `Element`s that are not valid according to the `View`. This filtering happens after the aggregation. + +### Locality groups + +Accumulo's ability to have a large number of different column families allows Gaffer to store lots of different types of data in the same table. Specifying the locality groups means that when a query for a particular group is made, graph elements from other groups do not need to be read. + +## Tests + +!!! warning + This section might not be fully up to date for Gaffer 2.0.0. To easily run integration tests against a cluster, you can now [use docker compose](https://github.com/gchq/gaffer-docker/tree/master/docker/gaffer-integration-tests). + +For the purposes of unit testing and small-scale examples, Gaffer offers the Store subclass [MiniAccumuloStore](https://github.com/gchq/Gaffer/blob/master/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/MiniAccumuloStore.java) and the `MiniAccumuloCluster`. + +By default all our tests use the MiniAccumuloStore. The MiniAccumuloStore automatically sets up or uses an existing MiniAccumuloCluster according to your store properties. + +Alongside the standard Accumulo properties, you also have the opportunity to add some extra ones for a MiniAccumuloStore: + +``` +accumulo.mini.directory=/path/to/directory +accumulo.mini.root.password=password +accumulo.mini.visibilities=vis1,vis2,publicVisibility,privateVisibility,public,private +``` +These properties are optional. By default the MiniAccumuloStore creates the cluster in a temporary directory, uses "password" as the root password and adds no extra visibilities to a user. + +Because the MiniAccumulo re-uses clusters to be efficient, if two tests use the same user with different visibilities, the second one will overwrite the first. Therefore it's advisable to use different users if you want a user with different visibilities. + +### Running the integration tests + +#### Running a Mini Accumulo Cluster manually + +Follow this [README.md](https://github.com/gchq/gaffer-tools/tree/master/mini-accumulo-cluster) in gaffer-tools on how to run a Mini Accumulo Cluster (with a shell) on your local machine. + +!!! note + When running a Mini Accumulo Cluster locally a `store.properties` file is generated, this can help identify the values you need to replace in the store.properties used for the integration tests below (such as the username, password, instance name and Zookeeper location). + +#### Setting up accumulo-store integration tests + +Update the following store properties files in src/test/resources/ to point to the location of the Accumulo store to test against: + +- [src/test/resources/store.properties](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/resources/store.properties) +- [src/test/resources/store2.properties](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/resources/store2.properties) +- [src/test/resources/accumuloStoreClassicKeys.properties](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/resources/accumuloStoreClassicKeys.properties) + +If you are running an Accumulo cluster locally, here is what an example test store.properties file should look like: + +```text +gaffer.store.class=uk.gov.gchq.gaffer.accumulostore.SingleUseAccumuloStore +gaffer.store.properties.class=uk.gov.gchq.gaffer.accumulostore.AccumuloProperties +accumulo.instance=instance +accumulo.user=root +accumulo.password=password +accumulo.zookeepers=localhost:58630 + +gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService +gaffer.store.job.tracker.enabled=true +gaffer.store.operation.declarations=ExportToOtherAuthorisedGraphOperationDeclarations.json,ExportToOtherGraphOperationDeclarations.json,ResultCacheExportOperations.json +``` + +Ensure that when running an Accumulo instance, the user specified by the `accumulo.user` property has the `System.CREATE_TABLE` and `System.CREATE_NAMESPACE` permissions ('root' user has these set by default) and the following scan authorisations: + +| Authorisation | Required by | +| ----------------- | ----------- | +| vis1 | [VisibilityIT](https://github.com/gchq/Gaffer/blob/develop/integration-test/src/test/java/uk/gov/gchq/gaffer/integration/impl/VisibilityIT.java) | +| vis2 | [VisibilityIT](https://github.com/gchq/Gaffer/blob/develop/integration-test/src/test/java/uk/gov/gchq/gaffer/integration/impl/VisibilityIT.java) | +| public | [SchemaHidingIT](https://github.com/gchq/Gaffer/blob/develop/core/graph/src/test/java/uk/gov/gchq/gaffer/integration/graph/SchemaHidingIT.java) | +| private | [ParameterizedLoaderIT](https://github.com/gchq/Gaffer/blob/develop/integration-test/src/test/java/uk/gov/gchq/gaffer/integration/impl/loader/ParameterizedLoaderIT.java#L63) +| publicVisibility | [AccumuloAggregationIT](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/AccumuloAggregationIT.java) | +| privateVisibility | [AccumuloAggregationIT](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/AccumuloAggregationIT.java) | + +You can set these scan authorisations via the Accumulo shell: + +e.g. if your store.properties have: `accumulo.user=root`, `accumulo.instance=instance` + +```sh +root@instance> setauths -u root -s vis1,vis2,publicVisibility,privateVisibility,public,private +``` + +Run the integration tests: + +```sh +mvn verify +``` diff --git a/docs/development-guide/project-structure/components/cache.md b/docs/development-guide/project-structure/components/cache.md new file mode 100644 index 0000000000..da049572d4 --- /dev/null +++ b/docs/development-guide/project-structure/components/cache.md @@ -0,0 +1,24 @@ +# Cache + +The [cache module](https://github.com/gchq/Gaffer/tree/master/core/cache) contains the `CacheServiceLoader` which is started when the store is initialised. The cache service loader can be called when a component needs access to short term data storage. +To get access to the cache service you need to call: +```java +CacheServiceLoader.getService(); +``` + +By default, there is no service loaded so if you're using a component that makes use of the `CacheServiceLoader`, be sure to specify the service class in the `store.properties` file. +See the [Stores reference guide](../../reference/stores-guide/stores.md#cache-configuration) for configuration info. + +If using an external cache service (anything found in the cache library) be sure to include the library as a dependency: +```xml + + uk.gov.gchq.gaffer + jcs-cache-service + ${gaffer.version} + +``` + +When run in a servlet context, the `CacheServiceLoader` should be shutdown gracefully by the `ServletLifecycleListener` found in the REST package. Do not trust the shutdown hook in a servlet context. +If running outside a servlet environment, you can either call shutdown on the cache service manually or use the shutdown hook upon initialisation of the cache service loader. + +For information on Gaffer caches and cache configuration, see [the cache section of the Stores Guide](../../reference/stores-guide/stores.md#caches). diff --git a/docs/development-guide/project-structure/components/components.md b/docs/development-guide/project-structure/components/components.md new file mode 100644 index 0000000000..eb4736dac3 --- /dev/null +++ b/docs/development-guide/project-structure/components/components.md @@ -0,0 +1,99 @@ +# Components/Maven Modules + +Gaffer has lots of components, which can be split into different categories. This section of the developer doc provides more detail on the most important components. + +## Key components + +For more developer information on these key components, see their associated page. + +- [Operation](operation.md): Classes for the Operation interfaces and core operation implementations. +- [Cache](cache.md): Classes for the Gaffer cache service. +- [Graph](graph.md): Contains the Gaffer `Graph` object and related utilities. +- [Data](data.md): Classes defining Gaffer's data objects: `Element`, `Edge` and `Entity`. +- [Store](store.md): Contains the Gaffer `Store` object and related classes. +- [Serialisation](serialisation.md): Contains classes for Serialisation in Gaffer. +- [Accumulo Store](accumulo-store.md): A store implemented using Apache Accumulo. +- [Core REST](core-rest.md): Classes which provide a Gaffer REST API using Jersey/JAX-RS. +- [Spring REST](spring-rest.md): Implementation of the Gaffer REST API deployed in a Spring Boot container. + +## Project Structure (Maven) + +Gaffer uses Maven and from this perspective the project is made up of multiple Maven modules. All components are Maven modules, but not all modules are components as some are just parent or aggregator POMs (see below). +Maven modules are not the same as Java modules (Java 9+), which Gaffer doesn't use or support. [See here for more info on multi-module builds and the Maven reactor](https://maven.apache.org/guides/mini/guide-multiple-modules.html). + +Gaffer's project structure involves three kinds of module/POM, mostly based on the [Maven packaging type](https://maven.apache.org/pom.html#Packaging). We can call these: + +- Parent/Aggregator modules (`pom`) +- JAR modules (`jar`) +- WAR only/Demo modules (`war`). + +Parent modules consist of a single POM file (parent POM) which can define various properties, dependencies and settings to be inherited by JAR modules or other parent modules. +These also specify the modules below the parent (Maven [aggregation](https://maven.apache.org/pom.html#aggregation-or-multi-module)). + +JAR modules are not inherited by other modules, and in addition to the POM they contain code which is compiled into artifacts (always `jar` and sometimes also `war`). Some contain only code used for demos. + +WAR only/Demo modules are not inherited by other modules, they contain only a POM and potentially config files and are used for either creating WAR archives for the REST API or are used for running demos using Maven plugins and other modules. + +Gaffer has 47 modules in total. When building the complete project, Maven automatically runs the build in a specific order because of dependencies between modules. + +??? tip "Finding and displaying POM structure" + The paths for all POMs in the Gaffer project can be seen by running `find . -name pom.xml` from the project/repository root. + + A simple text-based diagram of the Maven project structure can be generated and printed using `mvn org.qunix:structure-maven-plugin:modules -pl :gaffer2`. + +The diagram below shows all modules and their type (green diamond for Parent POM, blue for JAR and red for WAR/Demo): + +``` mermaid +graph LR + PP{uk.gov.gchq.gaffer:gaffer2} --> C{core} + PP --> I[integration-test]:::JAR + PP --> SI{store-implementation} + PP --> RI{rest-api} + PP --> L{library} + PP --> E{example} + C --> operation:::JAR + C --> cache:::JAR + C --> access:::JAR + C --> G[graph]:::JAR + C --> type:::JAR + C --> data:::JAR + C --> exception:::JAR + C --> store:::JAR + C --> common-util:::JAR + C --> serialisation:::JAR + SI --> accumulo-store:::JAR + SI --> map-store:::JAR + SI --> proxy-store:::JAR + SI --> federated-store:::JAR + RI --> spring-rest:::JAR + RI --> common-rest:::JAR + RI --> map-rest:::WarDemo + RI --> accumulo-rest:::WarDemo + RI --> core-rest:::JAR + L --> tinkerpop:::JAR + L --> sketches-library:::JAR + L --> CL{cache-library} + L --> hdfs-library:::JAR + L --> bitmap-library:::JAR + L --> time-library:::JAR + L --> flink-library:::JAR + L --> S{spark} + CL --> hazelcast-cache-service:::JAR + CL --> jcs-cache-service:::JAR + S --> spark-accumulo-library:::JAR + S --> spark-library:::JAR + E --> RT{road-traffic} + E --> B{basic} + E --> federated-demo:::JAR + RT --> road-traffic-model:::JAR + RT --> road-traffic-demo:::JAR + RT --> road-traffic-generators:::JAR + B --> basic-model:::JAR + B --> basic-rest:::WarDemo + + classDef parentPOM fill:lightgreen; + classDef JAR fill:lightblue; + classDef WarDemo fill:lightcoral; + + class PP,C,SI,RI,L,E,CL,S,RT,B parentPOM +``` diff --git a/docs/development-guide/project-structure/components/core-rest.md b/docs/development-guide/project-structure/components/core-rest.md new file mode 100644 index 0000000000..86c26f9249 --- /dev/null +++ b/docs/development-guide/project-structure/components/core-rest.md @@ -0,0 +1,72 @@ +# Core REST API + +The [Core REST API module](https://github.com/gchq/Gaffer/tree/master/rest-api/core-rest) contains a Gaffer REST API. + +Gaffer Stores have modules extending the core-rest and adds in the dependency for the Gaffer Store. So if you want to use the Accumulo Store REST API, you can use the `accumulo-rest` `.war`, or `map-rest` for the Map Store. + +For an example of using the core REST API please see the [example/road-traffic module](https://github.com/gchq/Gaffer/tree/master/example/road-traffic). + +## How to modify the Core REST API for your project + +You can easily make changes or additions to the core REST API for your project. You will need to create a new Maven module to build your core REST API. In your POM you should configure the `maven-dependency-plugin` to download the core Gaffer REST API `.war` and extract it. When Maven builds your module it will unpack the core war, add your files and repackage the war. If you wish to override a file in the core war then you can do this by including your own file with exactly the same name and path. + +Example `maven-dependency-plugin` configuration: +```xml + + src/main/java + + + org.apache.maven.plugins + maven-dependency-plugin + 3.5 + + + uk.gov.gchq.gaffer + core-rest + ${gaffer.version} + war + + + + + unpack + compile + + unpack + + + + + uk.gov.gchq.gaffer + core-rest + ${gaffer.version} + war + false + + ${project.build.directory}/${project.artifactId}-${project.version} + + + + + + + + + +``` + +So, if you want to change the CSS for the core REST API you can override the custom.css file: +``` +/src/main/webapp/css/custom.css +``` + +There are also various system properties you can use to configure to customise the Swagger UI. +For example: +``` +gaffer.properties.app.title=Road Traffic Example +gaffer.properties.app.description=Example using road traffic data +gaffer.properties.app.banner.description=DEMO +gaffer.properties.app.banner.colour=#1b75bb +gaffer.properties.app.logo.link=https://github.com/gchq/Gaffer +gaffer.properties.app.logo.src=images/iconCircle.png +``` diff --git a/docs/development-guide/project-structure/components/data.md b/docs/development-guide/project-structure/components/data.md new file mode 100644 index 0000000000..368be8eeff --- /dev/null +++ b/docs/development-guide/project-structure/components/data.md @@ -0,0 +1,28 @@ +# Data + +The [data module](https://github.com/gchq/Gaffer/tree/master/core/data) contains Gaffer's data objects: `Element`, `Edge` and `Entity`. + +It also contains the logic for processing these `Element`s - `ElementAggregator`, `ElementFilter` and `ElementTransformer`. + +## Functions and Predicates + +Gaffer makes use of Java 8's Function and Predicate interfaces to aggregate, transform and filter data. To allow these Function and Predicate classes to process tuples we make use of the [Koryphe](https://github.com/gchq/koryphe/tree/master) library. Koryphe allows us to wrap the Gaffer Elements in a tuple and pass it any Function or Predicate. + +You can use any of our implementations ([see reference pages](../../reference/intro.md)) or write your own. + +All the following classes will act on one or more Element identifiers (vertex/source/destination/directed) or properties. If you implement the Java 8 interfaces directly, you would need to add the following `JsonType` annotation to your class: + +```java +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "class") +``` + +Instead, we recommend you implement the Koryphe interfaces instead, which will add this annotation in for you. + +### Aggregation +Aggregation is done using a `KorypheBinaryOperator` (or just `BinaryOperator`), where T is the type of the property you are aggregating. For example: [Max](https://github.com/gchq/koryphe/blob/master/core/src/main/java/uk/gov/gchq/koryphe/impl/binaryoperator/Max.java). + +### Transforms +Transforms are applied using `KorypheFunction` (or just `Function`), where I is type of the input property and O is the type of the output value. If you want to transform multiple properties into a single new property then you can implement `KorypheFunction2` or `KoryphePredicate3`, etc. where R is the output value type. For example: [Concat](https://github.com/gchq/koryphe/blob/master/core/src/main/java/uk/gov/gchq/koryphe/impl/function/Concat.java). + +### Filtering +Filtering is applied using `KoryphePredicate` (or just `Predicate`), where T is type of the property you want to filter on. If you want to filter on multiple properties then you can implement `KoryphePredicate2` or `KoryphePredicate3` etc. For example: [Exists](https://github.com/gchq/koryphe/blob/master/core/src/main/java/uk/gov/gchq/koryphe/impl/predicate/Exists.java). diff --git a/docs/development-guide/project-structure/components/graph.md b/docs/development-guide/project-structure/components/graph.md new file mode 100644 index 0000000000..ffff6b9d3e --- /dev/null +++ b/docs/development-guide/project-structure/components/graph.md @@ -0,0 +1,107 @@ +# Graph + +The [graph module](https://github.com/gchq/Gaffer/tree/master/core/graph) contains the Gaffer `Graph` object and related utilities. This is the entry point (or proxy) for your chosen Gaffer store. + +The `Graph` separates the user from the underlying store. It holds a connection which acts as a proxy, delegating operations to the store. +It provides users with a single point of entry for executing operations on a store. This allows the underlying store to be swapped and the same operations can still be applied. + +When you instantiate a `Graph`, this doesn't mean you are creating an entirely new graph with its own data, you are simply creating a connection to a store where some data is held. + +To create an instance of `Graph`, we recommend you use the `Graph.Builder` class. This has several helpful methods to create the graph from various different sources. +But, essentially a graph requires just 3 things: some store properties, a schema and some graph specific configuration. + +See the [Graph Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/graph/package-summary.html) for further documentation. + +## Store Properties +The store properties tells the graph the type of store to connect to along with any required connection details. See the [Stores](../../reference/stores-guide/stores.md) reference page for more information on the different Stores for Gaffer. + +## Schema +The schema is passed to the store to instruct the store how to store and process the data. See [Schemas](https://gchq.github.io/gaffer-doc/v1docs/getting-started/developer-guide/schemas.html) for more information - v1 docs, to be updated and added to this documentation in future. + +## Graph Configuration +The graph configuration allows you to apply special customisations to the Graph instance. The only required field is the `graphId`. + +To create an instance of `GraphConfig` you can use the `GraphConfig.Builder` class, or create it using a json file. + +The `GraphConfig` can be configured with the following: + - `graphId` - The `graphId` is a String field that uniquely identifies a `Graph`. When backed by a Store like Accumulo, this `graphId` is used as the name of the Accumulo table for the `Graph`. + - `description` - a string describing the `Graph`. + - `view` - The `Graph View` allows a graph to be configured to only returned a subset of Elements when any Operation is executed. For example if you want your `Graph` to only show data that has a count more than 10 you could add a View to every operation you execute, or you can use this `Graph View` to apply the filter once and it would be merged into to all Operation Views so users only ever see this particular view of the data. + - `library` - This contains information about the `Schema` and `StoreProperties` to be used. + - `hooks` - A list of `GraphHook`s that will be triggered before, after and on failure when operations are executed on the `Graph`. See [GraphHooks](#graph-hooks) for more information. + +Here is an example of a `GraphConfig`: + +```java +new GraphConfig.Builder() + .config(new GraphConfig.Builder() + .graphId("exampleGraphId") + .build()) + .description("Example Graph description") + .view(new View.Builder() + .globalElements(new GlobalViewElementDefinition.Builder() + .postAggregationFilter(new ElementFilter.Builder() + .select("ExamplePropertyName") + .execute(new IsLessThan("10")) + .build()) + .build()) + .build()) + .library(new FileGraphLibrary()) + .addHook(new Log4jLogger()) + .build(); +``` + +and in json: + +```json +{ + "graphId": "exampleGraphId", + "description": "Example Graph description", + "view": { + "globalElements": [ + { + "postAggregationFilterFunctions": [ + { + "predicate": { + "class": "uk.gov.gchq.koryphe.impl.predicate.IsLessThan", + "orEqualTo": false, + "value": "10" + }, + "selection": ["ExamplePropertyName"] + } + ] + } + ] + }, + "library": { + "class": "uk.gov.gchq.gaffer.store.library.FileGraphLibrary" + }, + "hooks": [ + { + "class": "uk.gov.gchq.gaffer.graph.hook.Log4jLogger" + } + ] +} +``` + +## Graph Hooks +The `Graph` class is final and must be used when creating a new connection to a store. We want to ensure that all users have a common point of entry to Gaffer, so all users have to start by instantiating a `Graph`. +Initially this seems quite limiting, but to allow custom logic for different types of graphs we have added graph hooks. These graph hooks allow custom code to be run before and after an operation chain is executed. + +You can use hooks to do things like custom logging or special operation chain authorisation. To implement your own hook, just implement the `GraphHook` interface and register it with the graph when you build a `Graph` instance. +GraphHooks should be json serialisable and each hook should have a unit test that extends GraphHookTest. + +There are some graph hooks which are added by default if they aren't already present in the configuration. The NamedViewResolver and NamedOperationResolver (providing that NamedOperations are supported by the store) are added at the start of the list of hooks. +The third hook added is the FunctionAuthoriser, which is added at the end of the list - again assuming no hook is not present in the configuration. This hook stops users from using potentially dangerous functions in their operation chain. +If you want to disable this hook, you should overwrite it by adding an empty FunctionAuthoriser to your list of hooks. For example: + +```json +{ + "graphId": "example", + "hooks": [ + { + "class": "uk.gov.gchq.gaffer.graph.hook.FunctionAuthoriser" + } + ] +} +``` diff --git a/docs/development-guide/project-structure/components/integration-test.md b/docs/development-guide/project-structure/components/integration-test.md new file mode 100644 index 0000000000..486230793a --- /dev/null +++ b/docs/development-guide/project-structure/components/integration-test.md @@ -0,0 +1,6 @@ +# Integration Tests + +The [integration-test module](https://github.com/gchq/Gaffer/tree/master/integration-test) contains Gaffer's Integration Test suite for testing Store implementations. + +!!! note + This test framework is not a representative example of how to use the Gaffer framework. For an example of how the Gaffer framework can be used, please see the example road-traffic module. diff --git a/docs/development-guide/project-structure/components/libraries/bitmap.md b/docs/development-guide/project-structure/components/libraries/bitmap.md new file mode 100644 index 0000000000..459a9d9196 --- /dev/null +++ b/docs/development-guide/project-structure/components/libraries/bitmap.md @@ -0,0 +1,20 @@ +# Bitmap Library + +The [bitmap library](https://github.com/gchq/Gaffer/tree/master/library/bitmap-library) module contains various libraries for Bitmaps. + +In order to make use of the bitmap libraries you will need to include this library as a dependency: +```xml + + uk.gov.gchq.gaffer + bitmap-library + ${gaffer.version} + +``` + +## Registering + +You can register the BitmapJsonModules using the store or system property: `gaffer.serialiser.json.modules`. This property takes a CSV of classes, so you can use multiple json modules. + +``` +gaffer.serialiser.json.modules=uk.gov.gchq.gaffer.bitmap.serialisation.json.BitmapJsonModules +``` diff --git a/docs/development-guide/project-structure/components/libraries/flink.md b/docs/development-guide/project-structure/components/libraries/flink.md new file mode 100644 index 0000000000..8207bded0d --- /dev/null +++ b/docs/development-guide/project-structure/components/libraries/flink.md @@ -0,0 +1,19 @@ +# Flink Library + +The [flink library](https://github.com/gchq/Gaffer/tree/master/library/flink-library) module contains various libraries for using Apache Flink with Gaffer. + +In order to make use of the flink libraries you will need to include this library as a dependency: +```xml + + uk.gov.gchq.gaffer + flink-library + ${gaffer.version} + +``` + +For information on registering and using flink operations, see the [Flink Operations guide](../../../reference/operations-guide/flink.md). + +## I am getting errors when running the Flink operations on a cluster +This could be to do with the way the Gaffer Store class is serialised and distributed around the cluster. To distribute the job, Flink requires all of the components of the job to be Serializable. +The Gaffer Store class is not Serializable so instead we just Serialialize the graphId, Schema and Properties. Then when we require an instance of the Store class again we recreate it again with these parts. +This means that any files that are referenced in your StoreProperties must be available on all your nodes in your cluster. diff --git a/docs/development-guide/project-structure/components/libraries/sketches.md b/docs/development-guide/project-structure/components/libraries/sketches.md new file mode 100644 index 0000000000..4be4defa4b --- /dev/null +++ b/docs/development-guide/project-structure/components/libraries/sketches.md @@ -0,0 +1,14 @@ +# Sketches Library + +The [sketches library](https://github.com/gchq/Gaffer/tree/master/library/sketches-library) module contains various libraries for sketches. + +In order to make use of the sketches libraries you will need to include this library as a dependency: +```xml + + uk.gov.gchq.gaffer + sketches-library + ${gaffer.version} + +``` + +For information on configuring and using sketches, see the [Cardinality guide](../../../getting-started/guide/cardinality.md#how-to-add-cardinality-to-your-graph) (for configuring `SketchesJsonModules` expand "Additional config"). diff --git a/docs/development-guide/project-structure/components/libraries/spark.md b/docs/development-guide/project-structure/components/libraries/spark.md new file mode 100644 index 0000000000..6fbbbaf9e0 --- /dev/null +++ b/docs/development-guide/project-structure/components/libraries/spark.md @@ -0,0 +1,23 @@ +# Spark Library + +The [spark library](https://github.com/gchq/Gaffer/tree/master/library/spark/spark-library) contains various libraries for using Apache Spark with Gaffer. + +In order to make use of the spark libraries you will need to include this library as a dependency: +```xml + + uk.gov.gchq.gaffer + spark-library + ${gaffer.version} + +``` + +To use spark with Accumulo you will need to include this dependency: +```xml + + uk.gov.gchq.gaffer + spark-accumulo-library + ${gaffer.version} + +``` + +For information on registering and using spark operations, see the [Spark Operations guide](../../../reference/operations-guide/spark.md). diff --git a/docs/development-guide/project-structure/components/libraries/time.md b/docs/development-guide/project-structure/components/libraries/time.md new file mode 100644 index 0000000000..06128d8c15 --- /dev/null +++ b/docs/development-guide/project-structure/components/libraries/time.md @@ -0,0 +1,16 @@ +# Time Library + +The [time library](https://github.com/gchq/Gaffer/tree/master/library/time-library) module contains classes that represent concepts relating to time. + +For example, there is a class (`RBMBackedTimestampSet`) that can be used to represent a set of timestamps. Internally this stores its state in a Roaring Bitmap for efficiency reasons. +There is also a class that stores up to a maximum number N of timestamps. If more than N timestamps are added then a uniform random sample of the timestamps, of size at most N, is stored. Serialisers and aggregators for the above classes are provided. + +To use this library, you will need to include the following dependency: + +```xml + + uk.gov.gchq.gaffer + time-library + ${gaffer.version} + +``` diff --git a/docs/development-guide/project-structure/components/operation.md b/docs/development-guide/project-structure/components/operation.md new file mode 100644 index 0000000000..1047a8b927 --- /dev/null +++ b/docs/development-guide/project-structure/components/operation.md @@ -0,0 +1,82 @@ +# Operation + +The [operation module](https://github.com/gchq/Gaffer/tree/master/core/operation) contains the `Operation` interfaces and core operation implementations. + +It is assumed that all Gaffer graphs will be able to handle these core operations. + +An `Operation` implementation defines an operation to be processed on a graph, or on a set of results which are returned by another operation. An `Operation` class contains the configuration required to tell Gaffer how to carry out the operation. For example, the `AddElements` operation contains the elements to be added. +The `GetElements` operation contains the seeds to use to find elements in the graph and the filters to apply to the query. +The operation classes themselves should not contain the logic to carry out the operation (as this may vary between the different supported store types), just the configuration. + +For each operation, each Gaffer store will have an `OperationHandler`, where the processing logic is contained. This enables operations to be handled differently in each store. + +Operations can be chained together to form an `OperationChain`. When an operation chain is executed on a Gaffer graph the output of one operation is passed to the input of the next. + +An `OperationChain.Builder` is provided to help with constructing a valid operation chain - it ensures the output type of an operation matches the input type of the next. + +## How to write an Operation + +Operations should be written to be as generic as possible to allow them to be applied to different graphs/stores. + +Operations must be JSON serialisable in order to be used via the REST API - i.e. there must be a public constructor and all the fields should have getters and setters. + +Operation implementations need to implement the `Operation` interface and the extra interfaces they wish to make use of. For example an operation that takes a single input value should implement the `Input` interface. + +Here is a list of some of the common interfaces: + +- `uk.gov.gchq.gaffer.operation.io.Input` +- `uk.gov.gchq.gaffer.operation.io.Output` +- `uk.gov.gchq.gaffer.operation.io.InputOutput` - Use this instead of Input and Output if your operation takes both input and output. +- `uk.gov.gchq.gaffer.operation.io.MultiInput` - Use this in addition if your operation takes multiple inputs. This will help with JSON serialisation. +- `uk.gov.gchq.gaffer.operation.Validatable` +- `uk.gov.gchq.gaffer.operation.graph.OperationView` +- `uk.gov.gchq.gaffer.operation.graph.GraphFilters` +- `uk.gov.gchq.gaffer.operation.graph.SeededGraphFilters` + +Each operation implementation should have a corresponding unit test class that extends the `OperationTest` class. + +Operation implementations should override the close method and ensure all closeable fields are closed. + +The core Gaffer operations need to be registered in the `Store` class, with their respective handlers. The `StoreTest` unit test also needs similar information. + +### Annotations + +Any fields that are required should be annotated with the Required annotation. +As well, each `Operation` class now also requires the Since annotation, detailing to which version of Gaffer it was introduced. To demonstrate: +```java +@Since("1.4.0") +public class NewOperation extends Operation { + + @Required + private String requiredField; + ... +} +``` + +### Builder + +All implementations should also have a static inner `Builder` class that implements the required builders. For example: + +```java +public static class Builder extends Operation.BaseBuilder + implements InputOutput.Builder, CloseableIterable, Builder>, + MultiInput.Builder, + SeededGraphFilters.Builder, + Options.Builder { + public Builder() { + super(new GetElements()); + } +} +``` + +### Operation Scores + +For use with a `ScoreOperationChain`, some `Operation`s may require a custom way of calculating an associated score, therefore an implementation of the `ScoreResolver` interface may be required. +There is a `DefaultScoreResolver` to which the custom implementation should delegate, in a manner specific to the new Operation. For more info, see [ScoreOperationChain](/docs/reference/stores-guide/stores.md#scoreoperationchain) and [ScoreOperationChainExample](../../reference/operations-guide/misc.md#scoreoperationchain). + +### Documentation + +Class-level Javadoc should be provided for each `Operation`, giving a description of the functionality, and any configuration information that may not immediately be obvious. +Member-level Javadoc is not strictly necessary, but good practice for explanations/clarifications of complex methods. + +To assist users of the new Operation, it is best practice to provide documentation, and simple usage examples in [gaffer-doc](https://github.com/gchq/gaffer-doc). diff --git a/docs/development-guide/project-structure/components/serialisation.md b/docs/development-guide/project-structure/components/serialisation.md new file mode 100644 index 0000000000..16a938a60a --- /dev/null +++ b/docs/development-guide/project-structure/components/serialisation.md @@ -0,0 +1,7 @@ +# Serialisation + +The [serialisation module](https://github.com/gchq/Gaffer/tree/master/core/serialisation) contains the logic for converting an Object into serialised objects (normally a byte array). + +The main interface is Serialisation. We have provided a small set of serialisers for commonly used objects. The serialisers we have been designed to optimise speed and size. Serialisers can take arguments which may be mandatory depending on the serialiser used. + +It is important to choose your serialisers wisely as once your data is persisted using a chosen serialiser, there is no easy way of migrating your data into a different format. diff --git a/docs/development-guide/project-structure/components/spring-rest.md b/docs/development-guide/project-structure/components/spring-rest.md new file mode 100644 index 0000000000..f52ab96a57 --- /dev/null +++ b/docs/development-guide/project-structure/components/spring-rest.md @@ -0,0 +1,47 @@ +# Spring REST API + +The [Spring REST API module](https://github.com/gchq/Gaffer/tree/master/rest-api/spring-rest) is an implementation of the Gaffer REST API deployed in a Spring Boot container. +It is relatively new compared to the [core REST](./core-rest.md) war which uses Jersey/JAX-RS. + +The Spring REST should provide the following benefits: + +* Easier to extend through Spring Boot plugins +* Easier to add dependencies at deployment time (no need to re-build a `.war` file) +* Easier to deploy (you only need Java) + +However, going forward into Gaffer v2.0 we hope this to become the standard for how we build and deploy REST APIs. + +### Implemented Features +* Operations endpoint +* Graph configuration endpoint +* Properties endpoint +* Status endpoint + +### Features we're yet to implement +* Chunked endpoint + +### Features we don't plan to implement +* Custom Swagger UI with operation chain builder +* Supporting older versions of the API + + +### How to run +With Maven from the root of the project: +```bash +mvn spring-boot:run -pl :spring-rest -Pdemo +``` + +With Java using the 'exec' `.jar` directly: +``` +java \ +-Dgaffer.schemas=/path/to/schemas \ +-Dgaffer.storeProperties=/path/to/store.properties \ +-Dgaffer.graph.config=/path/to/graphConfig.json \ +-jar spring-rest-2.0.0-exec.jar +``` + +You can alternatively add the Gaffer system properties to your `application.properties` file. + +Once running, open the browser to `http://localhost:8080/rest`. + +You can change the context root by changing the `server.servlet.context-path` value in `application.properties`. \ No newline at end of file diff --git a/docs/development-guide/project-structure/components/store.md b/docs/development-guide/project-structure/components/store.md new file mode 100644 index 0000000000..578b5716c3 --- /dev/null +++ b/docs/development-guide/project-structure/components/store.md @@ -0,0 +1,15 @@ +# Store + +The [store module](https://github.com/gchq/Gaffer/tree/master/core/store) defines the API for Store implementations. The abstract Store class handles Operations by delegating the Operations to their registered handlers. + +See the [Store Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/package-summary.html) for further documentation. + +## Writing a Store + +When implementing a Store, the main task is to write handlers for the operations your Store chooses to support. This can be tricky, but the [Store Integration test suite](./integration-test.md) should be used by all Store implementations to validate these operation handlers. When writing these handlers you should implement `OperationHandler` or `OutputOperationHandler` depending on whether the operation has an output. + +Store implementations need to define a set of `StoreTraits`. These traits tell Gaffer the abilities the Store has. For example the ability to aggregate or filter elements. + +## Schema + +In addition to `OperationHandlers` the other large part of the store module is the Schema. The Schema is what defines what is in the Graph and how it should be persisted, compacted (summarised) and validated. diff --git a/docs/development-guide/project-structure/maven-dependencies/core.svg b/docs/development-guide/project-structure/maven-dependencies/core.svg new file mode 100644 index 0000000000..143d876923 --- /dev/null +++ b/docs/development-guide/project-structure/maven-dependencies/core.svg @@ -0,0 +1,307 @@ + + + + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT + + + +com.google.guava:guava:jar:30.1.1-jre:compile + +com.google.guava:guava:jar:30.1.1-jre:compile + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->com.google.guava:guava:jar:30.1.1-jre:compile + + + + + +org.slf4j:slf4j-api:jar:1.7.36:compile + +org.slf4j:slf4j-api:jar:1.7.36:compile + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.slf4j:slf4j-api:jar:1.7.36:compile + + + + + +org.slf4j:slf4j-reload4j:jar:1.7.36:compile + +org.slf4j:slf4j-reload4j:jar:1.7.36:compile + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.slf4j:slf4j-reload4j:jar:1.7.36:compile + + + + + +org.junit.jupiter:junit-jupiter:jar:5.9.0:test + +org.junit.jupiter:junit-jupiter:jar:5.9.0:test + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.junit.jupiter:junit-jupiter:jar:5.9.0:test + + + + + +org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test + +org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test + + + + + +org.assertj:assertj-core:jar:3.23.1:test + +org.assertj:assertj-core:jar:3.23.1:test + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.assertj:assertj-core:jar:3.23.1:test + + + + + +org.mockito:mockito-junit-jupiter:jar:4.6.1:test + +org.mockito:mockito-junit-jupiter:jar:4.6.1:test + + + +uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.mockito:mockito-junit-jupiter:jar:4.6.1:test + + + + + +com.google.guava:failureaccess:jar:1.0.1:compile + +com.google.guava:failureaccess:jar:1.0.1:compile + + + +com.google.guava:guava:jar:30.1.1-jre:compile->com.google.guava:failureaccess:jar:1.0.1:compile + + + + + +com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile + +com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile + + + +com.google.guava:guava:jar:30.1.1-jre:compile->com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile + + + + + +com.google.code.findbugs:jsr305:jar:3.0.2:compile + +com.google.code.findbugs:jsr305:jar:3.0.2:compile + + + +com.google.guava:guava:jar:30.1.1-jre:compile->com.google.code.findbugs:jsr305:jar:3.0.2:compile + + + + + +org.checkerframework:checker-qual:jar:3.8.0:compile + +org.checkerframework:checker-qual:jar:3.8.0:compile + + + +com.google.guava:guava:jar:30.1.1-jre:compile->org.checkerframework:checker-qual:jar:3.8.0:compile + + + + + +com.google.errorprone:error_prone_annotations:jar:2.5.1:compile + +com.google.errorprone:error_prone_annotations:jar:2.5.1:compile + + + +com.google.guava:guava:jar:30.1.1-jre:compile->com.google.errorprone:error_prone_annotations:jar:2.5.1:compile + + + + + +com.google.j2objc:j2objc-annotations:jar:1.3:compile + +com.google.j2objc:j2objc-annotations:jar:1.3:compile + + + +com.google.guava:guava:jar:30.1.1-jre:compile->com.google.j2objc:j2objc-annotations:jar:1.3:compile + + + + + +ch.qos.reload4j:reload4j:jar:1.2.18.3:compile + +ch.qos.reload4j:reload4j:jar:1.2.18.3:compile + + + +org.slf4j:slf4j-reload4j:jar:1.7.36:compile->ch.qos.reload4j:reload4j:jar:1.2.18.3:compile + + + + + +org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test + +org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test + + + +org.junit.jupiter:junit-jupiter:jar:5.9.0:test->org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test + + + + + +org.junit.jupiter:junit-jupiter-params:jar:5.9.0:test + +org.junit.jupiter:junit-jupiter-params:jar:5.9.0:test + + + +org.junit.jupiter:junit-jupiter:jar:5.9.0:test->org.junit.jupiter:junit-jupiter-params:jar:5.9.0:test + + + + + +org.junit.platform:junit-platform-engine:jar:1.9.0:test + +org.junit.platform:junit-platform-engine:jar:1.9.0:test + + + +org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test->org.junit.platform:junit-platform-engine:jar:1.9.0:test + + + + + +org.apiguardian:apiguardian-api:jar:1.1.2:test + +org.apiguardian:apiguardian-api:jar:1.1.2:test + + + +org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test->org.apiguardian:apiguardian-api:jar:1.1.2:test + + + + + +net.bytebuddy:byte-buddy:jar:1.12.10:test + +net.bytebuddy:byte-buddy:jar:1.12.10:test + + + +org.assertj:assertj-core:jar:3.23.1:test->net.bytebuddy:byte-buddy:jar:1.12.10:test + + + + + +org.mockito:mockito-core:jar:4.6.1:test + +org.mockito:mockito-core:jar:4.6.1:test + + + +org.mockito:mockito-junit-jupiter:jar:4.6.1:test->org.mockito:mockito-core:jar:4.6.1:test + + + + + +org.opentest4j:opentest4j:jar:1.2.0:test + +org.opentest4j:opentest4j:jar:1.2.0:test + + + +org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test->org.opentest4j:opentest4j:jar:1.2.0:test + + + + + +org.junit.platform:junit-platform-commons:jar:1.9.0:test + +org.junit.platform:junit-platform-commons:jar:1.9.0:test + + + +org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test->org.junit.platform:junit-platform-commons:jar:1.9.0:test + + + + + +net.bytebuddy:byte-buddy-agent:jar:1.12.10:test + +net.bytebuddy:byte-buddy-agent:jar:1.12.10:test + + + +org.mockito:mockito-core:jar:4.6.1:test->net.bytebuddy:byte-buddy-agent:jar:1.12.10:test + + + + + +org.objenesis:objenesis:jar:3.2:test + +org.objenesis:objenesis:jar:3.2:test + + + +org.mockito:mockito-core:jar:4.6.1:test->org.objenesis:objenesis:jar:3.2:test + + + + + diff --git a/docs/development-guide/project-structure/maven-dependencies/gaffer-complete.svg b/docs/development-guide/project-structure/maven-dependencies/gaffer-complete.svg new file mode 100644 index 0000000000..46b5076453 --- /dev/null +++ b/docs/development-guide/project-structure/maven-dependencies/gaffer-complete.svg @@ -0,0 +1,5772 @@ + + + + + + +gaffer2 + + + +uk.gov.gchq.gaffer:common-util:jar:compile + +uk.gov.gchq.gaffer +common-util +2.0.0 + + + +org.apache.commons:commons-lang3:jar:compile + +org.apache.commons +commons-lang3 +3.12.0 + + + +uk.gov.gchq.gaffer:common-util:jar:compile->org.apache.commons:commons-lang3:jar:compile + + + + + +org.reflections:reflections:jar:compile + +org.reflections +reflections +0.9.12 + + + +uk.gov.gchq.gaffer:common-util:jar:compile->org.reflections:reflections:jar:compile + + + + + +com.fasterxml.jackson.core:jackson-annotations:jar:compile + +com.fasterxml.jackson.core +jackson-annotations +2.13.5 + + + +uk.gov.gchq.gaffer:common-util:jar:compile->com.fasterxml.jackson.core:jackson-annotations:jar:compile + + + + + +com.fasterxml.jackson.core:jackson-databind:jar:compile + +com.fasterxml.jackson.core +jackson-databind +2.13.5 + + + +uk.gov.gchq.gaffer:common-util:jar:compile->com.fasterxml.jackson.core:jackson-databind:jar:compile + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile + +uk.gov.gchq.koryphe +core +2.5.2 +jdk8 + + + +uk.gov.gchq.gaffer:common-util:jar:compile->uk.gov.gchq.koryphe:core:jar:jdk8:compile + + + + + +com.google.guava:guava:jar:compile + +com.google.guava +guava +30.1.1-jre + + + +uk.gov.gchq.gaffer:common-util:jar:compile->com.google.guava:guava:jar:compile + + + + + +org.slf4j:slf4j-api:jar:compile + +org.slf4j +slf4j-api +1.7.36 + + + +uk.gov.gchq.gaffer:common-util:jar:compile->org.slf4j:slf4j-api:jar:compile + + + + + +org.slf4j:slf4j-reload4j:jar:compile + +org.slf4j +slf4j-reload4j +1.7.36 + + + +uk.gov.gchq.gaffer:common-util:jar:compile->org.slf4j:slf4j-reload4j:jar:compile + + + + + +org.javassist:javassist:jar:compile + +org.javassist +javassist +3.29.2-GA + + + +org.reflections:reflections:jar:compile->org.javassist:javassist:jar:compile + + + + + +com.fasterxml.jackson.core:jackson-databind:jar:compile->com.fasterxml.jackson.core:jackson-annotations:jar:compile + + + + + +com.fasterxml.jackson.core:jackson-core:jar:compile + +com.fasterxml.jackson.core +jackson-core +2.13.5 + + + +com.fasterxml.jackson.core:jackson-databind:jar:compile->com.fasterxml.jackson.core:jackson-core:jar:compile + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile->org.apache.commons:commons-lang3:jar:compile + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile->com.fasterxml.jackson.core:jackson-databind:jar:compile + + + + + +com.github.spotbugs:spotbugs-annotations:jar:compile + +com.github.spotbugs +spotbugs-annotations +4.7.3 + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile->com.github.spotbugs:spotbugs-annotations:jar:compile + + + + + +io.github.lukehutch:fast-classpath-scanner:jar:compile + +io.github.lukehutch +fast-classpath-scanner +2.10.0 + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile->io.github.lukehutch:fast-classpath-scanner:jar:compile + + + + + +commons-io:commons-io:jar:compile + +commons-io +commons-io +2.11.0 + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile->commons-io:commons-io:jar:compile + + + + + +org.apache.commons:commons-csv:jar:compile + +org.apache.commons +commons-csv +1.9.0 + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile->org.apache.commons:commons-csv:jar:compile + + + + + +commons-codec:commons-codec:jar:compile + +commons-codec +commons-codec +1.15 + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile->commons-codec:commons-codec:jar:compile + + + + + +org.json:json:jar:compile + +org.json +json +20190722 + + + +uk.gov.gchq.koryphe:core:jar:jdk8:compile->org.json:json:jar:compile + + + + + +com.google.guava:failureaccess:jar:compile + +com.google.guava +failureaccess +1.0.1 + + + +com.google.guava:guava:jar:compile->com.google.guava:failureaccess:jar:compile + + + + + +com.google.guava:listenablefuture:jar:compile + +com.google.guava +listenablefuture +9999.0-empty-to-avoid-conflict-with-guava + + + +com.google.guava:guava:jar:compile->com.google.guava:listenablefuture:jar:compile + + + + + +com.google.code.findbugs:jsr305:jar:compile + +com.google.code.findbugs +jsr305 +3.0.2 + + + +com.google.guava:guava:jar:compile->com.google.code.findbugs:jsr305:jar:compile + + + + + +org.checkerframework:checker-qual:jar:compile + +org.checkerframework +checker-qual +3.8.0 + + + +com.google.guava:guava:jar:compile->org.checkerframework:checker-qual:jar:compile + + + + + +com.google.errorprone:error_prone_annotations:jar:compile + +com.google.errorprone +error_prone_annotations +2.5.1 + + + +com.google.guava:guava:jar:compile->com.google.errorprone:error_prone_annotations:jar:compile + + + + + +com.google.j2objc:j2objc-annotations:jar:compile + +com.google.j2objc +j2objc-annotations +1.3 + + + +com.google.guava:guava:jar:compile->com.google.j2objc:j2objc-annotations:jar:compile + + + + + +ch.qos.reload4j:reload4j:jar:compile + +ch.qos.reload4j +reload4j +1.2.18.3 + + + +org.slf4j:slf4j-reload4j:jar:compile->ch.qos.reload4j:reload4j:jar:compile + + + + + +uk.gov.gchq.gaffer:exception:jar:compile + +uk.gov.gchq.gaffer +exception +2.0.0 + + + +uk.gov.gchq.gaffer:exception:jar:compile->uk.gov.gchq.gaffer:common-util:jar:compile + + + + + +uk.gov.gchq.gaffer:serialisation:jar:compile + +uk.gov.gchq.gaffer +serialisation +2.0.0 + + + +uk.gov.gchq.gaffer:serialisation:jar:compile->uk.gov.gchq.gaffer:exception:jar:compile + + + + + +org.apache.avro:avro:jar:compile + +org.apache.avro +avro +1.7.7 + + + +uk.gov.gchq.gaffer:serialisation:jar:compile->org.apache.avro:avro:jar:compile + + + + + +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:compile + +com.fasterxml.jackson.datatype +jackson-datatype-jsr310 +2.13.5 + + + +uk.gov.gchq.gaffer:serialisation:jar:compile->com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:compile + + + + + +org.codehaus.jackson:jackson-core-asl:jar:compile + +org.codehaus.jackson +jackson-core-asl +1.9.13 + + + +org.apache.avro:avro:jar:compile->org.codehaus.jackson:jackson-core-asl:jar:compile + + + + + +org.codehaus.jackson:jackson-mapper-asl:jar:compile + +org.codehaus.jackson +jackson-mapper-asl +1.9.13 + + + +org.apache.avro:avro:jar:compile->org.codehaus.jackson:jackson-mapper-asl:jar:compile + + + + + +com.thoughtworks.paranamer:paranamer:jar:compile + +com.thoughtworks.paranamer +paranamer +2.8 + + + +org.apache.avro:avro:jar:compile->com.thoughtworks.paranamer:paranamer:jar:compile + + + + + +org.xerial.snappy:snappy-java:jar:compile + +org.xerial.snappy +snappy-java +1.1.8.2 + + + +org.apache.avro:avro:jar:compile->org.xerial.snappy:snappy-java:jar:compile + + + + + +org.apache.commons:commons-compress:jar:compile + +org.apache.commons +commons-compress +1.21 + + + +org.apache.avro:avro:jar:compile->org.apache.commons:commons-compress:jar:compile + + + + + +org.tukaani:xz:jar:compile + +org.tukaani +xz +1.5 + + + +org.apache.avro:avro:jar:compile->org.tukaani:xz:jar:compile + + + + + +uk.gov.gchq.gaffer:type:jar:compile + +uk.gov.gchq.gaffer +type +2.0.0 + + + +uk.gov.gchq.gaffer:type:jar:compile->uk.gov.gchq.gaffer:serialisation:jar:compile + + + + + +uk.gov.gchq.gaffer:access:jar:compile + +uk.gov.gchq.gaffer +access +2.0.0 + + + +uk.gov.gchq.gaffer:access:jar:compile->uk.gov.gchq.gaffer:common-util:jar:compile + + + + + +uk.gov.gchq.gaffer:data:jar:compile + +uk.gov.gchq.gaffer +data +2.0.0 + + + +uk.gov.gchq.gaffer:data:jar:compile->uk.gov.gchq.gaffer:type:jar:compile + + + + + +uk.gov.gchq.gaffer:data:jar:compile->uk.gov.gchq.gaffer:access:jar:compile + + + + + +org.apache.commons:commons-collections4:jar:compile + +org.apache.commons +commons-collections4 +4.4 + + + +uk.gov.gchq.gaffer:data:jar:compile->org.apache.commons:commons-collections4:jar:compile + + + + + +uk.gov.gchq.gaffer:cache:jar:compile + +uk.gov.gchq.gaffer +cache +2.0.0 + + + +uk.gov.gchq.gaffer:cache:jar:compile->uk.gov.gchq.gaffer:serialisation:jar:compile + + + + + +uk.gov.gchq.gaffer:operation:jar:compile + +uk.gov.gchq.gaffer +operation +2.0.0 + + + +uk.gov.gchq.gaffer:operation:jar:compile->uk.gov.gchq.gaffer:data:jar:compile + + + + + +uk.gov.gchq.gaffer:operation:jar:compile->uk.gov.gchq.gaffer:cache:jar:compile + + + + + +uk.gov.gchq.gaffer:jcs-cache-service:jar:compile + +uk.gov.gchq.gaffer +jcs-cache-service +2.0.0 + + + +uk.gov.gchq.gaffer:jcs-cache-service:jar:compile->uk.gov.gchq.gaffer:cache:jar:compile + + + + + +org.apache.commons:commons-jcs-core:jar:compile + +org.apache.commons +commons-jcs-core +2.2.1 + + + +uk.gov.gchq.gaffer:jcs-cache-service:jar:compile->org.apache.commons:commons-jcs-core:jar:compile + + + + + +commons-logging:commons-logging:jar:compile + +commons-logging +commons-logging +1.2 + + + +org.apache.commons:commons-jcs-core:jar:compile->commons-logging:commons-logging:jar:compile + + + + + +uk.gov.gchq.gaffer:store:jar:compile + +uk.gov.gchq.gaffer +store +2.0.0 + + + +uk.gov.gchq.gaffer:store:jar:compile->uk.gov.gchq.gaffer:operation:jar:compile + + + + + +uk.gov.gchq.gaffer:graph:jar:compile + +uk.gov.gchq.gaffer +graph +2.0.0 + + + +uk.gov.gchq.gaffer:graph:jar:compile->uk.gov.gchq.gaffer:store:jar:compile + + + + + +uk.gov.gchq.gaffer:integration-test:jar:compile + +uk.gov.gchq.gaffer +integration-test +2.0.0 + + + +uk.gov.gchq.gaffer:integration-test:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile + + + + + +uk.gov.gchq.gaffer:common-util:test-jar:tests:compile + +uk.gov.gchq.gaffer +common-util +2.0.0 +.test-jar +tests + + + +uk.gov.gchq.gaffer:integration-test:jar:compile->uk.gov.gchq.gaffer:common-util:test-jar:tests:compile + + + + + +uk.gov.gchq.gaffer:data:test-jar:tests:compile + +uk.gov.gchq.gaffer +data +2.0.0 +.test-jar +tests + + + +uk.gov.gchq.gaffer:integration-test:jar:compile->uk.gov.gchq.gaffer:data:test-jar:tests:compile + + + + + +uk.gov.gchq.gaffer:store:test-jar:tests:compile + +uk.gov.gchq.gaffer +store +2.0.0 +.test-jar +tests + + + +uk.gov.gchq.gaffer:integration-test:jar:compile->uk.gov.gchq.gaffer:store:test-jar:tests:compile + + + + + +org.junit.platform:junit-platform-suite:jar:compile + +org.junit.platform +junit-platform-suite +1.9.0 + + + +uk.gov.gchq.gaffer:integration-test:jar:compile->org.junit.platform:junit-platform-suite:jar:compile + + + + + +org.junit.jupiter:junit-jupiter:jar:provided + +org.junit.jupiter +junit-jupiter +5.9.0 +(provided) + + + +uk.gov.gchq.gaffer:integration-test:jar:compile->org.junit.jupiter:junit-jupiter:jar:provided + + + + + +org.junit.jupiter:junit-jupiter-engine:jar:provided + +org.junit.jupiter +junit-jupiter-engine +5.9.0 +(provided) + + + +uk.gov.gchq.gaffer:integration-test:jar:compile->org.junit.jupiter:junit-jupiter-engine:jar:provided + + + + + +org.assertj:assertj-core:jar:provided + +org.assertj +assertj-core +3.23.1 +(provided) + + + +uk.gov.gchq.gaffer:integration-test:jar:compile->org.assertj:assertj-core:jar:provided + + + + + +uk.gov.gchq.gaffer:common-util:test-jar:tests:compile->org.reflections:reflections:jar:compile + + + + + +uk.gov.gchq.gaffer:common-util:test-jar:tests:compile->uk.gov.gchq.koryphe:core:jar:jdk8:compile + + + + + +uk.gov.gchq.gaffer:data:test-jar:tests:compile->uk.gov.gchq.gaffer:type:jar:compile + + + + + +uk.gov.gchq.gaffer:data:test-jar:tests:compile->uk.gov.gchq.gaffer:access:jar:compile + + + + + +uk.gov.gchq.gaffer:data:test-jar:tests:compile->org.apache.commons:commons-collections4:jar:compile + + + + + +uk.gov.gchq.gaffer:store:test-jar:tests:compile->uk.gov.gchq.gaffer:operation:jar:compile + + + + + +org.junit.platform:junit-platform-suite-api:jar:compile + +org.junit.platform +junit-platform-suite-api +1.9.0 + + + +org.junit.platform:junit-platform-commons:jar:compile + +org.junit.platform +junit-platform-commons +1.9.0 + + + +org.junit.platform:junit-platform-suite-api:jar:compile->org.junit.platform:junit-platform-commons:jar:compile + + + + + +org.junit.platform:junit-platform-suite:jar:compile->org.junit.platform:junit-platform-suite-api:jar:compile + + + + + +org.junit.jupiter:junit-jupiter-api:jar:provided + +org.junit.jupiter +junit-jupiter-api +5.9.0 +(provided) + + + +org.junit.jupiter:junit-jupiter:jar:provided->org.junit.jupiter:junit-jupiter-api:jar:provided + + + + + +org.junit.jupiter:junit-jupiter-params:jar:provided + +org.junit.jupiter +junit-jupiter-params +5.9.0 +(provided) + + + +org.junit.jupiter:junit-jupiter:jar:provided->org.junit.jupiter:junit-jupiter-params:jar:provided + + + + + +org.apiguardian:apiguardian-api:jar:compile + +org.apiguardian +apiguardian-api +1.1.2 + + + +org.junit.jupiter:junit-jupiter-engine:jar:provided->org.apiguardian:apiguardian-api:jar:compile + + + + + +net.bytebuddy:byte-buddy:jar:provided + +net.bytebuddy +byte-buddy +1.12.10 +(provided) + + + +org.assertj:assertj-core:jar:provided->net.bytebuddy:byte-buddy:jar:provided + + + + + +uk.gov.gchq.gaffer:sketches-library:jar:compile + +uk.gov.gchq.gaffer +sketches-library +2.0.0 + + + +uk.gov.gchq.gaffer:sketches-library:jar:compile->uk.gov.gchq.gaffer:data:jar:compile + + + + + +com.clearspring.analytics:stream:jar:compile + +com.clearspring.analytics +stream +2.7.0 + + + +uk.gov.gchq.gaffer:sketches-library:jar:compile->com.clearspring.analytics:stream:jar:compile + + + + + +com.yahoo.datasketches:sketches-core:jar:compile + +com.yahoo.datasketches +sketches-core +0.12.0 + + + +uk.gov.gchq.gaffer:sketches-library:jar:compile->com.yahoo.datasketches:sketches-core:jar:compile + + + + + +it.unimi.dsi:fastutil:jar:compile + +it.unimi.dsi +fastutil +6.5.7 + + + +com.clearspring.analytics:stream:jar:compile->it.unimi.dsi:fastutil:jar:compile + + + + + +com.yahoo.datasketches:memory:jar:compile + +com.yahoo.datasketches +memory +0.12.0 + + + +com.yahoo.datasketches:sketches-core:jar:compile->com.yahoo.datasketches:memory:jar:compile + + + + + +uk.gov.gchq.gaffer:hdfs-library:jar:compile + +uk.gov.gchq.gaffer +hdfs-library +2.0.0 + + + +uk.gov.gchq.gaffer:hdfs-library:jar:compile->uk.gov.gchq.gaffer:store:jar:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile + +org.apache.hadoop +hadoop-common +3.3.3 + + + +uk.gov.gchq.gaffer:hdfs-library:jar:compile->org.apache.hadoop:hadoop-common:jar:compile + + + + + +org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile + +org.apache.hadoop +hadoop-mapreduce-client-core +3.3.3 + + + +uk.gov.gchq.gaffer:hdfs-library:jar:compile->org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile + + + + + +org.apache.avro:avro-mapred:jar:hadoop2:compile + +org.apache.avro +avro-mapred +1.8.2 +hadoop2 + + + +uk.gov.gchq.gaffer:hdfs-library:jar:compile->org.apache.avro:avro-mapred:jar:hadoop2:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.commons:commons-lang3:jar:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile->com.fasterxml.jackson.core:jackson-databind:jar:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile->commons-io:commons-io:jar:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile->commons-codec:commons-codec:jar:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile->com.google.code.findbugs:jsr305:jar:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile->ch.qos.reload4j:reload4j:jar:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.avro:avro:jar:compile + + + + + +org.apache.hadoop:hadoop-common:jar:compile->commons-logging:commons-logging:jar:compile + + + + + +org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar:compile + +org.apache.hadoop.thirdparty +hadoop-shaded-protobuf_3_7 +1.1.1 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar:compile + + + + + +org.apache.hadoop:hadoop-annotations:jar:compile + +org.apache.hadoop +hadoop-annotations +3.3.3 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.hadoop:hadoop-annotations:jar:compile + + + + + +org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:compile + +org.apache.hadoop.thirdparty +hadoop-shaded-guava +1.1.1 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:compile + + + + + +commons-cli:commons-cli:jar:compile + +commons-cli +commons-cli +1.2 + + + +org.apache.hadoop:hadoop-common:jar:compile->commons-cli:commons-cli:jar:compile + + + + + +org.apache.commons:commons-math3:jar:compile + +org.apache.commons +commons-math3 +3.6.1 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.commons:commons-math3:jar:compile + + + + + +org.apache.httpcomponents:httpclient:jar:compile + +org.apache.httpcomponents +httpclient +4.5.13 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.httpcomponents:httpclient:jar:compile + + + + + +commons-net:commons-net:jar:compile + +commons-net +commons-net +3.6 + + + +org.apache.hadoop:hadoop-common:jar:compile->commons-net:commons-net:jar:compile + + + + + +commons-collections:commons-collections:jar:compile + +commons-collections +commons-collections +3.2.2 + + + +org.apache.hadoop:hadoop-common:jar:compile->commons-collections:commons-collections:jar:compile + + + + + +javax.servlet:javax.servlet-api:jar:compile + +javax.servlet +javax.servlet-api +3.1.0 + + + +org.apache.hadoop:hadoop-common:jar:compile->javax.servlet:javax.servlet-api:jar:compile + + + + + +jakarta.activation:jakarta.activation-api:jar:compile + +jakarta.activation +jakarta.activation-api +1.2.1 + + + +org.apache.hadoop:hadoop-common:jar:compile->jakarta.activation:jakarta.activation-api:jar:compile + + + + + +org.eclipse.jetty:jetty-server:jar:compile + +org.eclipse.jetty +jetty-server +9.4.43.v20210629 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.eclipse.jetty:jetty-server:jar:compile + + + + + +org.eclipse.jetty:jetty-util:jar:compile + +org.eclipse.jetty +jetty-util +9.4.43.v20210629 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.eclipse.jetty:jetty-util:jar:compile + + + + + +org.eclipse.jetty:jetty-servlet:jar:compile + +org.eclipse.jetty +jetty-servlet +9.4.43.v20210629 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.eclipse.jetty:jetty-servlet:jar:compile + + + + + +org.eclipse.jetty:jetty-webapp:jar:compile + +org.eclipse.jetty +jetty-webapp +9.4.43.v20210629 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.eclipse.jetty:jetty-webapp:jar:compile + + + + + +commons-beanutils:commons-beanutils:jar:compile + +commons-beanutils +commons-beanutils +1.9.4 + + + +org.apache.hadoop:hadoop-common:jar:compile->commons-beanutils:commons-beanutils:jar:compile + + + + + +org.apache.commons:commons-configuration2:jar:compile + +org.apache.commons +commons-configuration2 +2.5 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.commons:commons-configuration2:jar:compile + + + + + +org.apache.commons:commons-text:jar:compile + +org.apache.commons +commons-text +1.10.0 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.commons:commons-text:jar:compile + + + + + +com.google.re2j:re2j:jar:compile + +com.google.re2j +re2j +1.1 + + + +org.apache.hadoop:hadoop-common:jar:compile->com.google.re2j:re2j:jar:compile + + + + + +com.google.protobuf:protobuf-java:jar:compile + +com.google.protobuf +protobuf-java +2.5.0 + + + +org.apache.hadoop:hadoop-common:jar:compile->com.google.protobuf:protobuf-java:jar:compile + + + + + +com.google.code.gson:gson:jar:compile + +com.google.code.gson +gson +2.8.5 + + + +org.apache.hadoop:hadoop-common:jar:compile->com.google.code.gson:gson:jar:compile + + + + + +org.apache.hadoop:hadoop-auth:jar:compile + +org.apache.hadoop +hadoop-auth +3.3.3 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.hadoop:hadoop-auth:jar:compile + + + + + +com.jcraft:jsch:jar:compile + +com.jcraft +jsch +0.1.55 + + + +org.apache.hadoop:hadoop-common:jar:compile->com.jcraft:jsch:jar:compile + + + + + +org.apache.curator:curator-client:jar:compile + +org.apache.curator +curator-client +4.2.0 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.curator:curator-client:jar:compile + + + + + +org.apache.curator:curator-recipes:jar:compile + +org.apache.curator +curator-recipes +2.13.0 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.curator:curator-recipes:jar:compile + + + + + +org.apache.zookeeper:zookeeper:jar:compile + +org.apache.zookeeper +zookeeper +3.4.14 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.zookeeper:zookeeper:jar:compile + + + + + +org.apache.kerby:kerb-core:jar:compile + +org.apache.kerby +kerb-core +1.0.1 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.apache.kerby:kerb-core:jar:compile + + + + + +org.codehaus.woodstox:stax2-api:jar:compile + +org.codehaus.woodstox +stax2-api +4.2.1 + + + +org.apache.hadoop:hadoop-common:jar:compile->org.codehaus.woodstox:stax2-api:jar:compile + + + + + +com.fasterxml.woodstox:woodstox-core:jar:compile + +com.fasterxml.woodstox +woodstox-core +5.3.0 + + + +org.apache.hadoop:hadoop-common:jar:compile->com.fasterxml.woodstox:woodstox-core:jar:compile + + + + + +dnsjava:dnsjava:jar:compile + +dnsjava +dnsjava +2.1.7 + + + +org.apache.hadoop:hadoop-common:jar:compile->dnsjava:dnsjava:jar:compile + + + + + +org.apache.httpcomponents:httpcore:jar:compile + +org.apache.httpcomponents +httpcore +4.4.1 + + + +org.apache.httpcomponents:httpclient:jar:compile->org.apache.httpcomponents:httpcore:jar:compile + + + + + +org.eclipse.jetty:jetty-http:jar:compile + +org.eclipse.jetty +jetty-http +9.4.43.v20210629 + + + +org.eclipse.jetty:jetty-server:jar:compile->org.eclipse.jetty:jetty-http:jar:compile + + + + + +org.eclipse.jetty:jetty-io:jar:compile + +org.eclipse.jetty +jetty-io +9.4.43.v20210629 + + + +org.eclipse.jetty:jetty-server:jar:compile->org.eclipse.jetty:jetty-io:jar:compile + + + + + +org.eclipse.jetty:jetty-security:jar:compile + +org.eclipse.jetty +jetty-security +9.4.43.v20210629 + + + +org.eclipse.jetty:jetty-servlet:jar:compile->org.eclipse.jetty:jetty-security:jar:compile + + + + + +org.eclipse.jetty:jetty-util-ajax:jar:compile + +org.eclipse.jetty +jetty-util-ajax +9.4.43.v20210629 + + + +org.eclipse.jetty:jetty-servlet:jar:compile->org.eclipse.jetty:jetty-util-ajax:jar:compile + + + + + +org.eclipse.jetty:jetty-xml:jar:compile + +org.eclipse.jetty +jetty-xml +9.4.43.v20210629 + + + +org.eclipse.jetty:jetty-webapp:jar:compile->org.eclipse.jetty:jetty-xml:jar:compile + + + + + +com.nimbusds:nimbus-jose-jwt:jar:compile + +com.nimbusds +nimbus-jose-jwt +9.8.1 + + + +com.github.stephenc.jcip:jcip-annotations:jar:compile + +com.github.stephenc.jcip +jcip-annotations +1.0-1 + + + +com.nimbusds:nimbus-jose-jwt:jar:compile->com.github.stephenc.jcip:jcip-annotations:jar:compile + + + + + +org.apache.hadoop:hadoop-auth:jar:compile->com.nimbusds:nimbus-jose-jwt:jar:compile + + + + + +net.minidev:json-smart:jar:compile + +net.minidev +json-smart +2.4.7 + + + +org.apache.hadoop:hadoop-auth:jar:compile->net.minidev:json-smart:jar:compile + + + + + +org.apache.curator:curator-framework:jar:compile + +org.apache.curator +curator-framework +4.2.0 + + + +org.apache.hadoop:hadoop-auth:jar:compile->org.apache.curator:curator-framework:jar:compile + + + + + +org.apache.kerby:kerb-simplekdc:jar:compile + +org.apache.kerby +kerb-simplekdc +1.0.1 + + + +org.apache.hadoop:hadoop-auth:jar:compile->org.apache.kerby:kerb-simplekdc:jar:compile + + + + + +net.minidev:accessors-smart:jar:compile + +net.minidev +accessors-smart +2.4.7 + + + +org.ow2.asm:asm:jar:compile + +org.ow2.asm +asm +9.1 + + + +net.minidev:accessors-smart:jar:compile->org.ow2.asm:asm:jar:compile + + + + + +net.minidev:json-smart:jar:compile->net.minidev:accessors-smart:jar:compile + + + + + +org.apache.kerby:kerb-client:jar:compile + +org.apache.kerby +kerb-client +1.0.1 + + + +org.apache.kerby:kerby-config:jar:compile + +org.apache.kerby +kerby-config +1.0.1 + + + +org.apache.kerby:kerb-client:jar:compile->org.apache.kerby:kerby-config:jar:compile + + + + + +org.apache.kerby:kerb-common:jar:compile + +org.apache.kerby +kerb-common +1.0.1 + + + +org.apache.kerby:kerb-client:jar:compile->org.apache.kerby:kerb-common:jar:compile + + + + + +org.apache.kerby:kerb-util:jar:compile + +org.apache.kerby +kerb-util +1.0.1 + + + +org.apache.kerby:kerb-client:jar:compile->org.apache.kerby:kerb-util:jar:compile + + + + + +org.apache.kerby:token-provider:jar:compile + +org.apache.kerby +token-provider +1.0.1 + + + +org.apache.kerby:kerb-client:jar:compile->org.apache.kerby:token-provider:jar:compile + + + + + +org.apache.kerby:kerb-crypto:jar:compile + +org.apache.kerby +kerb-crypto +1.0.1 + + + +org.apache.kerby:kerb-common:jar:compile->org.apache.kerby:kerb-crypto:jar:compile + + + + + +org.apache.kerby:kerb-simplekdc:jar:compile->org.apache.kerby:kerb-client:jar:compile + + + + + +org.apache.kerby:kerb-admin:jar:compile + +org.apache.kerby +kerb-admin +1.0.1 + + + +org.apache.kerby:kerb-simplekdc:jar:compile->org.apache.kerby:kerb-admin:jar:compile + + + + + +org.apache.kerby:kerb-server:jar:compile + +org.apache.kerby +kerb-server +1.0.1 + + + +org.apache.kerby:kerb-identity:jar:compile + +org.apache.kerby +kerb-identity +1.0.1 + + + +org.apache.kerby:kerb-server:jar:compile->org.apache.kerby:kerb-identity:jar:compile + + + + + +org.apache.kerby:kerb-admin:jar:compile->org.apache.kerby:kerb-server:jar:compile + + + + + +org.apache.kerby:kerby-xdr:jar:compile + +org.apache.kerby +kerby-xdr +1.0.1 + + + +org.apache.kerby:kerb-admin:jar:compile->org.apache.kerby:kerby-xdr:jar:compile + + + + + +org.apache.curator:curator-recipes:jar:compile->org.apache.curator:curator-framework:jar:compile + + + + + +org.apache.zookeeper:zookeeper:jar:compile->com.github.spotbugs:spotbugs-annotations:jar:compile + + + + + +jline:jline:jar:compile + +jline +jline +2.11 + + + +org.apache.zookeeper:zookeeper:jar:compile->jline:jline:jar:compile + + + + + +org.apache.yetus:audience-annotations:jar:compile + +org.apache.yetus +audience-annotations +0.5.0 + + + +org.apache.zookeeper:zookeeper:jar:compile->org.apache.yetus:audience-annotations:jar:compile + + + + + +org.apache.kerby:kerby-pkix:jar:compile + +org.apache.kerby +kerby-pkix +1.0.1 + + + +org.apache.kerby:kerby-asn1:jar:compile + +org.apache.kerby +kerby-asn1 +1.0.1 + + + +org.apache.kerby:kerby-pkix:jar:compile->org.apache.kerby:kerby-asn1:jar:compile + + + + + +org.apache.kerby:kerby-util:jar:compile + +org.apache.kerby +kerby-util +1.0.1 + + + +org.apache.kerby:kerby-pkix:jar:compile->org.apache.kerby:kerby-util:jar:compile + + + + + +org.apache.kerby:kerb-core:jar:compile->org.apache.kerby:kerby-pkix:jar:compile + + + + + +org.eclipse.jetty.websocket:websocket-client:jar:compile + +org.eclipse.jetty.websocket +websocket-client +9.4.43.v20210629 + + + +org.eclipse.jetty.websocket:websocket-client:jar:compile->org.eclipse.jetty:jetty-io:jar:compile + + + + + +org.eclipse.jetty.websocket:websocket-client:jar:compile->org.eclipse.jetty:jetty-util:jar:compile + + + + + +org.eclipse.jetty:jetty-client:jar:compile + +org.eclipse.jetty +jetty-client +9.4.43.v20210629 + + + +org.eclipse.jetty.websocket:websocket-client:jar:compile->org.eclipse.jetty:jetty-client:jar:compile + + + + + +org.eclipse.jetty.websocket:websocket-common:jar:compile + +org.eclipse.jetty.websocket +websocket-common +9.4.43.v20210629 + + + +org.eclipse.jetty.websocket:websocket-client:jar:compile->org.eclipse.jetty.websocket:websocket-common:jar:compile + + + + + +org.eclipse.jetty:jetty-client:jar:compile->org.eclipse.jetty:jetty-http:jar:compile + + + + + +org.eclipse.jetty.websocket:websocket-api:jar:compile + +org.eclipse.jetty.websocket +websocket-api +9.4.43.v20210629 + + + +org.eclipse.jetty.websocket:websocket-common:jar:compile->org.eclipse.jetty.websocket:websocket-api:jar:compile + + + + + +org.apache.hadoop:hadoop-yarn-client:jar:compile + +org.apache.hadoop +hadoop-yarn-client +3.3.3 + + + +org.apache.hadoop:hadoop-yarn-client:jar:compile->org.eclipse.jetty.websocket:websocket-client:jar:compile + + + + + +org.apache.hadoop:hadoop-yarn-api:jar:compile + +org.apache.hadoop +hadoop-yarn-api +3.3.3 + + + +org.apache.hadoop:hadoop-yarn-client:jar:compile->org.apache.hadoop:hadoop-yarn-api:jar:compile + + + + + +org.jline:jline:jar:compile + +org.jline +jline +3.9.0 + + + +org.apache.hadoop:hadoop-yarn-client:jar:compile->org.jline:jline:jar:compile + + + + + +javax.xml.bind:jaxb-api:jar:compile + +javax.xml.bind +jaxb-api +2.2.11 + + + +org.apache.hadoop:hadoop-yarn-api:jar:compile->javax.xml.bind:jaxb-api:jar:compile + + + + + +javax.ws.rs:javax.ws.rs-api:jar:compile + +javax.ws.rs +javax.ws.rs-api +2.1.1 + + + +org.apache.hadoop:hadoop-yarn-api:jar:compile->javax.ws.rs:javax.ws.rs-api:jar:compile + + + + + +org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->org.apache.hadoop:hadoop-yarn-client:jar:compile + + + + + +org.apache.hadoop:hadoop-yarn-common:jar:compile + +org.apache.hadoop +hadoop-yarn-common +3.3.3 + + + +org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->org.apache.hadoop:hadoop-yarn-common:jar:compile + + + + + +org.apache.hadoop:hadoop-hdfs-client:jar:compile + +org.apache.hadoop +hadoop-hdfs-client +3.3.3 + + + +org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->org.apache.hadoop:hadoop-hdfs-client:jar:compile + + + + + +org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->javax.ws.rs:javax.ws.rs-api:jar:compile + + + + + +com.google.inject.extensions:guice-servlet:jar:compile + +com.google.inject.extensions +guice-servlet +4.0 + + + +org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->com.google.inject.extensions:guice-servlet:jar:compile + + + + + +org.apache.hadoop:hadoop-yarn-common:jar:compile->javax.xml.bind:jaxb-api:jar:compile + + + + + +com.google.inject:guice:jar:compile + +com.google.inject +guice +4.0 + + + +org.apache.hadoop:hadoop-yarn-common:jar:compile->com.google.inject:guice:jar:compile + + + + + +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile + +com.fasterxml.jackson.module +jackson-module-jaxb-annotations +2.13.5 + + + +org.apache.hadoop:hadoop-yarn-common:jar:compile->com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile + + + + + +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile + +com.fasterxml.jackson.jaxrs +jackson-jaxrs-json-provider +2.13.5 + + + +org.apache.hadoop:hadoop-yarn-common:jar:compile->com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile + + + + + +javax.activation:javax.activation-api:jar:compile + +javax.activation +javax.activation-api +1.2.0 + + + +javax.xml.bind:jaxb-api:jar:compile->javax.activation:javax.activation-api:jar:compile + + + + + +javax.inject:javax.inject:jar:compile + +javax.inject +javax.inject +1 + + + +com.google.inject:guice:jar:compile->javax.inject:javax.inject:jar:compile + + + + + +aopalliance:aopalliance:jar:compile + +aopalliance +aopalliance +1.0 + + + +com.google.inject:guice:jar:compile->aopalliance:aopalliance:jar:compile + + + + + +jakarta.xml.bind:jakarta.xml.bind-api:jar:compile + +jakarta.xml.bind +jakarta.xml.bind-api +2.3.3 + + + +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile->jakarta.xml.bind:jakarta.xml.bind-api:jar:compile + + + + + +jakarta.xml.bind:jakarta.xml.bind-api:jar:compile->jakarta.activation:jakarta.activation-api:jar:compile + + + + + +com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:compile + +com.fasterxml.jackson.jaxrs +jackson-jaxrs-base +2.13.5 + + + +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile->com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:compile + + + + + +com.squareup.okhttp:okhttp:jar:compile + +com.squareup.okhttp +okhttp +2.7.5 + + + +com.squareup.okio:okio:jar:compile + +com.squareup.okio +okio +1.6.0 + + + +com.squareup.okhttp:okhttp:jar:compile->com.squareup.okio:okio:jar:compile + + + + + +org.apache.hadoop:hadoop-hdfs-client:jar:compile->com.squareup.okhttp:okhttp:jar:compile + + + + + +org.apache.velocity:velocity:jar:compile + +org.apache.velocity +velocity +1.7 + + + +commons-lang:commons-lang:jar:compile + +commons-lang +commons-lang +2.4 + + + +org.apache.velocity:velocity:jar:compile->commons-lang:commons-lang:jar:compile + + + + + +org.apache.avro:avro-ipc:jar:compile + +org.apache.avro +avro-ipc +1.8.2 + + + +org.apache.avro:avro-ipc:jar:compile->org.apache.velocity:velocity:jar:compile + + + + + +org.apache.avro:avro-mapred:jar:hadoop2:compile->org.codehaus.jackson:jackson-core-asl:jar:compile + + + + + +org.apache.avro:avro-mapred:jar:hadoop2:compile->org.codehaus.jackson:jackson-mapper-asl:jar:compile + + + + + +org.apache.avro:avro-mapred:jar:hadoop2:compile->org.apache.avro:avro-ipc:jar:compile + + + + + +uk.gov.gchq.gaffer:spark-library:jar:compile + +uk.gov.gchq.gaffer +spark-library +2.0.0 + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile + + + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile + + + + + +com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile + +com.fasterxml.jackson.module +jackson-module-scala_2.12 +2.13.5 + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile + + + + + +org.apache.spark:spark-core_2.12:jar:compile + +org.apache.spark +spark-core_2.12 +3.0.3 + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->org.apache.spark:spark-core_2.12:jar:compile + + + + + +org.apache.spark:spark-sql_2.12:jar:compile + +org.apache.spark +spark-sql_2.12 +3.0.3 + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->org.apache.spark:spark-sql_2.12:jar:compile + + + + + +org.apache.spark:spark-graphx_2.12:jar:compile + +org.apache.spark +spark-graphx_2.12 +3.0.3 + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->org.apache.spark:spark-graphx_2.12:jar:compile + + + + + +org.apache.spark:spark-catalyst_2.12:jar:compile + +org.apache.spark +spark-catalyst_2.12 +3.0.3 + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->org.apache.spark:spark-catalyst_2.12:jar:compile + + + + + +graphframes:graphframes:jar:compile + +graphframes +graphframes +0.8.1-spark3.0-s_2.12 + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->graphframes:graphframes:jar:compile + + + + + +org.objenesis:objenesis:jar:compile + +org.objenesis +objenesis +3.2 + + + +uk.gov.gchq.gaffer:spark-library:jar:compile->org.objenesis:objenesis:jar:compile + + + + + +com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile->com.fasterxml.jackson.core:jackson-databind:jar:compile + + + + + +com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile->com.thoughtworks.paranamer:paranamer:jar:compile + + + + + +org.scala-lang:scala-library:jar:compile + +org.scala-lang +scala-library +2.12.15 + + + +com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile->org.scala-lang:scala-library:jar:compile + + + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.avro:avro-mapred:jar:hadoop2:compile + + + + + +com.twitter:chill_2.12:jar:compile + +com.twitter +chill_2.12 +0.7.6 + + + +org.apache.spark:spark-core_2.12:jar:compile->com.twitter:chill_2.12:jar:compile + + + + + +com.twitter:chill-java:jar:compile + +com.twitter +chill-java +0.7.6 + + + +org.apache.spark:spark-core_2.12:jar:compile->com.twitter:chill-java:jar:compile + + + + + +org.apache.xbean:xbean-asm7-shaded:jar:compile + +org.apache.xbean +xbean-asm7-shaded +4.15 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.xbean:xbean-asm7-shaded:jar:compile + + + + + +org.apache.hadoop:hadoop-client:jar:compile + +org.apache.hadoop +hadoop-client +3.3.3 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.hadoop:hadoop-client:jar:compile + + + + + +org.apache.spark:spark-launcher_2.12:jar:compile + +org.apache.spark +spark-launcher_2.12 +3.0.3 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-launcher_2.12:jar:compile + + + + + +org.apache.spark:spark-kvstore_2.12:jar:compile + +org.apache.spark +spark-kvstore_2.12 +3.0.3 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-kvstore_2.12:jar:compile + + + + + +org.apache.spark:spark-network-common_2.12:jar:compile + +org.apache.spark +spark-network-common_2.12 +3.0.3 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-network-common_2.12:jar:compile + + + + + +org.apache.spark:spark-network-shuffle_2.12:jar:compile + +org.apache.spark +spark-network-shuffle_2.12 +3.0.3 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-network-shuffle_2.12:jar:compile + + + + + +org.apache.spark:spark-unsafe_2.12:jar:compile + +org.apache.spark +spark-unsafe_2.12 +3.0.3 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-unsafe_2.12:jar:compile + + + + + +javax.activation:activation:jar:compile + +javax.activation +activation +1.1.1 + + + +org.apache.spark:spark-core_2.12:jar:compile->javax.activation:activation:jar:compile + + + + + +org.slf4j:jul-to-slf4j:jar:compile + +org.slf4j +jul-to-slf4j +1.7.36 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.slf4j:jul-to-slf4j:jar:compile + + + + + +org.slf4j:jcl-over-slf4j:jar:compile + +org.slf4j +jcl-over-slf4j +1.7.30 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.slf4j:jcl-over-slf4j:jar:compile + + + + + +com.ning:compress-lzf:jar:compile + +com.ning +compress-lzf +1.0.3 + + + +org.apache.spark:spark-core_2.12:jar:compile->com.ning:compress-lzf:jar:compile + + + + + +org.lz4:lz4-java:jar:compile + +org.lz4 +lz4-java +1.7.1 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.lz4:lz4-java:jar:compile + + + + + +com.github.luben:zstd-jni:jar:compile + +com.github.luben +zstd-jni +1.4.4-3 + + + +org.apache.spark:spark-core_2.12:jar:compile->com.github.luben:zstd-jni:jar:compile + + + + + +org.roaringbitmap:RoaringBitmap:jar:compile + +org.roaringbitmap +RoaringBitmap +0.5.11 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.roaringbitmap:RoaringBitmap:jar:compile + + + + + +org.scala-lang.modules:scala-xml_2.12:jar:compile + +org.scala-lang.modules +scala-xml_2.12 +1.2.0 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.scala-lang.modules:scala-xml_2.12:jar:compile + + + + + +org.scala-lang:scala-reflect:jar:compile + +org.scala-lang +scala-reflect +2.12.10 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.scala-lang:scala-reflect:jar:compile + + + + + +org.json4s:json4s-jackson_2.12:jar:compile + +org.json4s +json4s-jackson_2.12 +3.6.6 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.json4s:json4s-jackson_2.12:jar:compile + + + + + +org.glassfish.jersey.core:jersey-client:jar:compile + +org.glassfish.jersey.core +jersey-client +2.36 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.core:jersey-client:jar:compile + + + + + +org.glassfish.jersey.core:jersey-common:jar:compile + +org.glassfish.jersey.core +jersey-common +2.36 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.core:jersey-common:jar:compile + + + + + +org.glassfish.jersey.core:jersey-server:jar:compile + +org.glassfish.jersey.core +jersey-server +2.36 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.core:jersey-server:jar:compile + + + + + +org.glassfish.jersey.containers:jersey-container-servlet:jar:compile + +org.glassfish.jersey.containers +jersey-container-servlet +2.36 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.containers:jersey-container-servlet:jar:compile + + + + + +org.glassfish.jersey.containers:jersey-container-servlet-core:jar:compile + +org.glassfish.jersey.containers +jersey-container-servlet-core +2.36 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.containers:jersey-container-servlet-core:jar:compile + + + + + +org.glassfish.jersey.inject:jersey-hk2:jar:compile + +org.glassfish.jersey.inject +jersey-hk2 +2.36 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.inject:jersey-hk2:jar:compile + + + + + +io.netty:netty-all:jar:compile + +io.netty +netty-all +4.1.68.Final + + + +org.apache.spark:spark-core_2.12:jar:compile->io.netty:netty-all:jar:compile + + + + + +io.dropwizard.metrics:metrics-core:jar:compile + +io.dropwizard.metrics +metrics-core +4.1.1 + + + +org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-core:jar:compile + + + + + +io.dropwizard.metrics:metrics-jvm:jar:compile + +io.dropwizard.metrics +metrics-jvm +4.1.1 + + + +org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-jvm:jar:compile + + + + + +io.dropwizard.metrics:metrics-json:jar:compile + +io.dropwizard.metrics +metrics-json +4.1.1 + + + +org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-json:jar:compile + + + + + +io.dropwizard.metrics:metrics-graphite:jar:compile + +io.dropwizard.metrics +metrics-graphite +4.1.1 + + + +org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-graphite:jar:compile + + + + + +io.dropwizard.metrics:metrics-jmx:jar:compile + +io.dropwizard.metrics +metrics-jmx +4.1.1 + + + +org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-jmx:jar:compile + + + + + +org.apache.ivy:ivy:jar:compile + +org.apache.ivy +ivy +2.4.0 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.ivy:ivy:jar:compile + + + + + +oro:oro:jar:compile + +oro +oro +2.0.8 + + + +org.apache.spark:spark-core_2.12:jar:compile->oro:oro:jar:compile + + + + + +net.razorvine:pyrolite:jar:compile + +net.razorvine +pyrolite +4.30 + + + +org.apache.spark:spark-core_2.12:jar:compile->net.razorvine:pyrolite:jar:compile + + + + + +net.sf.py4j:py4j:jar:compile + +net.sf.py4j +py4j +0.10.9 + + + +org.apache.spark:spark-core_2.12:jar:compile->net.sf.py4j:py4j:jar:compile + + + + + +org.apache.spark:spark-tags_2.12:jar:compile + +org.apache.spark +spark-tags_2.12 +3.0.3 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-tags_2.12:jar:compile + + + + + +org.apache.commons:commons-crypto:jar:compile + +org.apache.commons +commons-crypto +1.1.0 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.apache.commons:commons-crypto:jar:compile + + + + + +org.spark-project.spark:unused:jar:compile + +org.spark-project.spark +unused +1.0.0 + + + +org.apache.spark:spark-core_2.12:jar:compile->org.spark-project.spark:unused:jar:compile + + + + + +com.esotericsoftware:kryo-shaded:jar:compile + +com.esotericsoftware +kryo-shaded +4.0.2 + + + +com.esotericsoftware:minlog:jar:compile + +com.esotericsoftware +minlog +1.3.0 + + + +com.esotericsoftware:kryo-shaded:jar:compile->com.esotericsoftware:minlog:jar:compile + + + + + +com.twitter:chill_2.12:jar:compile->com.esotericsoftware:kryo-shaded:jar:compile + + + + + +com.twitter:chill_2.12:jar:compile->com.twitter:chill-java:jar:compile + + + + + +org.apache.hadoop:hadoop-client:jar:compile->org.apache.hadoop:hadoop-common:jar:compile + + + + + +org.apache.hadoop:hadoop-client:jar:compile->org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile + + + + + +org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:compile + +org.apache.hadoop +hadoop-mapreduce-client-jobclient +3.3.3 + + + +org.apache.hadoop:hadoop-client:jar:compile->org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:compile + + + + + +org.apache.hadoop:hadoop-mapreduce-client-common:jar:provided + +org.apache.hadoop +hadoop-mapreduce-client-common +3.3.3 +(provided) + + + +org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:compile->org.apache.hadoop:hadoop-mapreduce-client-common:jar:provided + + + + + +org.fusesource.leveldbjni:leveldbjni-all:jar:compile + +org.fusesource.leveldbjni +leveldbjni-all +1.8 + + + +org.apache.spark:spark-kvstore_2.12:jar:compile->org.fusesource.leveldbjni:leveldbjni-all:jar:compile + + + + + +org.roaringbitmap:shims:jar:compile + +org.roaringbitmap +shims +0.7.45 + + + +org.roaringbitmap:RoaringBitmap:jar:compile->org.roaringbitmap:shims:jar:compile + + + + + +org.json4s:json4s-core_2.12:jar:compile + +org.json4s +json4s-core_2.12 +3.6.6 + + + +org.json4s:json4s-ast_2.12:jar:compile + +org.json4s +json4s-ast_2.12 +3.6.6 + + + +org.json4s:json4s-core_2.12:jar:compile->org.json4s:json4s-ast_2.12:jar:compile + + + + + +org.json4s:json4s-scalap_2.12:jar:compile + +org.json4s +json4s-scalap_2.12 +3.6.6 + + + +org.json4s:json4s-core_2.12:jar:compile->org.json4s:json4s-scalap_2.12:jar:compile + + + + + +org.json4s:json4s-jackson_2.12:jar:compile->org.json4s:json4s-core_2.12:jar:compile + + + + + +jakarta.ws.rs:jakarta.ws.rs-api:jar:compile + +jakarta.ws.rs +jakarta.ws.rs-api +2.1.6 + + + +org.glassfish.jersey.core:jersey-client:jar:compile->jakarta.ws.rs:jakarta.ws.rs-api:jar:compile + + + + + +org.glassfish.hk2.external:jakarta.inject:jar:compile + +org.glassfish.hk2.external +jakarta.inject +2.6.1 + + + +org.glassfish.jersey.core:jersey-client:jar:compile->org.glassfish.hk2.external:jakarta.inject:jar:compile + + + + + +org.glassfish.jersey.core:jersey-client:jar:compile->org.glassfish.jersey.core:jersey-common:jar:compile + + + + + +jakarta.annotation:jakarta.annotation-api:jar:compile + +jakarta.annotation +jakarta.annotation-api +1.3.5 + + + +org.glassfish.jersey.core:jersey-common:jar:compile->jakarta.annotation:jakarta.annotation-api:jar:compile + + + + + +org.glassfish.hk2:osgi-resource-locator:jar:compile + +org.glassfish.hk2 +osgi-resource-locator +1.0.3 + + + +org.glassfish.jersey.core:jersey-common:jar:compile->org.glassfish.hk2:osgi-resource-locator:jar:compile + + + + + +org.glassfish.jersey.core:jersey-server:jar:compile->org.glassfish.jersey.core:jersey-client:jar:compile + + + + + +org.glassfish.jersey.core:jersey-server:jar:compile->org.glassfish.jersey.core:jersey-common:jar:compile + + + + + +jakarta.validation:jakarta.validation-api:jar:compile + +jakarta.validation +jakarta.validation-api +2.0.2 + + + +org.glassfish.jersey.core:jersey-server:jar:compile->jakarta.validation:jakarta.validation-api:jar:compile + + + + + +org.glassfish.jersey.containers:jersey-container-servlet:jar:compile->org.glassfish.jersey.containers:jersey-container-servlet-core:jar:compile + + + + + +org.glassfish.hk2:hk2-locator:jar:compile + +org.glassfish.hk2 +hk2-locator +2.6.1 + + + +org.glassfish.hk2.external:aopalliance-repackaged:jar:compile + +org.glassfish.hk2.external +aopalliance-repackaged +2.6.1 + + + +org.glassfish.hk2:hk2-locator:jar:compile->org.glassfish.hk2.external:aopalliance-repackaged:jar:compile + + + + + +org.glassfish.hk2:hk2-api:jar:compile + +org.glassfish.hk2 +hk2-api +2.6.1 + + + +org.glassfish.hk2:hk2-locator:jar:compile->org.glassfish.hk2:hk2-api:jar:compile + + + + + +org.glassfish.hk2:hk2-utils:jar:compile + +org.glassfish.hk2 +hk2-utils +2.6.1 + + + +org.glassfish.hk2:hk2-locator:jar:compile->org.glassfish.hk2:hk2-utils:jar:compile + + + + + +org.glassfish.hk2:hk2-api:jar:compile->org.glassfish.hk2.external:aopalliance-repackaged:jar:compile + + + + + +org.glassfish.hk2:hk2-api:jar:compile->org.glassfish.hk2:hk2-utils:jar:compile + + + + + +org.glassfish.jersey.inject:jersey-hk2:jar:compile->org.javassist:javassist:jar:compile + + + + + +org.glassfish.jersey.inject:jersey-hk2:jar:compile->org.glassfish.jersey.core:jersey-common:jar:compile + + + + + +org.glassfish.jersey.inject:jersey-hk2:jar:compile->org.glassfish.hk2:hk2-locator:jar:compile + + + + + +com.univocity:univocity-parsers:jar:compile + +com.univocity +univocity-parsers +2.9.0 + + + +org.apache.spark:spark-sql_2.12:jar:compile->com.univocity:univocity-parsers:jar:compile + + + + + +org.apache.spark:spark-sketch_2.12:jar:compile + +org.apache.spark +spark-sketch_2.12 +3.0.3 + + + +org.apache.spark:spark-sql_2.12:jar:compile->org.apache.spark:spark-sketch_2.12:jar:compile + + + + + +org.apache.orc:orc-core:jar:compile + +org.apache.orc +orc-core +1.5.10 + + + +org.apache.spark:spark-sql_2.12:jar:compile->org.apache.orc:orc-core:jar:compile + + + + + +org.apache.orc:orc-mapreduce:jar:compile + +org.apache.orc +orc-mapreduce +1.5.10 + + + +org.apache.spark:spark-sql_2.12:jar:compile->org.apache.orc:orc-mapreduce:jar:compile + + + + + +org.apache.hive:hive-storage-api:jar:compile + +org.apache.hive +hive-storage-api +2.7.1 + + + +org.apache.spark:spark-sql_2.12:jar:compile->org.apache.hive:hive-storage-api:jar:compile + + + + + +org.apache.parquet:parquet-column:jar:compile + +org.apache.parquet +parquet-column +1.10.1 + + + +org.apache.spark:spark-sql_2.12:jar:compile->org.apache.parquet:parquet-column:jar:compile + + + + + +org.apache.parquet:parquet-hadoop:jar:compile + +org.apache.parquet +parquet-hadoop +1.10.1 + + + +org.apache.spark:spark-sql_2.12:jar:compile->org.apache.parquet:parquet-hadoop:jar:compile + + + + + +org.apache.orc:orc-core:jar:compile->com.google.protobuf:protobuf-java:jar:compile + + + + + +org.apache.orc:orc-core:jar:compile->commons-lang:commons-lang:jar:compile + + + + + +org.apache.orc:orc-shims:jar:compile + +org.apache.orc +orc-shims +1.5.10 + + + +org.apache.orc:orc-core:jar:compile->org.apache.orc:orc-shims:jar:compile + + + + + +io.airlift:aircompressor:jar:compile + +io.airlift +aircompressor +0.10 + + + +org.apache.orc:orc-core:jar:compile->io.airlift:aircompressor:jar:compile + + + + + +org.threeten:threeten-extra:jar:compile + +org.threeten +threeten-extra +1.5.0 + + + +org.apache.orc:orc-core:jar:compile->org.threeten:threeten-extra:jar:compile + + + + + +org.apache.parquet:parquet-common:jar:compile + +org.apache.parquet +parquet-common +1.10.1 + + + +org.apache.parquet:parquet-column:jar:compile->org.apache.parquet:parquet-common:jar:compile + + + + + +org.apache.parquet:parquet-encoding:jar:compile + +org.apache.parquet +parquet-encoding +1.10.1 + + + +org.apache.parquet:parquet-column:jar:compile->org.apache.parquet:parquet-encoding:jar:compile + + + + + +org.apache.parquet:parquet-format:jar:compile + +org.apache.parquet +parquet-format +2.4.0 + + + +org.apache.parquet:parquet-hadoop:jar:compile->org.apache.parquet:parquet-format:jar:compile + + + + + +org.apache.parquet:parquet-jackson:jar:compile + +org.apache.parquet +parquet-jackson +1.10.1 + + + +org.apache.parquet:parquet-hadoop:jar:compile->org.apache.parquet:parquet-jackson:jar:compile + + + + + +org.scalanlp:breeze_2.12:jar:compile + +org.scalanlp +breeze_2.12 +1.0 + + + +org.scalanlp:breeze-macros_2.12:jar:compile + +org.scalanlp +breeze-macros_2.12 +1.0 + + + +org.scalanlp:breeze_2.12:jar:compile->org.scalanlp:breeze-macros_2.12:jar:compile + + + + + +net.sf.opencsv:opencsv:jar:compile + +net.sf.opencsv +opencsv +2.3 + + + +org.scalanlp:breeze_2.12:jar:compile->net.sf.opencsv:opencsv:jar:compile + + + + + +com.github.wendykierp:JTransforms:jar:compile + +com.github.wendykierp +JTransforms +3.1 + + + +org.scalanlp:breeze_2.12:jar:compile->com.github.wendykierp:JTransforms:jar:compile + + + + + +org.typelevel:spire_2.12:jar:compile + +org.typelevel +spire_2.12 +0.17.0-M1 + + + +org.scalanlp:breeze_2.12:jar:compile->org.typelevel:spire_2.12:jar:compile + + + + + +org.scala-lang.modules:scala-collection-compat_2.12:jar:compile + +org.scala-lang.modules +scala-collection-compat_2.12 +2.1.1 + + + +org.scalanlp:breeze_2.12:jar:compile->org.scala-lang.modules:scala-collection-compat_2.12:jar:compile + + + + + +pl.edu.icm:JLargeArrays:jar:compile + +pl.edu.icm +JLargeArrays +1.5 + + + +com.github.wendykierp:JTransforms:jar:compile->pl.edu.icm:JLargeArrays:jar:compile + + + + + +org.typelevel:spire-macros_2.12:jar:compile + +org.typelevel +spire-macros_2.12 +0.17.0-M1 + + + +org.typelevel:spire_2.12:jar:compile->org.typelevel:spire-macros_2.12:jar:compile + + + + + +org.typelevel:spire-platform_2.12:jar:compile + +org.typelevel +spire-platform_2.12 +0.17.0-M1 + + + +org.typelevel:spire_2.12:jar:compile->org.typelevel:spire-platform_2.12:jar:compile + + + + + +org.typelevel:spire-util_2.12:jar:compile + +org.typelevel +spire-util_2.12 +0.17.0-M1 + + + +org.typelevel:spire_2.12:jar:compile->org.typelevel:spire-util_2.12:jar:compile + + + + + +org.typelevel:machinist_2.12:jar:compile + +org.typelevel +machinist_2.12 +0.6.8 + + + +org.typelevel:spire_2.12:jar:compile->org.typelevel:machinist_2.12:jar:compile + + + + + +org.typelevel:algebra_2.12:jar:compile + +org.typelevel +algebra_2.12 +2.0.0-M2 + + + +org.typelevel:spire_2.12:jar:compile->org.typelevel:algebra_2.12:jar:compile + + + + + +org.typelevel:cats-kernel_2.12:jar:compile + +org.typelevel +cats-kernel_2.12 +2.0.0-M4 + + + +org.typelevel:algebra_2.12:jar:compile->org.typelevel:cats-kernel_2.12:jar:compile + + + + + +org.apache.spark:spark-mllib-local_2.12:jar:compile + +org.apache.spark +spark-mllib-local_2.12 +3.0.3 + + + +org.apache.spark:spark-mllib-local_2.12:jar:compile->org.scalanlp:breeze_2.12:jar:compile + + + + + +org.apache.spark:spark-graphx_2.12:jar:compile->org.apache.spark:spark-mllib-local_2.12:jar:compile + + + + + +com.github.fommil.netlib:core:jar:compile + +com.github.fommil.netlib +core +1.1.2 + + + +org.apache.spark:spark-graphx_2.12:jar:compile->com.github.fommil.netlib:core:jar:compile + + + + + +net.sourceforge.f2j:arpack_combined_all:jar:compile + +net.sourceforge.f2j +arpack_combined_all +0.1 + + + +org.apache.spark:spark-graphx_2.12:jar:compile->net.sourceforge.f2j:arpack_combined_all:jar:compile + + + + + +org.apache.spark:spark-catalyst_2.12:jar:compile->commons-codec:commons-codec:jar:compile + + + + + +org.scala-lang.modules:scala-parser-combinators_2.12:jar:compile + +org.scala-lang.modules +scala-parser-combinators_2.12 +1.0.4 + + + +org.apache.spark:spark-catalyst_2.12:jar:compile->org.scala-lang.modules:scala-parser-combinators_2.12:jar:compile + + + + + +org.codehaus.janino:janino:jar:compile + +org.codehaus.janino +janino +3.0.16 + + + +org.apache.spark:spark-catalyst_2.12:jar:compile->org.codehaus.janino:janino:jar:compile + + + + + +org.codehaus.janino:commons-compiler:jar:compile + +org.codehaus.janino +commons-compiler +3.0.16 + + + +org.apache.spark:spark-catalyst_2.12:jar:compile->org.codehaus.janino:commons-compiler:jar:compile + + + + + +org.antlr:antlr4-runtime:jar:compile + +org.antlr +antlr4-runtime +4.7.1 + + + +org.apache.spark:spark-catalyst_2.12:jar:compile->org.antlr:antlr4-runtime:jar:compile + + + + + +org.apache.arrow:arrow-vector:jar:compile + +org.apache.arrow +arrow-vector +0.15.1 + + + +org.apache.spark:spark-catalyst_2.12:jar:compile->org.apache.arrow:arrow-vector:jar:compile + + + + + +org.apache.arrow:arrow-format:jar:compile + +org.apache.arrow +arrow-format +0.15.1 + + + +org.apache.arrow:arrow-vector:jar:compile->org.apache.arrow:arrow-format:jar:compile + + + + + +org.apache.arrow:arrow-memory:jar:compile + +org.apache.arrow +arrow-memory +0.15.1 + + + +org.apache.arrow:arrow-vector:jar:compile->org.apache.arrow:arrow-memory:jar:compile + + + + + +com.google.flatbuffers:flatbuffers-java:jar:compile + +com.google.flatbuffers +flatbuffers-java +1.9.0 + + + +org.apache.arrow:arrow-vector:jar:compile->com.google.flatbuffers:flatbuffers-java:jar:compile + + + + + +uk.gov.gchq.gaffer:map-store:jar:compile + +uk.gov.gchq.gaffer +map-store +2.0.0 + + + +uk.gov.gchq.gaffer:map-store:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile + + + + + +uk.gov.gchq.gaffer:map-store:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile + + + + + +uk.gov.gchq.gaffer:accumulo-store:jar:compile + +uk.gov.gchq.gaffer +accumulo-store +2.0.0 + + + +uk.gov.gchq.gaffer:accumulo-store:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile + + + + + +uk.gov.gchq.gaffer:accumulo-store:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile + + + + + +uk.gov.gchq.gaffer:accumulo-store:jar:compile->uk.gov.gchq.gaffer:hdfs-library:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile + +org.apache.accumulo +accumulo-core +2.0.1 + + + +uk.gov.gchq.gaffer:accumulo-store:jar:compile->org.apache.accumulo:accumulo-core:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.commons:commons-lang3:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->commons-io:commons-io:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.commons:commons-collections4:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->commons-logging:commons-logging:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.commons:commons-math3:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.commons:commons-configuration2:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->com.google.protobuf:protobuf-java:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->com.google.code.gson:gson:jar:compile + + + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.zookeeper:zookeeper:jar:compile + + + + + +com.beust:jcommander:jar:compile + +com.beust +jcommander +1.72 + + + +org.apache.accumulo:accumulo-core:jar:compile->com.beust:jcommander:jar:compile + + + + + +com.github.ben-manes.caffeine:caffeine:jar:compile + +com.github.ben-manes.caffeine +caffeine +2.7.0 + + + +org.apache.accumulo:accumulo-core:jar:compile->com.github.ben-manes.caffeine:caffeine:jar:compile + + + + + +org.apache.accumulo:accumulo-start:jar:compile + +org.apache.accumulo +accumulo-start +2.0.1 + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.accumulo:accumulo-start:jar:compile + + + + + +org.apache.hadoop:hadoop-client-api:jar:compile + +org.apache.hadoop +hadoop-client-api +3.1.1 + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.hadoop:hadoop-client-api:jar:compile + + + + + +org.apache.htrace:htrace-core:jar:compile + +org.apache.htrace +htrace-core +3.2.0-incubating + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.htrace:htrace-core:jar:compile + + + + + +org.apache.thrift:libthrift:jar:compile + +org.apache.thrift +libthrift +0.12.0 + + + +org.apache.accumulo:accumulo-core:jar:compile->org.apache.thrift:libthrift:jar:compile + + + + + +org.apache.accumulo:accumulo-start:jar:compile->commons-beanutils:commons-beanutils:jar:compile + + + + + +org.apache.commons:commons-vfs2:jar:compile + +org.apache.commons +commons-vfs2 +2.3 + + + +org.apache.accumulo:accumulo-start:jar:compile->org.apache.commons:commons-vfs2:jar:compile + + + + + +org.apache.thrift:libthrift:jar:compile->org.apache.httpcomponents:httpcore:jar:compile + + + + + +uk.gov.gchq.gaffer:spark-accumulo-library:jar:compile + +uk.gov.gchq.gaffer +spark-accumulo-library +2.0.0 + + + +uk.gov.gchq.gaffer:spark-accumulo-library:jar:compile->uk.gov.gchq.gaffer:spark-library:jar:compile + + + + + +uk.gov.gchq.gaffer:spark-accumulo-library:jar:compile->uk.gov.gchq.gaffer:accumulo-store:jar:compile + + + + + +io.netty:netty:jar:compile + +io.netty +netty +3.10.6.Final + + + +uk.gov.gchq.gaffer:spark-accumulo-library:jar:compile->io.netty:netty:jar:compile + + + + + +uk.gov.gchq.gaffer:bitmap-library:jar:compile + +uk.gov.gchq.gaffer +bitmap-library +2.0.0 + + + +uk.gov.gchq.gaffer:bitmap-library:jar:compile->uk.gov.gchq.gaffer:serialisation:jar:compile + + + + + +uk.gov.gchq.gaffer:bitmap-library:jar:compile->org.roaringbitmap:RoaringBitmap:jar:compile + + + + + +uk.gov.gchq.gaffer:hazelcast-cache-service:jar:compile + +uk.gov.gchq.gaffer +hazelcast-cache-service +2.0.0 + + + +uk.gov.gchq.gaffer:hazelcast-cache-service:jar:compile->uk.gov.gchq.gaffer:cache:jar:compile + + + + + +com.hazelcast:hazelcast:jar:compile + +com.hazelcast +hazelcast +5.3.0 + + + +uk.gov.gchq.gaffer:hazelcast-cache-service:jar:compile->com.hazelcast:hazelcast:jar:compile + + + + + +uk.gov.gchq.gaffer:flink-library:jar:compile + +uk.gov.gchq.gaffer +flink-library +2.0.0 + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->uk.gov.gchq.gaffer:store:jar:compile + + + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->jakarta.xml.bind:jakarta.xml.bind-api:jar:compile + + + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->org.scala-lang:scala-library:jar:compile + + + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->org.scala-lang.modules:scala-parser-combinators_2.12:jar:compile + + + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->org.objenesis:objenesis:jar:compile + + + + + +org.apache.flink:flink-java:jar:compile + +org.apache.flink +flink-java +1.7.2 + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->org.apache.flink:flink-java:jar:compile + + + + + +org.apache.flink:flink-streaming-java_2.12:jar:compile + +org.apache.flink +flink-streaming-java_2.12 +1.7.2 + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->org.apache.flink:flink-streaming-java_2.12:jar:compile + + + + + +org.apache.flink:flink-clients_2.12:jar:compile + +org.apache.flink +flink-clients_2.12 +1.7.2 + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->org.apache.flink:flink-clients_2.12:jar:compile + + + + + +org.apache.flink:flink-connector-kafka-0.10_2.12:jar:compile + +org.apache.flink +flink-connector-kafka-0.10_2.12 +1.7.2 + + + +uk.gov.gchq.gaffer:flink-library:jar:compile->org.apache.flink:flink-connector-kafka-0.10_2.12:jar:compile + + + + + +org.apache.flink:flink-core:jar:compile + +org.apache.flink +flink-core +1.7.2 + + + +org.apache.flink:flink-core:jar:compile->org.apache.commons:commons-compress:jar:compile + + + + + +org.apache.flink:flink-core:jar:compile->commons-collections:commons-collections:jar:compile + + + + + +org.apache.flink:flink-annotations:jar:compile + +org.apache.flink +flink-annotations +1.7.2 + + + +org.apache.flink:flink-core:jar:compile->org.apache.flink:flink-annotations:jar:compile + + + + + +org.apache.flink:flink-metrics-core:jar:compile + +org.apache.flink +flink-metrics-core +1.7.2 + + + +org.apache.flink:flink-core:jar:compile->org.apache.flink:flink-metrics-core:jar:compile + + + + + +com.esotericsoftware.kryo:kryo:jar:compile + +com.esotericsoftware.kryo +kryo +2.24.0 + + + +org.apache.flink:flink-core:jar:compile->com.esotericsoftware.kryo:kryo:jar:compile + + + + + +com.esotericsoftware.minlog:minlog:jar:compile + +com.esotericsoftware.minlog +minlog +1.2 + + + +com.esotericsoftware.kryo:kryo:jar:compile->com.esotericsoftware.minlog:minlog:jar:compile + + + + + +org.apache.flink:flink-java:jar:compile->org.apache.commons:commons-lang3:jar:compile + + + + + +org.apache.flink:flink-java:jar:compile->com.google.code.findbugs:jsr305:jar:compile + + + + + +org.apache.flink:flink-java:jar:compile->org.apache.commons:commons-math3:jar:compile + + + + + +org.apache.flink:flink-java:jar:compile->org.apache.flink:flink-core:jar:compile + + + + + +org.apache.flink:flink-shaded-asm:jar:compile + +org.apache.flink +flink-shaded-asm +5.0.4-5.0 + + + +org.apache.flink:flink-java:jar:compile->org.apache.flink:flink-shaded-asm:jar:compile + + + + + +org.apache.flink:force-shading:jar:compile + +org.apache.flink +force-shading +1.7.2 + + + +org.apache.flink:flink-java:jar:compile->org.apache.flink:force-shading:jar:compile + + + + + +org.apache.flink:flink-runtime_2.12:jar:compile + +org.apache.flink +flink-runtime_2.12 +1.7.2 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->commons-io:commons-io:jar:compile + + + + + +org.apache.flink:flink-runtime_2.12:jar:compile->com.twitter:chill_2.12:jar:compile + + + + + +org.apache.flink:flink-queryable-state-client-java_2.12:jar:compile + +org.apache.flink +flink-queryable-state-client-java_2.12 +1.7.2 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->org.apache.flink:flink-queryable-state-client-java_2.12:jar:compile + + + + + +org.apache.flink:flink-hadoop-fs:jar:compile + +org.apache.flink +flink-hadoop-fs +1.7.2 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->org.apache.flink:flink-hadoop-fs:jar:compile + + + + + +org.apache.flink:flink-shaded-netty:jar:compile + +org.apache.flink +flink-shaded-netty +4.1.24.Final-5.0 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->org.apache.flink:flink-shaded-netty:jar:compile + + + + + +org.apache.flink:flink-shaded-jackson:jar:compile + +org.apache.flink +flink-shaded-jackson +2.7.9-5.0 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->org.apache.flink:flink-shaded-jackson:jar:compile + + + + + +com.typesafe.akka:akka-actor_2.12:jar:compile + +com.typesafe.akka +akka-actor_2.12 +2.4.20 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->com.typesafe.akka:akka-actor_2.12:jar:compile + + + + + +com.typesafe.akka:akka-stream_2.12:jar:compile + +com.typesafe.akka +akka-stream_2.12 +2.4.20 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->com.typesafe.akka:akka-stream_2.12:jar:compile + + + + + +com.typesafe.akka:akka-protobuf_2.12:jar:compile + +com.typesafe.akka +akka-protobuf_2.12 +2.4.20 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->com.typesafe.akka:akka-protobuf_2.12:jar:compile + + + + + +com.typesafe.akka:akka-slf4j_2.12:jar:compile + +com.typesafe.akka +akka-slf4j_2.12 +2.4.20 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->com.typesafe.akka:akka-slf4j_2.12:jar:compile + + + + + +org.clapper:grizzled-slf4j_2.12:jar:compile + +org.clapper +grizzled-slf4j_2.12 +1.3.2 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->org.clapper:grizzled-slf4j_2.12:jar:compile + + + + + +com.github.scopt:scopt_2.12:jar:compile + +com.github.scopt +scopt_2.12 +3.5.0 + + + +org.apache.flink:flink-runtime_2.12:jar:compile->com.github.scopt:scopt_2.12:jar:compile + + + + + +com.typesafe:config:jar:compile + +com.typesafe +config +1.3.0 + + + +com.typesafe.akka:akka-actor_2.12:jar:compile->com.typesafe:config:jar:compile + + + + + +org.scala-lang.modules:scala-java8-compat_2.12:jar:compile + +org.scala-lang.modules +scala-java8-compat_2.12 +0.8.0 + + + +com.typesafe.akka:akka-actor_2.12:jar:compile->org.scala-lang.modules:scala-java8-compat_2.12:jar:compile + + + + + +org.reactivestreams:reactive-streams:jar:compile + +org.reactivestreams +reactive-streams +1.0.0 + + + +com.typesafe.akka:akka-stream_2.12:jar:compile->org.reactivestreams:reactive-streams:jar:compile + + + + + +com.typesafe:ssl-config-core_2.12:jar:compile + +com.typesafe +ssl-config-core_2.12 +0.2.1 + + + +com.typesafe.akka:akka-stream_2.12:jar:compile->com.typesafe:ssl-config-core_2.12:jar:compile + + + + + +org.apache.flink:flink-streaming-java_2.12:jar:compile->org.apache.flink:flink-runtime_2.12:jar:compile + + + + + +org.apache.flink:flink-shaded-guava:jar:compile + +org.apache.flink +flink-shaded-guava +18.0-5.0 + + + +org.apache.flink:flink-streaming-java_2.12:jar:compile->org.apache.flink:flink-shaded-guava:jar:compile + + + + + +org.apache.flink:flink-clients_2.12:jar:compile->commons-cli:commons-cli:jar:compile + + + + + +org.apache.flink:flink-optimizer_2.12:jar:compile + +org.apache.flink +flink-optimizer_2.12 +1.7.2 + + + +org.apache.flink:flink-clients_2.12:jar:compile->org.apache.flink:flink-optimizer_2.12:jar:compile + + + + + +org.apache.flink:flink-connector-kafka-0.9_2.12:jar:compile + +org.apache.flink +flink-connector-kafka-0.9_2.12 +1.7.2 + + + +org.apache.flink:flink-connector-kafka-base_2.12:jar:compile + +org.apache.flink +flink-connector-kafka-base_2.12 +1.7.2 + + + +org.apache.flink:flink-connector-kafka-0.9_2.12:jar:compile->org.apache.flink:flink-connector-kafka-base_2.12:jar:compile + + + + + +org.apache.flink:flink-connector-kafka-0.10_2.12:jar:compile->org.apache.flink:flink-connector-kafka-0.9_2.12:jar:compile + + + + + +uk.gov.gchq.gaffer:time-library:jar:compile + +uk.gov.gchq.gaffer +time-library +2.0.0 + + + +uk.gov.gchq.gaffer:time-library:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile + + + + + +uk.gov.gchq.gaffer:time-library:jar:compile->uk.gov.gchq.gaffer:bitmap-library:jar:compile + + + + + +uk.gov.gchq.gaffer:common-rest:jar:compile + +uk.gov.gchq.gaffer +common-rest +2.0.0 + + + +uk.gov.gchq.gaffer:common-rest:jar:compile->uk.gov.gchq.gaffer:map-store:jar:compile + + + + + +javax.annotation:javax.annotation-api:jar:compile + +javax.annotation +javax.annotation-api +1.3.2 + + + +uk.gov.gchq.gaffer:common-rest:jar:compile->javax.annotation:javax.annotation-api:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:war:compile + +uk.gov.gchq.gaffer +core-rest +2.0.0 +.war + + + +uk.gov.gchq.gaffer:core-rest:war:compile->com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:war:compile->com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.core:jersey-server:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.containers:jersey-container-servlet:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.inject:jersey-hk2:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:war:compile->uk.gov.gchq.gaffer:common-rest:jar:compile + + + + + +javax.servlet:javax.servlet-api:jar:provided + +javax.servlet +javax.servlet-api +3.1.0 +(provided) + + + +uk.gov.gchq.gaffer:core-rest:war:compile->javax.servlet:javax.servlet-api:jar:provided + + + + + +io.swagger:swagger-jaxrs:jar:compile + +io.swagger +swagger-jaxrs +1.6.6 + + + +uk.gov.gchq.gaffer:core-rest:war:compile->io.swagger:swagger-jaxrs:jar:compile + + + + + +io.swagger:swagger-annotations:jar:compile + +io.swagger +swagger-annotations +1.6.6 + + + +uk.gov.gchq.gaffer:core-rest:war:compile->io.swagger:swagger-annotations:jar:compile + + + + + +org.glassfish.jersey.media:jersey-media-multipart:jar:compile + +org.glassfish.jersey.media +jersey-media-multipart +2.36 + + + +uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.media:jersey-media-multipart:jar:compile + + + + + +org.glassfish.jersey.media:jersey-media-json-jackson:jar:compile + +org.glassfish.jersey.media +jersey-media-json-jackson +2.36 + + + +uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.media:jersey-media-json-jackson:jar:compile + + + + + +com.fasterxml.jackson.datatype:jackson-datatype-json-org:jar:compile + +com.fasterxml.jackson.datatype +jackson-datatype-json-org +2.13.5 + + + +uk.gov.gchq.gaffer:core-rest:war:compile->com.fasterxml.jackson.datatype:jackson-datatype-json-org:jar:compile + + + + + +com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:compile + +com.fasterxml.jackson.dataformat +jackson-dataformat-yaml +2.13.2 + + + +org.yaml:snakeyaml:jar:compile + +org.yaml +snakeyaml +1.30 + + + +com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:compile->org.yaml:snakeyaml:jar:compile + + + + + +io.swagger:swagger-core:jar:compile + +io.swagger +swagger-core +1.6.6 + + + +io.swagger:swagger-core:jar:compile->com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:compile + + + + + +io.swagger:swagger-models:jar:compile + +io.swagger +swagger-models +1.6.6 + + + +io.swagger:swagger-core:jar:compile->io.swagger:swagger-models:jar:compile + + + + + +javax.validation:validation-api:jar:compile + +javax.validation +validation-api +1.1.0.Final + + + +io.swagger:swagger-core:jar:compile->javax.validation:validation-api:jar:compile + + + + + +io.swagger:swagger-jaxrs:jar:compile->org.reflections:reflections:jar:compile + + + + + +io.swagger:swagger-jaxrs:jar:compile->io.swagger:swagger-core:jar:compile + + + + + +org.jvnet.mimepull:mimepull:jar:compile + +org.jvnet.mimepull +mimepull +1.9.13 + + + +org.glassfish.jersey.media:jersey-media-multipart:jar:compile->org.jvnet.mimepull:mimepull:jar:compile + + + + + +org.glassfish.jersey.ext:jersey-entity-filtering:jar:compile + +org.glassfish.jersey.ext +jersey-entity-filtering +2.36 + + + +org.glassfish.jersey.media:jersey-media-json-jackson:jar:compile->org.glassfish.jersey.ext:jersey-entity-filtering:jar:compile + + + + + +com.fasterxml.jackson.datatype:jackson-datatype-json-org:jar:compile->org.json:json:jar:compile + + + + + +uk.gov.gchq.gaffer:proxy-store:jar:compile + +uk.gov.gchq.gaffer +proxy-store +2.0.0 + + + +uk.gov.gchq.gaffer:proxy-store:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile + + + + + +uk.gov.gchq.gaffer:proxy-store:jar:compile->org.glassfish.jersey.core:jersey-client:jar:compile + + + + + +uk.gov.gchq.gaffer:federated-store:jar:compile + +uk.gov.gchq.gaffer +federated-store +2.0.0 + + + +uk.gov.gchq.gaffer:federated-store:jar:compile->uk.gov.gchq.gaffer:map-store:jar:compile + + + + + +uk.gov.gchq.gaffer:federated-store:jar:compile->uk.gov.gchq.gaffer:accumulo-store:jar:compile + + + + + +uk.gov.gchq.gaffer:federated-store:jar:compile->io.netty:netty:jar:compile + + + + + +uk.gov.gchq.gaffer:accumulo-rest:war:compile + +uk.gov.gchq.gaffer +accumulo-rest +2.0.0 +.war + + + +uk.gov.gchq.gaffer:accumulo-rest:war:compile->uk.gov.gchq.gaffer:accumulo-store:jar:compile + + + + + +uk.gov.gchq.gaffer:accumulo-rest:war:compile->io.netty:netty:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile + +uk.gov.gchq.gaffer +core-rest +2.0.0 +classes + + + +uk.gov.gchq.gaffer:accumulo-rest:war:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.core:jersey-server:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.containers:jersey-container-servlet:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.inject:jersey-hk2:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->uk.gov.gchq.gaffer:common-rest:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->io.swagger:swagger-jaxrs:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->io.swagger:swagger-annotations:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.media:jersey-media-multipart:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.media:jersey-media-json-jackson:jar:compile + + + + + +uk.gov.gchq.gaffer:core-rest:jar:classes:compile->com.fasterxml.jackson.datatype:jackson-datatype-json-org:jar:compile + + + + + +uk.gov.gchq.gaffer:map-rest:war:compile + +uk.gov.gchq.gaffer +map-rest +2.0.0 +.war + + + +uk.gov.gchq.gaffer:map-rest:war:compile->uk.gov.gchq.gaffer:map-store:jar:compile + + + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile + +uk.gov.gchq.gaffer +spring-rest +2.0.0 + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:jcs-cache-service:jar:compile + + + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->org.glassfish.jersey.inject:jersey-hk2:jar:compile + + + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:time-library:jar:compile + + + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:common-rest:jar:compile + + + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:proxy-store:jar:compile + + + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:federated-store:jar:compile + + + + + +org.springframework.boot:spring-boot-starter-web:jar:compile + +org.springframework.boot +spring-boot-starter-web +2.5.12 + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->org.springframework.boot:spring-boot-starter-web:jar:compile + + + + + +org.springdoc:springdoc-openapi-ui:jar:compile + +org.springdoc +springdoc-openapi-ui +1.6.9 + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->org.springdoc:springdoc-openapi-ui:jar:compile + + + + + +org.springframework:spring-core:jar:compile + +org.springframework +spring-core +5.3.18 + + + +uk.gov.gchq.gaffer:spring-rest:jar:compile->org.springframework:spring-core:jar:compile + + + + + +org.springframework.boot:spring-boot-starter:jar:compile + +org.springframework.boot +spring-boot-starter +2.5.12 + + + +org.springframework.boot:spring-boot-starter:jar:compile->jakarta.annotation:jakarta.annotation-api:jar:compile + + + + + +org.springframework.boot:spring-boot-starter:jar:compile->org.yaml:snakeyaml:jar:compile + + + + + +org.springframework.boot:spring-boot:jar:compile + +org.springframework.boot +spring-boot +2.5.12 + + + +org.springframework.boot:spring-boot-starter:jar:compile->org.springframework.boot:spring-boot:jar:compile + + + + + +org.springframework.boot:spring-boot-autoconfigure:jar:compile + +org.springframework.boot +spring-boot-autoconfigure +2.5.12 + + + +org.springframework.boot:spring-boot-starter:jar:compile->org.springframework.boot:spring-boot-autoconfigure:jar:compile + + + + + +org.springframework.boot:spring-boot-starter-logging:jar:compile + +org.springframework.boot +spring-boot-starter-logging +2.5.12 + + + +org.springframework.boot:spring-boot-starter:jar:compile->org.springframework.boot:spring-boot-starter-logging:jar:compile + + + + + +org.springframework.boot:spring-boot-starter-logging:jar:compile->org.slf4j:jul-to-slf4j:jar:compile + + + + + +org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework.boot:spring-boot-starter:jar:compile + + + + + +org.springframework.boot:spring-boot-starter-json:jar:compile + +org.springframework.boot +spring-boot-starter-json +2.5.12 + + + +org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework.boot:spring-boot-starter-json:jar:compile + + + + + +org.springframework.boot:spring-boot-starter-tomcat:jar:compile + +org.springframework.boot +spring-boot-starter-tomcat +2.5.12 + + + +org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework.boot:spring-boot-starter-tomcat:jar:compile + + + + + +org.springframework:spring-web:jar:compile + +org.springframework +spring-web +5.3.18 + + + +org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework:spring-web:jar:compile + + + + + +org.springframework:spring-webmvc:jar:compile + +org.springframework +spring-webmvc +5.3.18 + + + +org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework:spring-webmvc:jar:compile + + + + + +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:compile + +com.fasterxml.jackson.datatype +jackson-datatype-jdk8 +2.12.6 + + + +org.springframework.boot:spring-boot-starter-json:jar:compile->com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:compile + + + + + +com.fasterxml.jackson.module:jackson-module-parameter-names:jar:compile + +com.fasterxml.jackson.module +jackson-module-parameter-names +2.12.6 + + + +org.springframework.boot:spring-boot-starter-json:jar:compile->com.fasterxml.jackson.module:jackson-module-parameter-names:jar:compile + + + + + +org.apache.tomcat.embed:tomcat-embed-core:jar:compile + +org.apache.tomcat.embed +tomcat-embed-core +9.0.60 + + + +org.springframework.boot:spring-boot-starter-tomcat:jar:compile->org.apache.tomcat.embed:tomcat-embed-core:jar:compile + + + + + +org.apache.tomcat.embed:tomcat-embed-el:jar:compile + +org.apache.tomcat.embed +tomcat-embed-el +9.0.60 + + + +org.springframework.boot:spring-boot-starter-tomcat:jar:compile->org.apache.tomcat.embed:tomcat-embed-el:jar:compile + + + + + +org.apache.tomcat.embed:tomcat-embed-websocket:jar:compile + +org.apache.tomcat.embed +tomcat-embed-websocket +9.0.60 + + + +org.springframework.boot:spring-boot-starter-tomcat:jar:compile->org.apache.tomcat.embed:tomcat-embed-websocket:jar:compile + + + + + +org.springframework:spring-beans:jar:compile + +org.springframework +spring-beans +5.3.18 + + + +org.springframework:spring-web:jar:compile->org.springframework:spring-beans:jar:compile + + + + + +org.springframework:spring-aop:jar:compile + +org.springframework +spring-aop +5.3.18 + + + +org.springframework:spring-webmvc:jar:compile->org.springframework:spring-aop:jar:compile + + + + + +org.springframework:spring-context:jar:compile + +org.springframework +spring-context +5.3.18 + + + +org.springframework:spring-webmvc:jar:compile->org.springframework:spring-context:jar:compile + + + + + +org.springframework:spring-expression:jar:compile + +org.springframework +spring-expression +5.3.18 + + + +org.springframework:spring-webmvc:jar:compile->org.springframework:spring-expression:jar:compile + + + + + +io.swagger.core.v3:swagger-core:jar:compile + +io.swagger.core.v3 +swagger-core +2.2.0 + + + +io.swagger.core.v3:swagger-core:jar:compile->jakarta.validation:jakarta.validation-api:jar:compile + + + + + +io.swagger.core.v3:swagger-core:jar:compile->com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:compile + + + + + +io.swagger.core.v3:swagger-annotations:jar:compile + +io.swagger.core.v3 +swagger-annotations +2.2.0 + + + +io.swagger.core.v3:swagger-core:jar:compile->io.swagger.core.v3:swagger-annotations:jar:compile + + + + + +io.swagger.core.v3:swagger-models:jar:compile + +io.swagger.core.v3 +swagger-models +2.2.0 + + + +io.swagger.core.v3:swagger-core:jar:compile->io.swagger.core.v3:swagger-models:jar:compile + + + + + +org.springdoc:springdoc-openapi-common:jar:compile + +org.springdoc +springdoc-openapi-common +1.6.9 + + + +org.springdoc:springdoc-openapi-common:jar:compile->io.swagger.core.v3:swagger-core:jar:compile + + + + + +org.springdoc:springdoc-openapi-webmvc-core:jar:compile + +org.springdoc +springdoc-openapi-webmvc-core +1.6.9 + + + +org.springdoc:springdoc-openapi-webmvc-core:jar:compile->org.springdoc:springdoc-openapi-common:jar:compile + + + + + +org.springdoc:springdoc-openapi-ui:jar:compile->org.springdoc:springdoc-openapi-webmvc-core:jar:compile + + + + + +org.webjars:swagger-ui:jar:compile + +org.webjars +swagger-ui +4.11.1 + + + +org.springdoc:springdoc-openapi-ui:jar:compile->org.webjars:swagger-ui:jar:compile + + + + + +org.webjars:webjars-locator-core:jar:compile + +org.webjars +webjars-locator-core +0.46 + + + +org.springdoc:springdoc-openapi-ui:jar:compile->org.webjars:webjars-locator-core:jar:compile + + + + + +io.github.classgraph:classgraph:jar:compile + +io.github.classgraph +classgraph +4.8.147 + + + +org.springdoc:springdoc-openapi-ui:jar:compile->io.github.classgraph:classgraph:jar:compile + + + + + +org.springframework:spring-jcl:jar:compile + +org.springframework +spring-jcl +5.3.18 + + + +org.springframework:spring-core:jar:compile->org.springframework:spring-jcl:jar:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-model:jar:compile + +uk.gov.gchq.gaffer +road-traffic-model +2.0.0 + + + +uk.gov.gchq.gaffer:road-traffic-model:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-generators:jar:compile + +uk.gov.gchq.gaffer +road-traffic-generators +2.0.0 + + + +uk.gov.gchq.gaffer:road-traffic-generators:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-generators:jar:compile->uk.gov.gchq.gaffer:road-traffic-model:jar:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:war:compile + +uk.gov.gchq.gaffer +road-traffic-rest +2.0.0 +.war + + + +uk.gov.gchq.gaffer:road-traffic-rest:war:compile->uk.gov.gchq.gaffer:jcs-cache-service:jar:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:war:compile->javax.servlet:javax.servlet-api:jar:provided + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:war:compile->uk.gov.gchq.gaffer:proxy-store:jar:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:war:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:war:compile->uk.gov.gchq.gaffer:road-traffic-generators:jar:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-demo:war:compile + +uk.gov.gchq.gaffer +road-traffic-demo +2.0.0 +.war + + + +uk.gov.gchq.gaffer:road-traffic-demo:war:compile->uk.gov.gchq.gaffer:road-traffic-rest:war:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile + +uk.gov.gchq.gaffer +road-traffic-rest +2.0.0 +classes + + + +uk.gov.gchq.gaffer:road-traffic-demo:war:compile->uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile->uk.gov.gchq.gaffer:proxy-store:jar:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile + + + + + +uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile->uk.gov.gchq.gaffer:road-traffic-generators:jar:compile + + + + + +uk.gov.gchq.gaffer:basic-model:jar:compile + +uk.gov.gchq.gaffer +basic-model +2.0.0 + + + +uk.gov.gchq.gaffer:basic-model:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile + + + + + +uk.gov.gchq.gaffer:basic-rest:war:compile + +uk.gov.gchq.gaffer +basic-rest +2.0.0 +.war + + + +uk.gov.gchq.gaffer:basic-rest:war:compile->uk.gov.gchq.gaffer:jcs-cache-service:jar:compile + + + + + +uk.gov.gchq.gaffer:basic-rest:war:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile + + + + + +uk.gov.gchq.gaffer:federated-demo:war:compile + +uk.gov.gchq.gaffer +federated-demo +2.0.0 +.war + + + +uk.gov.gchq.gaffer:federated-demo:war:compile->uk.gov.gchq.gaffer:jcs-cache-service:jar:compile + + + + + +uk.gov.gchq.gaffer:federated-demo:war:compile->uk.gov.gchq.gaffer:federated-store:jar:compile + + + + + +uk.gov.gchq.gaffer:federated-demo:war:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile + + + + + diff --git a/docs/development-guide/project-structure/maven-dependencies/map-store.svg b/docs/development-guide/project-structure/maven-dependencies/map-store.svg new file mode 100644 index 0000000000..b5f433e2d5 --- /dev/null +++ b/docs/development-guide/project-structure/maven-dependencies/map-store.svg @@ -0,0 +1,1104 @@ + + + + + + +map-store + + + +uk.gov.gchq.gaffer:store:jar + +uk.gov.gchq.gaffer +store +2.0.0 + + + +uk.gov.gchq.gaffer:data:jar + +uk.gov.gchq.gaffer +data +2.0.0 + + + +uk.gov.gchq.gaffer:store:jar->uk.gov.gchq.gaffer:data:jar + + + + + +uk.gov.gchq.gaffer:operation:jar + +uk.gov.gchq.gaffer +operation +2.0.0 + + + +uk.gov.gchq.gaffer:store:jar->uk.gov.gchq.gaffer:operation:jar + + + + + +com.google.guava:guava:jar + +com.google.guava +guava +30.1.1-jre + + + +uk.gov.gchq.gaffer:store:jar->com.google.guava:guava:jar + + + + + +org.slf4j:slf4j-api:jar + +org.slf4j +slf4j-api +1.7.36 + + + +uk.gov.gchq.gaffer:store:jar->org.slf4j:slf4j-api:jar + + + + + +org.slf4j:slf4j-reload4j:jar + +org.slf4j +slf4j-reload4j +1.7.36 + + + +uk.gov.gchq.gaffer:store:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:data:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:data:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:data:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:exception:jar + +uk.gov.gchq.gaffer +exception +2.0.0 + + + +uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.gaffer:exception:jar + + + + + +uk.gov.gchq.gaffer:serialisation:jar + +uk.gov.gchq.gaffer +serialisation +2.0.0 + + + +uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.gaffer:serialisation:jar + + + + + +uk.gov.gchq.gaffer:access:jar + +uk.gov.gchq.gaffer +access +2.0.0 + + + +uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.gaffer:access:jar + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8 + +uk.gov.gchq.koryphe +core +2.5.2 +jdk8 + + + +uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.koryphe:core:jar:jdk8 + + + + + +uk.gov.gchq.gaffer:type:jar + +uk.gov.gchq.gaffer +type +2.0.0 + + + +uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.gaffer:type:jar + + + + + +org.apache.commons:commons-collections4:jar + +org.apache.commons +commons-collections4 +4.4 + + + +uk.gov.gchq.gaffer:data:jar->org.apache.commons:commons-collections4:jar + + + + + +uk.gov.gchq.gaffer:operation:jar->uk.gov.gchq.gaffer:data:jar + + + + + +uk.gov.gchq.gaffer:operation:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:operation:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:operation:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:cache:jar + +uk.gov.gchq.gaffer +cache +2.0.0 + + + +uk.gov.gchq.gaffer:operation:jar->uk.gov.gchq.gaffer:cache:jar + + + + + +uk.gov.gchq.gaffer:operation:jar->uk.gov.gchq.gaffer:access:jar + + + + + +com.google.code.findbugs:jsr305:jar + +com.google.code.findbugs +jsr305 +3.0.2 + + + +com.google.guava:guava:jar->com.google.code.findbugs:jsr305:jar + + + + + +com.google.guava:failureaccess:jar + +com.google.guava +failureaccess +1.0.1 + + + +com.google.guava:guava:jar->com.google.guava:failureaccess:jar + + + + + +com.google.guava:listenablefuture:jar + +com.google.guava +listenablefuture +9999.0-empty-to-avoid-conflict-with-guava + + + +com.google.guava:guava:jar->com.google.guava:listenablefuture:jar + + + + + +org.checkerframework:checker-qual:jar + +org.checkerframework +checker-qual +3.8.0 + + + +com.google.guava:guava:jar->org.checkerframework:checker-qual:jar + + + + + +com.google.errorprone:error_prone_annotations:jar + +com.google.errorprone +error_prone_annotations +2.5.1 + + + +com.google.guava:guava:jar->com.google.errorprone:error_prone_annotations:jar + + + + + +com.google.j2objc:j2objc-annotations:jar + +com.google.j2objc +j2objc-annotations +1.3 + + + +com.google.guava:guava:jar->com.google.j2objc:j2objc-annotations:jar + + + + + +org.slf4j:slf4j-reload4j:jar->org.slf4j:slf4j-api:jar + + + + + +ch.qos.reload4j:reload4j:jar + +ch.qos.reload4j +reload4j +1.2.18.3 + + + +org.slf4j:slf4j-reload4j:jar->ch.qos.reload4j:reload4j:jar + + + + + +uk.gov.gchq.gaffer:map-store:jar + +uk.gov.gchq.gaffer +map-store +2.0.0 + + + +uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:store:jar + + + + + +uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:data:jar + + + + + +uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:operation:jar + + + + + +uk.gov.gchq.gaffer:map-store:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:map-store:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:map-store:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:common-util:jar + +uk.gov.gchq.gaffer +common-util +2.0.0 + + + +uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:common-util:jar + + + + + +uk.gov.gchq.gaffer:graph:jar + +uk.gov.gchq.gaffer +graph +2.0.0 + + + +uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:graph:jar + + + + + +uk.gov.gchq.gaffer:sketches-library:jar + +uk.gov.gchq.gaffer +sketches-library +2.0.0 + + + +uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:sketches-library:jar + + + + + +uk.gov.gchq.gaffer:cache:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:cache:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:cache:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:cache:jar->uk.gov.gchq.gaffer:exception:jar + + + + + +uk.gov.gchq.gaffer:cache:jar->uk.gov.gchq.gaffer:serialisation:jar + + + + + +uk.gov.gchq.gaffer:exception:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:exception:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:exception:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:exception:jar->uk.gov.gchq.gaffer:common-util:jar + + + + + +org.apache.commons:commons-lang3:jar + +org.apache.commons +commons-lang3 +3.12.0 + + + +uk.gov.gchq.gaffer:exception:jar->org.apache.commons:commons-lang3:jar + + + + + +com.fasterxml.jackson.core:jackson-databind:jar + +com.fasterxml.jackson.core +jackson-databind +2.13.5 + + + +uk.gov.gchq.gaffer:exception:jar->com.fasterxml.jackson.core:jackson-databind:jar + + + + + +com.fasterxml.jackson.core:jackson-core:jar + +com.fasterxml.jackson.core +jackson-core +2.13.5 + + + +uk.gov.gchq.gaffer:exception:jar->com.fasterxml.jackson.core:jackson-core:jar + + + + + +uk.gov.gchq.gaffer:serialisation:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:serialisation:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:serialisation:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:serialisation:jar->uk.gov.gchq.gaffer:exception:jar + + + + + +uk.gov.gchq.gaffer:serialisation:jar->uk.gov.gchq.gaffer:common-util:jar + + + + + +uk.gov.gchq.gaffer:serialisation:jar->com.fasterxml.jackson.core:jackson-databind:jar + + + + + +org.apache.avro:avro:jar + +org.apache.avro +avro +1.8.2 + + + +uk.gov.gchq.gaffer:serialisation:jar->org.apache.avro:avro:jar + + + + + +uk.gov.gchq.gaffer:serialisation:jar->com.fasterxml.jackson.core:jackson-core:jar + + + + + +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar + +com.fasterxml.jackson.datatype +jackson-datatype-jsr310 +2.13.5 + + + +uk.gov.gchq.gaffer:serialisation:jar->com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar + + + + + +uk.gov.gchq.gaffer:access:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:access:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:access:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:access:jar->uk.gov.gchq.gaffer:common-util:jar + + + + + +uk.gov.gchq.gaffer:common-util:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:common-util:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:common-util:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:common-util:jar->uk.gov.gchq.koryphe:core:jar:jdk8 + + + + + +uk.gov.gchq.gaffer:common-util:jar->org.apache.commons:commons-lang3:jar + + + + + +com.fasterxml.jackson.core:jackson-annotations:jar + +com.fasterxml.jackson.core +jackson-annotations +2.13.5 + + + +uk.gov.gchq.gaffer:common-util:jar->com.fasterxml.jackson.core:jackson-annotations:jar + + + + + +uk.gov.gchq.gaffer:common-util:jar->com.fasterxml.jackson.core:jackson-databind:jar + + + + + +org.reflections:reflections:jar + +org.reflections +reflections +0.9.12 + + + +uk.gov.gchq.gaffer:common-util:jar->org.reflections:reflections:jar + + + + + +com.github.spotbugs:spotbugs-annotations:jar + +com.github.spotbugs +spotbugs-annotations +4.7.3 + + + +com.github.spotbugs:spotbugs-annotations:jar->com.google.code.findbugs:jsr305:jar + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8->com.google.guava:guava:jar + + +29.0-jre + + + +uk.gov.gchq.koryphe:core:jar:jdk8->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8->com.github.spotbugs:spotbugs-annotations:jar + + + + + +io.github.lukehutch:fast-classpath-scanner:jar + +io.github.lukehutch +fast-classpath-scanner +2.10.0 + + + +uk.gov.gchq.koryphe:core:jar:jdk8->io.github.lukehutch:fast-classpath-scanner:jar + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8->org.apache.commons:commons-lang3:jar + + + + + +commons-io:commons-io:jar + +commons-io +commons-io +2.11.0 + + + +uk.gov.gchq.koryphe:core:jar:jdk8->commons-io:commons-io:jar + + + + + +org.apache.commons:commons-csv:jar + +org.apache.commons +commons-csv +1.9.0 + + + +uk.gov.gchq.koryphe:core:jar:jdk8->org.apache.commons:commons-csv:jar + + + + + +commons-codec:commons-codec:jar + +commons-codec +commons-codec +1.15 + + + +uk.gov.gchq.koryphe:core:jar:jdk8->commons-codec:commons-codec:jar + + + + + +uk.gov.gchq.koryphe:core:jar:jdk8->com.fasterxml.jackson.core:jackson-annotations:jar + + +2.13.4 + + + +uk.gov.gchq.koryphe:core:jar:jdk8->com.fasterxml.jackson.core:jackson-databind:jar + + + + + +org.json:json:jar + +org.json +json +20211205 + + + +uk.gov.gchq.koryphe:core:jar:jdk8->org.json:json:jar + + + + + +com.fasterxml.jackson.core:jackson-databind:jar->com.fasterxml.jackson.core:jackson-annotations:jar + + + + + +com.fasterxml.jackson.core:jackson-databind:jar->com.fasterxml.jackson.core:jackson-core:jar + + + + + +org.apache.avro:avro:jar->org.slf4j:slf4j-api:jar + + +1.7.7 + + + +org.codehaus.jackson:jackson-core-asl:jar + +org.codehaus.jackson +jackson-core-asl +1.9.13 + + + +org.apache.avro:avro:jar->org.codehaus.jackson:jackson-core-asl:jar + + + + + +org.codehaus.jackson:jackson-mapper-asl:jar + +org.codehaus.jackson +jackson-mapper-asl +1.9.13 + + + +org.apache.avro:avro:jar->org.codehaus.jackson:jackson-mapper-asl:jar + + + + + +com.thoughtworks.paranamer:paranamer:jar + +com.thoughtworks.paranamer +paranamer +2.8 + + + +org.apache.avro:avro:jar->com.thoughtworks.paranamer:paranamer:jar + + + + + +org.xerial.snappy:snappy-java:jar + +org.xerial.snappy +snappy-java +1.1.1.3 + + + +org.apache.avro:avro:jar->org.xerial.snappy:snappy-java:jar + + + + + +org.apache.commons:commons-compress:jar + +org.apache.commons +commons-compress +1.8.1 + + + +org.apache.avro:avro:jar->org.apache.commons:commons-compress:jar + + + + + +org.tukaani:xz:jar + +org.tukaani +xz +1.5 + + + +org.apache.avro:avro:jar->org.tukaani:xz:jar + + + + + +org.codehaus.jackson:jackson-mapper-asl:jar->org.codehaus.jackson:jackson-core-asl:jar + + + + + +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar->com.fasterxml.jackson.core:jackson-annotations:jar + + + + + +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar->com.fasterxml.jackson.core:jackson-databind:jar + + + + + +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar->com.fasterxml.jackson.core:jackson-core:jar + + + + + +uk.gov.gchq.gaffer:type:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:type:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:type:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:type:jar->uk.gov.gchq.gaffer:serialisation:jar + + + + + +uk.gov.gchq.gaffer:type:jar->uk.gov.gchq.koryphe:core:jar:jdk8 + + + + + +org.javassist:javassist:jar + +org.javassist +javassist +3.29.2-GA + + + +org.reflections:reflections:jar->org.javassist:javassist:jar + + + + + +uk.gov.gchq.gaffer:graph:jar->uk.gov.gchq.gaffer:store:jar + + + + + +uk.gov.gchq.gaffer:graph:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:graph:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:graph:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:sketches-library:jar->uk.gov.gchq.gaffer:data:jar + + + + + +uk.gov.gchq.gaffer:sketches-library:jar->com.google.guava:guava:jar + + + + + +uk.gov.gchq.gaffer:sketches-library:jar->org.slf4j:slf4j-api:jar + + + + + +uk.gov.gchq.gaffer:sketches-library:jar->org.slf4j:slf4j-reload4j:jar + + + + + +uk.gov.gchq.gaffer:sketches-library:jar->uk.gov.gchq.gaffer:serialisation:jar + + + + + +uk.gov.gchq.gaffer:sketches-library:jar->uk.gov.gchq.koryphe:core:jar:jdk8 + + + + + +com.clearspring.analytics:stream:jar + +com.clearspring.analytics +stream +2.7.0 + + + +uk.gov.gchq.gaffer:sketches-library:jar->com.clearspring.analytics:stream:jar + + + + + +com.yahoo.datasketches:sketches-core:jar + +com.yahoo.datasketches +sketches-core +0.12.0 + + + +uk.gov.gchq.gaffer:sketches-library:jar->com.yahoo.datasketches:sketches-core:jar + + + + + +it.unimi.dsi:fastutil:jar + +it.unimi.dsi +fastutil +6.5.7 + + + +com.clearspring.analytics:stream:jar->it.unimi.dsi:fastutil:jar + + + + + +com.yahoo.datasketches:memory:jar + +com.yahoo.datasketches +memory +0.12.0 + + + +com.yahoo.datasketches:memory:jar->org.slf4j:slf4j-api:jar + + +1.7.25 + + + +com.yahoo.datasketches:sketches-core:jar->com.yahoo.datasketches:memory:jar + + + + + diff --git a/docs/development-guide/project-structure/maven-dependencies/spring.md b/docs/development-guide/project-structure/maven-dependencies/spring.md new file mode 100644 index 0000000000..8c4f605d52 --- /dev/null +++ b/docs/development-guide/project-structure/maven-dependencies/spring.md @@ -0,0 +1,105 @@ +# Managing dependencies in Gaffer + +Gaffer is a large project with a lot of dependency and transitive dependency management required. This page covers some ways to get information about and more easily visualise and understand the dependencies for given Maven modules (or even the whole project). + +For more information on how Maven handles dependencies, [see this Maven guide](https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html). + + +## Get a simple list of direct dependencies + +This outputs the direct (non-transitive) dependencies for all Maven modules in the project. For specific modules append `-pl :module-name` to the command. +``` +mvn dependency:list -DexcludeTransitive=true -Dsort=true +``` +If `-DexcludeTransitive=true` is not used it will also print transitive dependencies, but without showing where they come from. The tree option (explained below) is better for getting this kind of information. + +For more info, [see the Maven dependency plugin documentation for the 'list' goal](https://maven.apache.org/plugins/maven-dependency-plugin/list-mojo.html). + +### Output to File + +The user property `-DoutputFile=deps.txt` can be used to place output for each module into the root directory **of that module**. It is also possible to aggregate all results into a single file using `mvn dependency:list -B -DappendOutput=true -DoutputFile=$(pwd)/mvn-dep-list.txt`. + +However, any files created contain terminal escape characters for adding colour (even when using `-B` - this is likely a bug). Another option is to use `-B` to turn off colour, piping the output through `sort --unique` (which also aggregates the dependencies) and appending that to a file - instead of using the plugin's file output option. + + +## Get a tree showing Dependencies + +This produces a tree showing the **resolved** dependencies for all Maven modules in the project. For specific modules append `-pl :module-name` to the command (as before). +``` +mvn dependency:tree +``` + +To include **all** dependencies (resolved, transitive and duplicate) in the tree and to see information on versions which have been managed (via `dependencyManagement`), append `-Dverbose`. + +This verbose flag is very useful for finding out which dependencies are affected by version management and version conflicts. + +Here's a simplified tree output with the verbose output explained (expand the :material-plus-circle-outline: symbols): +```c +uk.gov.gchq.gaffer:core:pom:2.0.0 ++- org.slf4j:slf4j-api:jar:1.7.36:compile ++- org.slf4j:slf4j-reload4j:jar:1.7.36:compile +| +- (org.slf4j:slf4j-api:jar:1.7.36:compile - omitted for duplicate) // (1)! +| \- ch.qos.reload4j:reload4j:jar:1.2.18.3:compile (version managed from 1.2.19) // (2)! ++- org.junit.jupiter:junit-jupiter:jar:5.9.0:test +| +- org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test +\- org.mockito:mockito-junit-jupiter:jar:4.6.1:test + +- org.mockito:mockito-core:jar:4.6.1:test + \- (org.junit.jupiter:junit-jupiter-api:jar:5.8.2:test - omitted for conflict with 5.9.0) // (3)! +``` + +1. This dependency was omitted because it was already resolved (found) earlier. If a dependency with the same version is seen more than once, subsequent instances are ignored as **duplicates**. +2. This dependency has had its version changed (managed) from the version defined in the parent transitive dependency's POM to the version for this dependency given in the `dependencyManagement` section of our POM. The "version managed from" in the brackets is the version which would have been used if it hadn't been overridden by specifying a managed version. +3. This dependency was omitted because it was already resolved (found) earlier. If a dependency with different versions is seen more than once, subsequent instances are ignored as **conflicts**. In this case, the conflicting version is a transitive dependency and the resolved dependency is a direct dependency (declared in the module POM) which takes priority. + +This verbose output is also useful for discovering which dependencies need to have exclusions added when using [Maven dependency exclusions](https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html#dependency-exclusions). As an example, replacing Log4j with Reload4j can be done be excluding all Log4j dependencies and adding Reload4j as a project dependency (from a Java perspective this works because both use the same package name). This requires an exclusion section to be added for every dependency which includes Log4j as a transitive dependency. + +Without the verbose output, only a single resolved Log4j dependency is shown. In a project with 5 dependencies with Log4j as a sub-dependency, if only the resolved dependency were to be excluded, then another would become the resolved dependency. With this approach, the tree would need to be printed repeatedly until all these dependencies had been seen and excluded. With the verbose output these dependencies are all shown initially. + +For more info, [see the Maven dependency plugin documentation for the 'tree' goal](https://maven.apache.org/plugins/maven-dependency-plugin/tree-mojo.html). + +### Output to File + +Unlike `dependency:list`, the `tree` plugin goal does output to file correctly. This puts all tree output into a single file called `mvn-dep-tree.txt` in the current directory: +``` +mvn dependency:tree -DoutputFile=$(pwd)/mvn-dep-tree.txt -DappendOutput=true +``` + +## Plotting graphs of dependencies + +### Using the built-in Maven plugin + +In addition to text output, this can also produce `.dot` files which can be used to create dependency graph visualisations (in combination with [Graphviz](https://en.wikipedia.org/wiki/Graphviz)). The resulting files are simple and lack positioning information. They only work well if the number of dependencies is low (e.g. `-Dverbose=true` not used). + +This visualisation was produced using the plugin and [Graphviz `sfdp`](https://graphviz.org/docs/layouts/sfdp/). + +```bash +mvn dependency:tree -DoutputType=dot -DoutputFile=$(pwd)/core.dot -pl :core # Create the .dot file +sfdp -x -Goverlap=false -Gsplines=true -Tsvg core.dot -o core.svg # Create the SVG image +``` + +It has too much empty space to be very useful. + +![Image title](core.svg) + +### Using the Depgraph Maven plugin + +A much better tool for creating `.dot` files is the Depgraph Maven plugin. This creates graphs with optimised node positioning, better formatting and more options than the built-in plugin. + +The main options available are `showClassifiers`, `showDuplicates`, `showConflicts`, `showVersions`, `showGroupIds` & `showTypes`. All of these options are disabled by default, but have been enabled in the Gaffer POM. + +It's important to note that this plugin does not have an equivalent to the verbose mode of `dependency:tree`, and it does not have a way to shown that a dependency's version has been changed using dependency management. + +For more info on the plugin, see the [Depgraph GitHub](https://github.com/ferstl/depgraph-maven-plugin) and [plugin docs](https://ferstl.github.io/depgraph-maven-plugin/graph-mojo.html). + +An example is shown below. The `dot` tool is also [part of Graphviz](https://graphviz.org/docs/layouts/dot/). + +```bash +mvn depgraph:graph -pl :map-store -Dscope=compile -DoutputDirectory=$(pwd) +dot -Tsvg dependency-graph.dot -o filename.svg +``` + +The black dotted lines show duplicates and the red ones show conflicts which were ignored during dependency resolution. + +![Image title](map-store.svg) + +It's also possible to create an aggregated graph of all dependencies in a project using `depgraph:aggregate` (which doesn't support showing duplicates or conflicts). For Gaffer this creates [a very large image](gaffer-complete.svg). \ No newline at end of file diff --git a/docs/development-guide/remote-coding-environments.md b/docs/development-guide/remote-coding-environments.md new file mode 100644 index 0000000000..c1786964e2 --- /dev/null +++ b/docs/development-guide/remote-coding-environments.md @@ -0,0 +1,32 @@ +# Remote Coding Environments For Gaffer + +Gaffer is now configured for remote coding environemnts such as GitHub +Codespaces and Gitpod. This addition allows for an easier and faster way to +contribute to Gaffer with no manual setup or need to download dependencies to a +local machine. + +## GitHub Codespaces +To use GitHub Codespaces simply open the Gaffer repository, click the "Code" +button drop down, and then the option labeled "Create codespace on develop". + +This will launch a Codespaces environment with all the configuration needed to +contribute to Gaffer. See the [GitHub documentation for more information on +Codespaces] (https://github.com/features/codespaces). + +## Gitpod +To use Gitpod you can simply prefix any GitHub URL with **`gitpod.io/#`** and +follow the steps from there. You can also install the +[extension](https://www.gitpod.io/docs/configure/user-settings/browser-extension) +on your web browser. This adds a button to GitHub that does the prefixing for +you. + +Our custom GitPod configuration removes your Git commit email so you +will need to [re-configure your Git commit +email](https://www.gitpod.io/docs/configure/authentication/github). You can also +[configure your Git commit email to be a private GitHub email or a custom email +too.](https://www.gitpod.io/docs/configure/authentication#how-to-use-a-private-github-email-or-custom-email-for-git-commits). +Once done your environment will be all set to contribute to the Gaffer +repository. + +See the [Gitpod documentation for more +information.](https://www.gitpod.io/docs/introduction). \ No newline at end of file diff --git a/docs/development-guide/ways-of-working.md b/docs/development-guide/ways-of-working.md new file mode 100644 index 0000000000..86a03cce7f --- /dev/null +++ b/docs/development-guide/ways-of-working.md @@ -0,0 +1,91 @@ +# Ways of Working + +## Git branching model +We have adopted the GitFlow Branching Model in order to support both Gaffer v1 and v2: ![GitFlow Branching Model](https://nvie.com/img/git-model@2x.png) + +## Issues +Where possible a pull request should correlate to a single GitHub issue. An issue should relate to a single functional or non-functional change - changes to alter/improve other pieces of functionality should be addressed in a separate issue in order to keep reviews atomic. +The reasoning behind code changes should be documented in the GitHub issue. +All resolved issues should be included in the next GitHub milestone, this enables releases to be linked to the included issues. +If a code change requires users of Gaffer to make changes in order for them to adopt it then the issue should be labelled 'migration-required' and a comment should be added similar to: + +``` +### Migration Steps + +[Description of what needs to be done to adopt the code change with examples] +``` + +### Workflow +* Assign yourself to the issue +* Create a new branch off **develop** using pattern: `gh-[issue number]-[issue-title]` +* Commit your changes using descriptive commit titles +* Check and push your changes +* Create a pull request (PR) to merge your branch into **develop**, prefixing the PR title with "Gh-[issue number]: " +* If you named the branch and PR correctly, the PR should have "Resolve #[issue-number]" automatically added to the description after it is made. If it doesn't, then please add the issue it will resolve as a "Linked issue" +* If there is a significant change, please follow the same process to document the change in [gaffer-doc](https://github.com/gchq/gaffer-doc) +* The pull request will be reviewed and following any changes and approval your branch will be squashed and merged into **develop** +* Delete the branch +* The issue will be closed automatically + +## Pull Requests +Pull requests will undergo a review by a Gaffer committer to check the code changes are compliant with our coding style. This is a community so please be respectful of other members - offer encouragement, support and suggestions. + +As described in our git branching model - please raise pull requests to merge your changes in our **develop** branch. + +When pull requests are accepted, the reviewer should squash and merge them. This is because it keeps the **develop** branch clean and populated with only merge commits, rather than intermediate ones. As well as this, it makes everyone's job reviewing pull requests easier as any insecure and unreviewed intermediate commits are not included into the **develop** branch. + +Please agree to the [GCHQ OSS Contributor License Agreement](https://github.com/GovernmentCommunicationsHeadquarters/Gaffer/wiki/GCHQ-OSS-Contributor-License-Agreement-V1.0) before submitting a pull request. Signing the CLA is enforced by the cla-assistant. + +## Documentation +As mentioned before, any significant changes in a PR should be accompanied with an addition to Gaffer's documentation: [gaffer-doc](https://github.com/gchq/gaffer-doc). +Smaller changes should be self documented in the tests. With this approach, any large feature or change has user friendly documentation, whereas technical or implementation details are documented for developers by the tests. + +## Coding style +### Java +Please ensure your coding style is consistent with the rest of the Gaffer project and the [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html). Your changes should pass the checkstyle and spotless plugins that are part of the continuous integration pipeline and check for code formatting and licenses. Before you push your changes you can check the checkstyle plugin passes with `mvn checkstyle:check` and check the spotless plugin passes with `mvn spotless:check`. + +### Python +Please ensure your coding style is consistent with the rest of the Gaffer project and the [PEP 8 Style Guide](https://peps.python.org/pep-0008/). However, there are a few exceptions to the standards set by PEP8: +* Module level imports at the top of the file - this will not be enforced but is recommended where it does not cause issues with the code generated by Fishbowl. +* Max line length of 79 characters - the max line length that will be enforced in this project has been increased to 100 characters. + +Before you create a PR for your changes you can use [autopep8](https://github.com/hhatto/autopep8) to check and fix any styling issues. The following can be run which will take into account the rule exceptions mentioned above. +`autopep8 --exit-code -r -i -a -a --max-line-length 100 --ignore E402 .` + + +### Javadoc +Ensure your java code has sufficient javadocs explaining what the section of code does and the intended use of it. Javadocs should be used in addition to clean readable code. + +In particular: +* All public classes (not required for test classes unless an explanation of the testing is required) +* public methods (not required if the functionality is obvious from the method name) +* public constants (not required if the constant is obvious from the name) + +### Tests +* All new code should be unit tested. Where this is not possible the code should be invoked and the functionality should be tested in an integration test. In a small number of cases this will not be possible - instead steps to verify the code should be thoroughly documented. +* Tests should cover edge cases and exception cases as well as normal expected behavior. +* Keep each test decoupled and don't rely on tests running in a given order - don't save state between tests. +* For a given code change, aim to improve the code coverage. +* Unit test classes should test a single class and be named [testClass]Test. +* Integration test classes should be named [functionalityUnderTest]IT. +* Tests should be readable and self documenting. +* Each test should focus on testing one small piece of functionality invoked from a single method call. +* Tests should use JUnit 5 and assertJ. +* We suggest the following pattern: + + ```java + @Test + public void should[DoSomething|ReturnSomething] { + // Given + [Setup your test here] + + // When + [Invoke the test method] + + // Then + [assertThat the method did what was expected] + } + ``` + +## Gaffer 2 +During the Gaffer 2 development process there was a **v2-alpha** branch, which acted as the **develop** branch for changes staged for Gaffer 2. This branch is no longer in use and can be ignored. diff --git a/mkdocs.yml b/mkdocs.yml index d9f3e9fcfb..74ef859b6c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -118,6 +118,32 @@ nav: - 'Gremlin': - 'Gremlin Guide': 'user-guide/query/gremlin/gremlin.md' - 'Examples': 'user-guide/query/gremlin/examples.md' + - Development Guide: + - 'Getting Started': 'development-guide/getting-started.md' + - 'Ways of Working': 'development-guide/ways-of-working.md' + - 'Remote Coding Environments': 'development-guide/remote-coding-environments.md' + - 'Extending Gaffer': 'development-guide/extending-gaffer.md' + - Project Structure: + - Components: + - 'Components/Maven Modules': 'development-guide/project-structure/components/components.md' + - 'Cache': 'development-guide/project-structure/components/cache.md' + - 'Data': 'development-guide/project-structure/components/data.md' + - 'Graph': 'development-guide/project-structure/components/graph.md' + - 'Store': 'development-guide/project-structure/components/store.md' + - 'Operation': 'development-guide/project-structure/components/operation.md' + - 'Serialisation': 'development-guide/project-structure/components/serialisation.md' + - 'Integration Tests': 'development-guide/project-structure/components/integration-test.md' + - 'Core REST': 'development-guide/project-structure/components/core-rest.md' + - 'Spring REST': 'development-guide/project-structure/components/spring-rest.md' + - 'Accumulo Store': 'development-guide/project-structure/components/accumulo-store.md' + - Libraries: + - development-guide/project-structure/components/libraries/bitmap.md + - development-guide/project-structure/components/libraries/flink.md + - development-guide/project-structure/components/libraries/sketches.md + - development-guide/project-structure/components/libraries/spark.md + - development-guide/project-structure/components/libraries/time.md + - Maven Dependencies: + - 'Spring': 'development-guide/project-structure/maven-dependencies/spring.md' - 'Reference': - 'Reference Guide': 'reference/intro.md' - Stores: From 7a3feb5a3ecce7fd25b9645878b7ef1838c683ab Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:14:39 +0000 Subject: [PATCH 03/20] Added in new reference structure --- docs/reference/glossary.md | 0 docs/reference/javadoc.md | 0 mkdocs.yml | 2 ++ 3 files changed, 2 insertions(+) create mode 100644 docs/reference/glossary.md create mode 100644 docs/reference/javadoc.md diff --git a/docs/reference/glossary.md b/docs/reference/glossary.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/reference/javadoc.md b/docs/reference/javadoc.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mkdocs.yml b/mkdocs.yml index 74ef859b6c..9c70f0ce42 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -146,6 +146,8 @@ nav: - 'Spring': 'development-guide/project-structure/maven-dependencies/spring.md' - 'Reference': - 'Reference Guide': 'reference/intro.md' + - 'Glossary': 'reference/glossary.md' + - 'Javadoc': 'reference/javadoc.md' - Stores: - 'Store Guide': 'reference/stores-guide/stores.md' - 'Map Store': 'reference/stores-guide/map.md' From 32588a1fd70e592048862f268e5daffe225fc2b0 Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Mon, 4 Sep 2023 16:08:31 +0000 Subject: [PATCH 04/20] Added new administration guide structure --- .../gaffer-config/config.md | 0 .../gaffer-config/proxy.md | 0 .../administration-guide/gaffer-config/url.md | 0 .../gaffer-stores/accumulo-store.md} | 0 .../gaffer-stores/federated-store.md} | 0 .../gaffer-stores/map-store.md} | 0 .../gaffer-stores/proxy-store.md} | 0 .../gaffer-stores/store-guide.md} | 0 .../import-export-data.md | 0 docs/administration-guide/named-operations.md | 0 docs/administration-guide/operation-score.md | 0 docs/administration-guide/schema.md | 0 .../security/security-guide.md | 0 .../security/user-control.md | 0 .../where-to-run-gaffer/gaffer-docker.md | 0 mkdocs.yml | 20 +++++++++++++++++++ 16 files changed, 20 insertions(+) create mode 100644 docs/administration-guide/gaffer-config/config.md create mode 100644 docs/administration-guide/gaffer-config/proxy.md create mode 100644 docs/administration-guide/gaffer-config/url.md rename docs/{reference/stores-guide/accumulo.md => administration-guide/gaffer-stores/accumulo-store.md} (100%) rename docs/{reference/stores-guide/federated.md => administration-guide/gaffer-stores/federated-store.md} (100%) rename docs/{reference/stores-guide/map.md => administration-guide/gaffer-stores/map-store.md} (100%) rename docs/{reference/stores-guide/proxy.md => administration-guide/gaffer-stores/proxy-store.md} (100%) rename docs/{reference/stores-guide/stores.md => administration-guide/gaffer-stores/store-guide.md} (100%) create mode 100644 docs/administration-guide/import-export-data.md create mode 100644 docs/administration-guide/named-operations.md create mode 100644 docs/administration-guide/operation-score.md create mode 100644 docs/administration-guide/schema.md create mode 100644 docs/administration-guide/security/security-guide.md create mode 100644 docs/administration-guide/security/user-control.md create mode 100644 docs/administration-guide/where-to-run-gaffer/gaffer-docker.md diff --git a/docs/administration-guide/gaffer-config/config.md b/docs/administration-guide/gaffer-config/config.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/administration-guide/gaffer-config/proxy.md b/docs/administration-guide/gaffer-config/proxy.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/administration-guide/gaffer-config/url.md b/docs/administration-guide/gaffer-config/url.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/reference/stores-guide/accumulo.md b/docs/administration-guide/gaffer-stores/accumulo-store.md similarity index 100% rename from docs/reference/stores-guide/accumulo.md rename to docs/administration-guide/gaffer-stores/accumulo-store.md diff --git a/docs/reference/stores-guide/federated.md b/docs/administration-guide/gaffer-stores/federated-store.md similarity index 100% rename from docs/reference/stores-guide/federated.md rename to docs/administration-guide/gaffer-stores/federated-store.md diff --git a/docs/reference/stores-guide/map.md b/docs/administration-guide/gaffer-stores/map-store.md similarity index 100% rename from docs/reference/stores-guide/map.md rename to docs/administration-guide/gaffer-stores/map-store.md diff --git a/docs/reference/stores-guide/proxy.md b/docs/administration-guide/gaffer-stores/proxy-store.md similarity index 100% rename from docs/reference/stores-guide/proxy.md rename to docs/administration-guide/gaffer-stores/proxy-store.md diff --git a/docs/reference/stores-guide/stores.md b/docs/administration-guide/gaffer-stores/store-guide.md similarity index 100% rename from docs/reference/stores-guide/stores.md rename to docs/administration-guide/gaffer-stores/store-guide.md diff --git a/docs/administration-guide/import-export-data.md b/docs/administration-guide/import-export-data.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/administration-guide/named-operations.md b/docs/administration-guide/named-operations.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/administration-guide/operation-score.md b/docs/administration-guide/operation-score.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/administration-guide/schema.md b/docs/administration-guide/schema.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/administration-guide/security/security-guide.md b/docs/administration-guide/security/security-guide.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/administration-guide/security/user-control.md b/docs/administration-guide/security/user-control.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/administration-guide/where-to-run-gaffer/gaffer-docker.md b/docs/administration-guide/where-to-run-gaffer/gaffer-docker.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mkdocs.yml b/mkdocs.yml index 9c70f0ce42..a0f9459e0c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -144,6 +144,26 @@ nav: - development-guide/project-structure/components/libraries/time.md - Maven Dependencies: - 'Spring': 'development-guide/project-structure/maven-dependencies/spring.md' + - Administration Guide: + - 'How to Run Gaffer': + - 'Gaffer Docker': 'administration-guide/where-to-run-gaffer/gaffer-docker.md' + - 'Store Setup': + - 'Store Guide': 'administration-guide/gaffer-stores/store-guide.md' + - 'Accumulo Store': 'administration-guide/gaffer-stores/accumulo-store.md' + - 'Federated Store': 'administration-guide/gaffer-stores/federated-store.md' + - 'Map Store': 'administration-guide/gaffer-stores/map-store.md' + - 'Proxy Store': 'administration-guide/gaffer-stores/proxy-store.md' + - 'Gaffer Configuration': + - 'Configuration': 'administration-guide/gaffer-config/config.md' + - 'Proxy': 'administration-guide/gaffer-config/proxy.md' + - 'URL': 'administration-guide/gaffer-config/url.md' + - 'Schema': 'administration-guide/schema.md' + - 'Import/Export Data': 'administration-guide/import-export-data.md' + - 'Named Operations': 'administration-guide/named-operations.md' + - 'Operation Score': 'administration-guide/operation-score.md' + - 'Secuirty': + - 'Secuity Guide': 'administration-guide/security/security-guide.md' + - 'User Control': 'administration-guide/security/user-control.md' - 'Reference': - 'Reference Guide': 'reference/intro.md' - 'Glossary': 'reference/glossary.md' From 5e4ce5d1a0df673b7da0399de19c18da950f11db Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Mon, 4 Sep 2023 16:31:11 +0000 Subject: [PATCH 05/20] Added new change notes structure --- docs/change-notes/changelist/v2-changes.md | 130 +++++++++++++++++++ docs/change-notes/migrating-from-v1-to-v2.md | 0 mkdocs.yml | 4 + 3 files changed, 134 insertions(+) create mode 100644 docs/change-notes/changelist/v2-changes.md create mode 100644 docs/change-notes/migrating-from-v1-to-v2.md diff --git a/docs/change-notes/changelist/v2-changes.md b/docs/change-notes/changelist/v2-changes.md new file mode 100644 index 0000000000..dbcec62b14 --- /dev/null +++ b/docs/change-notes/changelist/v2-changes.md @@ -0,0 +1,130 @@ +# Gaffer 2 Changelist + +Below is a summary of changes that have been made in Gaffer version 2. + +### Accumulo 2 Support +The Accumulo store now supports Accumulo 2 and Hadoop 3 by default, with support for Accumulo 1 and Hadoop 2 retained. See the [Accumulo Migration page](accumulo-migration.md) for more information about this change. + +### Federated Store Improvements +The Federated Operation was added to greatly improve flexibility of using a Federated Store. +!!! danger "Breaking change" + To migrate, please see the [Federated Store Changes page](federation-changes.md). + +### Cache Improvements and fixes +All "caches" within Gaffer received a lot of bug fixes which should make them significantly more stable and consistent over time. This should improve usability of FederatedStores, NamedOperations and NamedViews. +!!! danger "Breaking change" + The cache will need to be reloaded, as the new internal cache interface has changed. To do this, export all of the contents of your cache, upgrade, then re-add everything manually. + +### Removal of Deprecated code +All of Gaffer 1's deprecated code has been removed. +!!! danger "Breaking change" + To migrate, please see the [deprecations](deprecations.md) page. + +### Dependency Upgrades +Dependencies have been updated, where possible to the latest version, removing vulnerabilities. +!!! danger "Breaking change" + You will need to migrate your dependencies to be compatible with Gaffer 2's new dependency versions. Please see the [dependencies](dependencies.md) page for full details. + +### Federated and Proxy store fixes +A lot of bugs have been fixed that should facilitate FederatedStores with ProxyStores in them. +!!! danger "Breaking change" + The unique store trait `DYNAMIC_SCHEMA` has been removed from Gaffer. Simply removing it from custom FederatedStore implementations should be an adequate fix. + +### Removal of CloseableIterable +The `CloseableIterable` class has been removed so Operations like `GetAllElements` now return an `Iterable` instead, but the result still implements `Closeable`. +!!! danger "Breaking change" + Everywhere `CloseableIterable` was used in client code should be replaced with an `Iterable`: + ```java + final CloseableIterable results = graph.execute(new GetAllElements(), USER); + ``` + ```java + final Iterable results = graph.execute(new GetAllElements(), USER); + ``` + +### Removal of HBase and Parquet stores +The HBase and Parquet stores have been removed from Gaffer in version 2. We made posts for both the [HBase](https://github.com/gchq/Gaffer/issues/2367) and [Parquet](https://github.com/gchq/Gaffer/discussions/2557) stores to understand the levels of usage. It was then decided to remove both stores as this would make introducing various improvements easier in the long term. HBase and Parquet remain available in Gaffer version 1. In the future, they could be reimplemented for Gaffer 2, though we do not plan to currently. +!!! danger "Breaking change" + We would recommend instead using an Accumulo Store. If you would like these store implementations in Gaffer 2, or any other potential store for that matter, please [make an issue](https://github.com/gchq/Gaffer/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.md&title=) on GitHub. + +### Gaffer now builds with Java 8 and Java 11 +There is now a maven profile that will swap dependency versions so you can build Gaffer with Java 11. The code has also been updated to build with both Java versions. + +### Accumulo Kerberos Authentication Support +The Accumulo store now supports authenticating to Accumulo and HDFS using Kerberos, in addition to username/password. For more information, see the [Kerberos support page](accumulo-kerberos.md). + +### CSV Import and Export +Basic support for importing and exporting [CSVs](../getting-started/guide/csv.md) has been added. + +### All operations can now be used within NamedOperations +Previously, `GetElementsBetweenSets` could not be used within a NamedOperation as it used `inputB`. `GetElementsBetweenSets` and `inputB` have both been deprecated and instead you should use `GetElementsBetweenSetsPairs`. +??? example + Old operation now deprecated: + ```json + { + "class": "GetElementsBetweenSets", + "input": [ + { + "class": "EntitySeed", + "vertex": "firstInput" + } + ], + "inputB": [ + { + "class": "EntitySeed", + "vertex": "secondInput" + } + ] + } + ``` + New operation that will work within NamedOperations: + ```json + { + "class": "GetElementsBetweenSetsPairs", + "input": { + "class": "Pair", + "first": { + "ArrayList" : [ + { + "class": "EntitySeed", + "vertex": "firstInput" + } + ] + }, + "second": { + "ArrayList" : [ + { + "class": "EntitySeed", + "vertex": "secondInput" + } + ] + } + } + } + ``` + +### Ability to set OperationDeclarations during AddGraph +This will mean subgraphs added to FederatedStores can have additional operation handlers set when they are added. You can directly provide the OperationsDeclarations json to the store properties with `gaffer.store.operation.declarations.json`. +??? example + ``` json + { + "class": "AddGraph", + "graphId": "myGraph", + "schema": {}, // (1)! + "storeProperties": { + "gaffer.store.class": "MapStore", + "gaffer.store.operation.declarations.json": { + "operations": [ + { + "operation": "ImportFromLocalFile", // (2)! + "handler": { + "class": "ImportFromLocalFileHandler" + } + } + ] + } + } + } + ``` + + 1. Schema left empty for brevity + 2. This example operation enables file import. Read more in the [CSV](../getting-started/guide/csv.md) docs. diff --git a/docs/change-notes/migrating-from-v1-to-v2.md b/docs/change-notes/migrating-from-v1-to-v2.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mkdocs.yml b/mkdocs.yml index a0f9459e0c..af069dd19e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -164,6 +164,10 @@ nav: - 'Secuirty': - 'Secuity Guide': 'administration-guide/security/security-guide.md' - 'User Control': 'administration-guide/security/user-control.md' + - Change Notes: + - 'Changelist': + - 'V2.0 Changes': 'change-notes/changelist/v2-changes.md' + - 'Migrating from V1 to V2': 'change-notes/migrating-from-v1-to-v2.md' - 'Reference': - 'Reference Guide': 'reference/intro.md' - 'Glossary': 'reference/glossary.md' From 7e091a5e767243fba194102117f72aa680cac9bf Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Mon, 4 Sep 2023 16:39:29 +0000 Subject: [PATCH 06/20] Corrected spelling mistakes --- mkdocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index af069dd19e..a65316d6b2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -161,8 +161,8 @@ nav: - 'Import/Export Data': 'administration-guide/import-export-data.md' - 'Named Operations': 'administration-guide/named-operations.md' - 'Operation Score': 'administration-guide/operation-score.md' - - 'Secuirty': - - 'Secuity Guide': 'administration-guide/security/security-guide.md' + - 'Security': + - 'Security Guide': 'administration-guide/security/security-guide.md' - 'User Control': 'administration-guide/security/user-control.md' - Change Notes: - 'Changelist': From 69028a37424882af29e6016bbdd2a2184587f0cc Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:09:50 +0000 Subject: [PATCH 07/20] Fixed paths and added gaffer docker doc --- .../where-to-run-gaffer/gaffer-docker.md | 37 +++++++++++++++++++ docs/index.md | 4 ++ mkdocs.yml | 8 +--- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/docs/administration-guide/where-to-run-gaffer/gaffer-docker.md b/docs/administration-guide/where-to-run-gaffer/gaffer-docker.md index e69de29bb2..4a983c548b 100644 --- a/docs/administration-guide/where-to-run-gaffer/gaffer-docker.md +++ b/docs/administration-guide/where-to-run-gaffer/gaffer-docker.md @@ -0,0 +1,37 @@ +# Gaffer Docker + +The [gaffer-docker](https://github.com/gchq/gaffer-docker) repository contains +all code needed to run Gaffer using Docker. + +All the files needed to get started using Gaffer in Docker are contained in the +['docker'](https://github.com/gchq/gaffer-docker/tree/develop/docker) +sub-folder. + +In this directory you can find the Dockerfiles and docker compose files for +building container images for: + +- [Gaffer](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer) +- [Gaffer's REST + API](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-rest) +- [Gaffer's Road Traffic + Example](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-road-traffic-loader) +- [HDFS](https://github.com/gchq/gaffer-docker/tree/develop/docker/hdfs) +- [Accumulo](https://github.com/gchq/gaffer-docker/tree/develop/docker/accumulo) +- [Gaffer's Integration + Tests](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-integration-tests) +- [gafferpy Jupyter + Notebook](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-pyspark-notebook) +- [Gaffer's JupyterHub Options + Server](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-jhub-options-server) +- [Spark](https://github.com/gchq/gaffer-docker/tree/develop/docker/spark-py) + +Each directory contains a README with more specific information on what these +images are for and how to build them. + +Please note that some of these containers will only be useful if utilised by the +Helm Charts under Kubernetes, and may not be possible to run on their own. + +## Requirements + +Before you can build and run these containers you will need to install Docker or +a compatible equivalent (e.g. Podman). diff --git a/docs/index.md b/docs/index.md index 5aa1f218f1..d1a7adf7c7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,6 +19,10 @@ Site Contents - [Gaffer 2.0](gaffer2.0/changelist/) - Information on the Gaffer 2.0 release, including changelogs, deprecation and migration details. - [Getting Started](getting-started/quickstart/) - Guides for using and deploying Gaffer. - [Developer Info](dev/development/) - Information about developing Gaffer itself. +- [User Guide](user-guide/gaffer-basics/what-is-gaffer/) - Information about how to use Gaffer. +- [Development Guide](development-guide/getting-started/) - Information about developing Gaffer itself. +- [Administration GUide](adiministration-guide/how-to-run-gaffer/gaffer-docker/) - Information about creating a gaffer instance. +- [Change Notes](change-notes/changelist/v2-changes/) - Information on the Gaffer 2.0 release, including changelogs, deprecation and migration details. - [Reference](reference/intro/) - Documentation for Gaffer Operations, Predicates, Functions, Binary Operators and Properties. Licence diff --git a/mkdocs.yml b/mkdocs.yml index a65316d6b2..298f7211be 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -101,7 +101,7 @@ nav: - User Guide: - 'Gaffer Basics': - 'What is Gaffer?': 'user-guide/gaffer-basics/what-is-gaffer.md' - - 'What is Graph?': 'user-guide/gaffer-basics/what-is-graph.md' + - 'What is a Graph?': 'user-guide/gaffer-basics/what-is-a-graph.md' - 'What is JSON?': 'user-guide/gaffer-basics/what-is-json.md' - 'What is Python?': 'user-guide/gaffer-basics/what-is-python.md' - 'Getting Started': @@ -172,12 +172,6 @@ nav: - 'Reference Guide': 'reference/intro.md' - 'Glossary': 'reference/glossary.md' - 'Javadoc': 'reference/javadoc.md' - - Stores: - - 'Store Guide': 'reference/stores-guide/stores.md' - - 'Map Store': 'reference/stores-guide/map.md' - - 'Accumulo Store': 'reference/stores-guide/accumulo.md' - - 'Proxy Store': 'reference/stores-guide/proxy.md' - - 'Federated Store': 'reference/stores-guide/federated.md' - Properties: - 'Properties Guide': 'reference/properties-guide/properties.md' - 'Basic Properties': 'reference/properties-guide/basic.md' From ea6cdbc1742e1d447819212261217b54eb63b56e Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Tue, 5 Sep 2023 15:01:39 +0000 Subject: [PATCH 08/20] Added example deployment to developer docs --- .../example-deployment/project-setup.md | 231 +++++++++++++++ .../running-the-deployment.md | 219 +++++++++++++++ .../example-deployment/using-the-api.md | 191 +++++++++++++ .../example-deployment/writing-the-schema.md | 264 ++++++++++++++++++ mkdocs.yml | 5 + 5 files changed, 910 insertions(+) create mode 100644 docs/development-guide/example-deployment/project-setup.md create mode 100644 docs/development-guide/example-deployment/running-the-deployment.md create mode 100644 docs/development-guide/example-deployment/using-the-api.md create mode 100644 docs/development-guide/example-deployment/writing-the-schema.md diff --git a/docs/development-guide/example-deployment/project-setup.md b/docs/development-guide/example-deployment/project-setup.md new file mode 100644 index 0000000000..1eb687b6c4 --- /dev/null +++ b/docs/development-guide/example-deployment/project-setup.md @@ -0,0 +1,231 @@ +# Example Deployment + +This guide will run through the start up and deployment of a basic Gaffer instance. It will cover +how to write a basic Gaffer Schema from scratch along with using the pre-made containers to run the +Gaffer rest API and Accumulo based data store. + +!!! warning + Please be aware that the example is only intended to demonstrate the core Gaffer concepts it is + not a production example. Various additional configuration to Accumulo and the HDFS set up would + be required to make this production ready and likely be specific to your infrastructure. + +## The Example Graph + +For this basic example we will attempt to recreate the graph in the following diagram consisting of +two nodes (vertexes) with one directed edge between them. + +```mermaid +graph LR + A(["Person + + name: marko + age: 29"]) + -- + "Created + weight: 0.4" + --> + B(["Software + + name: lop + lang: java"]) +``` + +This data describes one individual and a single piece of software that has been created by that +individual. The data will be loaded into the graph from a CSV file that follows the Neo4j export +syntax, this demonstrates how Gaffer can be used and how it can interact, model and query data from +other popular graph databases. Even with this basic graph we should be able to start building +queries to ask questions like "Who created the software called 'lop'?" and "How much did 'marko' +contribute to the software called 'lop'?" etc. + +To go with the diagram above the following CSV file (both raw and rendered are provided) represents +the graph in [Neo4j syntax](https://neo4j.com/labs/apoc/4.4/export/csv/#export-database-csv). + +!!! note "" + Please note that Gaffer often requires additional information about the data such as, + `:String` on the column headers to help with typing of the values. This is demonstrated below + in the raw file. There's more detail on this in the [OpenCypher documentation](../advanced-guide/import-export/csv.md#opencypher-formats). + +=== "Table" + | _id | name | age | lang | _labels | _start | _end | _type | weight | + |-----|-------|-----|------|----------|--------|------|---------|--------| + | v1 | marko | 29 | | Person | | | | | + | v2 | lop | | java | Software | | | | | + | e1 | | | | | v1 | v2 | Created | 0.4 | + +=== "CSV" + ```csv + _id,name:String,age:Int,lang:String,_labels,_start,_end,_type,weight:Float + v1,marko,29,,Person,,,, + v2,lop,,java,Software,,,, + e1,,,,,v1,v2,Created,0.4 + ``` + +## Project Setup + +First you must set up the files and directories you will need for the instance. As it stands there +are a couple of different ways to run a Gaffer project this example will use a logical structure +that suites a stand alone deployment consisting of the following file structure: + +!!! example "Example Gaffer project structure" + + !!! tip "" + Click the plus symbols for a brief description of each file + + ```yaml + ├── config + │ ├── accumulo + │ │ ├── accumulo-client.properties + │ │ ├── accumulo-env.sh + │ │ ├── accumulo.properties + │ │ ├── core-site.xml + │ │ └── log4j.properties + │ ├── gaffer + │ │ ├── application.properties #(1)! + │ │ ├── data #(2)! + │ │ │ ├── neo4jExport.csv + │ │ ├── graph + │ │ │ └── graphConfig.json #(3)! + │ │ ├── schema + │ │ │ ├── elements.json #(4)! + │ │ │ └── types.json #(5)! + │ │ └── store + │ │ ├── operationsDeclarations.json #(6)! + │ │ └── store.properties #(7)! + │ └── hdfs + │ ├── core-site.xml + │ ├── hdfs-site.xml + │ └── log4j.properties + └── docker-compose.yaml #(8)! + ``` + + 1. Properties file that generally sets the file locations of other Gaffer + configs e.g. schemas (note these are the absolute paths inside the + container). + 2. Any data files, e.g. CSV, to be made available to the Gaffer container. + 3. The main graph config file to set various properties of the overall graph. + 4. This file holds the schema outlining the elements in the graph, e.g. the + nodes (aka entities) and edges. + 5. This file defines the different data types in the graph and how they are + serialised to Java classes. + 6. Config file for additional Gaffer operations and set the class to handle + them on the store. + 7. The General store properties, sets up what store to use and any additional + configuration. + 8. This file controls which containers will be started up and the configuration + of them to ensure correct ports and files are available. + +All the files in the `config/accumulo/` and `config/hdfs/` directories will be copied directly from +the two locations in the Gaffer docker repo, +[here](https://github.com/gchq/gaffer-docker/tree/develop/docker/accumulo/conf-2.0.1) and +[here](https://github.com/gchq/gaffer-docker/tree/develop/docker/hdfs/conf). The configuration of +these are out of scope of this example but are covered in other sections of the documentation. The +main focus of this guide will be on the configuration files under the `config/gaffer/` directory. + +## Configuration Files + +There's a full [break down of Gaffer schema files on the next page](./writing-the-schema.md), this +section will instead cover the smaller additional config files that go along side the main Gaffer +schema to tweak other aspects of the graph. The location of these files will need to be volume +mounted into the container for them to be included in the deployment which is covered in more detail +[later in the guide](./running-the-deployment.md). + +!!! note + Many of these files have defaults already in the standard `gaffer-rest` container image, but its + useful to still include them in the project to allow easy configuration. + +### Application Properties + +This is probably the simplest configuration file in the Gaffer deployment. In general it borrows a +concept from [Spring Boot](https://docs.spring.io/spring-boot/docs/current/reference/html/application-properties.html) +to allow changing the context root and any properties related to Gaffer. In the example that follows +we use it to set the file location properties of where the other config files are (inside the +container). + +```properties title="application.properties" +gaffer.schemas=/gaffer/schema +gaffer.storeProperties=/gaffer/store/store.properties +gaffer.graph.config=/gaffer/graph/graphConfig.json +``` + +### Graph Configuration + +The graph configuration file is a JSON file that configures few bits of the Gaffer graph. Primarily +it is used to set the name and description along with any additional hooks to run before an operation +chain e.g. to impose limits on max results etc. For the example as, it is a very basic graph we just +set the name and short description. + +```json title="graphConfig.json" +{ + "graphId": "ExampleGraph", + "description": "An example graph" +} +``` + +### Store Properties + +The store properties file is used to configure how Gaffer will store its data. There are a few +different stores available for Gaffer, these are explained in more detail in the [reference +documentation](../../reference/stores-guide/stores.md), but by default you must provide a store +class and a store properties class. For this example we are using an Accumulo store as it is +recommended for efficient storage and retrieval of large data volumes. It's set up requires a few +custom properties which are outlined in the following file. + +```properties title="store.properties" +gaffer.store.class=uk.gov.gchq.gaffer.accumulostore.AccumuloStore +gaffer.store.properties.class=uk.gov.gchq.gaffer.accumulostore.AccumuloProperties + +# Accumulo specific config +accumulo.instance=accumulo +accumulo.zookeepers=zookeeper +accumulo.user=root +accumulo.password=secret + +# General store config +gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService +gaffer.store.job.tracker.enabled=true +gaffer.store.operation.declarations=/gaffer/store/operationsDeclarations.json +``` + +### Operations Declarations + +The operation declarations file is a way of enabling additional operations in Gaffer. By default +there are some built in operations already available (the rest API has a get all operations request +to see a list), but its likely you might want to enable others or add your own custom ones. As the +example will load its data from a local CSV file we can activate a couple of additional operations +using the following file. + +```json title="operationsDeclarations.json" +{ + "operations": [ + { + "operation": "uk.gov.gchq.gaffer.operation.impl.export.localfile.ImportFromLocalFile", + "handler": { + "class": "uk.gov.gchq.gaffer.store.operation.handler.export.localfile.ImportFromLocalFileHandler" + } + }, + { + "operation": "uk.gov.gchq.gaffer.operation.impl.export.localfile.ExportToLocalFile", + "handler": { + "class": "uk.gov.gchq.gaffer.store.operation.handler.export.localfile.ExportToLocalFileHandler" + } + } + ] +} +``` + +The two additional operations already exist in Gaffer (in the code base: +[ImportFromLocalFile](https://github.com/gchq/Gaffer/blob/develop/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/export/localfile/ImportFromLocalFile.java) +and +[ExportToLocalFile](https://github.com/gchq/Gaffer/blob/develop/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/export/localfile/ExportToLocalFile.java)), +what this file is doing is essentially activating them and setting the handler class for them. The +`ImportFromLocalFile` usage is demonstrated in the [using the API](./using-the-api.md) section to +load some data. + +This operation allows us to pass a local CSV file (in the container) which will be read line by line +and get a stream of the line strings. This is very useful when we start using Operation Chains as +we can pass this stream of data as the input to the next operation in the chain similar to shell +pipes. + +!!! note + The location of the file needs to be set via the store properties file using the + `gaffer.store.operation.declarations` property (see [previous section](#store-properties)). diff --git a/docs/development-guide/example-deployment/running-the-deployment.md b/docs/development-guide/example-deployment/running-the-deployment.md new file mode 100644 index 0000000000..10d492d455 --- /dev/null +++ b/docs/development-guide/example-deployment/running-the-deployment.md @@ -0,0 +1,219 @@ +# Running the Deployment + +To run the containers there are a couple of options but primarily the two main ways are with either +Docker Compose or Kubernetes (via Helm), the [gaffer-docker](https://github.com/gchq/gaffer-docker) +repository has some examples of how to run both. + +## Docker Compose + +For this example we will use a slightly modified version of the docker compose config file used in +the repository. + +??? example "docker-compose.yaml" + ```yaml + version: "3.7" + + services: + + zookeeper: + image: zookeeper:${ZOOKEEPER_VERSION} + healthcheck: + test: echo ruok | nc 127.0.0.1 2181 | grep imok + interval: 30s + timeout: 5s + retries: 3 + container_name: zookeeper + hostname: zookeeper + environment: + - ZOO_SERVERS=server.1=zookeeper:2888:3888;2181 + - ZOO_4LW_COMMANDS_WHITELIST=* + volumes: + - /data + - /datalog + + hdfs-namenode: + image: gchq/hdfs:${HADOOP_VERSION} + depends_on: + zookeeper: + condition: service_healthy + healthcheck: + test: curl -f http://localhost:9870 || exit 1 + interval: 30s + timeout: 10s + retries: 3 + command: namenode + container_name: hdfs-namenode + hostname: hdfs-namenode + environment: + - HADOOP_CONF_DIR=${HADOOP_CONF_DIR} + ports: + - 9870:9870 + volumes: + - ./configs/hdfs:${HADOOP_CONF_DIR}:ro + - /var/log/hadoop + - /data1 + - /data2 + + hdfs-datanode: + image: gchq/hdfs:${HADOOP_VERSION} + depends_on: + hdfs-namenode: + condition: service_healthy + command: datanode + container_name: hdfs-datanode + hostname: hdfs-datanode + environment: + - HADOOP_CONF_DIR=${HADOOP_CONF_DIR} + volumes: + - ./configs/hdfs:${HADOOP_CONF_DIR}:ro + - /var/log/hadoop + - /data1 + - /data2 + + accumulo-master: + image: gchq/gaffer:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} + depends_on: + hdfs-namenode: + condition: service_healthy + healthcheck: + test: cat /proc/net/tcp | grep 270F + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + build: + context: . + args: + GAFFER_VERSION: ${GAFFER_VERSION} + BASE_IMAGE_NAME: gchq/accumulo + BASE_IMAGE_TAG: ${ACCUMULO_VERSION} + command: master + container_name: accumulo-master + hostname: accumulo-master + environment: + - ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} + - HADOOP_USER_NAME=hadoop + volumes: + - ./configs/accumulo:${ACCUMULO_CONF_DIR}:ro + - /var/log/accumulo + + accumulo-tserver: + image: gchq/gaffer:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} + depends_on: + accumulo-master: + condition: service_healthy + healthcheck: + test: cat /proc/net/tcp | grep 270D + interval: 30s + timeout: 5s + retries: 3 + command: tserver + container_name: accumulo-tserver + hostname: accumulo-tserver + environment: + - ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} + - HADOOP_USER_NAME=hadoop + volumes: + - ./configs/accumulo:${ACCUMULO_CONF_DIR}:ro + - /var/log/accumulo + + accumulo-monitor: + image: gchq/gaffer:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} + depends_on: + accumulo-master: + condition: service_healthy + command: monitor + container_name: accumulo-monitor + hostname: accumulo-monitor + environment: + - ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} + - HADOOP_USER_NAME=hadoop + ports: + - 9995:9995 + volumes: + - ./configs/accumulo:${ACCUMULO_CONF_DIR}:ro + - /var/log/accumulo + + accumulo-gc: + image: gchq/gaffer:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} + depends_on: + accumulo-master: + condition: service_healthy + command: gc + container_name: accumulo-gc + hostname: accumulo-gc + environment: + - ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} + - HADOOP_USER_NAME=hadoop + volumes: + - ./configs/accumulo:${ACCUMULO_CONF_DIR}:ro + - /var/log/accumulo + + gaffer-rest: + image: gchq/gaffer-rest:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} + depends_on: + accumulo-tserver: + condition: service_healthy + ports: + - 8080:8080 + volumes: + - ./configs/gaffer/application.properties:/gaffer/config/application.properties:ro + - ./configs/gaffer/data:/gaffer/data:ro + - ./configs/gaffer/graph:/gaffer/graph:ro + - ./configs/gaffer/schema:/gaffer/schema:ro + - ./configs/gaffer/store:/gaffer/store:ro + ``` + +If you are not familiar with docker or docker compose there are plenty of +[resources](https://docs.docker.com/compose/) online to get up to speed but essentially, it is a +config file that will run up multiple containers with various bits of configuration which is a lot +easier than typing out multiple docker commands! + +Some key bits you may want to configure in the file are the shared volumes (under the `volumes` +section). The locations in the example file assume you use the project structure from the +[example setup](./project-setup.md) but if you change any of the locations then they will need +updating. + +To run up the cluster its as easy as running the following from the root of the project. + +```bash +docker compose up +``` + +The above configuration will start the following containers: + +- Zookeeper +- HDFS + - Datanode + - Namenode +- Accumulo + - Monitor + - GC + - tServer + - Master +- Gaffer Rest API + +The web UIs for nodes in cluster can then be accessed at the following addresses: + +- Access the HDFS NameNode web UI at: +- Access the Accumulo Monitor UI at: +- Access the Gaffer REST API at: + +## Environment Variables + +As you can probably see the example is using a few environment variables in the +`docker-compose.yaml` file, these set things such as the container versions and a couple of file +locations. The use of these variables are recommended as it can make it easier to update container +versions and other aspects of the containers. + +A basic set of these environment variables are shown below which can be saved in a `.env` file and +sourced before running the containers. + +```bash +ZOOKEEPER_VERSION="3.7.1" +GAFFER_VERSION="2.0.0" +ACCUMULO_VERSION="2.0.1" +HADOOP_VERSION="3.3.3" +ACCUMULO_CONF_DIR="/etc/accumulo/conf" +HADOOP_CONF_DIR="/etc/hadoop/conf" +``` diff --git a/docs/development-guide/example-deployment/using-the-api.md b/docs/development-guide/example-deployment/using-the-api.md new file mode 100644 index 0000000000..d592ff307c --- /dev/null +++ b/docs/development-guide/example-deployment/using-the-api.md @@ -0,0 +1,191 @@ +# Using the API + +As covered in the [Gaffer basics](../basics.md) the main POST request used in the API is +`/graph/operations/execute`. This part of the guide will cover the general usage of this part of the +API and walk through some general operations you might want to use to load data or query. + +!!! note + The [Gaffer operations reference guide](../../reference/operations-guide/operations.md) + has more detail on available operations. + +## Loading Data + +Gaffer supports various methods of loading data and depending on your use case you can even bypass +it all together to load directly into Accumulo. + +This example will focus on using the rest API to add the graph elements. In production this method +would not be recommended for large volumes of data. However, it is fine for smaller data sets and +generally can be done in a few stages outlined in the following diagram. + +```mermaid +flowchart LR + A(Raw Data) --> B(GenerateElements) + B --> C(AddElements) +``` + +The first stage is taking the raw input data and converting it into Gaffer elements via an element +generator class. Gaffer includes a few built in +[generators](../../reference/operations-guide/generate.md) but you can use a custom class or +pre-process the data before passing to Gaffer so that you're able to use a default generator. Once +the data has been converted to elements it needs to be added into the graph. To load elements there +is a standard `AddElements` operation which takes raw elements JSON as input and adds them into the +graph. + +!!! info + This is where the schema is used here to validate the elements are correct and conform before + adding. + +Using the example again we will demonstrate how we could write an operation chain to load the data +from the neo4j formatted CSV file. + +```json +{ + "class": "OperationChain", + "operations": [ + { + "class": "ImportFromLocalFile", + "filePath": "/gaffer/data/neo4jExport.csv" + }, + { + "class": "GenerateElements", + "elementGenerator": { + "class": "Neo4jCsvElementGenerator" + } + }, + { + "class": "AddElements" + } + ] +} +``` + +The operation chain above essentially mirrors the stages in the previous diagram. In the example +chain we first ingest the data via the `ImportFromLocalFile` class (an additional operation we added +via the [`operationsDeclarations.json`](./additional-config.md#operations-declarations)), which +streams the data from the CSV file into the next `GenerateElements` operation. + +For the generator we have selected the built in `Neo4jCsvElementGenerator` class, this is already +set up to be able to parse a correctly formatted neo4j exported CSV into Gaffer elements via the +schema. If you are curious as to what the output of each operation is you can try run a subset of +this chain to see how the data changes on each one, the output should be returned back to you in the +server response section of the Swagger API. + +## Querying Data + +Once data is loaded in the graph its now possible to start querying the data to gain insight and +perform analytics. Querying in Gaffer can get fairly complex but generally simple queries are made +up of two parts; a `Get` Operation and a `View`. + +Starting with the `Get` operation, say we want to get all nodes and edges based on their ID. To do +this we can use the `GetElements` operation and set the `Seed` to the entity (e.g. node) or edge +where we want to start the search. To demonstrate this on the example graph we can attempt to get +all entities and edges associated with the `Person` node with ID `v1`. + +The result from this query should return the node associated with the `v1` id along with any edges +on this node, which in this case is just one + +=== "Input Query" + ```json + { + "class": "GetElements", + "input": [ + { + "class": "EntitySeed", + "vertex": "v1" + } + ] + } + ``` + +=== "Example Result" + + ```json + [ + { + "class": "uk.gov.gchq.gaffer.data.element.Entity", + "group": "Person", + "vertex": "v1", + "properties": { + "name": "marko", + "age": 29 + } + }, + { + "class": "uk.gov.gchq.gaffer.data.element.Edge", + "group": "Created", + "source": "v1", + "destination": "v2", + "directed": true, + "matchedVertex": "SOURCE", + "properties": { + "weight": { + "java.lang.Float": 0.4 + } + } + } + ] + ``` + +### Filtering Data + +The final bit of querying this guide will go into is how to apply a `View` to a returned set of +elements. A `View` in Gaffer allows you to filter, aggregate, transform and just generally +manipulate the results. In general a `View` has the following possible use cases: + +- **Filtering** - General filtering on elements based on predicates. Filtering can be applied + pre-aggregation, post aggregation and post transformation. + +- **Aggregation** - This is to control how similar elements are aggregated together. You can provide + a subset of the schema `groupBy` properties and override the aggregation functions. + +- **Transformation** - Transformations can be applied by providing Functions to transform properties + and vertex values. This is a powerful feature, you can override the existing values or you can + transform and save the new value into a new transient property. + +- **Property Removal** - The relevant properties you want to be returned can be controlled. You can + use either `properties` or `excludeProperties` to define the list of properties to be included + or excluded. + +Taking the example from the previous section we will demonstrate general filtering on a query. As +before, the query returns the node `v1` and any edges associated with it. We will now filter it to +include only edges where the weight is over a certain value. In this scenario it is analogous to +asking, *"get all the `Created` edges on node `v1` that have a `weight` greater than 0.3"*. + +=== "Filter Query" + + ```json + { + "class": "GetElements", + "input": [ + { + "class": "EntitySeed", + "vertex": "v1" + } + ], + "view": { + "edges": { + "Created": { + "preAggregationFilterFunctions": [ + { + "selection": [ + "weight" + ], + "predicate": { + "class": "IsMoreThan", + "orEqualTo": false, + "value": { + "Float": 0.3 + } + } + } + ] + } + } + } + } + ``` + +!!! tip + As you can see filtering is based around predicates which are similar to if else statements in + traditional programming. For a full list of available predicates refer to the reference + [documentation](../../reference/predicates-guide/predicates.md). diff --git a/docs/development-guide/example-deployment/writing-the-schema.md b/docs/development-guide/example-deployment/writing-the-schema.md new file mode 100644 index 0000000000..0be48ba33c --- /dev/null +++ b/docs/development-guide/example-deployment/writing-the-schema.md @@ -0,0 +1,264 @@ +# Writing the Schema + +In Gaffer JSON based schemas need to be written upfront to model and understand how to load and +treat the data in the graph. These schemas define all aspects of the nodes and edges in the graph, +and can even be used to automatically do basic analysis or aggregation on queries and ingested data. + +For reference, this guide will use the same CSV data set from the [project setup](./project-setup.md#the-example-graph) page. + +=== "Table" + | _id | name | age | lang | _labels | _start | _end | _type | weight | + |-----|-------|-----|------|----------|--------|------|---------|--------| + | v1 | marko | 29 | | Person | | | | | + | v2 | lop | | java | Software | | | | | + | e1 | | | | | v1 | v2 | Created | 0.4 | + +=== "CSV" + ```csv + _id,name:String,age:Int,lang:String,_labels,_start,_end,_type,weight:Float + v1,marko,29,,Person,,,, + v2,lop,,java,Software,,,, + e1,,,,,v1,v2,Created,0.4 + ``` + +## Elements Schema + +In Gaffer an element refers to any object in the graph, i.e. your nodes (vertexes) and edges. To set +up a graph we need to tell Gaffer what objects are in the graph and the properties they have. The +standard way to do this is a JSON config file in the schema directory. The filename can just be +called something like `elements.json`, the name is not special as all files under the `schema` +directory will be merged into a master schema but its still recommended to use an appropriate name. + +As covered in the [Gaffer basics](../basics.md), to write a schema you can see that there are some +required fields, but largely a schema is highly specific to your input data. + +Starting with the `entities` from the example, we can see there will be two distinct types of nodes +in the graph; one representing a `Person` and another for `Software`. These can be added into the +schema to give something like the the following: + +!!! info "" + + The types here such as `id.person.string` are covered in the [next section](#types-schema). + +```json +{ + "entities": { + "Person": { + "description": "Entity representing a person vertex", + "vertex": "id.person.string" + }, + "Software": { + "description": "Entity representing a software vertex", + "vertex": "id.software.string" + } + } +} +``` + +From the basic schema you can see that we have added two entity types for the graph. For now, each +`entity` just contains a short description and a type associated to the `vertex` key. The type here +is just a place holder but it has been named appropriately as it's assumed that we will just use the +string representation of the node's id (this will be defined in the `types.json` later in the +guide). + +Expanding on the basic schema we will now add the `edges` to the graph. As the example graph is +small we only need to add one edge - the `Created` edge. This is a directed edge that connects a +`Person` to a `Software` and can be defined as the following. + +```json +{ + "edges": { + "Created": { + "source": "id.person.string", + "destination": "id.software.string", + "directed": "true" + } + }, + "entities": { + "Person": { + "description": "Entity representing a person vertex", + "vertex": "id.person.string" + }, + "Software": { + "description": "Entity representing a software vertex", + "vertex": "id.software.string" + } + } +} +``` + +As discussed in the [basics guide](../basics.md), edges have some mandatory fields. Starting with +the `source` and `destination` fields, these must match the types associated with the vertex field +in the relevant entities. From the example, we can see that the source of a `Created` edge is a +`Person` so we will use the placeholder type we set as the `vertex` field which is +`id.person.string`. Similarly the destination is a `Software` node so we will use its placeholder of +`id.software.string`. + +We must also set whether an edge is directed or not, in this case it is as only a person can create +software not the other way around. To set this we will use the `true` type, but note that this is a +placeholder and must still be defined in the types.json. + +Continuing with the example, the nodes and edges also have some properties associated with each such +as name, age etc. These can also be added to the schema using a properties map to result in the +extended schema below. + +```json +{ + "edges": { + "Created": { + "source": "id.person.string", + "destination": "id.software.string", + "directed": "true", + "aggregate": "false", + "properties": { + "weight": "property.float" + } + } + }, + "entities": { + "Person": { + "description": "Entity representing a person vertex", + "vertex": "id.person.string", + "aggregate": "false", + "properties": { + "name": "property.string", + "age": "property.integer" + } + }, + "Software": { + "description": "Entity representing a software vertex", + "vertex": "id.software.string", + "aggregate": "false", + "properties": { + "name": "property.string", + "lang": "property.string" + } + } + } +} +``` + +!!! note + Take note of the `"aggregate": "false"` setting, this skips any ingest aggregation as it is not + required and out of scope of this example. All entity property types must have an aggregation + function in Gaffer unless this option is added. Aggregation is fairly advanced topic in Gaffer + but very powerful it is covered in more depth later in the documentation. + +## Types Schema + +The other schema that now needs to be written is the types schema. As you have seen in the elements +schema there are some place holder types added as the values for many of the keys. These types work +similarly to if you have ever programmed in a strongly typed language, they are essentially the +wrapper for the value to encapsulate it. + +Now starting with the types for the nodes/vertexes, we used two placeholder types, one for the +`Person` entity and one for the `Software` entity. From the example CSV you can see there is a `_id` +column that uses a string identifier that is used for the ID of the node (this will also be used by +the `edge` to identify the source and destination). We will define a type for each node ID using the +standard java `String` class to encapsulate it, this leads to a basic `type.json` like the +following. + +```json +{ + "types": { + "id.person.string": { + "description": "A basic type to hold the string id of a person entity", + "class": "java.lang.String" + }, + "id.software.string": { + "description": "A basic type to hold the string id of a person entity", + "class": "java.lang.String" + } + } +} +``` + +The next set of types that need defining are, the ones used for the properties that are attached to +the nodes/entities. Again we need to take a look back at what our input data looks like, in the CSV +file we can see there are three different types that are used for the properties which are analogous +to a `String`, an `Integer` and a `Float`. + +!!! tip + Of course technically, all of these properties could be encapsulated in a string but, assigning + a relevant type allows some additional type specific features when doing things like grouping + and aggregation as it would in traditional programming. + +If we make a type for each of the possible properties using the standard Java classes we end up with +the following. + +```json +{ + "types": { + "id.person.string": { + "description": "A basic type to hold the string id of a person entity", + "class": "java.lang.String" + }, + "id.software.string": { + "description": "A basic type to hold the string id of a person entity", + "class": "java.lang.String" + }, + "property.string": { + "description": "A type to hold string properties of entities", + "class": "java.lang.String" + }, + "property.integer": { + "description": "A basic type to hold integer properties of entities", + "class": "java.lang.Integer" + }, + "property.float": { + "description": "A basic type to hold float properties of entities", + "class": "java.lang.Float" + } + } +} +``` + +The final thing that we need to add to the schema is a type for the `true` Boolean value that's used +by the directed field of the edge element. This leaves us with the complete list of types for this +example. + +```json +{ + "types": { + "id.person.string": { + "description": "A basic type to hold the string id of a person entity", + "class": "java.lang.String" + }, + "id.software.string": { + "description": "A basic type to hold the string id of a person entity", + "class": "java.lang.String" + }, + "property.string": { + "description": "A type to hold string properties of entities", + "class": "java.lang.String" + }, + "property.integer": { + "description": "A basic type to hold integer properties of entities", + "class": "java.lang.Integer" + }, + "property.float": { + "description": "A basic type to hold float properties of entities", + "class": "java.lang.Float" + }, + "true": { + "description": "A simple boolean that must always be true.", + "class": "java.lang.Boolean", + "validateFunctions": [ + { + "class": "uk.gov.gchq.koryphe.impl.predicate.IsTrue" + } + ] + } + } +} +``` + +As you can see the Boolean value also demonstrates the validation feature which allows for +validation of any values using the type. In this example it verifies its true but you could also +check it exists, see if its less than another value etc. or even run your own custom validator +class. + +!!! tip + The Koryphe module provides lots of default functions that can be used to validate and aggregate + data, see the [predicate reference guide](../../reference/predicates-guide/koryphe-predicates.md) + for more information. diff --git a/mkdocs.yml b/mkdocs.yml index 298f7211be..91d93fa019 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -120,6 +120,11 @@ nav: - 'Examples': 'user-guide/query/gremlin/examples.md' - Development Guide: - 'Getting Started': 'development-guide/getting-started.md' + - 'Example Deployment': + - 'Project Setup': 'development-guide/example-deployment/project-setup.md' + - 'Writing the Schema': 'development-guide/example-deployment/writing-the-schema.md' + - 'Running the Deployment': 'development-guide/example-deployment/running-the-deployment.md' + - 'Using the API': 'development-guide/example-deployment/using-the-api.md' - 'Ways of Working': 'development-guide/ways-of-working.md' - 'Remote Coding Environments': 'development-guide/remote-coding-environments.md' - 'Extending Gaffer': 'development-guide/extending-gaffer.md' From aa85ef32c9d54e48a3489c5e2e1ade39cd990107 Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:41:42 +0000 Subject: [PATCH 09/20] Moved old v2 docs to new structure --- .../kubernetes-guide/add-libraries.md | 70 +++ .../change-accumulo-passwords.md | 36 ++ .../kubernetes-guide/change-graph-metadata.md | 63 ++ .../kubernetes-guide/deploy-empty-graph.md | 73 +++ .../kubernetes-guide/deploy-schema.md | 68 +++ .../kubernetes-guide/kubernetes-guide.md | 40 ++ docs/change-notes/migrating-from-v1-to-v2.md | 0 .../accumulo-kerberos.md | 93 +++ .../accumulo-migration.md | 27 + .../migrating-from-v1-to-v2/dependencies.md | 35 ++ .../migrating-from-v1-to-v2/deprecations.md | 563 ++++++++++++++++++ .../federation-changes.md | 240 ++++++++ .../migrating-from-v1-to-v2/log4j.md | 30 + docs/development-guide/getting-started.md | 35 ++ .../gaffer-basics/what-is-cardinality.md | 316 ++++++++++ mkdocs.yml | 20 +- 16 files changed, 1706 insertions(+), 3 deletions(-) create mode 100644 docs/administration-guide/where-to-run-gaffer/kubernetes-guide/add-libraries.md create mode 100644 docs/administration-guide/where-to-run-gaffer/kubernetes-guide/change-accumulo-passwords.md create mode 100644 docs/administration-guide/where-to-run-gaffer/kubernetes-guide/change-graph-metadata.md create mode 100644 docs/administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-empty-graph.md create mode 100644 docs/administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-schema.md create mode 100644 docs/administration-guide/where-to-run-gaffer/kubernetes-guide/kubernetes-guide.md delete mode 100644 docs/change-notes/migrating-from-v1-to-v2.md create mode 100644 docs/change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md create mode 100644 docs/change-notes/migrating-from-v1-to-v2/accumulo-migration.md create mode 100644 docs/change-notes/migrating-from-v1-to-v2/dependencies.md create mode 100644 docs/change-notes/migrating-from-v1-to-v2/deprecations.md create mode 100644 docs/change-notes/migrating-from-v1-to-v2/federation-changes.md create mode 100644 docs/change-notes/migrating-from-v1-to-v2/log4j.md create mode 100644 docs/user-guide/gaffer-basics/what-is-cardinality.md diff --git a/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/add-libraries.md b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/add-libraries.md new file mode 100644 index 0000000000..77d4f934fb --- /dev/null +++ b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/add-libraries.md @@ -0,0 +1,70 @@ +# Adding your own libraries and functions + +By default with the Gaffer deployment you get access to the: + +- Sketches library +- Time library +- Bitmap Library +- JCS cache library + +If you want more libraries than this (either one of ours of one of your own) you will need to customise the docker images and use them in place of the defaults. + +You will need a [basic Gaffer instance deployed on Kubernetes](deploy-empty-graph.md). + +## Add Extra Libraries to Gaffer REST + +At the moment, Gaffer uses a runnable jar file located at `/gaffer/jars`. When it runs it includes the `/gaffer/jars/lib` on the classpath. There is nothing in there by default because all the dependencies are bundled in to the JAR. However, if you wanted to add your own jars, you can do it like this: + +```Dockerfile +FROM gchq/gaffer-rest:latest +COPY ./my-custom-lib:1.0-SNAPSHOT.jar /gaffer/jars/lib/ +``` + +Build the image using: + +```bash +docker build -t custom-rest:latest . +``` + +## Add the extra libraries to the Accumulo image + +Gaffer's Accumulo image includes support for the following Gaffer libraries: + +- The Bitmap Library +- The Sketches Library +- The Time Library + +In order to push down any extra value objects and filters to Accumulo that are not in those libraries, we have to add the jars to the accumulo `/lib/ext directory`. Here is an example `Dockerfile`: + +```Dockerfile +FROM gchq/gaffer:latest +COPY ./my-library-1.0-SNAPSHOT.jar /opt/accumulo/lib/ext +``` + +Then build the image + +```bash +docker build -t custom-gaffer-accumulo:latest . +``` + +# Switch the images in the deployment + +You will need a way of making the custom images visible to the kubernetes cluster. Once visible you can switch them out. Create a `custom-images.yaml` file with the following contents: + +```yaml +api: + image: + repository: custom-rest + tag: latest + +accumulo: + image: + repository: custom-gaffer-accumulo + tag: latest +``` + +To switch them run: + +```bash +helm upgrade my-graph gaffer-docker/gaffer -f custom-images.yaml --reuse-values +``` diff --git a/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/change-accumulo-passwords.md b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/change-accumulo-passwords.md new file mode 100644 index 0000000000..a5e2316858 --- /dev/null +++ b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/change-accumulo-passwords.md @@ -0,0 +1,36 @@ +# Changing the Accumulo Passwords + +When deploying Accumulo - either as part of a Gaffer stack or as a standalone, the passwords for all the users and the instance.secret are set to default values and should be changed. The instance.secret cannot be changed once deployed as it is used in initalisation. + +When deploying the Accumulo helm chart, the following values are set. If you are using the Gaffer helm chart with the Accumulo integration, the values will be prefixed with "accumulo": + +| Name | value | default value | +| -------------------- | --------------------------------------------- | ------------- | +| Instance Secret | `config.accumuloSite."instance.secret"` | "DEFAULT" | +| Root password | `config.userManagement.rootPassword` | "root" | +| Tracer user password | `config.userManagement.users.tracer.password` | "tracer" | + +When you deploy the Gaffer Helm chart with Accumulo, a "gaffer" user with a password of "gaffer" is used by default following the same pattern as the tracer user. + +So to install a new Gaffer with Accumulo store, create an `accumulo-passwords.yaml` with the following contents: + +```yaml +accumulo: + enabled: true + config: + accumuloSite: + instance.secret: "changeme" + userManagement: + rootPassword: "changeme" + users: + tracer: + password: "changme" + gaffer: + password: "changeme" +``` + +You can install the graph with: + +```bash +helm install my-graph gaffer-docker/gaffer -f accumulo-passwords.yaml +``` diff --git a/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/change-graph-metadata.md b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/change-graph-metadata.md new file mode 100644 index 0000000000..603db49c99 --- /dev/null +++ b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/change-graph-metadata.md @@ -0,0 +1,63 @@ +# Changing the Graph ID and Description + +By default, the default Gaffer deployment ships with the Graph name "simpleGraph" and description "A graph for demo purposes" These are just placeholders and can be overwritten. This guide will show you how. + +The first thing you will need to do is [deploy an empty graph](deploy-empty-graph.md). + +## Changing the description + +Create a file called `graph-meta.yaml`. We will use this file to add our description and graph ID. Changing the description is as easy as changing the `graph.config.description` value. + +```yaml +graph: + config: + description: "My graph description" +``` + +## Deploy the new description + +Upgrade your deployment using helm: + +```bash +helm upgrade my-graph gaffer-docker/gaffer -f graph-metadata.yaml --reuse-values +``` + +The `--reuse-values` argument means we do not override any passwords that we set in the initial construction. + +You can see you new description if you to the Swagger UI and call the `/graph/config/description` endpoint. + +## Updating the Graph ID + +This may be simple or complicated depending on your store type. If you are using the Map or Federated store, you can just set the `graph.config.graphId` value in the same way. Though if you are using a MapStore, the graph will be emptied as a result. + +However, if you are using the Accumulo store, updating the graph Id is a little more complicated since the Graph Id corresponds to an Accumulo table. We have to change the gaffer users permissions to read and write to that table. To do that update the graph-meta.yaml file with the following contents: + +```yaml +graph: + config: + graphId: "MyGraph" + description: "My Graph description" + +accumulo: + config: + userManagement: + users: + gaffer: + permissions: + table: + MyGraph: + - READ + - WRITE + - BULK_IMPORT + - ALTER_TABLE +``` + +## Deploy your changes + +Upgrade your deployment using Helm. + +```bash +helm upgrade my-graph gaffer-docker/gaffer -f graph-metadata.yaml --reuse-values +``` + +If you take a look at Accumulo monitor, you will see your new Accumulo table. diff --git a/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-empty-graph.md b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-empty-graph.md new file mode 100644 index 0000000000..966a17f05c --- /dev/null +++ b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-empty-graph.md @@ -0,0 +1,73 @@ +# How to deploy a simple graph + +This guide will describe how to deploy a simple empty graph with the minimum configuration. + +You will need: + +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +- [helm](https://github.com/helm/helm/releases) +- A Kubernetes cluster (local or remote) +- An ingress controller running (for accessing UIs) + +## Add the Gaffer Docker repo + +To start with, you should add the Gaffer Docker repo to your helm repos. This will save the need for cloning this Git repository. If you have already done this you can skip this step. + +```bash +helm repo add gaffer-docker https://gchq.github.io/gaffer-docker +``` + +## Choose the store + +Gaffer can be backed with a number of different technologies to back its store. Which one you want depends on the use case but as a rule of thumb: + +- If you want just something to spin up quickly at small scale and are not worried about persistence, use the MapStore. +- If you want to back it with a key value datastore, you can deploy the Accumulo Store. +- If you want to join two or more graphs together to query them as one, you will want to use the Federated Store. + +### Deploy the MapStore + +The MapStore is just an in-memory store that can be used for demos or if you need something small scale short-term. It is our default store so there is no need for any extra configuration. + +You can install a MapStore by just running: + +``` +helm install my-graph gaffer-docker/gaffer +``` + +### Deploy the Accumulo Store + +If you want to deploy an Accumulo Store with your graph, it is relatively easy to do so with some small additional configuration. Create a file called `accumulo.yaml` and add the following: + +```yaml +accumulo: + enabled: true +``` + +By default, the Gaffer user is created with a password of "gaffer" the CREATE_TABLE system permission with full access to the simpleGraph table which is coupled to the graphId. All the default Accumulo passwords are in place so if you were to deploy this in production, you should consider changing the [default accumulo passwords](change-accumulo-passwords.md). + +You can stand up the accumulo store by running: + +```bash +helm install my-graph gaffer-docker/gaffer -f accumulo.yaml +``` + +### Deploy the Federated Store + +If you want to deploy the Federated Store, all that you really need to do is set the `store.properties`. To do this add the following to a `federated.yaml` file: + +```yaml +graph: + storeProperties: + gaffer.store.class: uk.gov.gchq.gaffer.federatedstore.FederatedStore + gaffer.store.properties.class: uk.gov.gchq.gaffer.federatedstore.FederatedStoreProperties + gaffer.serialiser.json.modules: uk.gov.gchq.gaffer.sketches.serialisation.json.SketchesJsonModules +``` + +The addition of the `SketchesJsonModules` is just to ensure that if the FederatedStore was connecting to a store which used sketches, they could be rendered nicely in json. + +We can create the graph with: + +```bash +helm install federated gaffer-docker/gaffer -f federated.yaml +``` diff --git a/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-schema.md b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-schema.md new file mode 100644 index 0000000000..fa80719ef2 --- /dev/null +++ b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-schema.md @@ -0,0 +1,68 @@ +# How to deploy your own schema + +Gaffer uses schema files to describe the data contained in a Graph. This guide will tell you how to deploy your own schemas with a Gaffer Graph. + +You will first need [a basic Gaffer instance deployed on Kubernetes] (deploy-empty-graph.md). + +Once you have that deployed we can change the schema. + +## Edit the schema + +If you run a GetSchema operation against the graph, you will notice that the count property is of type `java.lang.Integer` - change that property to be of type `java.lang.Long`. + +The easiest way to deploy a schema file is to use helms `--set-file` option which lets you set a value from the contents of a file. + +??? example "Example of a `schema.json` file" + + ```json + { + "edges": { + "BasicEdge": { + "source": "vertex", + "destination": "vertex", + "directed": "true", + "properties": { + "count": "count" + } + } + }, + "entities": { + "BasicEntity": { + "vertex": "vertex", + "properties": { + "count": "count" + } + } + }, + "types": { + "vertex": { + "class": "java.lang.String" + }, + "count": { + "class": "java.lang.Long", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Sum" + } + }, + "true": { + "description": "A simple boolean that must always be true.", + "class": "java.lang.Boolean", + "validateFunctions": [ + { "class": "uk.gov.gchq.koryphe.impl.predicate.IsTrue" } + ] + } + } + } + ``` + +## Update deployment with the new schema + +For our deployment to pick up the changes, we need to run a helm upgrade: + +```bash +helm upgrade my-graph gaffer-docker/gaffer --set-file graph.schema."schema\.json"=./schema.json --reuse-values +``` + +The `--reuse-values` argument tells helm to re-use the passwords that we defined earlier. + +Now if we inspect the schema, you will see that the `count` property has changed to a `Long`. diff --git a/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/kubernetes-guide.md b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/kubernetes-guide.md new file mode 100644 index 0000000000..5fd7acce24 --- /dev/null +++ b/docs/administration-guide/where-to-run-gaffer/kubernetes-guide/kubernetes-guide.md @@ -0,0 +1,40 @@ +# Gaffer in Kubernetes + +The [gaffer-docker](https://github.com/gchq/gaffer-docker) repository contains all code needed to run Gaffer using Docker and Kubernetes. + +All the files needed to get started using Gaffer in Kubernetes are contained in the ['kubernetes'](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes) sub-folder of the [gaffer-docker](https://github.com/gchq/gaffer-docker) repository. +In this directory you can find the Helm charts required to deploy various applications onto Kubernetes clusters. + +The Helm charts and associated information for each application can be found in the following places: + +- [Gaffer](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/gaffer) +- [Example Gaffer Graph of Road Traffic Data](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/gaffer-road-traffic) +- [JupyterHub with Gaffer Integrations](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/gaffer-jhub) +- [HFDS](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/hdfs) +- [Accumulo](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/accumulo) + +These charts can be accessed by cloning our repository or by using our Helm repo hosted on our [Github Pages Site](https://gchq.github.io/gaffer-docker/). + +## Requirements + +To deploy these applications, you'll need access to a suitable Kubernetes distribution. + +You will also need to install a container management engine, for example Docker or Podman, to build, run and manage your containers. + +## Adding this repo to Helm + +To add the gaffer-docker repo to helm run: + +```bash +helm repo add gaffer-docker https://gchq.github.io/gaffer-docker +``` + +## How to Guides + +There are a number of guides to help you deploy Gaffer on Kubernetes. It is important you look at these before you get started as they provide the initial steps for running these applications. + +* [Deploy a simple empty graph](deploy-empty-graph.md) +* [Add your schema](deploy-schema.md) +* [Change the graph ID and description](change-graph-metadata.md) +* [Adding your own libraries and functions](add-libraries.md) +* [Changing passwords for the Accumulo store](change-accumulo-passwords.md) \ No newline at end of file diff --git a/docs/change-notes/migrating-from-v1-to-v2.md b/docs/change-notes/migrating-from-v1-to-v2.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md b/docs/change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md new file mode 100644 index 0000000000..a3e3cbb414 --- /dev/null +++ b/docs/change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md @@ -0,0 +1,93 @@ +# Accumulo Kerberos Support + +This page contains information on Kerberos Authentication support for Gaffer's Accumulo Store. This functionality was introduced in version `2.0.0-alpha-0.3.1` of Gaffer. + +## Using the Accumulo Store with Kerberos + +### Prerequisites +To use Gaffer's Accumulo Store with Kerberos authentication: + +- The Accumulo cluster to connect with must be correctly configured to use Kerberos. +- A principal for the system/host Gaffer will be running on must be created in the Key Distribution Center (KDC) database. +- The Gaffer principal should use the standard [`primary/instance@realm`](https://web.mit.edu/kerberos/krb5-1.5/krb5-1.5.4/doc/krb5-user/What-is-a-Kerberos-Principal_003f.html) format. Using principals without an _instance_ qualification has not been tested. +- A keytab for the Gaffer principal must be created and transferred to the Gaffer host. +- The Gaffer principal must have been added as an Accumulo user with suitable permissions granted. +- Kerberos client utilities should be installed on the host and `krb5.conf` must be correctly configured. +- An Accumulo client configuration should be available on the host and contain the correct options to enable Kerberos. +- The Gaffer store.properties should state that Kerberos is to be used, specify the principal name and the keytab path. + +The sections below cover some of these points in more detail. + +### Accumulo user for Gaffer + +When Kerberos is used with Accumulo, [any client with a principal can connect without requiring an Accumulo user to have been created previously](https://accumulo.apache.org/docs/2.x/security/kerberos#kerberosauthenticator). This works by creating an Accumulo user automatically when a new client connects. These users are not granted any permissions. + +Users can still be created manually via the Accumulo shell, with Gaffer's full principal (with all components) given as the username. Permissions to create and read tables can then be granted to this user. If this isn't done, Accumulo will create the user automatically when Gaffer first connects. In this case Gaffer will fail to start as the required permissions will not have been granted - they can then be granted via the shell and Gaffer restarted. + +### Accumulo Client configuration +Depending on the version of Accumulo used, an [`accumulo-client.properties` (2.x)](https://accumulo.apache.org/docs/2.x/security/kerberos#configuration) or [`client.conf` (1.x)](https://accumulo.apache.org/1.10/accumulo_user_manual.html#_configuration_3) must be populated as described in the respective version of the Accumulo documentation. The only value which needs to be altered is the Kerberos server primary. This should reflect the primary part of the principals used by the Accumulo cluster. + +The location of this config file can be specified using the `ACCUMULO_CLIENT_CONF_PATH` environment variable. If this is not set, then [default paths will be checked](https://accumulo.apache.org/docs/2.x/apidocs/org/apache/accumulo/core/client/ClientConfiguration.html#loadDefault()). + +Other than this file, Accumulo libraries and configuration files do not need to be installed on the Gaffer host. + +### Gaffer `store.properties` configuration +In addition to the usual [Accumulo Store settings](../reference/stores-guide/accumulo.md#properties-file), these extra options must be specified for Kerberos: +``` +accumulo.kerberos.enable=true +accumulo.kerberos.principal=gaffer/host.domain@REALM.NAME +accumulo.kerberos.keytab=/gaffer/config/gaffer.keytab +``` +The `accumulo.username` and `accumulo.password` values do not need to be set and are ignored when `accumulo.kerberos.enable` is true. + +The `kinit` Kerberos command does not need to be used, although it might be useful for ensuring the client principal works correctly. All Kerberos ticket management, renewal and re-login is handled automatically. + +### Specifying a different `krb5.conf` +If the `krb5.conf` in the default system location is not suitable, or if it's stored in a non-standard location, then +custom a custom `krb5.conf` location can be specified when starting Gaffer by setting the system property value `java.security.krb5.conf`. The simplest way to do this is by using the option flag `-Djava.security.krb5.conf=/my/path/to/krb5.conf` when launching the Gaffer JAR. + +## Federation Considerations +Due to the way Kerberos is implemented in Accumulo, it is not possible for Gaffer to use multiple principals at the same time. For the `FederatedStore`, this prevents adding graphs which are on different Accumulo clusters, if those clusters require different principals. In practice this is unlikely to be a problem, as different Accumulo clusters would only need separate client principals if they were on separate Kerberos Realms or using different KDCs. + +This only impacts Accumulo clusters which require Kerberos. It doesn't impact on adding graphs which are stored in clusters using basic authentication and not Kerberos. Nor does it affect adding graphs from a Kerberos cluster and also adding graphs from a non Kerberos cluster in the same `FederatedStore`. + +If this limitation is a problem, it can be worked around by running additional Gaffer instances and connecting to them using a `ProxyStore` in the `FederatedStore`, rather than connecting directly using an `AccumuloStore`. + +## HDFS Considerations +When using the [`AddElementsFromHdfs`](https://gchq.github.io/gaffer-doc/v1docs/getting-started/operations/addelementsfromhdfs.html) operation Gaffer acts as a HDFS client. When Kerberos is used ([Hadoop Secure Mode](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html)), HDFS clients must have [native libraries](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/NativeLibraries.html) installed and configured correctly; else Hadoop will raise a Runtime Exception stating that "Secure IO is not possible without native code extensions". + +The HDFS client also requires the Hadoop configuration files `core-site.xml` and `hdfs-site.xml` to both be present and configured as below. The location of these files can be specified using the `HADOOP_CONF_DIR` environment variable. + +```xml + + + hadoop.security.authentication + kerberos + + + hadoop.security.authorization + true + +``` + +In particular, `hdfs-site.xml` requires the `yarn.resourcemanager.principal` property to be set to the HDFS client principal - should be the same one as in the Gaffer Store properties. If this is missing Hadoop will fail to connect and raise an IO Exception with "Can't get Master Kerberos principal for use as renewer". + +```xml + + + yarn.resourcemanager.principal + primary/instance@realm + +``` + +Note that the `core-site.xml` and `hdfs-site.xml` files are **only** required if `AddElementsFromHdfs` is going to be used. For Accumulo connections the Hadoop properties (from `core-site.xml`) used for enabling Kerberos are set automatically in Gaffer's connection code. + +## Spark Accumulo Library +The [Spark Accumulo Library](https://github.com/gchq/Gaffer/tree/master/library/spark) has not yet been updated to support Kerberos. This prevents [Spark Operations](https://gchq.github.io/gaffer-doc/v1docs/getting-started/spark-operations/contents.html) from being used with an `AccumuloStore` which has Kerberos authentication enabled. It is on the backlog for support to be added in future. + +## Troubleshooting +Kerberos is not easy to configure and familiarity with Kerberos concepts is recommended. There are [some useful links to introductory information in the Accumulo Kerberos docs](https://accumulo.apache.org/docs/2.x/security/kerberos#overview). + +Improperly configured DNS will cause problems with Kerberos. Ensure all hostnames used in Principals resolved correctly, include reverse lookup. Due to how the system's hostname is used by the Hadoop Kerberos libraries, a mismatch between the configured hostname and the hostname resolved by a reverse lookup can prevent authentication from working correctly. + +Various environment variables can be set for debugging Kerberos, [see the Hadoop docs for more information](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html#Troubleshooting). These variables are applicable to Accumulo ([see docs](https://accumulo.apache.org/docs/2.x/security/kerberos#debugging)) because its Kerberos implementation uses Hadoop libraries. The Gaffer logging level (set in `log4.xml`) should be increased to at least `INFO` when using these environment variables. diff --git a/docs/change-notes/migrating-from-v1-to-v2/accumulo-migration.md b/docs/change-notes/migrating-from-v1-to-v2/accumulo-migration.md new file mode 100644 index 0000000000..15a406e68b --- /dev/null +++ b/docs/change-notes/migrating-from-v1-to-v2/accumulo-migration.md @@ -0,0 +1,27 @@ +# Accumulo Migration + +This page contains information on changes to the Accumulo/Hadoop versions supported by Gaffer and how to continue using the previously supported versions. + +## Accumulo 2 & Hadoop 3 become default versions + +From the `2.0.0-alpha-0.3` release of Gaffer, the default version of Accumulo has been upgraded to [Accumulo 2.0.1](https://accumulo.apache.org/release/accumulo-2.0.1/). Hadoop has also been upgraded to the latest version (currently 3.3.3). This is because Hadoop 2.x is not compatible with Accumulo 2. + +## Retained support for Accumulo 1 & Hadoop 2 + +Support for certain versions of Accumulo 1 and Hadoop 2 (specifically 1.9.3 & 2.6.5) has been retained and can be enabled by using a Maven profile when building from source (see below). This facilitates testing with these versions and creates shaded JARs (e.g. spring-rest exec, accumulo-store iterators) with the appropriate versions of supporting libraries. As described in the source docs, other versions of Accumulo 1.x and Hadoop 2.x might also work. + +### Availability of Gaffer artifacts supporting Accumulo 1 + +The shaded JARs differ based on the versions of the bundled libraries and only the default version (Accumulo 2.0.1) is published to the Maven Central repository. The 'legacy' version must be built locally. + +### Building Gaffer with the 'legacy' profile + +To build Gaffer using Accumulo 1.9.3 and Hadoop 2.6.5, the 'legacy' Maven profile needs to be used. This is enabled by supplying `-Dlegacy=true` as an extra argument at the command line when running Maven. For example, `mvn clean install -Pcoverage -Dlegacy=true` will perform a full build/test of Gaffer with this profile enabled. Java 11 cannot be used with this profile because only Hadoop 3.3.0 and higher support it. + +With the 'legacy' Maven profile active, the filenames of all shaded JARs produced are appended with `-legacy`. This is to differentiate them from the default shaded JARs which contain different libraries and different library versions. A default Gaffer Accumulo REST API JAR will not work with an Accumulo 1 cluster, and the 'legacy' version will not work with Accumulo 2 because the bundled libraries are specific to the version of Accumulo. + +## Migrating from Accumulo 1 to 2 + +See [the Accumulo documentation](https://accumulo.apache.org/docs/2.x/administration/upgrading) for guidance on upgrading from Accumulo 1 to 2. Of particular significance is the [deprecation of the dynamic reloading classpath directory functionality](https://accumulo.apache.org/release/accumulo-2.0.0/#removed-default-dynamic-reloading-classpath-directory-libext) in Accumulo 2. This affects where and how the Gaffer iterators JAR can be installed. See the Accumulo store documentation for these installation details. + +Otherwise, no Accumulo specific Gaffer configuration needs to be changed and migrating from Accumulo 1 to 2 should be as simple as swapping the Gaffer dependency versions/JARs, although this has not been actively tested. diff --git a/docs/change-notes/migrating-from-v1-to-v2/dependencies.md b/docs/change-notes/migrating-from-v1-to-v2/dependencies.md new file mode 100644 index 0000000000..99d7b66d60 --- /dev/null +++ b/docs/change-notes/migrating-from-v1-to-v2/dependencies.md @@ -0,0 +1,35 @@ +# Dependency Upgrades + +This page lists the dependencies that have been upgraded as part of Gaffer 2. + + - Assertj: 3.20.2 -> 3.23.1 + - Avro: 1.7.7 -> 1.8.2 + - Commons-codec: 1.6 -> 1.15 + - Commons-csv: 1.4 -> 1.10.0 + - Commons-io: 2.7 -> 2.11.0 + - Commons-jcs-core: 2.1 -> 2.2.1 + - Commons-lang: 3.3.2 -> 3.12.0 + - Commons-logging: 1.1.3 -> 1.2 + - Commons-math3: 3.4.1 -> 3.6.1 + - Commons-math: 2.1 -> 2.2 + - Curator: 2.6.0 -> 2.13.0 + - Flink: 1.4.1 -> 1.7.2 + - Graphframes: 0.4.0 -> 0.8.1 + - Guava: 13.0.1 -> 30.1.1 + - Hadoop: 2.6.5 -> 3.3.3 + - Hazelcast: 3.8 -> 5.3.0 + - Jackson: 2.6.5 -> 2.13.5 + - Javassist: 3.19.0-GA -> 3.28.0-GA + - Jersey: 2.25 -> 2.36 + - Jersey: 2.25 -> 2.36 + - Junit5: 5.6.0 -> 5.9.0 + - Kafka: 0.10.0.0 -> 0.10.2.2 + - Koryphe: 1.14.0 -> 2.1.0 + - Log4j: 1.2.17 -> Reload4j: 1.2.18.3 + - Mockito: 3.3.3 -> 4.6.1 + - Paranamer: 2.6 -> 2.8 + - Reflections: 0.9.10 -> 0.9.12 + - Slf4j: 1.7.25 -> 1.7.36 + - Spark 2.3.2 -> 3.0.3 + - Spring API Swagger: 2.6.0 -> 3.0.0 + - Spring Boot: 1.3.2 -> 2.5.12 diff --git a/docs/change-notes/migrating-from-v1-to-v2/deprecations.md b/docs/change-notes/migrating-from-v1-to-v2/deprecations.md new file mode 100644 index 0000000000..0319e5506f --- /dev/null +++ b/docs/change-notes/migrating-from-v1-to-v2/deprecations.md @@ -0,0 +1,563 @@ +# Deprecations + +This page describes deprecated code which has been removed in Gaffer 2 and how to migrate to better equivalents. Each heading for a section below refers to a classname from `uk.gov.gchq.gaffer` where there have been changes or where that class has been removed entirely. The section headings link to the code on GitHub for that class (as of the Gaffer 1.21.1 release). + +Deprecations impacting the serialisers used in schemas are listed first, followed by [changes to Seed Matching and changes to Traits](#changes-to-seed-matching-and-traits). Other deprecations are then [listed in alphabetical order](#all-other-deprecations). + +## Serialisers + +### Migrating away from deprecated Serialisers +Various deprecated serialisers have been removed completely (details below). If any of these are being used in an existing schema, a new graph and schema will need to be created (see below for replacement serialisers to use) and data from existing graphs migrated. Data will need to be migrated (export and reimport) from graphs using deprecated serialisers before upgrading to Gaffer v2. + +It is essential to migrate data stored using deprecated serialisers. Simply replacing these serialisers is not enough because this **will prevent existing data from being read** and potentially put the backing store into a **corrupted state**. + +### Preservation of ordering +When using an ordered store (such as Accumulo), all serialisers used on vertices must preserve order. As such, `compactRaw` serialisers (which do not preserve order) cannot be used on vertices in ordered stores. + +However, when preserving order is not required, such as for properties, `CompactRaw` serialisers are the most effective solution and should always be used. Using an ordered serialiser on a property would reduce performance without providing any benefit. [See the schemas documentation for more detail](https://gchq.github.io/gaffer-doc/v1docs/getting-started/developer-guide/schemas.html#serialisers). + +### Removed Serialisers + +#### [`serialisation.implementation.raw.RawDateSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawDateSerialiser.java) and [`serialisation.DateSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/DateSerialiser.java) +Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedDateSerialiser` instead - note that this will preserve order. Neither of these replacement serialisers implement `.deserialiseString(String)`, instead use `new Date(Long.parseLong(String))` in place of this. + +#### [`serialisation.implementation.raw.RawDoubleSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawDoubleSerialiser.java) and [`serialisation.DoubleSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/DoubleSerialiser.java) +Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedDoubleSerialiser` instead - note that this will preserve order. Neither of these replacement serialisers implement `.deserialiseString(String)`, instead use `Double.parseDouble(String)` in place of this. + +#### [`serialisation.implementation.raw.RawFloatSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawFloatSerialiser.java) and [`serialisation.FloatSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/FloatSerialiser.java) +Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedFloatSerialiser` instead - note that this will preserve order. + +#### [`serialisation.IntegerSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/IntegerSerialiser.java) and [`serialisation.implementation.raw.RawIntegerSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawIntegerSerialiser.java) +Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedIntegerSerialiser` instead if you need order preserved (e.g. vertex types). If object ordering definitely does **not** need to be preserved (e.g. only property types), `uk.gov.gchq.gaffer.serialisation.implementation.raw.CompactRawIntegerSerialiser` should be used instead. Neither of these replacement serialisers implement `.deserialiseString(String)`, instead use `Integer.parseInt(String)` in place of this. + +#### [`serialisation.LongSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/LongSerialiser.java) and [`serialisation.implementation.raw.RawLongSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawLongSerialiser.java) +Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedLongSerialiser` instead if you need order preserved (e.g. vertex types). If object ordering definitely does **not** need to be preserved (e.g. only property types), `uk.gov.gchq.gaffer.serialisation.implementation.raw.CompactRawLongSerialiser` could also be used instead. + +### Changed Serialisers +#### [`serialisation.ToBytesSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/ToBytesSerialiser.java) and [`serialisation.ToBytesViaStringDeserialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/ToBytesViaStringDeserialiser.java) +In both serialisers, the method `deserialise(byte[])` has been marked as deprecated. It cannot be deleted as it is needed to implement the Serialiser interface. It is recommended for speed/performance to use the other implementation with an offset and a length: `deserialise(byte[], int, int)`. + +## Removal of Seed Matching + +### [`operation.SeedMatching`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/SeedMatching.java) +SeedMatching has been removed from Gaffer. This was previously used in [get](../reference/operations-guide/get.md) operations, like [`GetElements`](../reference/operations-guide/get.md#getelements), to select whether you wanted your results to contain only Elements that are the same type as the seed, or both Edges and Entities. For more info, see the Gaffer 1.X docs page on [SeedMatching](https://gchq.github.io/gaffer-doc/v1docs/getting-started/user-guide/filtering.html#seedmatching). As described in the Gaffer 1.X docs, `SeedMatching` can be replaced with a `View`. The default behaviour in Gaffer is the same as if you used `seed_matching="RELATED"`, so **if this is the case, there is no migration required**. However, if you used `seed_matching="EQUAL"`, you will need to migrate to a `View`. +??? example "SeedMatching migration with EdgeSeeds" + + Where SeedMatching was used to only get back Edges from EdgeSeeds + + === "Java" + + ``` java + final GetElements getEdgesWithSeedMatching = new GetElements.Builder() + .input(new EdgeSeed("source", "dest", true)) + .seedMatching(SeedMatching.SeedMatchingType.EQUAL) + .build(); + ``` + + === "JSON" + + ``` json + { + "class" : "GetElements", + "input" : [ { + "class" : "EdgeSeed", + "source" : "source", + "destination" : "dest", + "matchedVertex" : "SOURCE", + "directedType" : "DIRECTED" + } ], + "seedMatching" : "EQUAL" + } + ``` + + === "Python" + + ``` python + g.GetElements( + input=[ + g.EdgeSeed( + source="source", + destination="dest", + directed_type="DIRECTED", + matched_vertex="SOURCE" + ) + ], + seed_matching="EQUAL" + ) + ``` + + You should instead specify that in a View + + === "Java" + + ``` java + final GetElements getEdgesWithoutSeedMatching = new GetElements.Builder() + .input(new EdgeSeed("source", "dest", true)) + .view(new View.Builder() + .edge("relevantEdgeGroup") + .build()) + .build(); + ``` + + === "JSON" + + ``` json + { + "class" : "GetElements", + "input" : [ { + "class" : "EdgeSeed", + "source" : "source", + "destination" : "dest", + "matchedVertex" : "SOURCE", + "directedType" : "DIRECTED" + } ], + "view" : { + "edges" : { + "relevantEdgeGroup" : { } + } + } + } + ``` + + === "Python" + + ``` python + g.GetElements( + input=[ + g.EdgeSeed( + source="source", + destination="dest", + directed_type="DIRECTED", + matched_vertex="SOURCE" + ) + ], + view=g.View( + edges=[g.ElementDefinition(group="relevantEdgeGroup")] + ) + ) + ``` + +??? example "SeedMatching migration with EntitySeeds" + + Where SeedMatching was used to only get back Entities from EntitySeeds + + === "Java" + + ``` java + final GetElements getEntitiesWithSeedMatching = new GetElements.Builder() + .input(new EntitySeed("vertex")) + .seedMatching(SeedMatching.SeedMatchingType.EQUAL) + .build(); + ``` + + === "JSON" + + ``` json + { + "class" : "GetElements", + "input" : [ { + "class" : "EntitySeed", + "vertex" : "vertex" + } ], + "seedMatching" : "EQUAL" + } + ``` + + === "Python" + + ``` python + g.GetElements( + input=[ + g.EntitySeed( + vertex="vertex" + ) + ], + seed_matching="EQUAL" + ) + ``` + + You should instead specify that in a View + + === "Java" + + ``` java + final GetElements getEntitiesWithoutSeedMatching = new GetElements.Builder() + .input(new EntitySeed("vertex")) + .view(new View.Builder() + .entity("relevantEntityGroup") + .build()) + .build(); + ``` + + === "JSON" + + ``` json + { + "class" : "GetElements", + "input" : [ { + "class" : "EntitySeed", + "vertex" : "vertex" + } ], + "view" : { + "entities" : { + "relevantEntityGroup" : { } + } + } + } + ``` + + === "Python" + + ``` python + g.GetElements( + input=[ + g.EntitySeed( + vertex="vertex" + ) + ], + view=g.View( + entities=[g.ElementDefinition(group="relevantEntityGroup")] + ) + ) + ``` + +??? example "SeedMatching migration with EdgeSeeds and EntitySeeds" + + Where SeedMatching was used to only get back only Edges from the provided EdgeSeeds and only Entities from the provided EntitySeeds + + === "Java" + + ``` java + final GetElements getBothWithSeedMatching = new GetElements.Builder() + .input(new EntitySeed("vertex"), new EdgeSeed("source", "dest", true)) + .seedMatching(SeedMatching.SeedMatchingType.EQUAL) + .build(); + ``` + + === "JSON" + + ``` json + { + "class" : "GetElements", + "input" : [ { + "class" : "EntitySeed", + "vertex" : "vertex" + }, { + "class" : "EdgeSeed", + "source" : "source", + "destination" : "dest", + "matchedVertex" : "SOURCE", + "directedType" : "DIRECTED" + } ], + "seedMatching" : "EQUAL" + } + ``` + + === "Python" + + ``` python + g.GetElements( + input=[ + g.EntitySeed( + vertex="vertex" + ), + g.EdgeSeed( + source="source", + destination="dest", + directed_type="DIRECTED", + matched_vertex="SOURCE" + ) + ], + seed_matching="EQUAL" + ) + ``` + + You will instead need to perform multiple Operations and combine the results. To perform the above operation, you would have to combine both previous examples. + + +## Changes to Store Traits + +### [`store.Store`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java) +- The method `getTraits()` has been removed. Use `Store.execute(Operation, Context)` with the `GetTraits` operation instead. +- The method `hasTrait(StoreTrait)` has been removed. Use `Store.execute(Operation, Context)` with the `HasTrait` operation instead. + +### [`federatedstore.FederatedGraphStorage`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedGraphStorage.java) +- The method `getTraits(GetTraits, Context)` has been removed. Use `Store.execute(Operation, Context)` with the `GetTraits` operation instead. + +### [`federatedstore.FederatedStore`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedStore.java) +- The methods `getTraits()` and `getTraits(GetTraits, Context)` have been removed. Use `Store.execute(Operation, Context)` with the `GetTraits` operation instead. + +## Changes to Schemas + +Deprecated methods in `store.schema.TypeDefinition` and `store.schema.Schema` have been removed ([see below sections](#storeschematypedefinition)). + +### Elements Schema + +Specifying a property name to use as a time stamp is now set under `config`. + +Previously set at the top level: +``` +{ + "entities": { + ... + }, + "edges": { + ... + }, + "timestampProperty": "timestamp" +} +``` + +Now set in `config`: +``` +{ + "entities": { + ... + }, + "edges": { + ... + }, + "config": { + "timestampProperty": "timestamp" + } +} +``` + +### Types Schema + +Specifying a serialiser class using deprecated `serialiserClass` is no longer possible. Instead, use `class` in `serialiser`. Deprecated `vertexSerialiserClass` has also been removed. + +Old, deprecated, and now removed approach: +``` +{ + "types": { + "example.map": { + "description": "Map type description", + "class": "java.util.LinkedHashMap", + "serialiserClass": "uk.gov.gchq.gaffer.serialisation.implementation.MapSerialiser" + } + } +} +``` + +Now set in `serialiser`: +``` +{ + "types": { + "example.map": { + "description": "Map type description", + "class": "java.util.LinkedHashMap", + "serialiser": { + "class": "uk.gov.gchq.gaffer.serialisation.implementation.MapSerialiser" + } + } + } +} +``` + +## All other Deprecations + +### [`accumulostore.AccumuloProperties`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloProperties.java) +- The `TABLE` setting/variable plus the methods `getTable()` and `setTable(String)` have been removed. For `getTable()`, [uk.gov.gchq.gaffer.accumulostore.getTableName()](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/accumulostore/AccumuloStore.html#getTableName--) could be used instead. +- A `graphId` should be supplied instead of setting `TABLE` directly. + +### [`accumulostore.MockAccumuloStore`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/MockAccumuloStore.java) +- This class has been removed. +- For in memory graphs, use `uk.gov.gchq.gaffer.mapstore.MapStore` instead. +- For Accumulo specific store tests, use `uk.gov.gchq.gaffer.accumulostore.MiniAccumuloStore` instead. + +### [`commonutil.TestTypes`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/common-util/src/test/java/uk/gov/gchq/gaffer/commonutil/TestTypes.java) +- This class has been removed. +- Use the equivalent `TestTypes` class in the store module `uk.gov.gchq.gaffer.store.TestTypes` instead. + +### [`commonutil.CommonConstants`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/CommonConstants.java) +- This class has been removed as it was redundant. +- For the `UTF-8` constant use `StandardCharsets.UTF_8.name()` from built in Java libraries. +- The above also applies to the `ISO_8859_1` constant from this class. +- This also allows for more robust error handing, [this commit is an example of a change implementing this](https://github.com/gchq/Gaffer/commit/3999890f3aebadbc5384695123af89a4d9053333#diff-faa371afa5e6d548ff73af92f1f148870424e7858e3fef43b3d25bc5b740c013). + +### [`data.elementdefinition.view.NamedViewDetail`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/data/src/main/java/uk/gov/gchq/gaffer/data/elementdefinition/view/NamedViewDetail.java) +- The method `hasWriteAccess(String userId, Set opAuths, String adminAuth)` has been removed. +- Use `hasWriteAccess(User user, String adminAuth)` instead. + +### [`data.elementdefinition.view.ViewElementDefinition`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/data/src/main/java/uk/gov/gchq/gaffer/data/elementdefinition/view/ViewElementDefinition.java) +- The method `setAggregator(ElementAggregator aggregator)` has been removed. +- A `ViewElementDefinition` should be constructed using the builder `uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition.Builder` instead. + +??? example + + Where `setAggregator` was used previously + ``` java + final ViewElementDefinition elementDef = new ViewElementDefinition(); + elementDef.setAggregator(myElementAggregator); + ``` + + You should now use the Builder + ``` java + final ViewElementDefinition elementDef = new ViewElementDefinition.Builder() + .aggregator(myElementAggregator) + .build(); + ``` + +### [`federatedstore.FederatedAccess`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedAccess.java) +- The method `isAddingUser(User)` has been removed. +- Use `hasReadAccess(User user, String adminAuth)`/`hasWriteAccess(User user, String adminAuth)` instead. + +### [`federatedstore.FederatedGraphStorage`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedGraphStorage.java) +- The methods `getAllIdsAsAdmin()`, `getAllGraphAndAccessAsAdmin(List)` and `changeGraphAccessAsAdmin(String, FederatedAccess)` have all been removed. +- The method `remove(String graphId)` has been removed. The following can be used instead: + - `remove(String graphId, User user)` + - `remove(String graphId, User user, String adminAuth)` + - `remove(String graphId, Predicate>> entryPredicateForGraphRemoval)` + +### [`federatedstore.FederatedStore`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedStore.java) +- The method `updateOperationForGraph(Operation, Graph)` has been removed. Use `FederatedStoreUtil.updateOperationForGraph(Operation, Graph)` instead. +- The method `addGraphs(Set graphAuths, String addingUserId, GraphSerialisable... graphs)` has been removed. The following can be used instead: + - `addGraphs(Set graphAuths, String addingUserId, boolean isPublic, GraphSerialisable... graphs)` + - `addGraphs(Set graphAuths, String addingUserId, boolean isPublic, boolean disabledByDefault, GraphSerialisable... graphs)` + - `addGraphs(Set graphAuths, String addingUserId, boolean isPublic, boolean disabledByDefault, AccessPredicate readAccessPredicate, AccessPredicate writeAccessPredicate, GraphSerialisable... graphs)` + - `addGraphs(FederatedAccess access, GraphSerialisable... graphs)` + +### [`federatedstore.operation.RemoveGraph`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/operation/RemoveGraph.java) +- The method `Builder.setGraphId(String graphId)` has been removed. +- Use `Builder.graphId(String graphId)` which has identical behaviour instead. + +??? example + + Where `Builder.setGraphId` was used previously + ``` java + final RemoveGraph removeGraphOp = new RemoveGraph.Builder() + .setGraphId("myGraph") + .build(); + ``` + + You should now use `Builder.graphId` + ``` java + final RemoveGraph removeGraphOp = new RemoveGraph.Builder() + .graphId("myGraph") + .build(); + ``` + +### [`graph.Graph`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java) +- The methods `Builder.graphId`, `Builder.library`, `Builder.view`, `Builder.addHook`, `Builder.addHooks` have all been removed in all forms. +- Instead of using these methods, use `.config()` to set the `graphConfig`. + +??? example + + Where the graph config was added using the `Graph.Builder` before + ``` java + final Graph myGraph = new Graph.Builder() + .graphId("myGraph") + .library(myGraphLibrary) + .view(myView) + .addHook(customHook) + .addSchema(mySchema) + .storeProperties(storeProperties) + .build(); + ``` + + You should now use the `GraphConfig.Builder` + ``` java + final Graph myGraph = new Graph.Builder() + .config(new GraphConfig.Builder() + .graphId("myGraph") + .library(myGraphLibrary) + .view(myView) + .addHook(customHook) + .build()) + .addSchema(mySchema) // (1)! + .storeProperties(storeProperties) // (2)! + .build(); + ``` + + 1. Schemas are not part of the GraphConfig + 2. StoreProperties are not part of the GraphConfig + +### [`hdfs.operation.MapReduce`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/library/hdfs-library/src/main/java/uk/gov/gchq/gaffer/hdfs/operation/MapReduce.java) +- The methods `getNumReduceTasks()` and `setNumReduceTasks(Integer)` have been removed. +- Gaffer’s operations that inherit `MapReduce` did not make use of `numReduceTasks`, either setting it to a constant number in the `JobFactory` or using Accumulo to automatically set the number (recommended for performance) and using min/max to keep it within a range. Therefore, `numReduceTasks`, `getNumReduceTasks` and `setNumReduceTasks` have been removed from this interface. + +### [`hdfs.operation.AddElementsFromHdfs`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/library/hdfs-library/src/main/java/uk/gov/gchq/gaffer/hdfs/operation/AddElementsFromHdfs.java) +- The methods `getNumReduceTasks()` and `setNumReduceTasks(Integer)` have been removed. +- The number of reduce tasks should not be set. By default the number of reduce tasks should match the number of tablets. Use minimum and maximum reduce tasks to specify boundaries for the number of reduce tasks. + +### [`hdfs.operation.SampleDataForSplitPoints`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/library/hdfs-library/src/main/java/uk/gov/gchq/gaffer/hdfs/operation/SampleDataForSplitPoints.java) +- The methods `getNumReduceTasks()` and `setNumReduceTasks(Integer)` have been removed. +- These methods were not required as `NumReduceTasks` was always set to 1 in any case. + +### [`jobtracker.JobDetail`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/jobtracker/JobDetail.java) +- The constructors which took `userId` as a `String` have been removed. +- Instead, a `User` (`uk.gov.gchq.gaffer.user.User`) should be used in its place. See the [Builder for User](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/user/User.Builder.html). +- `getUserId` and `setUserId` have also been removed. For getting the `UserId`, `getUser().getUserId()` can be used instead. See the [Javadoc for User](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/user/User.html#getUserId--). + +### [`jsonserialisation.JSONSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/jsonserialisation/JSONSerialiser.java) +- The method `update(String jsonSerialiserClass, String jsonSerialiserModules)` has been removed. +- Use `update(String jsonSerialiserClass, String jsonSerialiserModules, Boolean strictJson)` instead. Passing `strictJson` as `null` will result in the same behaviour. + +### [`operation.Operation`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/Operation.java) +- The method `asOperationChain(Operation operation)` has been removed. +- Use `OperationChain.wrap` with the `Operation` instead. + +### [`operation.impl.GetWalks`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/GetWalks.java) +- The method `Builder.operation` has been removed. +- Use the vararg method `Builder.addOperations` instead. + +### [`operation.impl.SplitStore`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/SplitStore.java) +- This class has been removed. +- It is replaced by `SplitStoreFromFile` which is identical except in name. + +### [`operation.impl.join.methods.JoinFunction`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/join/methods/JoinFunction.java) +- The method `join(Iterable keys, String keyName, String matchingValuesName, Match match, Boolean flatten)` which was not implemented has been removed. + +### [`rest.SystemProperty`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/rest-api/common-rest/src/main/java/uk/gov/gchq/gaffer/rest/SystemProperty.java) +- `GRAPH_ID`, `GRAPH_HOOKS_PATH`, `GRAPH_LIBRARY_PATH` and `GRAPH_LIBRARY_CONFIG` have been removed. +- These config options have been removed in favour of providing a `graphConfig` JSON and using `GRAPH_CONFIG_PATH` instead. + +### [`rest.service.v2.example.ExamplesFactory`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/example/ExamplesFactory.java) +- This class has been removed. +- It is replaced by `uk.gov.gchq.gaffer.rest.factory.ExamplesFactory`, which can be used instead. + +### [`store.StoreProperties`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/StoreProperties.java) +- StoreProperties ID (`gaffer.store.id`) and related methods (`getId()`, `setId(String)`) have been removed. +- The ID of the store properties is instead directly set in the `GraphLibrary` when adding the `StoreProperties` with `GraphLibrary.add(String graphId, String schemaId, Schema schema, String propertiesId, StoreProperties properties)`. +- See the [Javadoc for GraphLibrary](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/library/GraphLibrary.html) for more detail. +- If you aren't using a `GraphLibrary`, this change shouldn't affect you as store properties ID is only used in `GraphLibrary`. + +### [`store.Context`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/Context.java) +- The private constructor `Context(User user, Map config, String jobId)` has been removed; along with the `jobId(String)` method. +- Use `Context(User user, Map config)` instead. This does not support supplying the Job ID, this will be set automatically. To get the Job ID use `.getJobId()`. + +### [`store.schema.TypeDefinition`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/schema/TypeDefinition.java) +- The method `getSerialiserClass()` has been removed. Instead, use `getSerialiser()` with `.getClass()` and related methods. +- The method `setSerialiserClass(String)` has been removed. Instead, set the Serialiser directly using `setSerialiser(Serialiser)`. + +### [`store.schema.Schema`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/schema/Schema.java) +- Schema ID (`gaffer.store.id`) and related methods have been removed. The ID is now defined in `GraphLibrary` when adding the schema. +- `timestampProperty` and related methods have been removed. Instead, this is specified by setting `"config": {"timestampProperty": "timestamp"}` (where `"timestamp"` is the property name to use as a time stamp) in the Schema. See [this example schema](https://github.com/gchq/Gaffer/blob/gaffer2-2.0.0-alpha-0.1/core/store/src/test/resources/schema-nested/elements.json) for more info. +- The method `getVertexSerialiserClass()` has been removed. It can be replaced by calling `vertexSerialiser.getClass()` and converting the result as appropriate, e.g. `getVertexSerialiserClass()` used `SimpleClassNameIdResolver.getSimpleClassName(vertexSerialiser.getClass())`. + +### [`store.library.GraphLibrary`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/library/GraphLibrary.java) +- The method `addSchema(Schema schema)` has been removed. Use `addSchema(String id, Schema schema)` instead. +- The method `addProperties(StoreProperties properties)` has been removed. Use `addProperties(String id, StoreProperties properties)` instead. +- Both of these now require the ID to be supplied. + +### [`store.operation.OperationChainValidator`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/OperationChainValidator.java) +- The method `validateViews(Operation op, ValidationResult validationResult, Schema schemaNotUsed, Store store)` has been removed. Use `validateViews(Operation op, User user, Store store, ValidationResult validationResult)` instead, passing `user` as `null` will result in the same behaviour. +- The method `validateComparables(Operation op, ValidationResult validationResult, Schema schemaNotUsed, Store store)` has been removed. Use `validateComparables(Operation op, User user, Store store, ValidationResult validationResult)` instead, passing `user` as `null` will result in the same behaviour. + +### [`store.operation.handler.named.cache.NamedViewCache`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/named/cache/NamedViewCache.java) +- The method `deleteNamedView(String name)` has been removed. Use `deleteNamedView(String name, User user)` instead, passing `user` as `null` will result in the same behaviour. +- The method `getNamedView(String name)` has been removed. Use `getNamedView(String name, User user)` instead. +- The method `getAllNamedViews()` has been removed. Use `getAllNamedViews(User user)` instead. + +### [`types.IntegerFreqMap`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/type/src/main/java/uk/gov/gchq/gaffer/types/IntegerFreqMap.java) +- This class has been removed. +- Use `uk.gov.gchq.gaffer.types.FreqMap` instead, this is identical except for using Long rather than Integer. + +#### [`types.function.IntegerFreqMapAggregator`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/type/src/main/java/uk/gov/gchq/gaffer/types/function/IntegerFreqMapAggregator.java) +- This class has been removed. +- Use `uk.gov.gchq.gaffer.types.function.FreqMapAggregator` instead. + +#### [`serialisation.IntegerFreqMapSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/type/src/main/java/uk/gov/gchq/gaffer/serialisation/IntegerFreqMapSerialiser.java) +- This class has been removed. +- Use `uk.gov.gchq.gaffer.serialisation.FreqMapSerialiser` instead. diff --git a/docs/change-notes/migrating-from-v1-to-v2/federation-changes.md b/docs/change-notes/migrating-from-v1-to-v2/federation-changes.md new file mode 100644 index 0000000000..0da482e74a --- /dev/null +++ b/docs/change-notes/migrating-from-v1-to-v2/federation-changes.md @@ -0,0 +1,240 @@ +# Federated Store Changes + +This page contains information on the changes to Gaffer's Federated Store. This functionality was introduced in version `2.0.0-alpha-0.4` of Gaffer. +The main changes were the addition of the [Federated Operation](#the-federated-operation), and a change to how results are [merged](#default-results-merging) by default. + +## The Federated Operation + +The `FederatedOperationChain` was removed and replaced with a new Operation, the `FederatedOperation`. This was added to improve the control you have over how operations are federated. +The Federated Operation has 3 key parameters: `operation`, `graphIds` and `mergeFunction`: +``` json +{ + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "uk.gov.gchq.gaffer.operation.impl.get.GetAllElements" + }, + "graphIds": [ "graphA", "graphB" ], + "mergeFunction": { + "class": "uk.gov.gchq.gaffer.federatedstore.util.ConcatenateMergeFunction" + } +} +``` + +### Required parameter: operation + +This is the Operation you wish to be federated to the subgraphs. This can be a single Operation or an OperationChain. If you use an OperationChain, then the whole chain will be sent to the subgraphs. + +### Optional parameter: graphIds + +This is a list of graph IDs which you want to send the operation to. + +If the user does not specify `graphIds` in the Operation, then the `storeConfiguredGraphIds` for that store will be used. If the admin has not configured the `storeConfiguredGraphIds` then all graphIds will be used. + +For information on sending different operations in one chain to different subgraphs, see [below](#breaking-change-removal-of-federatedoperationchain). + +### Optional parameter: mergeFunction + +The `mergeFunction` parameter is the Function you want to use when merging the results from the subgraphs. + +If the user does not specify a `mergeFunction` then it will be selected from the `storeConfiguredMergeFunctions` for that store. If the admin has not configured the `storeConfiguredMergeFunctions`, it will contain pre-populated `mergeFunctions`. Lastly, if a suitable `mergeFunction` is not found then a default `ConcatenateMergeFunction` is used. + +For example, when GetElements is used as the operation inside a FederatedOperation and the user hasn't specified a `mergeFunction`, the pre-populated `ApplyViewToElementsFunction` will be selected from `storeConfiguredMergeFunctions`, unless the admin configured it to use something else. + + +## Migrating to a FederatedOperation + +Previously, graphIds were selected in queries with the now deprecated option: `gaffer.federatedstore.operation.graphIds`. This is being supported while users migrate to using a FederatedOperation. + +### Sending an Operation to specific stores + +As mentioned, the `gaffer.federatedstore.operation.graphIds` option is still being supported so if you have an Operation using that option, it will continue to work. +Despite the option still being supported, we recommend you migrate to using a FederatedOperation. + +The `gaffer.federatedstore.operation.graphIds` option does not work an OperationChain. Previously, if you wanted to send an entire OperationChain to specific graphs, then you had to use a FederatedOperationChain. This has been replaced by a FederatedOperation with an OperationChain as the payload. For migration, see [below](#breaking-change-removal-of-federatedoperationchain). + +#### Deprecated graphIds option on a single Operation +```json +{ + "class": "uk.gov.gchq.gaffer.operation.impl.get.GetAllElements", + "options": { + "gaffer.federatedstore.operation.graphIds": "graphA" + } +} +``` + +#### New FederatedOperation graphIds on a single Operation +``` json +{ + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "uk.gov.gchq.gaffer.operation.impl.get.GetAllElements" + }, + "graphIds": [ "graphA" ] +} +``` + +#### Deprecated graphIds option inside an OperationChain +```json +{ + "class": "uk.gov.gchq.gaffer.operation.OperationChain", + "operations": [ + { + "class": "ExampleOperation1", + "options": { + "gaffer.federatedstore.operation.graphIds": "graphA" + } + }, + { + "class": "ExampleOperation2", + "options": { + "gaffer.federatedstore.operation.graphIds": "graphB" + } + } + ] +} +``` + +#### New FederatedOperation graphIds inside an OperationChain +```json +{ + "class": "uk.gov.gchq.gaffer.operation.OperationChain", + "operations": [ + { + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "ExampleOperation1" + }, + "graphIds": [ "graphA" ] + }, + { + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "ExampleOperation2" + }, + "graphIds": [ "graphB" ] + } + ] +} +``` + +## Breaking change: Removal of FederatedOperationChain + +The FederatedOperationChain has been removed, and where you would have used it before you should instead use a FederatedOperation with an OperationChain inside. + +This is useful if you have an OperationChain and want to send different parts of the chain to different subgraphs. + +#### Individually sending a sequence of Operations to a subgraph +You could send a sequence of operations within one chain to the same subgraph using `graphIds`, however, this is not always efficient: +```json +{ + "class": "uk.gov.gchq.gaffer.operation.OperationChain", + "operations": [ + { + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "ExampleOperation1" + }, + "graphIds": [ "graphA" ] + }, + { + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "ExampleOperation2" + }, + "graphIds": [ "graphA" ] + }, + { + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "ExampleOperation3" + }, + "graphIds": [ "graphB" ] + } + ] +} +``` + +#### Removed FederatedOperationChain sending a sequence of operations to a subgraph +It is more efficient to group together sequences of Operations that will go to the same subgraph. +This used to be done with a FederatedOperationChain: +```json +{ + "class": "uk.gov.gchq.gaffer.operation.OperationChain", + "operations": [ + { + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperationChain", + "operations": { + [ + "class": "ExampleOperation1", + "class": "ExampleOperation2" + ] + }, + "options": { + "gaffer.federatedstore.operation.graphIds": "graphA" + } + }, + { + "class": "ExampleOperation3", + "options": { + "gaffer.federatedstore.operation.graphIds": "graphB" + } + } + ] +} +``` + + +#### New FederatedOperation sending a sequence of operations to a subgraph +Now you should instead wrap an OperationChain inside a FederatedOperation: +```json +{ + "class": "uk.gov.gchq.gaffer.operation.OperationChain", + "operations": [ + { + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "uk.gov.gchq.gaffer.operation.OperationChain", + "operations": { + [ + "class": "ExampleOperation1", + "class": "ExampleOperation2" + ] + } + }, + "graphIds": [ "graphA" ] + }, + { + "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", + "operation": { + "class": "ExampleOperation3" + }, + "graphIds": [ "graphB" ] + } + ] +} +``` + +## Default results merging + +As described above, FederatedStores now have `storeConfiguredMergeFunctions` that dictate how the FederatedStore will merge results from different subgraphs dependent on the Operation. + +In places, these new defaults do differ from previous behaviour, hence results will too. This can be overriden on a per Operation basis using the `mergeFunction` parameter described above, or a per store basis by overriding `storeConfiguredMergeFunctions`. +The previous behaviour was that all Operation results were concatenated together, this is now a mergeFunction within Gaffer called `ConcatenateMergeFunction`. Therefore, if you wanted a FederatedOperation to use this old behaviour, you can set the `mergeFunction` to `ConcatenateMergeFunction` (as shown [above](#the-federated-operation)). + +### New Merge function examples + +By default, `GetElements` results will be merged with `ApplyViewToElementsFunction`. This uses the View from the operation and applies it to all of the results, meaning the results are now re-aggregated and re-filtered using the Schema, locally in the FederatedStore. This makes the results look like they came from one graph, rather than getting back a list of Elements from different subgraphs. + +By default, `GetTraits` results will be merged with `CollectionIntersect`. This returns the intersection of common store traits from the subgraphs. This behaviour is the same, but now it can be overriden. + +By default, `GetSchema` results will be merged with `MergeSchema`. This returns an aggregated schema from the subgraphs, unless there is a conflict. This behaviour is the same, but now it can be overriden. For example, you may wish to use the `ConcatenateMergeFunction` if there is a schema conflict. + +### Default storeConfiguredMergeFunctions + +| Operation | Merge function | +|-------------------|-----------------------------| +| GetElements | ApplyViewToElementsFunction | +| GetAllElements | ApplyViewToElementsFunction | +| GetSchema | MergeSchema | +| GetTraits | CollectionIntersect | +| others | ConcatenateMergeFunction | diff --git a/docs/change-notes/migrating-from-v1-to-v2/log4j.md b/docs/change-notes/migrating-from-v1-to-v2/log4j.md new file mode 100644 index 0000000000..333d509b0d --- /dev/null +++ b/docs/change-notes/migrating-from-v1-to-v2/log4j.md @@ -0,0 +1,30 @@ +# Log4j in Gaffer + +This page contains information on how logging is done in Gaffer and on previous use of Log4j in Gaffer. + +## Log4j Version + +Log4j version 1 (1.2.17), was used by Gaffer versions 1.21 and below. From Gaffer 1.22, Log4j was replaced with Reload4j. **The newer version of Log4j, Log4j2 - which is susceptible to the major Log4Shell attack, has never been used by Gaffer or its dependencies.** + +## How Logging is done + +Gaffer uses SLF4J ([Simple Logging Facade for Java](https://www.slf4j.org/)) for all logging. This is a framework/abstraction layer which allows for different loggers to be used ([known as bindings](https://www.slf4j.org/manual.html#swapping)). The binding used by Gaffer is `org.slf4j:slf4j-reload4j:jar:1.7.36`. + +## Impact of Log4j removal on projects incorporating Gaffer + +Gaffer now uses Reload4j via SLF4J. This may impact projects which are using Gaffer if they are using Log4j directly or through a transitive dependency. To help avoid dependency conflicts, we have configured `maven-enforcer-plugin` to block use of Log4j with Gaffer. If you are using Gaffer in your project and your build fails because of this plugin, you will need to add a dependency exclusion to any dependencies which depend transitively on Log4j. These can be found by using the Maven dependency tree (ideally in verbose mode). + +## Dependencies of Gaffer using Log4j 1.2.17 + +Some major Gaffer dependencies (listed below) use Log4j internally (either directly or through SLF4J). From Gaffer version 1.22 these transitive dependencies are excluded and replaced with Reload4j, such that Log4j does not appear on the classpath at all. + +- GCHQ Koryphe 1.14.0 - Uses SLF4J with Log4j. +- Apache HBase 1.3.0 - Multiple artefacts used from the group `org.apache.hbase`. All depend directly on Log4j. +- Apache Hadoop 2.6.5 - Multiple artefacts used from the group `org.apache.hadoop`. All depend directly on Log4j. +- Apache Accumulo 1.9.3 - Multiple artefacts used from the group `org.apache.accumulo`. All depend directly on Log4j. +- Apache Kafka 0.10.0.0 - Artefact depends indirectly on Log4j through a sub dependency (`com.101tec:zkclient`). +- Apache Spark 2.3.2 - Artefact depends directly on Log4j. + +## Log4j Vulnerabilities + +Current vulnerabilities in Log4j 1.12.17 relate to the JDBC, SMTP and JMS appenders, the JMS Sink and the Socket Server. Gaffer never used any of this. In its default configuration, we don't believe Gaffer is vulnerable to any of these problems. If the Log4j configuration is altered, changes could be made which may cause Gaffer to be vulnerable to one or more of the above vulnerabilities. Standard security processes to prevent unauthorised access and modification of configuration files should preclude this possibility. diff --git a/docs/development-guide/getting-started.md b/docs/development-guide/getting-started.md index e69de29bb2..8e1be9c9c3 100644 --- a/docs/development-guide/getting-started.md +++ b/docs/development-guide/getting-started.md @@ -0,0 +1,35 @@ +# Development + +!!! info "Work in Progress" + + This page is under construction. To propose additions or changes, please use the pencil button above on the right. + +## Source Control + +Development of Gaffer is done on the +[GCHQ/Gaffer](https://github.com/gchq/Gaffer) GitHub repository, or other Gaffer +GitHub repositories under the [GCHQ +Organization](https://github.com/orgs/gchq/repositories). + +## Building Gaffer + +### Build Instructions + +The latest instructions for building Gaffer [are in our +README](https://github.com/gchq/Gaffer/blob/develop/README.md#building-and-deploying). + +### Supported Platforms + +A recent Linux distribution is recommended, although it should be possible to +build Gaffer on any system which has the latest version of Java 8 or 11 (the +Gaffer codebase uses Java 8). Running tests on Windows is not recommended due to +complexities with native libraries. + +## Contributing + +We welcome contributions to the project. See our [ways of working for more +detail](ways-of-working.md). All contributors must sign the [GCHQ Contributor +Licence Agreement](https://cla-assistant.io/gchq/Gaffer). + +You can quickly and easily contribute towards Gaffer using a [remote coding +environment](remote-coding-environments.md) such as GitHub Codespaces or Gitpod. \ No newline at end of file diff --git a/docs/user-guide/gaffer-basics/what-is-cardinality.md b/docs/user-guide/gaffer-basics/what-is-cardinality.md new file mode 100644 index 0000000000..9aa2f0bcd8 --- /dev/null +++ b/docs/user-guide/gaffer-basics/what-is-cardinality.md @@ -0,0 +1,316 @@ +# Cardinality + +This page describes what cardinality is, how to add it to your Gaffer graph and how to use it in +Gaffer queries. + +## What is Cardinality? + +In Gaffer, cardinality represents the number of unique vertices that are connected to a given +vertex. + +``` mermaid +graph TD + 1(1, cardinality=4) --> 2 + 1 --> 3 + 1 --> 4 + 1 --> 5 + 3(3, cardinality=2) --> 5 + 3 --> 5 + 3 --> 5 + 3 --> 5 +``` + +For very large graphs, updating this number accurately would be very costly in compute. This is +because for each new Edge that is added, we would have to check both connected Entities to see if +they are already connected to the other Entity, and this could be costly for Entities with a high +cardinality. Instead, Gaffer uses approximate cardinality making use of a [HyperLogLog +Sketch](https://datasketches.apache.org/docs/HLL/HLL.html), which estimates the cardinality with +relatively low error. In Gaffer, where you see the term "cardinality" used, it is referring to this +approximate cardinality backed by a sketch. + +## How to add cardinality to your graph + +You will need to add a property to your schema that will represent the approximate cardinality of an +Entity. This property is usually added to a specific Entity group that exists solely to represent +the Cardinality of a given vertex value. An example of the schema changes can be seen in the +[advanced properties guide](../../reference/properties-guide/advanced.md#hllsketch). If you are +using an Accumulo or Map store as your data store, this should be all that is needed. However, if +you are using a custom store, or a custom rest API, some additional config is needed. + +!!! tip + It is often useful keep track of cardinality per edge group. This is usually done with an edge + group property which is group in the `groupBy`. + + ``` json + { + "entities": { + "cardinality": { + "vertex": "vertex.string", + "properties": { + "approxCardinality": "hll", + "edgeGroup": "set" + }, + "groupBy": [ + "edgeGroup" + ] + } + } + } + ``` + +??? info "Additional config" + If you are using a custom data store, you will need to make sure the `SketchesJsonModules` is + added to your store properties. This can be easily done by changing the `store.properties` file, + as shown below. Alternatively, it can be hardcoded into the store, like in the + [AccumuloProperties](https://github.com/gchq/Gaffer/blob/a91ce4cd1e04dd0a2dfdf9633b513768fccd3507/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloProperties.java#L468). + + ```properties + gaffer.serialiser.json.modules=uk.gov.gchq.gaffer.sketches.serialisation.json.SketchesJsonModules + ``` + + If you are using a custom data store, or you not using the standard spring-rest Gaffer rest API, + then you will also need to ensure that the `sketches-library` dependency is added to your + `pom.xml` for the store and/or rest API. + + ```xml + + uk.gov.gchq.gaffer + sketches-library + ${gaffer.version} + + ``` + +## How to add data with cardinality in your schema + +There are two main methods of adding cardinality elements in Gaffer. The first is to do it manually +when you add your edges, the second is to use a generator that can do it for you. + +### Manually adding cardinality entities + +Adding a cardinality entity between vertex 'A' and 'B' using `AddElements`. For more examples of +different property types, see the sketches [dev +docs](../../dev/rest-api-sketches.md#primitive-data-types-over-json) page. + +=== "Java" + + ``` java + final HllSketch hll = new HllSketch(10); //(1)! + hll.update("B"); //(2)! + new AddElements.Builder() + .input(new Entity.Builder() + .group("cardinality") + .vertex("A") + .property("approxCardinality", hll) //(3)! + .build()) + .build(); + ``` + + 1. Create the HllSketch object and define the precision values. By default, logK = 10. + 2. Update the sketch with the connected entity's vertex value, this adds it to the HllSketch's + bit hash. + 3. When adding the cardinality entity, set the property. + +=== "JSON" + + ``` json + { + "class": "AddElements", + "input": [{ + "class": "Entity", + "vertex": "A", + "group": "cardinality", + "properties": { + "approxCardinality": { + "org.apache.datasketches.hll.HllSketch": { + "values": ["B"] //(1)! + } + } + } + }] + } + ``` + + 1. You can directly add the values in the json and the deserialiser and aggregator will ensure + it is properly added to the object. + +=== "Python" + + ```python + g.AddElements( + input=[ + g.Entity( + vertex="A", + group="cardinality", + properties={ + "hll": g.hll_sketch(["B"]) #(1)! + } + ) + ] + ) + ``` + + 1. The `g.hll_sketch` helper function lets you directly add the values. The deserialiser and + aggregator will ensure it is properly added to the object. + +### Automatically adding cardinality entities + +Rather than using the `AddElements` operation to manually add cardinality entities, you can add them +automatically when you add edges to the graph using the +[`HllSketchEntityGenerator`](https://github.com/gchq/Gaffer/blob/a91ce4cd1e04dd0a2dfdf9633b513768fccd3507/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/datasketches/cardinality/HllSketchEntityGenerator.java). +This will take an edge and return the same edge, as well as two cardinality entities, each +representing the additional cardinality link between the two vertices in the edge. + +=== "Java" + + ```java + new OperationChain.Builder() + .first(new GenerateElements.Builder() + .input(new Edge("edgeGroup1", "A", "B", true)) //(1)! + .generator(new HllSketchEntityGenerator() //(2)! + .cardinalityPropertyName("approxCardinality") //(3)! + .group("cardinality") //(4)! + .edgeGroupProperty("edgeGroup") //(5)! + .propertiesToCopy(...) //(6)! + ) + .build()) + .then(new AddElements()) //(7)! + .build(); + ``` + + 1. The input of edges is added to the OperationChain. Here we make one Edge between "A" and "B", + with group = "edgeGroup". + 2. The input is streamed into the `HllSketchEntityGenerator`, which will return the edge as well + as the two cardinality entities. + 3. The name of the property where we should store the cardinality. + 4. The group which the cardinality entity should be put into. + 5. If you track cardinality per edge group using a set property (as described + [above](#how-to-add-cardinality-to-your-graph)), which property should track this. + 6. Any properties from the edge to copy onto the cardinality entity. + 7. The results are streamed into `AddElements` to be added to the graph. + +=== "JSON" + + ```json + { + "class": "OperationChain", + "operations": [ + { + "class": "GenerateElements", + "input": [{ + "class": "Edge", + "group": "edgeGroup1", + "source": "A", + "destination": "B", + "directed": true + }], + "elementGenerator": { + "class": "HllSketchEntityGenerator", + "cardinalityPropertyName": "approxCardinality", + "edgeGroupProperty": "edgeGroup", + "group": "cardinality" + } + }, + { + "class": "AddElements" + } + ] + } + ``` + +=== "Python" + + ```python + g.OperationChain([ + g.GenerateElements( + input=[g.Edge("edgeGroup1", "A", "B", True)], + element_generator=g.HllSketchEntityGenerator( + cardinality_property_name="approxCardinality", + group="cardinality", + edge_group_property="edgeGroup" + ) + ), + g.AddElements() + ]) + ``` + +## How to get cardinality back using Gaffer queries + +Depending on how you query Gaffer, approximate cardinality will be displayed in results in different ways. + +=== "Java" + + ```java + final GetElements query = new GetElements.Builder() + .input(new EntitySeed("A")) + .build(); + + final Element element; + try (final CloseableIterable elements = graph.execute(query, user)) { + element = elements.iterator().next(); + } + + final HllSketch hllSketch = (HllSketch) element.getProperty("approxCardinality"); + final double approxCardinality = hllSketch.getEstimate(); + final String cardinalityEstimate = "Entity A has approximate cardinality " + approxCardinality; + ``` + + Result: + + ```txt + Entity A has approximate cardinality 1.0 + ``` + +=== "JSON" + + ```json + { + "class": "GetElements", + "input": [ + { + "class": "EntitySeed", + "vertex": "A" + } + ] + } + ``` + + Result: + + ```json + [{ + "class": "Entity", + "group": "cardinality", + "vertex": "A", + "properties": { + "approxCardinality": { + "org.apache.datasketches.hll.HllSketch": { + "bytes": "AgEHCgMIAQBejtgF", "cardinality": 1.0 + } + } + } + }] + ``` + +=== "Python" + + ```python + g.GetElements(g.EntitySeed("A")) + ``` + + Result: + + ```python + [{ + 'class': 'uk.gov.gchq.gaffer.data.element.Entity', + 'group': 'BasicEntity', + 'vertex': 'A', + 'properties': { + 'approxCardinality': { + 'org.apache.datasketches.hll.HllSketch': { + 'bytes': 'AgEHCgMIAQBejtgF', + 'cardinality': 1.0 + } + } + } + }] + ``` diff --git a/mkdocs.yml b/mkdocs.yml index 91d93fa019..4f78befea6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -104,6 +104,7 @@ nav: - 'What is a Graph?': 'user-guide/gaffer-basics/what-is-a-graph.md' - 'What is JSON?': 'user-guide/gaffer-basics/what-is-json.md' - 'What is Python?': 'user-guide/gaffer-basics/what-is-python.md' + - 'What is Cardinality?': 'user-guide/gaffer-basics/what-is-cardinality.md' - 'Getting Started': - 'API': 'user-guide/getting-started/api.md' - 'Schema': 'user-guide/getting-started/schema.md' @@ -150,8 +151,15 @@ nav: - Maven Dependencies: - 'Spring': 'development-guide/project-structure/maven-dependencies/spring.md' - Administration Guide: - - 'How to Run Gaffer': - - 'Gaffer Docker': 'administration-guide/where-to-run-gaffer/gaffer-docker.md' + - 'Getting Started': + - 'Docker': 'administration-guide/where-to-run-gaffer/gaffer-docker.md' + - 'Kubernetes': + - 'Kubernetes Guide': 'administration-guide/where-to-run-gaffer/kubernetes-guide/kubernetes-guide.md' + - 'Deploy an Empty Graph': 'administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-empty-graph.md' + - 'Add your Schema': 'administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-schema.md' + - 'Change the Graph ID and Description': 'administration-guide/where-to-run-gaffer/kubernetes-guide/change-graph-metadata.md' + - 'Adding your Own Libraries and Functions': 'administration-guide/where-to-run-gaffer/kubernetes-guide/add-libraries.md' + - 'Changing Accumulo Passwords': 'administration-guide/where-to-run-gaffer/kubernetes-guide/change-accumulo-passwords.md' - 'Store Setup': - 'Store Guide': 'administration-guide/gaffer-stores/store-guide.md' - 'Accumulo Store': 'administration-guide/gaffer-stores/accumulo-store.md' @@ -172,7 +180,13 @@ nav: - Change Notes: - 'Changelist': - 'V2.0 Changes': 'change-notes/changelist/v2-changes.md' - - 'Migrating from V1 to V2': 'change-notes/migrating-from-v1-to-v2.md' + - 'Migrating from V1 to V2': + - 'Depreciations': 'change-notes/migrating-from-v1-to-v2/' + - 'Dependencies': 'change-notes/migrating-from-v1-to-v2/' + - 'Accumulo Migration': 'change-notes/migrating-from-v1-to-v2/' + - 'Federation Changes': 'change-notes/migrating-from-v1-to-v2/' + - 'Accumulo Kerberos': 'change-notes/migrating-from-v1-to-v2/' + - 'Log4j in Gaffer': 'change-notes/migrating-from-v1-to-v2/' - 'Reference': - 'Reference Guide': 'reference/intro.md' - 'Glossary': 'reference/glossary.md' From 7481d8ff250215a5a8a7bd701d9c0b23e5e44f7a Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Wed, 6 Sep 2023 11:00:10 +0000 Subject: [PATCH 10/20] Removed unwanted pages and corrected paths --- docs/development-guide/rest-api-sketches.md | 236 ++++++++++++++++++ ...affer-basics.md => what-is-aggregation.md} | 0 .../getting-started/getting-started.md | 0 docs/user-guide/query/query.md | 0 mkdocs.yml | 14 +- 5 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 docs/development-guide/rest-api-sketches.md rename docs/user-guide/gaffer-basics/{gaffer-basics.md => what-is-aggregation.md} (100%) delete mode 100644 docs/user-guide/getting-started/getting-started.md delete mode 100644 docs/user-guide/query/query.md diff --git a/docs/development-guide/rest-api-sketches.md b/docs/development-guide/rest-api-sketches.md new file mode 100644 index 0000000000..5b6ba6055f --- /dev/null +++ b/docs/development-guide/rest-api-sketches.md @@ -0,0 +1,236 @@ +# Using Sketches with the REST API + +This page explains some nuances and special steps required when using classes from the Sketches library with the REST API. If you just want to know how to use the sketches libraries to +use cardinality, see the [cardinality](../getting-started/guide/cardinality.md) docs page. + +## Sketches Library + +To learn more about the Sketches library see [advanced properties](../../reference/properties-guide/advanced) reference page. +The sketches library is included by default with the Map and Accumulo stores. This is because the `sketches-library` is a dependency in each of +the respective store modules' poms. As well as this, the serialisation is handled by the fact the +[SketchesJsonModules](https://github.com/gchq/Gaffer/blob/gaffer2-2.0.0/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/serialisation/json/SketchesJsonModules.java) +is returned by the `getJsonSerialiserModules` method in both the +[Map](https://github.com/gchq/Gaffer/blob/gaffer2-2.0.0/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/MapStoreProperties.java#L127) +and [Accumulo](https://github.com/gchq/Gaffer/blob/gaffer2-2.0.0/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloProperties.java#L468) +property classes. The modules are then loaded by the [JSONSerialiser](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/jsonserialisation/JSONSerialiser.html) +and used during the deserialisation of the REST JSON queries. + +## HyperLogLog sketches + +Gaffer currently supports the [Datasketches HllSketch](https://github.com/apache/datasketches-java/blob/master/src/main/java/org/apache/datasketches/hll/HllSketch.java) and [Clearspring HyperLogLogPlus](https://github.com/addthis/stream-lib/blob/master/src/main/java/com/clearspring/analytics/stream/cardinality/HyperLogLogPlus.java) algorithms. The Clearspring HyperLogLogPlus has been **deprecated in Gaffer** and we recommend the Datasketches HllSketch to users for the reasons described in the [advanced properties guide](../reference/properties-guide/advanced.md#hyperloglogplus). + +The `HllSketch` and `HyperLogLogPlus` sketches can be used to store an approximation of +cardinality of an element. The JSON of the query is converted to Java +objects during deserialisation using the `JSONSerialiser`. During the +deserialisation, the sketch's JSON representation is converted to a Java +object using the `ObjectMapper` module which uses the relevant deserialiser ( +[HyperLogLogPlusJsonDeserialiser](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/clearspring/cardinality/serialisation/json/HyperLogLogPlusJsonDeserialiser.java) or [HllSketchJsonDeserialiser](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/datasketches/cardinality/serialisation/json/HllSketchJsonDeserialiser.java)). + +## Creating cardinality values over JSON + +When adding or updating a cardinality object over the rest api, you specify the vertex values to add to the sketch. +This is done by either using the `offers` field with `HyperLogLogPlus`, or the `values` field with `HllSketch`. +The HyperLogLog object is then instantiated and updated with +the values. The object can then be serialised and stored in the datastore. +The vertex object is serialised using the `toString` representation of the object. +???+ note + As the algorithms use the `toString` method, any user defined type + introduced must override the `toString` method returning meaningful string + value representing the object rather than the default class instance + identifier. User defined types can be introduced by either adding further + [types](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/types/package-summary.html) + to Gaffer or by adding a jar with the extra type(s) to the Gaffer + classpath on startup. + +Depending on whether you are using `HyperLogLogPlus` or `HllSketch`, either the +[`HyperLogLogPlusWithOffers`](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/clearspring/cardinality/serialisation/json/HyperLogLogPlusWithOffers.java) or the +[`HllSketchWithValues`](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/datasketches/cardinality/serialisation/json/HllSketchWithValues.java) +respectively is responsible for the JSON deserialisation. +The helper classes wrap the underlying sketch and includes the following annotation on +the `offers`/`values` field: + +```java +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "class") +``` + +This signals to the Jackson `ObjectMapper` that it needs to look for the +`class` field in each object and translate to the correct object type. + +### Primitive data types over JSON +Primitive types are converted to the correct format by Jackson +`ObjectMapper` automatically. Here are some examples of the values: + +=== "String" + `"values": ["valueA", "value2",...]` + +=== "Long" + `"values": [1, 2,...]` + +=== "Double" + `"values": [1.1, 2.2,...]` + +### Non-primitive data types over JSON +In order to convert non-primitive vertex values (like `TypeSubTypeValue`) to Java objects, the JSON values need to contain the special field **class** +containing the class name of the object. The `deserialiser` uses this `class` +field when deserialising using the [JSONSerialiser](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/jsonserialisation/JSONSerialiser.html) +`deserialise` method. + +Here are the Gaffer user defined types: + +=== "FreqMap" + ```json + "values": [ + { + "class": "uk.gov.gchq.gaffer.types.FreqMap", + "test": 1 + }, + ... + ] + ``` + +=== "CustomMap" + ```json + "values": [ + { + "class": "uk.gov.gchq.gaffer.types.CustomMap", + "keySerialiser": { + "class": "uk.gov.gchq.gaffer.serialisation.implementation.BooleanSerialiser" + }, + "valueSerialiser": { + "class": "uk.gov.gchq.gaffer.serialisation.implementation.BooleanSerialiser" + }, + "jsonStorage": [] + }, + ... + ] + ``` + +=== "TypeValue" + ```json + "values": [ + { + "class" : "uk.gov.gchq.gaffer.types.TypeValue", + "type" : "type", + "value" : "value" + }, + ... + ] + ``` + +=== "TypeSubTypeValue" + ```json + "values": [ + { + "class" : "uk.gov.gchq.gaffer.types.TypeSubTypeValue", + "type" : "type", + "subType" : "subType", + "value" : "value" + }, + ... + ] + ``` + +???+ note + The subclass fields must also have the `class` field set (for + example, the `keySerialiser` in the `CustomMap` type) if not a standard Java Object + so that the Jackson `ObjectMapper` knows how to convert the correct values + to Java objects. + +#### Composing using Java + +If you are composing the `HllSketch` with values using Java, before +converting to JSON and sending via REST, you need ensure that the `values` +objects are translated to JSON with the correct `class` field added. +To make sure of this, you could add the `sketches-library` JAR and use the +[HllSketchWithValues](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/datasketches/cardinality/serialisation/json/HllSketchWithValues.java) +object to construct your query (or the equivalent for HyperLogLogPlus). +This way you know that all the objects have the +correct field added. You can then convert the `HllSketchWithValues` to +JSON using the +[JSONSerialiser](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/jsonserialisation/JSONSerialiser.html) +`serialisation` method: +```java +final HllSketchWithValues hllSketchWithValues = JSONSerialiser.deserialise(treeNode.toString(), HllSketchWithValues.class); +``` +If you want to create your own class instead, rather than using +`HllSketchWithValues`, ensure +that the `values` list has the correct annotation so the `class` is added on +conversion using by the Jackson `ObjectMapper`: + +```java +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "class") +private List values = new ArrayList<>(); +``` + +#### Composing using Python + +An example of using Python to add a `HyperLogLogPlus` property with a `TypeSubTypeValue` offer: +```python +g.AddElements( + input=[ + g.Entity( + vertex="A", + group="cardinality", + properties={ + "hllp": g.hyper_log_log_plus([ + { + "class" : "uk.gov.gchq.gaffer.types.TypeSubTypeValue", + "type" : "t", + "subType" : "st", + "value" : "B" + } + ]) + } + ) + ] + ) +``` + +An example of using Python to add a `HllSketch` property with a `TypeSubTypeValue` offer: +```python +g.AddElements( + input=[ + g.Entity( + vertex="A", + group="cardinality", + properties={ + "hllSketch": g.hll_sketch([ + { + "class" : "uk.gov.gchq.gaffer.types.TypeSubTypeValue", + "type" : "t", + "subType" : "st", + "value" : "B" + } + ]) + } + ) + ] + ) +``` + +### Adding user defined vertex types into offers + +To add a user defined type you must ensure that: + +- the type is on the Gaffer classpath +- the type must override the `toString` method +- the type contains the correct annotations if you are converting from Java to + JSON before sending via REST + +The following user defined type example features the annotation required as +well as the `@Override` of the `toString` method: + +```java +@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "class") +public class ExampleType implements Comparable, Serializable { + + private String value...; + + // getters and setters + + @Override + public String toString() { + return ...; + } +} +``` diff --git a/docs/user-guide/gaffer-basics/gaffer-basics.md b/docs/user-guide/gaffer-basics/what-is-aggregation.md similarity index 100% rename from docs/user-guide/gaffer-basics/gaffer-basics.md rename to docs/user-guide/gaffer-basics/what-is-aggregation.md diff --git a/docs/user-guide/getting-started/getting-started.md b/docs/user-guide/getting-started/getting-started.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/user-guide/query/query.md b/docs/user-guide/query/query.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/mkdocs.yml b/mkdocs.yml index 4f78befea6..44d477cc3d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -105,6 +105,7 @@ nav: - 'What is JSON?': 'user-guide/gaffer-basics/what-is-json.md' - 'What is Python?': 'user-guide/gaffer-basics/what-is-python.md' - 'What is Cardinality?': 'user-guide/gaffer-basics/what-is-cardinality.md' + - 'What is Aggregation?': 'user-guide/gaffer-basics/what-is-aggregation.md' - 'Getting Started': - 'API': 'user-guide/getting-started/api.md' - 'Schema': 'user-guide/getting-started/schema.md' @@ -129,6 +130,7 @@ nav: - 'Ways of Working': 'development-guide/ways-of-working.md' - 'Remote Coding Environments': 'development-guide/remote-coding-environments.md' - 'Extending Gaffer': 'development-guide/extending-gaffer.md' + - 'Sketches Custom Deserialisation': 'development-guide/rest-api-sketches.md' - Project Structure: - Components: - 'Components/Maven Modules': 'development-guide/project-structure/components/components.md' @@ -181,12 +183,12 @@ nav: - 'Changelist': - 'V2.0 Changes': 'change-notes/changelist/v2-changes.md' - 'Migrating from V1 to V2': - - 'Depreciations': 'change-notes/migrating-from-v1-to-v2/' - - 'Dependencies': 'change-notes/migrating-from-v1-to-v2/' - - 'Accumulo Migration': 'change-notes/migrating-from-v1-to-v2/' - - 'Federation Changes': 'change-notes/migrating-from-v1-to-v2/' - - 'Accumulo Kerberos': 'change-notes/migrating-from-v1-to-v2/' - - 'Log4j in Gaffer': 'change-notes/migrating-from-v1-to-v2/' + - 'Deprecations': 'change-notes/migrating-from-v1-to-v2/deprecations.md' + - 'Dependencies': 'change-notes/migrating-from-v1-to-v2/dependencies.md' + - 'Accumulo Migration': 'change-notes/migrating-from-v1-to-v2/accumulo-migration.md' + - 'Federation Changes': 'change-notes/migrating-from-v1-to-v2/federation-changes.md' + - 'Accumulo Kerberos': 'change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md' + - 'Log4j in Gaffer': 'change-notes/migrating-from-v1-to-v2/log4j.md' - 'Reference': - 'Reference Guide': 'reference/intro.md' - 'Glossary': 'reference/glossary.md' From 29bc5e76fe4e33ad2ac4b3bf4cccf515e415b3e7 Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Thu, 7 Sep 2023 09:49:25 +0000 Subject: [PATCH 11/20] Removed unwanted sections and files --- .../gaffer-stores/accumulo-store.md | 2 +- docs/dev/components/accumulo-store.md | 213 - docs/dev/components/cache.md | 24 - docs/dev/components/components.md | 99 - docs/dev/components/core-rest.md | 72 - docs/dev/components/data.md | 28 - docs/dev/components/graph.md | 107 - docs/dev/components/integration-test.md | 6 - docs/dev/components/libraries/bitmap.md | 20 - docs/dev/components/libraries/flink.md | 19 - docs/dev/components/libraries/sketches.md | 14 - docs/dev/components/libraries/spark.md | 23 - docs/dev/components/libraries/time.md | 16 - docs/dev/components/operation.md | 82 - docs/dev/components/serialisation.md | 7 - docs/dev/components/spring-rest.md | 47 - docs/dev/components/store.md | 15 - docs/dev/development.md | 28 - docs/dev/docker.md | 25 - docs/dev/kubernetes-guide/add-libraries.md | 70 - .../change-accumulo-passwords.md | 36 - .../kubernetes-guide/change-graph-metadata.md | 63 - .../kubernetes-guide/deploy-empty-graph.md | 73 - docs/dev/kubernetes-guide/deploy-schema.md | 68 - docs/dev/kubernetes-guide/kubernetes.md | 40 - docs/dev/managing-dependencies/core.svg | 307 - .../managing-dependencies/gaffer-complete.svg | 5772 ----------------- .../managing-dependencies.md | 105 - docs/dev/managing-dependencies/map-store.svg | 1104 ---- docs/dev/remote-coding-environments.md | 32 - docs/dev/rest-api-sketches.md | 236 - docs/dev/ways-of-working.md | 91 - docs/gaffer2.0/accumulo-kerberos.md | 93 - docs/gaffer2.0/accumulo-migration.md | 27 - docs/gaffer2.0/changelist.md | 130 - docs/gaffer2.0/dependencies.md | 35 - docs/gaffer2.0/deprecations.md | 563 -- docs/gaffer2.0/federation-changes.md | 240 - docs/gaffer2.0/log4j.md | 30 - .../advanced-guide/aggregation.md | 7 - .../advanced-guide/cardinality.md | 316 - .../advanced-guide/import-export/csv.md | 201 - docs/getting-started/basics.md | 216 - .../example-deployment/project-setup.md | 231 - .../running-the-deployment.md | 219 - .../example-deployment/using-the-api.md | 191 - .../example-deployment/writing-the-schema.md | 264 - docs/getting-started/quickstart.md | 69 - mkdocs.yml | 53 - 49 files changed, 1 insertion(+), 11728 deletions(-) delete mode 100644 docs/dev/components/accumulo-store.md delete mode 100644 docs/dev/components/cache.md delete mode 100644 docs/dev/components/components.md delete mode 100644 docs/dev/components/core-rest.md delete mode 100644 docs/dev/components/data.md delete mode 100644 docs/dev/components/graph.md delete mode 100644 docs/dev/components/integration-test.md delete mode 100644 docs/dev/components/libraries/bitmap.md delete mode 100644 docs/dev/components/libraries/flink.md delete mode 100644 docs/dev/components/libraries/sketches.md delete mode 100644 docs/dev/components/libraries/spark.md delete mode 100644 docs/dev/components/libraries/time.md delete mode 100644 docs/dev/components/operation.md delete mode 100644 docs/dev/components/serialisation.md delete mode 100644 docs/dev/components/spring-rest.md delete mode 100644 docs/dev/components/store.md delete mode 100644 docs/dev/development.md delete mode 100644 docs/dev/docker.md delete mode 100644 docs/dev/kubernetes-guide/add-libraries.md delete mode 100644 docs/dev/kubernetes-guide/change-accumulo-passwords.md delete mode 100644 docs/dev/kubernetes-guide/change-graph-metadata.md delete mode 100644 docs/dev/kubernetes-guide/deploy-empty-graph.md delete mode 100644 docs/dev/kubernetes-guide/deploy-schema.md delete mode 100644 docs/dev/kubernetes-guide/kubernetes.md delete mode 100644 docs/dev/managing-dependencies/core.svg delete mode 100644 docs/dev/managing-dependencies/gaffer-complete.svg delete mode 100644 docs/dev/managing-dependencies/managing-dependencies.md delete mode 100644 docs/dev/managing-dependencies/map-store.svg delete mode 100644 docs/dev/remote-coding-environments.md delete mode 100644 docs/dev/rest-api-sketches.md delete mode 100644 docs/dev/ways-of-working.md delete mode 100644 docs/gaffer2.0/accumulo-kerberos.md delete mode 100644 docs/gaffer2.0/accumulo-migration.md delete mode 100644 docs/gaffer2.0/changelist.md delete mode 100644 docs/gaffer2.0/dependencies.md delete mode 100644 docs/gaffer2.0/deprecations.md delete mode 100644 docs/gaffer2.0/federation-changes.md delete mode 100644 docs/gaffer2.0/log4j.md delete mode 100644 docs/getting-started/advanced-guide/aggregation.md delete mode 100644 docs/getting-started/advanced-guide/cardinality.md delete mode 100644 docs/getting-started/advanced-guide/import-export/csv.md delete mode 100644 docs/getting-started/basics.md delete mode 100644 docs/getting-started/example-deployment/project-setup.md delete mode 100644 docs/getting-started/example-deployment/running-the-deployment.md delete mode 100644 docs/getting-started/example-deployment/using-the-api.md delete mode 100644 docs/getting-started/example-deployment/writing-the-schema.md delete mode 100644 docs/getting-started/quickstart.md diff --git a/docs/administration-guide/gaffer-stores/accumulo-store.md b/docs/administration-guide/gaffer-stores/accumulo-store.md index e3d5495854..026319c2ba 100644 --- a/docs/administration-guide/gaffer-stores/accumulo-store.md +++ b/docs/administration-guide/gaffer-stores/accumulo-store.md @@ -56,7 +56,7 @@ accumulo.user=myUser accumulo.password=myPassword ``` -When using Kerberos authentication, the username and password are not used, alternative properties are used to configure Kerberos. See the [Accumulo Kerberos guide for more information](../../gaffer2.0/accumulo-kerberos.md). +When using Kerberos authentication, the username and password are not used, alternative properties are used to configure Kerberos. See the [Accumulo Kerberos guide for more information](../../change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md). Note that if the graph does not exist, it will be created when a `Graph` object is instantiated using these properties, the schema and the graph ID (given when the graph is created in Java or via a `graphConfig.json`). In this case the user must have permission to create a table. If the graph already exists (based on the graph ID) then the user simply needs permission to read the table. For information about protecting data via setting the visibility, see [Visibilty](#visibility). diff --git a/docs/dev/components/accumulo-store.md b/docs/dev/components/accumulo-store.md deleted file mode 100644 index 4fb633a477..0000000000 --- a/docs/dev/components/accumulo-store.md +++ /dev/null @@ -1,213 +0,0 @@ -# Accumulo Store Implementation - -The [accumulo-store module](https://github.com/gchq/Gaffer/tree/master/core/store) is an implementation of the Store API which uses Apache Accumulo. - -This page contains brief details on the internal implementation of the `AccumuloStore`. For information on configuring and using this store, see [the Accumulo Store reference page](../../reference/stores-guide/accumulo.md). - -## Introduction - -It is assumed that the reader has some familiarity with the design of Accumulo (see the [Design page in Accumulo's Docs](https://accumulo.apache.org/docs/2.x/getting-started/design)). - -The important features for Gaffer are: - -- Accumulo stores data in key-value pairs. A key has multiple parts, namely a row ID, a column family, a column qualifier, a column visibility, and a timestamp. Each of these is simply a byte array, with the exception of the timestamp which is a long. A value is simply a byte array. -- Data in Accumulo is stored ordered by key. Keys are stored sorted by increasing row ID, then column family, then column qualifier, then column visibility, then by decreasing timestamp. -- Accumulo allows locality groups to be set which group together column families. This means that scans that only need to read certain column families can skip families they do not need to read. -- Accumulo allows data to be tagged with a visibility which restricts which users can view it. -- Accumulo allows the user to configure iterators that run at scan time, at compaction time or both. Gaffer adds iterators to scans to filter data. It uses compaction time iterators to persistently aggregate the properties of elements together, and to continually validate data. -- Accumulo provides an `InputFormat` that allows data to be retrieved via MapReduce jobs. - -The core of the functionality is implemented in the key-packages, the iterators and the retrievers. Each of these is described in some detail below. - -## Key-packages - -As noted in the [Key-packages section of the Accumulo Store reference](../../reference/stores-guide/accumulo.md#key-packages), key-packages are responsible for converting `Element`s to and from key-value pairs, for creating ranges of keys containing all data relevant to a particular query, and for configuring the Iterators. Gaffer provides two key-packages: `ByteEntityKeyPackage` and `ClassicKeyPackage`. Advanced users are able to create their own key-packages if they wish - see [options for future key-packages](#options-for-future-key-packages) for some ideas. - -Before these key-packages are described, we review the main design goals: - -- To be able to retrieve all `Edge`s for a vertex by seeking to a single point in the table and scanning forwards. -- To be able to retrieve all `Entity`s for a vertex by seeking to a single point in the table, and reading only relevant key-value pairs, i.e. not reading any of the `Edge`s associated to the vertex. -- A vertex should be uniquely identified by its serialised value. It should not be necessary to consult an external source to find the value that identifies a vertex. In particular unlike most graph databases we do not use longs to identify vertices. -- To ensure that there are no "fat" rows, i.e. that there are not very large numbers of key-value pairs with the same row-key. -- To allow efficient aggregation of properties. - -Both key-packages convert an `Entity` into a single Accumulo key-value pair and an `Edge` into two key-value pairs. The row ID (also known as the row-key) of the key-value formed from the `Entity` is the vertex serialised to a byte array, followed by a flag to indicate that this is an `Entity`. This allows the `Entity`s associated to a vertex to be quickly retrieved. It is necessary to store each `Edge` as two key-values so that it can found from both the source vertex and the destination vertex: one key-value has a row ID consisting of the source vertex serialised to a byte array, followed by a delimiter, followed by the destination vertex serialised to a byte array; the other key-value has the opposite, with the destination vertex followed by the source vertex. A flag is also stored to indicate which of these two versions the key is so that the original `Edge` can be recreated. - -An important feature of the row IDs created by both key-packages is that it is possible to create ranges of keys that either only contain the `Entity`s or only contain the `Edge`s or contain both. This means that if, for example, a user states that they only want to retrieve the `Entity`s for a particular vertex then only relevant key-value pairs need to be read. In the case of a high-degree vertex, this means that queries for just the `Entity`s will still be very quick. - -The two key-packages differ in subtle details of how the row ID is created. In the following descriptions the notation "(serialised_vertex)" refers to the vertex serialised to a byte array with any occurrences of the zero byte removed. This is necessary so that the zero byte delimiter can be used to separate different parts of the row-key. The zero bytes are removed in such a way that the original byte array can be recreated, and so that ordering is preserved. - -### `ClassicKeyPackage` details - -The `ClassicKeyPackage` constructs the following Accumulo key-value pair for an `Entity`: - -| Row ID | Column Family | Column Qualifier | Visibility | Timestamp | Value | -| ------------------- | ------------- | ------------------- | ------------------- | --------- | -------------------- | -| (serialised_vertex) | group | group by properties | visibility property | timestamp | all other properties | - -The following Accumulo key-value pairs are created for an `Edge`: - -| Row ID | Column Family | Column Qualifier | Visibility | Timestamp | Value | -| ------------------------------------------------------------ | ------------- | ------------------- | ------------------- | --------- | -------------------- | -| (serialised_source_vertex)0(serialised_destination_vertex)0x | group | group by properties | visibility property | timestamp | all other properties | -| (serialised_destination_vertex)0(serialised_source_vertex)0y | group | group by properties | visibility property | timestamp | all other properties | - -If the `Edge` is undirected then `x` and `y` are both 1 for both key-values. If the `Edge` is directed then `x` is 2 and `y` is 3. - -This is very similar to the design of the key-value pairs in version 1 of Gaffer, with the exception that version 1 did not store a delimiter or flag at the end of the row-key for an `Entity`. This necessitated a scan of the row-key counting the number of delimiters to determine whether it was an `Entity` or `Edge`. If it is an `Entity` the vertex could be created directly from the row-key. For the `ClassicKeyPackage`, this scan is not needed but an array copy of the row-key minus the delimiter and flag is needed. In practice, the difference in performance between the two is likely to be negligible. - -### `ByteEntityKeyPackage` details - -The ByteEntity key-package constructs the following Accumulo key-value pair for an `Entity`: - -| Row ID | Column Family | Column Qualifier | Visibility | Timestamp | Value | -| ------------------- | ------------- | ------------------- | ------------------- | --------- | -------------------- | -| (serialised_vertex)01 | group | group by properties | visibility property | timestamp | all other properties | - -In the row ID the 0 is a delimiter to split the serialised vertex from the 1. The 1 indicates that this is an `Entity`. By having this flag at the end of the row id it is easy to determine if the key relates to an `Entity` or an `Edge`. - -The following Accumulo key-value pairs are created for an `Edge`: - -| Row ID | Column Family | Column Qualifier | Visibility | Timestamp | Value | -| -------------------------------------------------------------- | ------------- | ------------------- | ------------------- | --------- | -------------------- | -| (serialised_source_vertex)0x0(serialised_destination_vertex)0x | group | group by properties | visibility property | timestamp | all other properties | -| (serialised_destination_vertex)0y0(serialised_source_vertex)0y | group | group by properties | visibility property | timestamp | all other properties | - -If the `Edge` is undirected then both `x` and `y` are 4. If the `Edge` is directed then `x` is 2 and `y` is 3. - -The flag is repeated twice to allow filters that need to know whether the key corresponds to a `Entity` or an `Edge` to avoid having to fully deserialise the row ID. For a query such as find all out-going edges from this vertex, the flag that is directly after the source vertex can be used to restrict the range of row IDs queried for. - -Note that in a range query filtering to restrict the results to say only out-going edges happens in an iterator. - -### Options for future key-packages - -Numerous variations on the above key-packages could be implemented. These would generally improve the performance for some types of query, at the expense of decreasing the performance for other types of query. Some examples are: - -- The row-keys could be sharded. The current design is optimised for retrieving all `Edge`s for a given vertex, when there are relatively few such `Edge`s. If there are a million edges for a vertex then all of these have to be read by a small number of tablet servers (typically one, unless the range spans multiple tablets). This limits the query performance. An alternative approach is to introduce a shard key at the start of the row-key to cause different edges for the same vertex to be spread uniformly across the table. This would increase the parallelism for queries which would lead to better performance when large numbers of edges need to be retrieved for a vertex. The trade-off is that all queries would need to query all shards which would reduce the performance when a vertex has only a small number of edges. -- If there are a very large number of `Edge`s with the same source, destination and group-by properties then this could cause unbalanced tablets. A sharding scheme similar to the above would deal with this. -- Remove the flag at the end of the row-key that indicates whether it corresponds to an `Entity` or an `Edge`. This is used to quickly determine whether it is an `Entity` or an `Edge`. This is actually superfluous information as the group is stored in the column family and that indicates whether the key-value is an `Entity` or an `Edge`. Storing the flag there creates the need for an array copy when an `Entity` is created from the key-value. Instead of storing the group string in the column family, two bytes could be stored. The first would indicate whether this is an `Entity` or an `Edge`, and if an `Edge` whether it needs reversing or not; the second would indicate what group it is. -- Store each group in a separate table. This should slightly improve the performance of queries that only require a subset of the groups, especially if the query scans lots of data (as Accumulo's locality groups are set in the above key-packages the performance improvement will probably be minor). It would worsen the query performance when multiple groups are being retrieved. -- If the vertices serialise to a fixed length, or if a maximum length is known, then the row-keys could be of fixed length. This would eliminate the need for the use of delimiters which forces the escaping of the zero byte inside the serialised value. This would potentially provide a small increase in ingest and query speed. - -## Iterators - -Gaffer makes substantial use of Accumulo's iterator functionality to perform permanent aggregation and validation of data at compaction time, and filtering and aggregation at query time. See the [Iterators](https://accumulo.apache.org/docs/2.x/development/iterators) section of Accumulo's Docs for more information on iterators. - -The following subsections describes the iterators that are used in Gaffer. They are listed in decreasing order of priority, i.e. the first iterator runs first. The text in brackets after the name of the iterator gives the scopes that the iterator is applied in. Some iterators that are only used for very specific operations are not listed here. - -### `AggregatorIterator` (compaction, scan) - -This iterator aggregates together all properties that are not group-by properties for `Element`s that are otherwise identical. As the non-group-by properties are stored in the `Value` this means that all `Value`s for identical keys are merged together. - -### `ValidatorFilter` (compaction, scan) - -The `ValidatorFilter` iterator validates every `Element` using the validation logic defined in the schema. When this is run during a compaction it causes invalid data to be deleted. This is typically used to delete data that is older than a certain date. - -### `ClassicEdgeDirectedUndirectedFilterIterator` (scan) - -!!! note - This is only used in the `ClassicKeyPackage`. - -This is used to filter out edges that are not required because the user has specified filters relating to edge direction (outgoing or incoming) and edge "directedness" (directed or undirected) in their query. Note that it is possible to ask for various combinations of these, e.g.: - -- Directed edges only: if the seed is A then directed edges A->B and B->A would be returned, but an undirected edge A-B wouldn't be. -- Directed outgoing edges only: if the seed is A then a directed edge A->B would be returned, but a directed edge B->A wouldn't be, nor would an undirected edge A-B. -- Directed incoming edges only: if the seed is A then a directed edge B->A would be returned, but a directed edge A->B wouldn't be, nor would an undirected edge A-B. -- Undirected edges only: if the seed is A then an undirected edge A-B would be returned, but directed edges A->B and B->A wouldn't be. -- Undirected outgoing edges only: if the seed is A then an undirected edge A-B would be returned, but directed edges A->B and B->A wouldn't be. -- Undirected incoming edges only: if the seed is A then an undirected edge A-B would be returned, but directed edges A->B and B->A wouldn't be. - -In the latter two examples, note that an undirected edge A-B is defined to be both outgoing from, and incoming to, both A and B. - -### `ElementPreAggregationFilter` (scan) - -This iterator filters out `Element`s that are not valid according to the `View`. This filtering happens before the aggregation. - -### `CoreKeyGroupByAggregatorIterator` (scan) - -This iterator aggregates together all properties according to the group-by in the view. - -### `ElementPostAggregationFilter` (scan) - -This iterator filters out `Element`s that are not valid according to the `View`. This filtering happens after the aggregation. - -### Locality groups - -Accumulo's ability to have a large number of different column families allows Gaffer to store lots of different types of data in the same table. Specifying the locality groups means that when a query for a particular group is made, graph elements from other groups do not need to be read. - -## Tests - -!!! warning - This section might not be fully up to date for Gaffer 2.0.0. To easily run integration tests against a cluster, you can now [use docker compose](https://github.com/gchq/gaffer-docker/tree/master/docker/gaffer-integration-tests). - -For the purposes of unit testing and small-scale examples, Gaffer offers the Store subclass [MiniAccumuloStore](https://github.com/gchq/Gaffer/blob/master/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/MiniAccumuloStore.java) and the `MiniAccumuloCluster`. - -By default all our tests use the MiniAccumuloStore. The MiniAccumuloStore automatically sets up or uses an existing MiniAccumuloCluster according to your store properties. - -Alongside the standard Accumulo properties, you also have the opportunity to add some extra ones for a MiniAccumuloStore: - -``` -accumulo.mini.directory=/path/to/directory -accumulo.mini.root.password=password -accumulo.mini.visibilities=vis1,vis2,publicVisibility,privateVisibility,public,private -``` -These properties are optional. By default the MiniAccumuloStore creates the cluster in a temporary directory, uses "password" as the root password and adds no extra visibilities to a user. - -Because the MiniAccumulo re-uses clusters to be efficient, if two tests use the same user with different visibilities, the second one will overwrite the first. Therefore it's advisable to use different users if you want a user with different visibilities. - -### Running the integration tests - -#### Running a Mini Accumulo Cluster manually - -Follow this [README.md](https://github.com/gchq/gaffer-tools/tree/master/mini-accumulo-cluster) in gaffer-tools on how to run a Mini Accumulo Cluster (with a shell) on your local machine. - -!!! note - When running a Mini Accumulo Cluster locally a `store.properties` file is generated, this can help identify the values you need to replace in the store.properties used for the integration tests below (such as the username, password, instance name and Zookeeper location). - -#### Setting up accumulo-store integration tests - -Update the following store properties files in src/test/resources/ to point to the location of the Accumulo store to test against: - -- [src/test/resources/store.properties](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/resources/store.properties) -- [src/test/resources/store2.properties](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/resources/store2.properties) -- [src/test/resources/accumuloStoreClassicKeys.properties](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/resources/accumuloStoreClassicKeys.properties) - -If you are running an Accumulo cluster locally, here is what an example test store.properties file should look like: - -```text -gaffer.store.class=uk.gov.gchq.gaffer.accumulostore.SingleUseAccumuloStore -gaffer.store.properties.class=uk.gov.gchq.gaffer.accumulostore.AccumuloProperties -accumulo.instance=instance -accumulo.user=root -accumulo.password=password -accumulo.zookeepers=localhost:58630 - -gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService -gaffer.store.job.tracker.enabled=true -gaffer.store.operation.declarations=ExportToOtherAuthorisedGraphOperationDeclarations.json,ExportToOtherGraphOperationDeclarations.json,ResultCacheExportOperations.json -``` - -Ensure that when running an Accumulo instance, the user specified by the `accumulo.user` property has the `System.CREATE_TABLE` and `System.CREATE_NAMESPACE` permissions ('root' user has these set by default) and the following scan authorisations: - -| Authorisation | Required by | -| ----------------- | ----------- | -| vis1 | [VisibilityIT](https://github.com/gchq/Gaffer/blob/develop/integration-test/src/test/java/uk/gov/gchq/gaffer/integration/impl/VisibilityIT.java) | -| vis2 | [VisibilityIT](https://github.com/gchq/Gaffer/blob/develop/integration-test/src/test/java/uk/gov/gchq/gaffer/integration/impl/VisibilityIT.java) | -| public | [SchemaHidingIT](https://github.com/gchq/Gaffer/blob/develop/core/graph/src/test/java/uk/gov/gchq/gaffer/integration/graph/SchemaHidingIT.java) | -| private | [ParameterizedLoaderIT](https://github.com/gchq/Gaffer/blob/develop/integration-test/src/test/java/uk/gov/gchq/gaffer/integration/impl/loader/ParameterizedLoaderIT.java#L63) -| publicVisibility | [AccumuloAggregationIT](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/AccumuloAggregationIT.java) | -| privateVisibility | [AccumuloAggregationIT](https://github.com/gchq/Gaffer/blob/develop/store-implementation/accumulo-store/src/test/java/uk/gov/gchq/gaffer/accumulostore/integration/AccumuloAggregationIT.java) | - -You can set these scan authorisations via the Accumulo shell: - -e.g. if your store.properties have: `accumulo.user=root`, `accumulo.instance=instance` - -```sh -root@instance> setauths -u root -s vis1,vis2,publicVisibility,privateVisibility,public,private -``` - -Run the integration tests: - -```sh -mvn verify -``` diff --git a/docs/dev/components/cache.md b/docs/dev/components/cache.md deleted file mode 100644 index da049572d4..0000000000 --- a/docs/dev/components/cache.md +++ /dev/null @@ -1,24 +0,0 @@ -# Cache - -The [cache module](https://github.com/gchq/Gaffer/tree/master/core/cache) contains the `CacheServiceLoader` which is started when the store is initialised. The cache service loader can be called when a component needs access to short term data storage. -To get access to the cache service you need to call: -```java -CacheServiceLoader.getService(); -``` - -By default, there is no service loaded so if you're using a component that makes use of the `CacheServiceLoader`, be sure to specify the service class in the `store.properties` file. -See the [Stores reference guide](../../reference/stores-guide/stores.md#cache-configuration) for configuration info. - -If using an external cache service (anything found in the cache library) be sure to include the library as a dependency: -```xml - - uk.gov.gchq.gaffer - jcs-cache-service - ${gaffer.version} - -``` - -When run in a servlet context, the `CacheServiceLoader` should be shutdown gracefully by the `ServletLifecycleListener` found in the REST package. Do not trust the shutdown hook in a servlet context. -If running outside a servlet environment, you can either call shutdown on the cache service manually or use the shutdown hook upon initialisation of the cache service loader. - -For information on Gaffer caches and cache configuration, see [the cache section of the Stores Guide](../../reference/stores-guide/stores.md#caches). diff --git a/docs/dev/components/components.md b/docs/dev/components/components.md deleted file mode 100644 index eb4736dac3..0000000000 --- a/docs/dev/components/components.md +++ /dev/null @@ -1,99 +0,0 @@ -# Components/Maven Modules - -Gaffer has lots of components, which can be split into different categories. This section of the developer doc provides more detail on the most important components. - -## Key components - -For more developer information on these key components, see their associated page. - -- [Operation](operation.md): Classes for the Operation interfaces and core operation implementations. -- [Cache](cache.md): Classes for the Gaffer cache service. -- [Graph](graph.md): Contains the Gaffer `Graph` object and related utilities. -- [Data](data.md): Classes defining Gaffer's data objects: `Element`, `Edge` and `Entity`. -- [Store](store.md): Contains the Gaffer `Store` object and related classes. -- [Serialisation](serialisation.md): Contains classes for Serialisation in Gaffer. -- [Accumulo Store](accumulo-store.md): A store implemented using Apache Accumulo. -- [Core REST](core-rest.md): Classes which provide a Gaffer REST API using Jersey/JAX-RS. -- [Spring REST](spring-rest.md): Implementation of the Gaffer REST API deployed in a Spring Boot container. - -## Project Structure (Maven) - -Gaffer uses Maven and from this perspective the project is made up of multiple Maven modules. All components are Maven modules, but not all modules are components as some are just parent or aggregator POMs (see below). -Maven modules are not the same as Java modules (Java 9+), which Gaffer doesn't use or support. [See here for more info on multi-module builds and the Maven reactor](https://maven.apache.org/guides/mini/guide-multiple-modules.html). - -Gaffer's project structure involves three kinds of module/POM, mostly based on the [Maven packaging type](https://maven.apache.org/pom.html#Packaging). We can call these: - -- Parent/Aggregator modules (`pom`) -- JAR modules (`jar`) -- WAR only/Demo modules (`war`). - -Parent modules consist of a single POM file (parent POM) which can define various properties, dependencies and settings to be inherited by JAR modules or other parent modules. -These also specify the modules below the parent (Maven [aggregation](https://maven.apache.org/pom.html#aggregation-or-multi-module)). - -JAR modules are not inherited by other modules, and in addition to the POM they contain code which is compiled into artifacts (always `jar` and sometimes also `war`). Some contain only code used for demos. - -WAR only/Demo modules are not inherited by other modules, they contain only a POM and potentially config files and are used for either creating WAR archives for the REST API or are used for running demos using Maven plugins and other modules. - -Gaffer has 47 modules in total. When building the complete project, Maven automatically runs the build in a specific order because of dependencies between modules. - -??? tip "Finding and displaying POM structure" - The paths for all POMs in the Gaffer project can be seen by running `find . -name pom.xml` from the project/repository root. - - A simple text-based diagram of the Maven project structure can be generated and printed using `mvn org.qunix:structure-maven-plugin:modules -pl :gaffer2`. - -The diagram below shows all modules and their type (green diamond for Parent POM, blue for JAR and red for WAR/Demo): - -``` mermaid -graph LR - PP{uk.gov.gchq.gaffer:gaffer2} --> C{core} - PP --> I[integration-test]:::JAR - PP --> SI{store-implementation} - PP --> RI{rest-api} - PP --> L{library} - PP --> E{example} - C --> operation:::JAR - C --> cache:::JAR - C --> access:::JAR - C --> G[graph]:::JAR - C --> type:::JAR - C --> data:::JAR - C --> exception:::JAR - C --> store:::JAR - C --> common-util:::JAR - C --> serialisation:::JAR - SI --> accumulo-store:::JAR - SI --> map-store:::JAR - SI --> proxy-store:::JAR - SI --> federated-store:::JAR - RI --> spring-rest:::JAR - RI --> common-rest:::JAR - RI --> map-rest:::WarDemo - RI --> accumulo-rest:::WarDemo - RI --> core-rest:::JAR - L --> tinkerpop:::JAR - L --> sketches-library:::JAR - L --> CL{cache-library} - L --> hdfs-library:::JAR - L --> bitmap-library:::JAR - L --> time-library:::JAR - L --> flink-library:::JAR - L --> S{spark} - CL --> hazelcast-cache-service:::JAR - CL --> jcs-cache-service:::JAR - S --> spark-accumulo-library:::JAR - S --> spark-library:::JAR - E --> RT{road-traffic} - E --> B{basic} - E --> federated-demo:::JAR - RT --> road-traffic-model:::JAR - RT --> road-traffic-demo:::JAR - RT --> road-traffic-generators:::JAR - B --> basic-model:::JAR - B --> basic-rest:::WarDemo - - classDef parentPOM fill:lightgreen; - classDef JAR fill:lightblue; - classDef WarDemo fill:lightcoral; - - class PP,C,SI,RI,L,E,CL,S,RT,B parentPOM -``` diff --git a/docs/dev/components/core-rest.md b/docs/dev/components/core-rest.md deleted file mode 100644 index 86c26f9249..0000000000 --- a/docs/dev/components/core-rest.md +++ /dev/null @@ -1,72 +0,0 @@ -# Core REST API - -The [Core REST API module](https://github.com/gchq/Gaffer/tree/master/rest-api/core-rest) contains a Gaffer REST API. - -Gaffer Stores have modules extending the core-rest and adds in the dependency for the Gaffer Store. So if you want to use the Accumulo Store REST API, you can use the `accumulo-rest` `.war`, or `map-rest` for the Map Store. - -For an example of using the core REST API please see the [example/road-traffic module](https://github.com/gchq/Gaffer/tree/master/example/road-traffic). - -## How to modify the Core REST API for your project - -You can easily make changes or additions to the core REST API for your project. You will need to create a new Maven module to build your core REST API. In your POM you should configure the `maven-dependency-plugin` to download the core Gaffer REST API `.war` and extract it. When Maven builds your module it will unpack the core war, add your files and repackage the war. If you wish to override a file in the core war then you can do this by including your own file with exactly the same name and path. - -Example `maven-dependency-plugin` configuration: -```xml - - src/main/java - - - org.apache.maven.plugins - maven-dependency-plugin - 3.5 - - - uk.gov.gchq.gaffer - core-rest - ${gaffer.version} - war - - - - - unpack - compile - - unpack - - - - - uk.gov.gchq.gaffer - core-rest - ${gaffer.version} - war - false - - ${project.build.directory}/${project.artifactId}-${project.version} - - - - - - - - - -``` - -So, if you want to change the CSS for the core REST API you can override the custom.css file: -``` -/src/main/webapp/css/custom.css -``` - -There are also various system properties you can use to configure to customise the Swagger UI. -For example: -``` -gaffer.properties.app.title=Road Traffic Example -gaffer.properties.app.description=Example using road traffic data -gaffer.properties.app.banner.description=DEMO -gaffer.properties.app.banner.colour=#1b75bb -gaffer.properties.app.logo.link=https://github.com/gchq/Gaffer -gaffer.properties.app.logo.src=images/iconCircle.png -``` diff --git a/docs/dev/components/data.md b/docs/dev/components/data.md deleted file mode 100644 index 368be8eeff..0000000000 --- a/docs/dev/components/data.md +++ /dev/null @@ -1,28 +0,0 @@ -# Data - -The [data module](https://github.com/gchq/Gaffer/tree/master/core/data) contains Gaffer's data objects: `Element`, `Edge` and `Entity`. - -It also contains the logic for processing these `Element`s - `ElementAggregator`, `ElementFilter` and `ElementTransformer`. - -## Functions and Predicates - -Gaffer makes use of Java 8's Function and Predicate interfaces to aggregate, transform and filter data. To allow these Function and Predicate classes to process tuples we make use of the [Koryphe](https://github.com/gchq/koryphe/tree/master) library. Koryphe allows us to wrap the Gaffer Elements in a tuple and pass it any Function or Predicate. - -You can use any of our implementations ([see reference pages](../../reference/intro.md)) or write your own. - -All the following classes will act on one or more Element identifiers (vertex/source/destination/directed) or properties. If you implement the Java 8 interfaces directly, you would need to add the following `JsonType` annotation to your class: - -```java -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "class") -``` - -Instead, we recommend you implement the Koryphe interfaces instead, which will add this annotation in for you. - -### Aggregation -Aggregation is done using a `KorypheBinaryOperator` (or just `BinaryOperator`), where T is the type of the property you are aggregating. For example: [Max](https://github.com/gchq/koryphe/blob/master/core/src/main/java/uk/gov/gchq/koryphe/impl/binaryoperator/Max.java). - -### Transforms -Transforms are applied using `KorypheFunction` (or just `Function`), where I is type of the input property and O is the type of the output value. If you want to transform multiple properties into a single new property then you can implement `KorypheFunction2` or `KoryphePredicate3`, etc. where R is the output value type. For example: [Concat](https://github.com/gchq/koryphe/blob/master/core/src/main/java/uk/gov/gchq/koryphe/impl/function/Concat.java). - -### Filtering -Filtering is applied using `KoryphePredicate` (or just `Predicate`), where T is type of the property you want to filter on. If you want to filter on multiple properties then you can implement `KoryphePredicate2` or `KoryphePredicate3` etc. For example: [Exists](https://github.com/gchq/koryphe/blob/master/core/src/main/java/uk/gov/gchq/koryphe/impl/predicate/Exists.java). diff --git a/docs/dev/components/graph.md b/docs/dev/components/graph.md deleted file mode 100644 index ffff6b9d3e..0000000000 --- a/docs/dev/components/graph.md +++ /dev/null @@ -1,107 +0,0 @@ -# Graph - -The [graph module](https://github.com/gchq/Gaffer/tree/master/core/graph) contains the Gaffer `Graph` object and related utilities. This is the entry point (or proxy) for your chosen Gaffer store. - -The `Graph` separates the user from the underlying store. It holds a connection which acts as a proxy, delegating operations to the store. -It provides users with a single point of entry for executing operations on a store. This allows the underlying store to be swapped and the same operations can still be applied. - -When you instantiate a `Graph`, this doesn't mean you are creating an entirely new graph with its own data, you are simply creating a connection to a store where some data is held. - -To create an instance of `Graph`, we recommend you use the `Graph.Builder` class. This has several helpful methods to create the graph from various different sources. -But, essentially a graph requires just 3 things: some store properties, a schema and some graph specific configuration. - -See the [Graph Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/graph/package-summary.html) for further documentation. - -## Store Properties -The store properties tells the graph the type of store to connect to along with any required connection details. See the [Stores](../../reference/stores-guide/stores.md) reference page for more information on the different Stores for Gaffer. - -## Schema -The schema is passed to the store to instruct the store how to store and process the data. See [Schemas](https://gchq.github.io/gaffer-doc/v1docs/getting-started/developer-guide/schemas.html) for more information - v1 docs, to be updated and added to this documentation in future. - -## Graph Configuration -The graph configuration allows you to apply special customisations to the Graph instance. The only required field is the `graphId`. - -To create an instance of `GraphConfig` you can use the `GraphConfig.Builder` class, or create it using a json file. - -The `GraphConfig` can be configured with the following: - - `graphId` - The `graphId` is a String field that uniquely identifies a `Graph`. When backed by a Store like Accumulo, this `graphId` is used as the name of the Accumulo table for the `Graph`. - - `description` - a string describing the `Graph`. - - `view` - The `Graph View` allows a graph to be configured to only returned a subset of Elements when any Operation is executed. For example if you want your `Graph` to only show data that has a count more than 10 you could add a View to every operation you execute, or you can use this `Graph View` to apply the filter once and it would be merged into to all Operation Views so users only ever see this particular view of the data. - - `library` - This contains information about the `Schema` and `StoreProperties` to be used. - - `hooks` - A list of `GraphHook`s that will be triggered before, after and on failure when operations are executed on the `Graph`. See [GraphHooks](#graph-hooks) for more information. - -Here is an example of a `GraphConfig`: - -```java -new GraphConfig.Builder() - .config(new GraphConfig.Builder() - .graphId("exampleGraphId") - .build()) - .description("Example Graph description") - .view(new View.Builder() - .globalElements(new GlobalViewElementDefinition.Builder() - .postAggregationFilter(new ElementFilter.Builder() - .select("ExamplePropertyName") - .execute(new IsLessThan("10")) - .build()) - .build()) - .build()) - .library(new FileGraphLibrary()) - .addHook(new Log4jLogger()) - .build(); -``` - -and in json: - -```json -{ - "graphId": "exampleGraphId", - "description": "Example Graph description", - "view": { - "globalElements": [ - { - "postAggregationFilterFunctions": [ - { - "predicate": { - "class": "uk.gov.gchq.koryphe.impl.predicate.IsLessThan", - "orEqualTo": false, - "value": "10" - }, - "selection": ["ExamplePropertyName"] - } - ] - } - ] - }, - "library": { - "class": "uk.gov.gchq.gaffer.store.library.FileGraphLibrary" - }, - "hooks": [ - { - "class": "uk.gov.gchq.gaffer.graph.hook.Log4jLogger" - } - ] -} -``` - -## Graph Hooks -The `Graph` class is final and must be used when creating a new connection to a store. We want to ensure that all users have a common point of entry to Gaffer, so all users have to start by instantiating a `Graph`. -Initially this seems quite limiting, but to allow custom logic for different types of graphs we have added graph hooks. These graph hooks allow custom code to be run before and after an operation chain is executed. - -You can use hooks to do things like custom logging or special operation chain authorisation. To implement your own hook, just implement the `GraphHook` interface and register it with the graph when you build a `Graph` instance. -GraphHooks should be json serialisable and each hook should have a unit test that extends GraphHookTest. - -There are some graph hooks which are added by default if they aren't already present in the configuration. The NamedViewResolver and NamedOperationResolver (providing that NamedOperations are supported by the store) are added at the start of the list of hooks. -The third hook added is the FunctionAuthoriser, which is added at the end of the list - again assuming no hook is not present in the configuration. This hook stops users from using potentially dangerous functions in their operation chain. -If you want to disable this hook, you should overwrite it by adding an empty FunctionAuthoriser to your list of hooks. For example: - -```json -{ - "graphId": "example", - "hooks": [ - { - "class": "uk.gov.gchq.gaffer.graph.hook.FunctionAuthoriser" - } - ] -} -``` diff --git a/docs/dev/components/integration-test.md b/docs/dev/components/integration-test.md deleted file mode 100644 index 486230793a..0000000000 --- a/docs/dev/components/integration-test.md +++ /dev/null @@ -1,6 +0,0 @@ -# Integration Tests - -The [integration-test module](https://github.com/gchq/Gaffer/tree/master/integration-test) contains Gaffer's Integration Test suite for testing Store implementations. - -!!! note - This test framework is not a representative example of how to use the Gaffer framework. For an example of how the Gaffer framework can be used, please see the example road-traffic module. diff --git a/docs/dev/components/libraries/bitmap.md b/docs/dev/components/libraries/bitmap.md deleted file mode 100644 index 459a9d9196..0000000000 --- a/docs/dev/components/libraries/bitmap.md +++ /dev/null @@ -1,20 +0,0 @@ -# Bitmap Library - -The [bitmap library](https://github.com/gchq/Gaffer/tree/master/library/bitmap-library) module contains various libraries for Bitmaps. - -In order to make use of the bitmap libraries you will need to include this library as a dependency: -```xml - - uk.gov.gchq.gaffer - bitmap-library - ${gaffer.version} - -``` - -## Registering - -You can register the BitmapJsonModules using the store or system property: `gaffer.serialiser.json.modules`. This property takes a CSV of classes, so you can use multiple json modules. - -``` -gaffer.serialiser.json.modules=uk.gov.gchq.gaffer.bitmap.serialisation.json.BitmapJsonModules -``` diff --git a/docs/dev/components/libraries/flink.md b/docs/dev/components/libraries/flink.md deleted file mode 100644 index 8207bded0d..0000000000 --- a/docs/dev/components/libraries/flink.md +++ /dev/null @@ -1,19 +0,0 @@ -# Flink Library - -The [flink library](https://github.com/gchq/Gaffer/tree/master/library/flink-library) module contains various libraries for using Apache Flink with Gaffer. - -In order to make use of the flink libraries you will need to include this library as a dependency: -```xml - - uk.gov.gchq.gaffer - flink-library - ${gaffer.version} - -``` - -For information on registering and using flink operations, see the [Flink Operations guide](../../../reference/operations-guide/flink.md). - -## I am getting errors when running the Flink operations on a cluster -This could be to do with the way the Gaffer Store class is serialised and distributed around the cluster. To distribute the job, Flink requires all of the components of the job to be Serializable. -The Gaffer Store class is not Serializable so instead we just Serialialize the graphId, Schema and Properties. Then when we require an instance of the Store class again we recreate it again with these parts. -This means that any files that are referenced in your StoreProperties must be available on all your nodes in your cluster. diff --git a/docs/dev/components/libraries/sketches.md b/docs/dev/components/libraries/sketches.md deleted file mode 100644 index 4be4defa4b..0000000000 --- a/docs/dev/components/libraries/sketches.md +++ /dev/null @@ -1,14 +0,0 @@ -# Sketches Library - -The [sketches library](https://github.com/gchq/Gaffer/tree/master/library/sketches-library) module contains various libraries for sketches. - -In order to make use of the sketches libraries you will need to include this library as a dependency: -```xml - - uk.gov.gchq.gaffer - sketches-library - ${gaffer.version} - -``` - -For information on configuring and using sketches, see the [Cardinality guide](../../../getting-started/guide/cardinality.md#how-to-add-cardinality-to-your-graph) (for configuring `SketchesJsonModules` expand "Additional config"). diff --git a/docs/dev/components/libraries/spark.md b/docs/dev/components/libraries/spark.md deleted file mode 100644 index 6fbbbaf9e0..0000000000 --- a/docs/dev/components/libraries/spark.md +++ /dev/null @@ -1,23 +0,0 @@ -# Spark Library - -The [spark library](https://github.com/gchq/Gaffer/tree/master/library/spark/spark-library) contains various libraries for using Apache Spark with Gaffer. - -In order to make use of the spark libraries you will need to include this library as a dependency: -```xml - - uk.gov.gchq.gaffer - spark-library - ${gaffer.version} - -``` - -To use spark with Accumulo you will need to include this dependency: -```xml - - uk.gov.gchq.gaffer - spark-accumulo-library - ${gaffer.version} - -``` - -For information on registering and using spark operations, see the [Spark Operations guide](../../../reference/operations-guide/spark.md). diff --git a/docs/dev/components/libraries/time.md b/docs/dev/components/libraries/time.md deleted file mode 100644 index 06128d8c15..0000000000 --- a/docs/dev/components/libraries/time.md +++ /dev/null @@ -1,16 +0,0 @@ -# Time Library - -The [time library](https://github.com/gchq/Gaffer/tree/master/library/time-library) module contains classes that represent concepts relating to time. - -For example, there is a class (`RBMBackedTimestampSet`) that can be used to represent a set of timestamps. Internally this stores its state in a Roaring Bitmap for efficiency reasons. -There is also a class that stores up to a maximum number N of timestamps. If more than N timestamps are added then a uniform random sample of the timestamps, of size at most N, is stored. Serialisers and aggregators for the above classes are provided. - -To use this library, you will need to include the following dependency: - -```xml - - uk.gov.gchq.gaffer - time-library - ${gaffer.version} - -``` diff --git a/docs/dev/components/operation.md b/docs/dev/components/operation.md deleted file mode 100644 index 1047a8b927..0000000000 --- a/docs/dev/components/operation.md +++ /dev/null @@ -1,82 +0,0 @@ -# Operation - -The [operation module](https://github.com/gchq/Gaffer/tree/master/core/operation) contains the `Operation` interfaces and core operation implementations. - -It is assumed that all Gaffer graphs will be able to handle these core operations. - -An `Operation` implementation defines an operation to be processed on a graph, or on a set of results which are returned by another operation. An `Operation` class contains the configuration required to tell Gaffer how to carry out the operation. For example, the `AddElements` operation contains the elements to be added. -The `GetElements` operation contains the seeds to use to find elements in the graph and the filters to apply to the query. -The operation classes themselves should not contain the logic to carry out the operation (as this may vary between the different supported store types), just the configuration. - -For each operation, each Gaffer store will have an `OperationHandler`, where the processing logic is contained. This enables operations to be handled differently in each store. - -Operations can be chained together to form an `OperationChain`. When an operation chain is executed on a Gaffer graph the output of one operation is passed to the input of the next. - -An `OperationChain.Builder` is provided to help with constructing a valid operation chain - it ensures the output type of an operation matches the input type of the next. - -## How to write an Operation - -Operations should be written to be as generic as possible to allow them to be applied to different graphs/stores. - -Operations must be JSON serialisable in order to be used via the REST API - i.e. there must be a public constructor and all the fields should have getters and setters. - -Operation implementations need to implement the `Operation` interface and the extra interfaces they wish to make use of. For example an operation that takes a single input value should implement the `Input` interface. - -Here is a list of some of the common interfaces: - -- `uk.gov.gchq.gaffer.operation.io.Input` -- `uk.gov.gchq.gaffer.operation.io.Output` -- `uk.gov.gchq.gaffer.operation.io.InputOutput` - Use this instead of Input and Output if your operation takes both input and output. -- `uk.gov.gchq.gaffer.operation.io.MultiInput` - Use this in addition if your operation takes multiple inputs. This will help with JSON serialisation. -- `uk.gov.gchq.gaffer.operation.Validatable` -- `uk.gov.gchq.gaffer.operation.graph.OperationView` -- `uk.gov.gchq.gaffer.operation.graph.GraphFilters` -- `uk.gov.gchq.gaffer.operation.graph.SeededGraphFilters` - -Each operation implementation should have a corresponding unit test class that extends the `OperationTest` class. - -Operation implementations should override the close method and ensure all closeable fields are closed. - -The core Gaffer operations need to be registered in the `Store` class, with their respective handlers. The `StoreTest` unit test also needs similar information. - -### Annotations - -Any fields that are required should be annotated with the Required annotation. -As well, each `Operation` class now also requires the Since annotation, detailing to which version of Gaffer it was introduced. To demonstrate: -```java -@Since("1.4.0") -public class NewOperation extends Operation { - - @Required - private String requiredField; - ... -} -``` - -### Builder - -All implementations should also have a static inner `Builder` class that implements the required builders. For example: - -```java -public static class Builder extends Operation.BaseBuilder - implements InputOutput.Builder, CloseableIterable, Builder>, - MultiInput.Builder, - SeededGraphFilters.Builder, - Options.Builder { - public Builder() { - super(new GetElements()); - } -} -``` - -### Operation Scores - -For use with a `ScoreOperationChain`, some `Operation`s may require a custom way of calculating an associated score, therefore an implementation of the `ScoreResolver` interface may be required. -There is a `DefaultScoreResolver` to which the custom implementation should delegate, in a manner specific to the new Operation. For more info, see [ScoreOperationChain](/docs/reference/stores-guide/stores.md#scoreoperationchain) and [ScoreOperationChainExample](../../reference/operations-guide/misc.md#scoreoperationchain). - -### Documentation - -Class-level Javadoc should be provided for each `Operation`, giving a description of the functionality, and any configuration information that may not immediately be obvious. -Member-level Javadoc is not strictly necessary, but good practice for explanations/clarifications of complex methods. - -To assist users of the new Operation, it is best practice to provide documentation, and simple usage examples in [gaffer-doc](https://github.com/gchq/gaffer-doc). diff --git a/docs/dev/components/serialisation.md b/docs/dev/components/serialisation.md deleted file mode 100644 index 16a938a60a..0000000000 --- a/docs/dev/components/serialisation.md +++ /dev/null @@ -1,7 +0,0 @@ -# Serialisation - -The [serialisation module](https://github.com/gchq/Gaffer/tree/master/core/serialisation) contains the logic for converting an Object into serialised objects (normally a byte array). - -The main interface is Serialisation. We have provided a small set of serialisers for commonly used objects. The serialisers we have been designed to optimise speed and size. Serialisers can take arguments which may be mandatory depending on the serialiser used. - -It is important to choose your serialisers wisely as once your data is persisted using a chosen serialiser, there is no easy way of migrating your data into a different format. diff --git a/docs/dev/components/spring-rest.md b/docs/dev/components/spring-rest.md deleted file mode 100644 index f52ab96a57..0000000000 --- a/docs/dev/components/spring-rest.md +++ /dev/null @@ -1,47 +0,0 @@ -# Spring REST API - -The [Spring REST API module](https://github.com/gchq/Gaffer/tree/master/rest-api/spring-rest) is an implementation of the Gaffer REST API deployed in a Spring Boot container. -It is relatively new compared to the [core REST](./core-rest.md) war which uses Jersey/JAX-RS. - -The Spring REST should provide the following benefits: - -* Easier to extend through Spring Boot plugins -* Easier to add dependencies at deployment time (no need to re-build a `.war` file) -* Easier to deploy (you only need Java) - -However, going forward into Gaffer v2.0 we hope this to become the standard for how we build and deploy REST APIs. - -### Implemented Features -* Operations endpoint -* Graph configuration endpoint -* Properties endpoint -* Status endpoint - -### Features we're yet to implement -* Chunked endpoint - -### Features we don't plan to implement -* Custom Swagger UI with operation chain builder -* Supporting older versions of the API - - -### How to run -With Maven from the root of the project: -```bash -mvn spring-boot:run -pl :spring-rest -Pdemo -``` - -With Java using the 'exec' `.jar` directly: -``` -java \ --Dgaffer.schemas=/path/to/schemas \ --Dgaffer.storeProperties=/path/to/store.properties \ --Dgaffer.graph.config=/path/to/graphConfig.json \ --jar spring-rest-2.0.0-exec.jar -``` - -You can alternatively add the Gaffer system properties to your `application.properties` file. - -Once running, open the browser to `http://localhost:8080/rest`. - -You can change the context root by changing the `server.servlet.context-path` value in `application.properties`. \ No newline at end of file diff --git a/docs/dev/components/store.md b/docs/dev/components/store.md deleted file mode 100644 index 578b5716c3..0000000000 --- a/docs/dev/components/store.md +++ /dev/null @@ -1,15 +0,0 @@ -# Store - -The [store module](https://github.com/gchq/Gaffer/tree/master/core/store) defines the API for Store implementations. The abstract Store class handles Operations by delegating the Operations to their registered handlers. - -See the [Store Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/package-summary.html) for further documentation. - -## Writing a Store - -When implementing a Store, the main task is to write handlers for the operations your Store chooses to support. This can be tricky, but the [Store Integration test suite](./integration-test.md) should be used by all Store implementations to validate these operation handlers. When writing these handlers you should implement `OperationHandler` or `OutputOperationHandler` depending on whether the operation has an output. - -Store implementations need to define a set of `StoreTraits`. These traits tell Gaffer the abilities the Store has. For example the ability to aggregate or filter elements. - -## Schema - -In addition to `OperationHandlers` the other large part of the store module is the Schema. The Schema is what defines what is in the Graph and how it should be persisted, compacted (summarised) and validated. diff --git a/docs/dev/development.md b/docs/dev/development.md deleted file mode 100644 index d4429dd12e..0000000000 --- a/docs/dev/development.md +++ /dev/null @@ -1,28 +0,0 @@ -# Development - -!!! info "Work in Progress" - - This page is under construction. To propose additions or changes, please use the pencil button above on the right. - -## Source Control - -Development of Gaffer is done on the [GCHQ/Gaffer](https://github.com/gchq/Gaffer) GitHub repository, or other Gaffer GitHub repositories under the [GCHQ Organization](https://github.com/orgs/gchq/repositories). - -## Building Gaffer - -### Build Instructions - -The latest instructions for building Gaffer [are in our README](https://github.com/gchq/Gaffer/blob/develop/README.md#building-and-deploying). - -### Supported Platforms - -A recent Linux distribution is recommended, although it should be possible to build Gaffer on any system which has the latest version of Java 8 or 11 (the Gaffer codebase uses Java 8). Running tests on Windows is not recommended due to complexities with native libraries. - -## Contributing - -We welcome contributions to the project. See our [ways of working for more -detail](ways-of-working.md). All contributors must sign the [GCHQ Contributor -Licence Agreement](https://cla-assistant.io/gchq/Gaffer). - -You can quickly and easily contribute towards Gaffer using a [remote coding -environment](remote-coding-environments.md) such as GitHub Codespaces or Gitpod. \ No newline at end of file diff --git a/docs/dev/docker.md b/docs/dev/docker.md deleted file mode 100644 index 5a4a2a49f6..0000000000 --- a/docs/dev/docker.md +++ /dev/null @@ -1,25 +0,0 @@ -# Gaffer Docker - -The [gaffer-docker](https://github.com/gchq/gaffer-docker) repository contains all code needed to run Gaffer using Docker. - -All the files needed to get started using Gaffer in Docker are contained in the ['docker'](https://github.com/gchq/gaffer-docker/tree/develop/docker) sub-folder. - -In this directory you can find the Dockerfiles and docker compose files for building container images for: - -- [Gaffer](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer) -- [Gaffer's REST API](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-rest) -- [Gaffer's Road Traffic Example](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-road-traffic-loader) -- [HDFS](https://github.com/gchq/gaffer-docker/tree/develop/docker/hdfs) -- [Accumulo](https://github.com/gchq/gaffer-docker/tree/develop/docker/accumulo) -- [Gaffer's Integration Tests](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-integration-tests) -- [gafferpy Jupyter Notebook](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-pyspark-notebook) -- [Gaffer's JupyterHub Options Server](https://github.com/gchq/gaffer-docker/tree/develop/docker/gaffer-jhub-options-server) -- [Spark](https://github.com/gchq/gaffer-docker/tree/develop/docker/spark-py) - -Each directory contains a README with more specific information on what these images are for and how to build them. - -Please note that some of these containers will only be useful if utilised by the Helm Charts under Kubernetes, and may not be possible to run on their own. - -## Requirements - -Before you can build and run these containers you will need to install Docker or a compatible equivalent (e.g. Podman). diff --git a/docs/dev/kubernetes-guide/add-libraries.md b/docs/dev/kubernetes-guide/add-libraries.md deleted file mode 100644 index 77d4f934fb..0000000000 --- a/docs/dev/kubernetes-guide/add-libraries.md +++ /dev/null @@ -1,70 +0,0 @@ -# Adding your own libraries and functions - -By default with the Gaffer deployment you get access to the: - -- Sketches library -- Time library -- Bitmap Library -- JCS cache library - -If you want more libraries than this (either one of ours of one of your own) you will need to customise the docker images and use them in place of the defaults. - -You will need a [basic Gaffer instance deployed on Kubernetes](deploy-empty-graph.md). - -## Add Extra Libraries to Gaffer REST - -At the moment, Gaffer uses a runnable jar file located at `/gaffer/jars`. When it runs it includes the `/gaffer/jars/lib` on the classpath. There is nothing in there by default because all the dependencies are bundled in to the JAR. However, if you wanted to add your own jars, you can do it like this: - -```Dockerfile -FROM gchq/gaffer-rest:latest -COPY ./my-custom-lib:1.0-SNAPSHOT.jar /gaffer/jars/lib/ -``` - -Build the image using: - -```bash -docker build -t custom-rest:latest . -``` - -## Add the extra libraries to the Accumulo image - -Gaffer's Accumulo image includes support for the following Gaffer libraries: - -- The Bitmap Library -- The Sketches Library -- The Time Library - -In order to push down any extra value objects and filters to Accumulo that are not in those libraries, we have to add the jars to the accumulo `/lib/ext directory`. Here is an example `Dockerfile`: - -```Dockerfile -FROM gchq/gaffer:latest -COPY ./my-library-1.0-SNAPSHOT.jar /opt/accumulo/lib/ext -``` - -Then build the image - -```bash -docker build -t custom-gaffer-accumulo:latest . -``` - -# Switch the images in the deployment - -You will need a way of making the custom images visible to the kubernetes cluster. Once visible you can switch them out. Create a `custom-images.yaml` file with the following contents: - -```yaml -api: - image: - repository: custom-rest - tag: latest - -accumulo: - image: - repository: custom-gaffer-accumulo - tag: latest -``` - -To switch them run: - -```bash -helm upgrade my-graph gaffer-docker/gaffer -f custom-images.yaml --reuse-values -``` diff --git a/docs/dev/kubernetes-guide/change-accumulo-passwords.md b/docs/dev/kubernetes-guide/change-accumulo-passwords.md deleted file mode 100644 index a5e2316858..0000000000 --- a/docs/dev/kubernetes-guide/change-accumulo-passwords.md +++ /dev/null @@ -1,36 +0,0 @@ -# Changing the Accumulo Passwords - -When deploying Accumulo - either as part of a Gaffer stack or as a standalone, the passwords for all the users and the instance.secret are set to default values and should be changed. The instance.secret cannot be changed once deployed as it is used in initalisation. - -When deploying the Accumulo helm chart, the following values are set. If you are using the Gaffer helm chart with the Accumulo integration, the values will be prefixed with "accumulo": - -| Name | value | default value | -| -------------------- | --------------------------------------------- | ------------- | -| Instance Secret | `config.accumuloSite."instance.secret"` | "DEFAULT" | -| Root password | `config.userManagement.rootPassword` | "root" | -| Tracer user password | `config.userManagement.users.tracer.password` | "tracer" | - -When you deploy the Gaffer Helm chart with Accumulo, a "gaffer" user with a password of "gaffer" is used by default following the same pattern as the tracer user. - -So to install a new Gaffer with Accumulo store, create an `accumulo-passwords.yaml` with the following contents: - -```yaml -accumulo: - enabled: true - config: - accumuloSite: - instance.secret: "changeme" - userManagement: - rootPassword: "changeme" - users: - tracer: - password: "changme" - gaffer: - password: "changeme" -``` - -You can install the graph with: - -```bash -helm install my-graph gaffer-docker/gaffer -f accumulo-passwords.yaml -``` diff --git a/docs/dev/kubernetes-guide/change-graph-metadata.md b/docs/dev/kubernetes-guide/change-graph-metadata.md deleted file mode 100644 index 603db49c99..0000000000 --- a/docs/dev/kubernetes-guide/change-graph-metadata.md +++ /dev/null @@ -1,63 +0,0 @@ -# Changing the Graph ID and Description - -By default, the default Gaffer deployment ships with the Graph name "simpleGraph" and description "A graph for demo purposes" These are just placeholders and can be overwritten. This guide will show you how. - -The first thing you will need to do is [deploy an empty graph](deploy-empty-graph.md). - -## Changing the description - -Create a file called `graph-meta.yaml`. We will use this file to add our description and graph ID. Changing the description is as easy as changing the `graph.config.description` value. - -```yaml -graph: - config: - description: "My graph description" -``` - -## Deploy the new description - -Upgrade your deployment using helm: - -```bash -helm upgrade my-graph gaffer-docker/gaffer -f graph-metadata.yaml --reuse-values -``` - -The `--reuse-values` argument means we do not override any passwords that we set in the initial construction. - -You can see you new description if you to the Swagger UI and call the `/graph/config/description` endpoint. - -## Updating the Graph ID - -This may be simple or complicated depending on your store type. If you are using the Map or Federated store, you can just set the `graph.config.graphId` value in the same way. Though if you are using a MapStore, the graph will be emptied as a result. - -However, if you are using the Accumulo store, updating the graph Id is a little more complicated since the Graph Id corresponds to an Accumulo table. We have to change the gaffer users permissions to read and write to that table. To do that update the graph-meta.yaml file with the following contents: - -```yaml -graph: - config: - graphId: "MyGraph" - description: "My Graph description" - -accumulo: - config: - userManagement: - users: - gaffer: - permissions: - table: - MyGraph: - - READ - - WRITE - - BULK_IMPORT - - ALTER_TABLE -``` - -## Deploy your changes - -Upgrade your deployment using Helm. - -```bash -helm upgrade my-graph gaffer-docker/gaffer -f graph-metadata.yaml --reuse-values -``` - -If you take a look at Accumulo monitor, you will see your new Accumulo table. diff --git a/docs/dev/kubernetes-guide/deploy-empty-graph.md b/docs/dev/kubernetes-guide/deploy-empty-graph.md deleted file mode 100644 index 966a17f05c..0000000000 --- a/docs/dev/kubernetes-guide/deploy-empty-graph.md +++ /dev/null @@ -1,73 +0,0 @@ -# How to deploy a simple graph - -This guide will describe how to deploy a simple empty graph with the minimum configuration. - -You will need: - -- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -- [helm](https://github.com/helm/helm/releases) -- A Kubernetes cluster (local or remote) -- An ingress controller running (for accessing UIs) - -## Add the Gaffer Docker repo - -To start with, you should add the Gaffer Docker repo to your helm repos. This will save the need for cloning this Git repository. If you have already done this you can skip this step. - -```bash -helm repo add gaffer-docker https://gchq.github.io/gaffer-docker -``` - -## Choose the store - -Gaffer can be backed with a number of different technologies to back its store. Which one you want depends on the use case but as a rule of thumb: - -- If you want just something to spin up quickly at small scale and are not worried about persistence, use the MapStore. -- If you want to back it with a key value datastore, you can deploy the Accumulo Store. -- If you want to join two or more graphs together to query them as one, you will want to use the Federated Store. - -### Deploy the MapStore - -The MapStore is just an in-memory store that can be used for demos or if you need something small scale short-term. It is our default store so there is no need for any extra configuration. - -You can install a MapStore by just running: - -``` -helm install my-graph gaffer-docker/gaffer -``` - -### Deploy the Accumulo Store - -If you want to deploy an Accumulo Store with your graph, it is relatively easy to do so with some small additional configuration. Create a file called `accumulo.yaml` and add the following: - -```yaml -accumulo: - enabled: true -``` - -By default, the Gaffer user is created with a password of "gaffer" the CREATE_TABLE system permission with full access to the simpleGraph table which is coupled to the graphId. All the default Accumulo passwords are in place so if you were to deploy this in production, you should consider changing the [default accumulo passwords](change-accumulo-passwords.md). - -You can stand up the accumulo store by running: - -```bash -helm install my-graph gaffer-docker/gaffer -f accumulo.yaml -``` - -### Deploy the Federated Store - -If you want to deploy the Federated Store, all that you really need to do is set the `store.properties`. To do this add the following to a `federated.yaml` file: - -```yaml -graph: - storeProperties: - gaffer.store.class: uk.gov.gchq.gaffer.federatedstore.FederatedStore - gaffer.store.properties.class: uk.gov.gchq.gaffer.federatedstore.FederatedStoreProperties - gaffer.serialiser.json.modules: uk.gov.gchq.gaffer.sketches.serialisation.json.SketchesJsonModules -``` - -The addition of the `SketchesJsonModules` is just to ensure that if the FederatedStore was connecting to a store which used sketches, they could be rendered nicely in json. - -We can create the graph with: - -```bash -helm install federated gaffer-docker/gaffer -f federated.yaml -``` diff --git a/docs/dev/kubernetes-guide/deploy-schema.md b/docs/dev/kubernetes-guide/deploy-schema.md deleted file mode 100644 index fa80719ef2..0000000000 --- a/docs/dev/kubernetes-guide/deploy-schema.md +++ /dev/null @@ -1,68 +0,0 @@ -# How to deploy your own schema - -Gaffer uses schema files to describe the data contained in a Graph. This guide will tell you how to deploy your own schemas with a Gaffer Graph. - -You will first need [a basic Gaffer instance deployed on Kubernetes] (deploy-empty-graph.md). - -Once you have that deployed we can change the schema. - -## Edit the schema - -If you run a GetSchema operation against the graph, you will notice that the count property is of type `java.lang.Integer` - change that property to be of type `java.lang.Long`. - -The easiest way to deploy a schema file is to use helms `--set-file` option which lets you set a value from the contents of a file. - -??? example "Example of a `schema.json` file" - - ```json - { - "edges": { - "BasicEdge": { - "source": "vertex", - "destination": "vertex", - "directed": "true", - "properties": { - "count": "count" - } - } - }, - "entities": { - "BasicEntity": { - "vertex": "vertex", - "properties": { - "count": "count" - } - } - }, - "types": { - "vertex": { - "class": "java.lang.String" - }, - "count": { - "class": "java.lang.Long", - "aggregateFunction": { - "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Sum" - } - }, - "true": { - "description": "A simple boolean that must always be true.", - "class": "java.lang.Boolean", - "validateFunctions": [ - { "class": "uk.gov.gchq.koryphe.impl.predicate.IsTrue" } - ] - } - } - } - ``` - -## Update deployment with the new schema - -For our deployment to pick up the changes, we need to run a helm upgrade: - -```bash -helm upgrade my-graph gaffer-docker/gaffer --set-file graph.schema."schema\.json"=./schema.json --reuse-values -``` - -The `--reuse-values` argument tells helm to re-use the passwords that we defined earlier. - -Now if we inspect the schema, you will see that the `count` property has changed to a `Long`. diff --git a/docs/dev/kubernetes-guide/kubernetes.md b/docs/dev/kubernetes-guide/kubernetes.md deleted file mode 100644 index 5fd7acce24..0000000000 --- a/docs/dev/kubernetes-guide/kubernetes.md +++ /dev/null @@ -1,40 +0,0 @@ -# Gaffer in Kubernetes - -The [gaffer-docker](https://github.com/gchq/gaffer-docker) repository contains all code needed to run Gaffer using Docker and Kubernetes. - -All the files needed to get started using Gaffer in Kubernetes are contained in the ['kubernetes'](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes) sub-folder of the [gaffer-docker](https://github.com/gchq/gaffer-docker) repository. -In this directory you can find the Helm charts required to deploy various applications onto Kubernetes clusters. - -The Helm charts and associated information for each application can be found in the following places: - -- [Gaffer](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/gaffer) -- [Example Gaffer Graph of Road Traffic Data](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/gaffer-road-traffic) -- [JupyterHub with Gaffer Integrations](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/gaffer-jhub) -- [HFDS](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/hdfs) -- [Accumulo](https://github.com/gchq/gaffer-docker/tree/develop/kubernetes/accumulo) - -These charts can be accessed by cloning our repository or by using our Helm repo hosted on our [Github Pages Site](https://gchq.github.io/gaffer-docker/). - -## Requirements - -To deploy these applications, you'll need access to a suitable Kubernetes distribution. - -You will also need to install a container management engine, for example Docker or Podman, to build, run and manage your containers. - -## Adding this repo to Helm - -To add the gaffer-docker repo to helm run: - -```bash -helm repo add gaffer-docker https://gchq.github.io/gaffer-docker -``` - -## How to Guides - -There are a number of guides to help you deploy Gaffer on Kubernetes. It is important you look at these before you get started as they provide the initial steps for running these applications. - -* [Deploy a simple empty graph](deploy-empty-graph.md) -* [Add your schema](deploy-schema.md) -* [Change the graph ID and description](change-graph-metadata.md) -* [Adding your own libraries and functions](add-libraries.md) -* [Changing passwords for the Accumulo store](change-accumulo-passwords.md) \ No newline at end of file diff --git a/docs/dev/managing-dependencies/core.svg b/docs/dev/managing-dependencies/core.svg deleted file mode 100644 index 143d876923..0000000000 --- a/docs/dev/managing-dependencies/core.svg +++ /dev/null @@ -1,307 +0,0 @@ - - - - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT - - - -com.google.guava:guava:jar:30.1.1-jre:compile - -com.google.guava:guava:jar:30.1.1-jre:compile - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->com.google.guava:guava:jar:30.1.1-jre:compile - - - - - -org.slf4j:slf4j-api:jar:1.7.36:compile - -org.slf4j:slf4j-api:jar:1.7.36:compile - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.slf4j:slf4j-api:jar:1.7.36:compile - - - - - -org.slf4j:slf4j-reload4j:jar:1.7.36:compile - -org.slf4j:slf4j-reload4j:jar:1.7.36:compile - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.slf4j:slf4j-reload4j:jar:1.7.36:compile - - - - - -org.junit.jupiter:junit-jupiter:jar:5.9.0:test - -org.junit.jupiter:junit-jupiter:jar:5.9.0:test - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.junit.jupiter:junit-jupiter:jar:5.9.0:test - - - - - -org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test - -org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test - - - - - -org.assertj:assertj-core:jar:3.23.1:test - -org.assertj:assertj-core:jar:3.23.1:test - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.assertj:assertj-core:jar:3.23.1:test - - - - - -org.mockito:mockito-junit-jupiter:jar:4.6.1:test - -org.mockito:mockito-junit-jupiter:jar:4.6.1:test - - - -uk.gov.gchq.gaffer:core:pom:2.0.0-alpha-1.6-SNAPSHOT->org.mockito:mockito-junit-jupiter:jar:4.6.1:test - - - - - -com.google.guava:failureaccess:jar:1.0.1:compile - -com.google.guava:failureaccess:jar:1.0.1:compile - - - -com.google.guava:guava:jar:30.1.1-jre:compile->com.google.guava:failureaccess:jar:1.0.1:compile - - - - - -com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile - -com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile - - - -com.google.guava:guava:jar:30.1.1-jre:compile->com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile - - - - - -com.google.code.findbugs:jsr305:jar:3.0.2:compile - -com.google.code.findbugs:jsr305:jar:3.0.2:compile - - - -com.google.guava:guava:jar:30.1.1-jre:compile->com.google.code.findbugs:jsr305:jar:3.0.2:compile - - - - - -org.checkerframework:checker-qual:jar:3.8.0:compile - -org.checkerframework:checker-qual:jar:3.8.0:compile - - - -com.google.guava:guava:jar:30.1.1-jre:compile->org.checkerframework:checker-qual:jar:3.8.0:compile - - - - - -com.google.errorprone:error_prone_annotations:jar:2.5.1:compile - -com.google.errorprone:error_prone_annotations:jar:2.5.1:compile - - - -com.google.guava:guava:jar:30.1.1-jre:compile->com.google.errorprone:error_prone_annotations:jar:2.5.1:compile - - - - - -com.google.j2objc:j2objc-annotations:jar:1.3:compile - -com.google.j2objc:j2objc-annotations:jar:1.3:compile - - - -com.google.guava:guava:jar:30.1.1-jre:compile->com.google.j2objc:j2objc-annotations:jar:1.3:compile - - - - - -ch.qos.reload4j:reload4j:jar:1.2.18.3:compile - -ch.qos.reload4j:reload4j:jar:1.2.18.3:compile - - - -org.slf4j:slf4j-reload4j:jar:1.7.36:compile->ch.qos.reload4j:reload4j:jar:1.2.18.3:compile - - - - - -org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test - -org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test - - - -org.junit.jupiter:junit-jupiter:jar:5.9.0:test->org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test - - - - - -org.junit.jupiter:junit-jupiter-params:jar:5.9.0:test - -org.junit.jupiter:junit-jupiter-params:jar:5.9.0:test - - - -org.junit.jupiter:junit-jupiter:jar:5.9.0:test->org.junit.jupiter:junit-jupiter-params:jar:5.9.0:test - - - - - -org.junit.platform:junit-platform-engine:jar:1.9.0:test - -org.junit.platform:junit-platform-engine:jar:1.9.0:test - - - -org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test->org.junit.platform:junit-platform-engine:jar:1.9.0:test - - - - - -org.apiguardian:apiguardian-api:jar:1.1.2:test - -org.apiguardian:apiguardian-api:jar:1.1.2:test - - - -org.junit.jupiter:junit-jupiter-engine:jar:5.9.0:test->org.apiguardian:apiguardian-api:jar:1.1.2:test - - - - - -net.bytebuddy:byte-buddy:jar:1.12.10:test - -net.bytebuddy:byte-buddy:jar:1.12.10:test - - - -org.assertj:assertj-core:jar:3.23.1:test->net.bytebuddy:byte-buddy:jar:1.12.10:test - - - - - -org.mockito:mockito-core:jar:4.6.1:test - -org.mockito:mockito-core:jar:4.6.1:test - - - -org.mockito:mockito-junit-jupiter:jar:4.6.1:test->org.mockito:mockito-core:jar:4.6.1:test - - - - - -org.opentest4j:opentest4j:jar:1.2.0:test - -org.opentest4j:opentest4j:jar:1.2.0:test - - - -org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test->org.opentest4j:opentest4j:jar:1.2.0:test - - - - - -org.junit.platform:junit-platform-commons:jar:1.9.0:test - -org.junit.platform:junit-platform-commons:jar:1.9.0:test - - - -org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test->org.junit.platform:junit-platform-commons:jar:1.9.0:test - - - - - -net.bytebuddy:byte-buddy-agent:jar:1.12.10:test - -net.bytebuddy:byte-buddy-agent:jar:1.12.10:test - - - -org.mockito:mockito-core:jar:4.6.1:test->net.bytebuddy:byte-buddy-agent:jar:1.12.10:test - - - - - -org.objenesis:objenesis:jar:3.2:test - -org.objenesis:objenesis:jar:3.2:test - - - -org.mockito:mockito-core:jar:4.6.1:test->org.objenesis:objenesis:jar:3.2:test - - - - - diff --git a/docs/dev/managing-dependencies/gaffer-complete.svg b/docs/dev/managing-dependencies/gaffer-complete.svg deleted file mode 100644 index 46b5076453..0000000000 --- a/docs/dev/managing-dependencies/gaffer-complete.svg +++ /dev/null @@ -1,5772 +0,0 @@ - - - - - - -gaffer2 - - - -uk.gov.gchq.gaffer:common-util:jar:compile - -uk.gov.gchq.gaffer -common-util -2.0.0 - - - -org.apache.commons:commons-lang3:jar:compile - -org.apache.commons -commons-lang3 -3.12.0 - - - -uk.gov.gchq.gaffer:common-util:jar:compile->org.apache.commons:commons-lang3:jar:compile - - - - - -org.reflections:reflections:jar:compile - -org.reflections -reflections -0.9.12 - - - -uk.gov.gchq.gaffer:common-util:jar:compile->org.reflections:reflections:jar:compile - - - - - -com.fasterxml.jackson.core:jackson-annotations:jar:compile - -com.fasterxml.jackson.core -jackson-annotations -2.13.5 - - - -uk.gov.gchq.gaffer:common-util:jar:compile->com.fasterxml.jackson.core:jackson-annotations:jar:compile - - - - - -com.fasterxml.jackson.core:jackson-databind:jar:compile - -com.fasterxml.jackson.core -jackson-databind -2.13.5 - - - -uk.gov.gchq.gaffer:common-util:jar:compile->com.fasterxml.jackson.core:jackson-databind:jar:compile - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile - -uk.gov.gchq.koryphe -core -2.5.2 -jdk8 - - - -uk.gov.gchq.gaffer:common-util:jar:compile->uk.gov.gchq.koryphe:core:jar:jdk8:compile - - - - - -com.google.guava:guava:jar:compile - -com.google.guava -guava -30.1.1-jre - - - -uk.gov.gchq.gaffer:common-util:jar:compile->com.google.guava:guava:jar:compile - - - - - -org.slf4j:slf4j-api:jar:compile - -org.slf4j -slf4j-api -1.7.36 - - - -uk.gov.gchq.gaffer:common-util:jar:compile->org.slf4j:slf4j-api:jar:compile - - - - - -org.slf4j:slf4j-reload4j:jar:compile - -org.slf4j -slf4j-reload4j -1.7.36 - - - -uk.gov.gchq.gaffer:common-util:jar:compile->org.slf4j:slf4j-reload4j:jar:compile - - - - - -org.javassist:javassist:jar:compile - -org.javassist -javassist -3.29.2-GA - - - -org.reflections:reflections:jar:compile->org.javassist:javassist:jar:compile - - - - - -com.fasterxml.jackson.core:jackson-databind:jar:compile->com.fasterxml.jackson.core:jackson-annotations:jar:compile - - - - - -com.fasterxml.jackson.core:jackson-core:jar:compile - -com.fasterxml.jackson.core -jackson-core -2.13.5 - - - -com.fasterxml.jackson.core:jackson-databind:jar:compile->com.fasterxml.jackson.core:jackson-core:jar:compile - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile->org.apache.commons:commons-lang3:jar:compile - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile->com.fasterxml.jackson.core:jackson-databind:jar:compile - - - - - -com.github.spotbugs:spotbugs-annotations:jar:compile - -com.github.spotbugs -spotbugs-annotations -4.7.3 - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile->com.github.spotbugs:spotbugs-annotations:jar:compile - - - - - -io.github.lukehutch:fast-classpath-scanner:jar:compile - -io.github.lukehutch -fast-classpath-scanner -2.10.0 - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile->io.github.lukehutch:fast-classpath-scanner:jar:compile - - - - - -commons-io:commons-io:jar:compile - -commons-io -commons-io -2.11.0 - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile->commons-io:commons-io:jar:compile - - - - - -org.apache.commons:commons-csv:jar:compile - -org.apache.commons -commons-csv -1.9.0 - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile->org.apache.commons:commons-csv:jar:compile - - - - - -commons-codec:commons-codec:jar:compile - -commons-codec -commons-codec -1.15 - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile->commons-codec:commons-codec:jar:compile - - - - - -org.json:json:jar:compile - -org.json -json -20190722 - - - -uk.gov.gchq.koryphe:core:jar:jdk8:compile->org.json:json:jar:compile - - - - - -com.google.guava:failureaccess:jar:compile - -com.google.guava -failureaccess -1.0.1 - - - -com.google.guava:guava:jar:compile->com.google.guava:failureaccess:jar:compile - - - - - -com.google.guava:listenablefuture:jar:compile - -com.google.guava -listenablefuture -9999.0-empty-to-avoid-conflict-with-guava - - - -com.google.guava:guava:jar:compile->com.google.guava:listenablefuture:jar:compile - - - - - -com.google.code.findbugs:jsr305:jar:compile - -com.google.code.findbugs -jsr305 -3.0.2 - - - -com.google.guava:guava:jar:compile->com.google.code.findbugs:jsr305:jar:compile - - - - - -org.checkerframework:checker-qual:jar:compile - -org.checkerframework -checker-qual -3.8.0 - - - -com.google.guava:guava:jar:compile->org.checkerframework:checker-qual:jar:compile - - - - - -com.google.errorprone:error_prone_annotations:jar:compile - -com.google.errorprone -error_prone_annotations -2.5.1 - - - -com.google.guava:guava:jar:compile->com.google.errorprone:error_prone_annotations:jar:compile - - - - - -com.google.j2objc:j2objc-annotations:jar:compile - -com.google.j2objc -j2objc-annotations -1.3 - - - -com.google.guava:guava:jar:compile->com.google.j2objc:j2objc-annotations:jar:compile - - - - - -ch.qos.reload4j:reload4j:jar:compile - -ch.qos.reload4j -reload4j -1.2.18.3 - - - -org.slf4j:slf4j-reload4j:jar:compile->ch.qos.reload4j:reload4j:jar:compile - - - - - -uk.gov.gchq.gaffer:exception:jar:compile - -uk.gov.gchq.gaffer -exception -2.0.0 - - - -uk.gov.gchq.gaffer:exception:jar:compile->uk.gov.gchq.gaffer:common-util:jar:compile - - - - - -uk.gov.gchq.gaffer:serialisation:jar:compile - -uk.gov.gchq.gaffer -serialisation -2.0.0 - - - -uk.gov.gchq.gaffer:serialisation:jar:compile->uk.gov.gchq.gaffer:exception:jar:compile - - - - - -org.apache.avro:avro:jar:compile - -org.apache.avro -avro -1.7.7 - - - -uk.gov.gchq.gaffer:serialisation:jar:compile->org.apache.avro:avro:jar:compile - - - - - -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:compile - -com.fasterxml.jackson.datatype -jackson-datatype-jsr310 -2.13.5 - - - -uk.gov.gchq.gaffer:serialisation:jar:compile->com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:compile - - - - - -org.codehaus.jackson:jackson-core-asl:jar:compile - -org.codehaus.jackson -jackson-core-asl -1.9.13 - - - -org.apache.avro:avro:jar:compile->org.codehaus.jackson:jackson-core-asl:jar:compile - - - - - -org.codehaus.jackson:jackson-mapper-asl:jar:compile - -org.codehaus.jackson -jackson-mapper-asl -1.9.13 - - - -org.apache.avro:avro:jar:compile->org.codehaus.jackson:jackson-mapper-asl:jar:compile - - - - - -com.thoughtworks.paranamer:paranamer:jar:compile - -com.thoughtworks.paranamer -paranamer -2.8 - - - -org.apache.avro:avro:jar:compile->com.thoughtworks.paranamer:paranamer:jar:compile - - - - - -org.xerial.snappy:snappy-java:jar:compile - -org.xerial.snappy -snappy-java -1.1.8.2 - - - -org.apache.avro:avro:jar:compile->org.xerial.snappy:snappy-java:jar:compile - - - - - -org.apache.commons:commons-compress:jar:compile - -org.apache.commons -commons-compress -1.21 - - - -org.apache.avro:avro:jar:compile->org.apache.commons:commons-compress:jar:compile - - - - - -org.tukaani:xz:jar:compile - -org.tukaani -xz -1.5 - - - -org.apache.avro:avro:jar:compile->org.tukaani:xz:jar:compile - - - - - -uk.gov.gchq.gaffer:type:jar:compile - -uk.gov.gchq.gaffer -type -2.0.0 - - - -uk.gov.gchq.gaffer:type:jar:compile->uk.gov.gchq.gaffer:serialisation:jar:compile - - - - - -uk.gov.gchq.gaffer:access:jar:compile - -uk.gov.gchq.gaffer -access -2.0.0 - - - -uk.gov.gchq.gaffer:access:jar:compile->uk.gov.gchq.gaffer:common-util:jar:compile - - - - - -uk.gov.gchq.gaffer:data:jar:compile - -uk.gov.gchq.gaffer -data -2.0.0 - - - -uk.gov.gchq.gaffer:data:jar:compile->uk.gov.gchq.gaffer:type:jar:compile - - - - - -uk.gov.gchq.gaffer:data:jar:compile->uk.gov.gchq.gaffer:access:jar:compile - - - - - -org.apache.commons:commons-collections4:jar:compile - -org.apache.commons -commons-collections4 -4.4 - - - -uk.gov.gchq.gaffer:data:jar:compile->org.apache.commons:commons-collections4:jar:compile - - - - - -uk.gov.gchq.gaffer:cache:jar:compile - -uk.gov.gchq.gaffer -cache -2.0.0 - - - -uk.gov.gchq.gaffer:cache:jar:compile->uk.gov.gchq.gaffer:serialisation:jar:compile - - - - - -uk.gov.gchq.gaffer:operation:jar:compile - -uk.gov.gchq.gaffer -operation -2.0.0 - - - -uk.gov.gchq.gaffer:operation:jar:compile->uk.gov.gchq.gaffer:data:jar:compile - - - - - -uk.gov.gchq.gaffer:operation:jar:compile->uk.gov.gchq.gaffer:cache:jar:compile - - - - - -uk.gov.gchq.gaffer:jcs-cache-service:jar:compile - -uk.gov.gchq.gaffer -jcs-cache-service -2.0.0 - - - -uk.gov.gchq.gaffer:jcs-cache-service:jar:compile->uk.gov.gchq.gaffer:cache:jar:compile - - - - - -org.apache.commons:commons-jcs-core:jar:compile - -org.apache.commons -commons-jcs-core -2.2.1 - - - -uk.gov.gchq.gaffer:jcs-cache-service:jar:compile->org.apache.commons:commons-jcs-core:jar:compile - - - - - -commons-logging:commons-logging:jar:compile - -commons-logging -commons-logging -1.2 - - - -org.apache.commons:commons-jcs-core:jar:compile->commons-logging:commons-logging:jar:compile - - - - - -uk.gov.gchq.gaffer:store:jar:compile - -uk.gov.gchq.gaffer -store -2.0.0 - - - -uk.gov.gchq.gaffer:store:jar:compile->uk.gov.gchq.gaffer:operation:jar:compile - - - - - -uk.gov.gchq.gaffer:graph:jar:compile - -uk.gov.gchq.gaffer -graph -2.0.0 - - - -uk.gov.gchq.gaffer:graph:jar:compile->uk.gov.gchq.gaffer:store:jar:compile - - - - - -uk.gov.gchq.gaffer:integration-test:jar:compile - -uk.gov.gchq.gaffer -integration-test -2.0.0 - - - -uk.gov.gchq.gaffer:integration-test:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile - - - - - -uk.gov.gchq.gaffer:common-util:test-jar:tests:compile - -uk.gov.gchq.gaffer -common-util -2.0.0 -.test-jar -tests - - - -uk.gov.gchq.gaffer:integration-test:jar:compile->uk.gov.gchq.gaffer:common-util:test-jar:tests:compile - - - - - -uk.gov.gchq.gaffer:data:test-jar:tests:compile - -uk.gov.gchq.gaffer -data -2.0.0 -.test-jar -tests - - - -uk.gov.gchq.gaffer:integration-test:jar:compile->uk.gov.gchq.gaffer:data:test-jar:tests:compile - - - - - -uk.gov.gchq.gaffer:store:test-jar:tests:compile - -uk.gov.gchq.gaffer -store -2.0.0 -.test-jar -tests - - - -uk.gov.gchq.gaffer:integration-test:jar:compile->uk.gov.gchq.gaffer:store:test-jar:tests:compile - - - - - -org.junit.platform:junit-platform-suite:jar:compile - -org.junit.platform -junit-platform-suite -1.9.0 - - - -uk.gov.gchq.gaffer:integration-test:jar:compile->org.junit.platform:junit-platform-suite:jar:compile - - - - - -org.junit.jupiter:junit-jupiter:jar:provided - -org.junit.jupiter -junit-jupiter -5.9.0 -(provided) - - - -uk.gov.gchq.gaffer:integration-test:jar:compile->org.junit.jupiter:junit-jupiter:jar:provided - - - - - -org.junit.jupiter:junit-jupiter-engine:jar:provided - -org.junit.jupiter -junit-jupiter-engine -5.9.0 -(provided) - - - -uk.gov.gchq.gaffer:integration-test:jar:compile->org.junit.jupiter:junit-jupiter-engine:jar:provided - - - - - -org.assertj:assertj-core:jar:provided - -org.assertj -assertj-core -3.23.1 -(provided) - - - -uk.gov.gchq.gaffer:integration-test:jar:compile->org.assertj:assertj-core:jar:provided - - - - - -uk.gov.gchq.gaffer:common-util:test-jar:tests:compile->org.reflections:reflections:jar:compile - - - - - -uk.gov.gchq.gaffer:common-util:test-jar:tests:compile->uk.gov.gchq.koryphe:core:jar:jdk8:compile - - - - - -uk.gov.gchq.gaffer:data:test-jar:tests:compile->uk.gov.gchq.gaffer:type:jar:compile - - - - - -uk.gov.gchq.gaffer:data:test-jar:tests:compile->uk.gov.gchq.gaffer:access:jar:compile - - - - - -uk.gov.gchq.gaffer:data:test-jar:tests:compile->org.apache.commons:commons-collections4:jar:compile - - - - - -uk.gov.gchq.gaffer:store:test-jar:tests:compile->uk.gov.gchq.gaffer:operation:jar:compile - - - - - -org.junit.platform:junit-platform-suite-api:jar:compile - -org.junit.platform -junit-platform-suite-api -1.9.0 - - - -org.junit.platform:junit-platform-commons:jar:compile - -org.junit.platform -junit-platform-commons -1.9.0 - - - -org.junit.platform:junit-platform-suite-api:jar:compile->org.junit.platform:junit-platform-commons:jar:compile - - - - - -org.junit.platform:junit-platform-suite:jar:compile->org.junit.platform:junit-platform-suite-api:jar:compile - - - - - -org.junit.jupiter:junit-jupiter-api:jar:provided - -org.junit.jupiter -junit-jupiter-api -5.9.0 -(provided) - - - -org.junit.jupiter:junit-jupiter:jar:provided->org.junit.jupiter:junit-jupiter-api:jar:provided - - - - - -org.junit.jupiter:junit-jupiter-params:jar:provided - -org.junit.jupiter -junit-jupiter-params -5.9.0 -(provided) - - - -org.junit.jupiter:junit-jupiter:jar:provided->org.junit.jupiter:junit-jupiter-params:jar:provided - - - - - -org.apiguardian:apiguardian-api:jar:compile - -org.apiguardian -apiguardian-api -1.1.2 - - - -org.junit.jupiter:junit-jupiter-engine:jar:provided->org.apiguardian:apiguardian-api:jar:compile - - - - - -net.bytebuddy:byte-buddy:jar:provided - -net.bytebuddy -byte-buddy -1.12.10 -(provided) - - - -org.assertj:assertj-core:jar:provided->net.bytebuddy:byte-buddy:jar:provided - - - - - -uk.gov.gchq.gaffer:sketches-library:jar:compile - -uk.gov.gchq.gaffer -sketches-library -2.0.0 - - - -uk.gov.gchq.gaffer:sketches-library:jar:compile->uk.gov.gchq.gaffer:data:jar:compile - - - - - -com.clearspring.analytics:stream:jar:compile - -com.clearspring.analytics -stream -2.7.0 - - - -uk.gov.gchq.gaffer:sketches-library:jar:compile->com.clearspring.analytics:stream:jar:compile - - - - - -com.yahoo.datasketches:sketches-core:jar:compile - -com.yahoo.datasketches -sketches-core -0.12.0 - - - -uk.gov.gchq.gaffer:sketches-library:jar:compile->com.yahoo.datasketches:sketches-core:jar:compile - - - - - -it.unimi.dsi:fastutil:jar:compile - -it.unimi.dsi -fastutil -6.5.7 - - - -com.clearspring.analytics:stream:jar:compile->it.unimi.dsi:fastutil:jar:compile - - - - - -com.yahoo.datasketches:memory:jar:compile - -com.yahoo.datasketches -memory -0.12.0 - - - -com.yahoo.datasketches:sketches-core:jar:compile->com.yahoo.datasketches:memory:jar:compile - - - - - -uk.gov.gchq.gaffer:hdfs-library:jar:compile - -uk.gov.gchq.gaffer -hdfs-library -2.0.0 - - - -uk.gov.gchq.gaffer:hdfs-library:jar:compile->uk.gov.gchq.gaffer:store:jar:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile - -org.apache.hadoop -hadoop-common -3.3.3 - - - -uk.gov.gchq.gaffer:hdfs-library:jar:compile->org.apache.hadoop:hadoop-common:jar:compile - - - - - -org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile - -org.apache.hadoop -hadoop-mapreduce-client-core -3.3.3 - - - -uk.gov.gchq.gaffer:hdfs-library:jar:compile->org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile - - - - - -org.apache.avro:avro-mapred:jar:hadoop2:compile - -org.apache.avro -avro-mapred -1.8.2 -hadoop2 - - - -uk.gov.gchq.gaffer:hdfs-library:jar:compile->org.apache.avro:avro-mapred:jar:hadoop2:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.commons:commons-lang3:jar:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile->com.fasterxml.jackson.core:jackson-databind:jar:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile->commons-io:commons-io:jar:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile->commons-codec:commons-codec:jar:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile->com.google.code.findbugs:jsr305:jar:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile->ch.qos.reload4j:reload4j:jar:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.avro:avro:jar:compile - - - - - -org.apache.hadoop:hadoop-common:jar:compile->commons-logging:commons-logging:jar:compile - - - - - -org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar:compile - -org.apache.hadoop.thirdparty -hadoop-shaded-protobuf_3_7 -1.1.1 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7:jar:compile - - - - - -org.apache.hadoop:hadoop-annotations:jar:compile - -org.apache.hadoop -hadoop-annotations -3.3.3 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.hadoop:hadoop-annotations:jar:compile - - - - - -org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:compile - -org.apache.hadoop.thirdparty -hadoop-shaded-guava -1.1.1 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.hadoop.thirdparty:hadoop-shaded-guava:jar:compile - - - - - -commons-cli:commons-cli:jar:compile - -commons-cli -commons-cli -1.2 - - - -org.apache.hadoop:hadoop-common:jar:compile->commons-cli:commons-cli:jar:compile - - - - - -org.apache.commons:commons-math3:jar:compile - -org.apache.commons -commons-math3 -3.6.1 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.commons:commons-math3:jar:compile - - - - - -org.apache.httpcomponents:httpclient:jar:compile - -org.apache.httpcomponents -httpclient -4.5.13 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.httpcomponents:httpclient:jar:compile - - - - - -commons-net:commons-net:jar:compile - -commons-net -commons-net -3.6 - - - -org.apache.hadoop:hadoop-common:jar:compile->commons-net:commons-net:jar:compile - - - - - -commons-collections:commons-collections:jar:compile - -commons-collections -commons-collections -3.2.2 - - - -org.apache.hadoop:hadoop-common:jar:compile->commons-collections:commons-collections:jar:compile - - - - - -javax.servlet:javax.servlet-api:jar:compile - -javax.servlet -javax.servlet-api -3.1.0 - - - -org.apache.hadoop:hadoop-common:jar:compile->javax.servlet:javax.servlet-api:jar:compile - - - - - -jakarta.activation:jakarta.activation-api:jar:compile - -jakarta.activation -jakarta.activation-api -1.2.1 - - - -org.apache.hadoop:hadoop-common:jar:compile->jakarta.activation:jakarta.activation-api:jar:compile - - - - - -org.eclipse.jetty:jetty-server:jar:compile - -org.eclipse.jetty -jetty-server -9.4.43.v20210629 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.eclipse.jetty:jetty-server:jar:compile - - - - - -org.eclipse.jetty:jetty-util:jar:compile - -org.eclipse.jetty -jetty-util -9.4.43.v20210629 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.eclipse.jetty:jetty-util:jar:compile - - - - - -org.eclipse.jetty:jetty-servlet:jar:compile - -org.eclipse.jetty -jetty-servlet -9.4.43.v20210629 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.eclipse.jetty:jetty-servlet:jar:compile - - - - - -org.eclipse.jetty:jetty-webapp:jar:compile - -org.eclipse.jetty -jetty-webapp -9.4.43.v20210629 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.eclipse.jetty:jetty-webapp:jar:compile - - - - - -commons-beanutils:commons-beanutils:jar:compile - -commons-beanutils -commons-beanutils -1.9.4 - - - -org.apache.hadoop:hadoop-common:jar:compile->commons-beanutils:commons-beanutils:jar:compile - - - - - -org.apache.commons:commons-configuration2:jar:compile - -org.apache.commons -commons-configuration2 -2.5 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.commons:commons-configuration2:jar:compile - - - - - -org.apache.commons:commons-text:jar:compile - -org.apache.commons -commons-text -1.10.0 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.commons:commons-text:jar:compile - - - - - -com.google.re2j:re2j:jar:compile - -com.google.re2j -re2j -1.1 - - - -org.apache.hadoop:hadoop-common:jar:compile->com.google.re2j:re2j:jar:compile - - - - - -com.google.protobuf:protobuf-java:jar:compile - -com.google.protobuf -protobuf-java -2.5.0 - - - -org.apache.hadoop:hadoop-common:jar:compile->com.google.protobuf:protobuf-java:jar:compile - - - - - -com.google.code.gson:gson:jar:compile - -com.google.code.gson -gson -2.8.5 - - - -org.apache.hadoop:hadoop-common:jar:compile->com.google.code.gson:gson:jar:compile - - - - - -org.apache.hadoop:hadoop-auth:jar:compile - -org.apache.hadoop -hadoop-auth -3.3.3 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.hadoop:hadoop-auth:jar:compile - - - - - -com.jcraft:jsch:jar:compile - -com.jcraft -jsch -0.1.55 - - - -org.apache.hadoop:hadoop-common:jar:compile->com.jcraft:jsch:jar:compile - - - - - -org.apache.curator:curator-client:jar:compile - -org.apache.curator -curator-client -4.2.0 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.curator:curator-client:jar:compile - - - - - -org.apache.curator:curator-recipes:jar:compile - -org.apache.curator -curator-recipes -2.13.0 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.curator:curator-recipes:jar:compile - - - - - -org.apache.zookeeper:zookeeper:jar:compile - -org.apache.zookeeper -zookeeper -3.4.14 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.zookeeper:zookeeper:jar:compile - - - - - -org.apache.kerby:kerb-core:jar:compile - -org.apache.kerby -kerb-core -1.0.1 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.apache.kerby:kerb-core:jar:compile - - - - - -org.codehaus.woodstox:stax2-api:jar:compile - -org.codehaus.woodstox -stax2-api -4.2.1 - - - -org.apache.hadoop:hadoop-common:jar:compile->org.codehaus.woodstox:stax2-api:jar:compile - - - - - -com.fasterxml.woodstox:woodstox-core:jar:compile - -com.fasterxml.woodstox -woodstox-core -5.3.0 - - - -org.apache.hadoop:hadoop-common:jar:compile->com.fasterxml.woodstox:woodstox-core:jar:compile - - - - - -dnsjava:dnsjava:jar:compile - -dnsjava -dnsjava -2.1.7 - - - -org.apache.hadoop:hadoop-common:jar:compile->dnsjava:dnsjava:jar:compile - - - - - -org.apache.httpcomponents:httpcore:jar:compile - -org.apache.httpcomponents -httpcore -4.4.1 - - - -org.apache.httpcomponents:httpclient:jar:compile->org.apache.httpcomponents:httpcore:jar:compile - - - - - -org.eclipse.jetty:jetty-http:jar:compile - -org.eclipse.jetty -jetty-http -9.4.43.v20210629 - - - -org.eclipse.jetty:jetty-server:jar:compile->org.eclipse.jetty:jetty-http:jar:compile - - - - - -org.eclipse.jetty:jetty-io:jar:compile - -org.eclipse.jetty -jetty-io -9.4.43.v20210629 - - - -org.eclipse.jetty:jetty-server:jar:compile->org.eclipse.jetty:jetty-io:jar:compile - - - - - -org.eclipse.jetty:jetty-security:jar:compile - -org.eclipse.jetty -jetty-security -9.4.43.v20210629 - - - -org.eclipse.jetty:jetty-servlet:jar:compile->org.eclipse.jetty:jetty-security:jar:compile - - - - - -org.eclipse.jetty:jetty-util-ajax:jar:compile - -org.eclipse.jetty -jetty-util-ajax -9.4.43.v20210629 - - - -org.eclipse.jetty:jetty-servlet:jar:compile->org.eclipse.jetty:jetty-util-ajax:jar:compile - - - - - -org.eclipse.jetty:jetty-xml:jar:compile - -org.eclipse.jetty -jetty-xml -9.4.43.v20210629 - - - -org.eclipse.jetty:jetty-webapp:jar:compile->org.eclipse.jetty:jetty-xml:jar:compile - - - - - -com.nimbusds:nimbus-jose-jwt:jar:compile - -com.nimbusds -nimbus-jose-jwt -9.8.1 - - - -com.github.stephenc.jcip:jcip-annotations:jar:compile - -com.github.stephenc.jcip -jcip-annotations -1.0-1 - - - -com.nimbusds:nimbus-jose-jwt:jar:compile->com.github.stephenc.jcip:jcip-annotations:jar:compile - - - - - -org.apache.hadoop:hadoop-auth:jar:compile->com.nimbusds:nimbus-jose-jwt:jar:compile - - - - - -net.minidev:json-smart:jar:compile - -net.minidev -json-smart -2.4.7 - - - -org.apache.hadoop:hadoop-auth:jar:compile->net.minidev:json-smart:jar:compile - - - - - -org.apache.curator:curator-framework:jar:compile - -org.apache.curator -curator-framework -4.2.0 - - - -org.apache.hadoop:hadoop-auth:jar:compile->org.apache.curator:curator-framework:jar:compile - - - - - -org.apache.kerby:kerb-simplekdc:jar:compile - -org.apache.kerby -kerb-simplekdc -1.0.1 - - - -org.apache.hadoop:hadoop-auth:jar:compile->org.apache.kerby:kerb-simplekdc:jar:compile - - - - - -net.minidev:accessors-smart:jar:compile - -net.minidev -accessors-smart -2.4.7 - - - -org.ow2.asm:asm:jar:compile - -org.ow2.asm -asm -9.1 - - - -net.minidev:accessors-smart:jar:compile->org.ow2.asm:asm:jar:compile - - - - - -net.minidev:json-smart:jar:compile->net.minidev:accessors-smart:jar:compile - - - - - -org.apache.kerby:kerb-client:jar:compile - -org.apache.kerby -kerb-client -1.0.1 - - - -org.apache.kerby:kerby-config:jar:compile - -org.apache.kerby -kerby-config -1.0.1 - - - -org.apache.kerby:kerb-client:jar:compile->org.apache.kerby:kerby-config:jar:compile - - - - - -org.apache.kerby:kerb-common:jar:compile - -org.apache.kerby -kerb-common -1.0.1 - - - -org.apache.kerby:kerb-client:jar:compile->org.apache.kerby:kerb-common:jar:compile - - - - - -org.apache.kerby:kerb-util:jar:compile - -org.apache.kerby -kerb-util -1.0.1 - - - -org.apache.kerby:kerb-client:jar:compile->org.apache.kerby:kerb-util:jar:compile - - - - - -org.apache.kerby:token-provider:jar:compile - -org.apache.kerby -token-provider -1.0.1 - - - -org.apache.kerby:kerb-client:jar:compile->org.apache.kerby:token-provider:jar:compile - - - - - -org.apache.kerby:kerb-crypto:jar:compile - -org.apache.kerby -kerb-crypto -1.0.1 - - - -org.apache.kerby:kerb-common:jar:compile->org.apache.kerby:kerb-crypto:jar:compile - - - - - -org.apache.kerby:kerb-simplekdc:jar:compile->org.apache.kerby:kerb-client:jar:compile - - - - - -org.apache.kerby:kerb-admin:jar:compile - -org.apache.kerby -kerb-admin -1.0.1 - - - -org.apache.kerby:kerb-simplekdc:jar:compile->org.apache.kerby:kerb-admin:jar:compile - - - - - -org.apache.kerby:kerb-server:jar:compile - -org.apache.kerby -kerb-server -1.0.1 - - - -org.apache.kerby:kerb-identity:jar:compile - -org.apache.kerby -kerb-identity -1.0.1 - - - -org.apache.kerby:kerb-server:jar:compile->org.apache.kerby:kerb-identity:jar:compile - - - - - -org.apache.kerby:kerb-admin:jar:compile->org.apache.kerby:kerb-server:jar:compile - - - - - -org.apache.kerby:kerby-xdr:jar:compile - -org.apache.kerby -kerby-xdr -1.0.1 - - - -org.apache.kerby:kerb-admin:jar:compile->org.apache.kerby:kerby-xdr:jar:compile - - - - - -org.apache.curator:curator-recipes:jar:compile->org.apache.curator:curator-framework:jar:compile - - - - - -org.apache.zookeeper:zookeeper:jar:compile->com.github.spotbugs:spotbugs-annotations:jar:compile - - - - - -jline:jline:jar:compile - -jline -jline -2.11 - - - -org.apache.zookeeper:zookeeper:jar:compile->jline:jline:jar:compile - - - - - -org.apache.yetus:audience-annotations:jar:compile - -org.apache.yetus -audience-annotations -0.5.0 - - - -org.apache.zookeeper:zookeeper:jar:compile->org.apache.yetus:audience-annotations:jar:compile - - - - - -org.apache.kerby:kerby-pkix:jar:compile - -org.apache.kerby -kerby-pkix -1.0.1 - - - -org.apache.kerby:kerby-asn1:jar:compile - -org.apache.kerby -kerby-asn1 -1.0.1 - - - -org.apache.kerby:kerby-pkix:jar:compile->org.apache.kerby:kerby-asn1:jar:compile - - - - - -org.apache.kerby:kerby-util:jar:compile - -org.apache.kerby -kerby-util -1.0.1 - - - -org.apache.kerby:kerby-pkix:jar:compile->org.apache.kerby:kerby-util:jar:compile - - - - - -org.apache.kerby:kerb-core:jar:compile->org.apache.kerby:kerby-pkix:jar:compile - - - - - -org.eclipse.jetty.websocket:websocket-client:jar:compile - -org.eclipse.jetty.websocket -websocket-client -9.4.43.v20210629 - - - -org.eclipse.jetty.websocket:websocket-client:jar:compile->org.eclipse.jetty:jetty-io:jar:compile - - - - - -org.eclipse.jetty.websocket:websocket-client:jar:compile->org.eclipse.jetty:jetty-util:jar:compile - - - - - -org.eclipse.jetty:jetty-client:jar:compile - -org.eclipse.jetty -jetty-client -9.4.43.v20210629 - - - -org.eclipse.jetty.websocket:websocket-client:jar:compile->org.eclipse.jetty:jetty-client:jar:compile - - - - - -org.eclipse.jetty.websocket:websocket-common:jar:compile - -org.eclipse.jetty.websocket -websocket-common -9.4.43.v20210629 - - - -org.eclipse.jetty.websocket:websocket-client:jar:compile->org.eclipse.jetty.websocket:websocket-common:jar:compile - - - - - -org.eclipse.jetty:jetty-client:jar:compile->org.eclipse.jetty:jetty-http:jar:compile - - - - - -org.eclipse.jetty.websocket:websocket-api:jar:compile - -org.eclipse.jetty.websocket -websocket-api -9.4.43.v20210629 - - - -org.eclipse.jetty.websocket:websocket-common:jar:compile->org.eclipse.jetty.websocket:websocket-api:jar:compile - - - - - -org.apache.hadoop:hadoop-yarn-client:jar:compile - -org.apache.hadoop -hadoop-yarn-client -3.3.3 - - - -org.apache.hadoop:hadoop-yarn-client:jar:compile->org.eclipse.jetty.websocket:websocket-client:jar:compile - - - - - -org.apache.hadoop:hadoop-yarn-api:jar:compile - -org.apache.hadoop -hadoop-yarn-api -3.3.3 - - - -org.apache.hadoop:hadoop-yarn-client:jar:compile->org.apache.hadoop:hadoop-yarn-api:jar:compile - - - - - -org.jline:jline:jar:compile - -org.jline -jline -3.9.0 - - - -org.apache.hadoop:hadoop-yarn-client:jar:compile->org.jline:jline:jar:compile - - - - - -javax.xml.bind:jaxb-api:jar:compile - -javax.xml.bind -jaxb-api -2.2.11 - - - -org.apache.hadoop:hadoop-yarn-api:jar:compile->javax.xml.bind:jaxb-api:jar:compile - - - - - -javax.ws.rs:javax.ws.rs-api:jar:compile - -javax.ws.rs -javax.ws.rs-api -2.1.1 - - - -org.apache.hadoop:hadoop-yarn-api:jar:compile->javax.ws.rs:javax.ws.rs-api:jar:compile - - - - - -org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->org.apache.hadoop:hadoop-yarn-client:jar:compile - - - - - -org.apache.hadoop:hadoop-yarn-common:jar:compile - -org.apache.hadoop -hadoop-yarn-common -3.3.3 - - - -org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->org.apache.hadoop:hadoop-yarn-common:jar:compile - - - - - -org.apache.hadoop:hadoop-hdfs-client:jar:compile - -org.apache.hadoop -hadoop-hdfs-client -3.3.3 - - - -org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->org.apache.hadoop:hadoop-hdfs-client:jar:compile - - - - - -org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->javax.ws.rs:javax.ws.rs-api:jar:compile - - - - - -com.google.inject.extensions:guice-servlet:jar:compile - -com.google.inject.extensions -guice-servlet -4.0 - - - -org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile->com.google.inject.extensions:guice-servlet:jar:compile - - - - - -org.apache.hadoop:hadoop-yarn-common:jar:compile->javax.xml.bind:jaxb-api:jar:compile - - - - - -com.google.inject:guice:jar:compile - -com.google.inject -guice -4.0 - - - -org.apache.hadoop:hadoop-yarn-common:jar:compile->com.google.inject:guice:jar:compile - - - - - -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile - -com.fasterxml.jackson.module -jackson-module-jaxb-annotations -2.13.5 - - - -org.apache.hadoop:hadoop-yarn-common:jar:compile->com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile - - - - - -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile - -com.fasterxml.jackson.jaxrs -jackson-jaxrs-json-provider -2.13.5 - - - -org.apache.hadoop:hadoop-yarn-common:jar:compile->com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile - - - - - -javax.activation:javax.activation-api:jar:compile - -javax.activation -javax.activation-api -1.2.0 - - - -javax.xml.bind:jaxb-api:jar:compile->javax.activation:javax.activation-api:jar:compile - - - - - -javax.inject:javax.inject:jar:compile - -javax.inject -javax.inject -1 - - - -com.google.inject:guice:jar:compile->javax.inject:javax.inject:jar:compile - - - - - -aopalliance:aopalliance:jar:compile - -aopalliance -aopalliance -1.0 - - - -com.google.inject:guice:jar:compile->aopalliance:aopalliance:jar:compile - - - - - -jakarta.xml.bind:jakarta.xml.bind-api:jar:compile - -jakarta.xml.bind -jakarta.xml.bind-api -2.3.3 - - - -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile->jakarta.xml.bind:jakarta.xml.bind-api:jar:compile - - - - - -jakarta.xml.bind:jakarta.xml.bind-api:jar:compile->jakarta.activation:jakarta.activation-api:jar:compile - - - - - -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:compile - -com.fasterxml.jackson.jaxrs -jackson-jaxrs-base -2.13.5 - - - -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile->com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar:compile - - - - - -com.squareup.okhttp:okhttp:jar:compile - -com.squareup.okhttp -okhttp -2.7.5 - - - -com.squareup.okio:okio:jar:compile - -com.squareup.okio -okio -1.6.0 - - - -com.squareup.okhttp:okhttp:jar:compile->com.squareup.okio:okio:jar:compile - - - - - -org.apache.hadoop:hadoop-hdfs-client:jar:compile->com.squareup.okhttp:okhttp:jar:compile - - - - - -org.apache.velocity:velocity:jar:compile - -org.apache.velocity -velocity -1.7 - - - -commons-lang:commons-lang:jar:compile - -commons-lang -commons-lang -2.4 - - - -org.apache.velocity:velocity:jar:compile->commons-lang:commons-lang:jar:compile - - - - - -org.apache.avro:avro-ipc:jar:compile - -org.apache.avro -avro-ipc -1.8.2 - - - -org.apache.avro:avro-ipc:jar:compile->org.apache.velocity:velocity:jar:compile - - - - - -org.apache.avro:avro-mapred:jar:hadoop2:compile->org.codehaus.jackson:jackson-core-asl:jar:compile - - - - - -org.apache.avro:avro-mapred:jar:hadoop2:compile->org.codehaus.jackson:jackson-mapper-asl:jar:compile - - - - - -org.apache.avro:avro-mapred:jar:hadoop2:compile->org.apache.avro:avro-ipc:jar:compile - - - - - -uk.gov.gchq.gaffer:spark-library:jar:compile - -uk.gov.gchq.gaffer -spark-library -2.0.0 - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile - - - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile - - - - - -com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile - -com.fasterxml.jackson.module -jackson-module-scala_2.12 -2.13.5 - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile - - - - - -org.apache.spark:spark-core_2.12:jar:compile - -org.apache.spark -spark-core_2.12 -3.0.3 - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->org.apache.spark:spark-core_2.12:jar:compile - - - - - -org.apache.spark:spark-sql_2.12:jar:compile - -org.apache.spark -spark-sql_2.12 -3.0.3 - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->org.apache.spark:spark-sql_2.12:jar:compile - - - - - -org.apache.spark:spark-graphx_2.12:jar:compile - -org.apache.spark -spark-graphx_2.12 -3.0.3 - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->org.apache.spark:spark-graphx_2.12:jar:compile - - - - - -org.apache.spark:spark-catalyst_2.12:jar:compile - -org.apache.spark -spark-catalyst_2.12 -3.0.3 - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->org.apache.spark:spark-catalyst_2.12:jar:compile - - - - - -graphframes:graphframes:jar:compile - -graphframes -graphframes -0.8.1-spark3.0-s_2.12 - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->graphframes:graphframes:jar:compile - - - - - -org.objenesis:objenesis:jar:compile - -org.objenesis -objenesis -3.2 - - - -uk.gov.gchq.gaffer:spark-library:jar:compile->org.objenesis:objenesis:jar:compile - - - - - -com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile->com.fasterxml.jackson.core:jackson-databind:jar:compile - - - - - -com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile->com.thoughtworks.paranamer:paranamer:jar:compile - - - - - -org.scala-lang:scala-library:jar:compile - -org.scala-lang -scala-library -2.12.15 - - - -com.fasterxml.jackson.module:jackson-module-scala_2.12:jar:compile->org.scala-lang:scala-library:jar:compile - - - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.avro:avro-mapred:jar:hadoop2:compile - - - - - -com.twitter:chill_2.12:jar:compile - -com.twitter -chill_2.12 -0.7.6 - - - -org.apache.spark:spark-core_2.12:jar:compile->com.twitter:chill_2.12:jar:compile - - - - - -com.twitter:chill-java:jar:compile - -com.twitter -chill-java -0.7.6 - - - -org.apache.spark:spark-core_2.12:jar:compile->com.twitter:chill-java:jar:compile - - - - - -org.apache.xbean:xbean-asm7-shaded:jar:compile - -org.apache.xbean -xbean-asm7-shaded -4.15 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.xbean:xbean-asm7-shaded:jar:compile - - - - - -org.apache.hadoop:hadoop-client:jar:compile - -org.apache.hadoop -hadoop-client -3.3.3 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.hadoop:hadoop-client:jar:compile - - - - - -org.apache.spark:spark-launcher_2.12:jar:compile - -org.apache.spark -spark-launcher_2.12 -3.0.3 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-launcher_2.12:jar:compile - - - - - -org.apache.spark:spark-kvstore_2.12:jar:compile - -org.apache.spark -spark-kvstore_2.12 -3.0.3 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-kvstore_2.12:jar:compile - - - - - -org.apache.spark:spark-network-common_2.12:jar:compile - -org.apache.spark -spark-network-common_2.12 -3.0.3 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-network-common_2.12:jar:compile - - - - - -org.apache.spark:spark-network-shuffle_2.12:jar:compile - -org.apache.spark -spark-network-shuffle_2.12 -3.0.3 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-network-shuffle_2.12:jar:compile - - - - - -org.apache.spark:spark-unsafe_2.12:jar:compile - -org.apache.spark -spark-unsafe_2.12 -3.0.3 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-unsafe_2.12:jar:compile - - - - - -javax.activation:activation:jar:compile - -javax.activation -activation -1.1.1 - - - -org.apache.spark:spark-core_2.12:jar:compile->javax.activation:activation:jar:compile - - - - - -org.slf4j:jul-to-slf4j:jar:compile - -org.slf4j -jul-to-slf4j -1.7.36 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.slf4j:jul-to-slf4j:jar:compile - - - - - -org.slf4j:jcl-over-slf4j:jar:compile - -org.slf4j -jcl-over-slf4j -1.7.30 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.slf4j:jcl-over-slf4j:jar:compile - - - - - -com.ning:compress-lzf:jar:compile - -com.ning -compress-lzf -1.0.3 - - - -org.apache.spark:spark-core_2.12:jar:compile->com.ning:compress-lzf:jar:compile - - - - - -org.lz4:lz4-java:jar:compile - -org.lz4 -lz4-java -1.7.1 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.lz4:lz4-java:jar:compile - - - - - -com.github.luben:zstd-jni:jar:compile - -com.github.luben -zstd-jni -1.4.4-3 - - - -org.apache.spark:spark-core_2.12:jar:compile->com.github.luben:zstd-jni:jar:compile - - - - - -org.roaringbitmap:RoaringBitmap:jar:compile - -org.roaringbitmap -RoaringBitmap -0.5.11 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.roaringbitmap:RoaringBitmap:jar:compile - - - - - -org.scala-lang.modules:scala-xml_2.12:jar:compile - -org.scala-lang.modules -scala-xml_2.12 -1.2.0 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.scala-lang.modules:scala-xml_2.12:jar:compile - - - - - -org.scala-lang:scala-reflect:jar:compile - -org.scala-lang -scala-reflect -2.12.10 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.scala-lang:scala-reflect:jar:compile - - - - - -org.json4s:json4s-jackson_2.12:jar:compile - -org.json4s -json4s-jackson_2.12 -3.6.6 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.json4s:json4s-jackson_2.12:jar:compile - - - - - -org.glassfish.jersey.core:jersey-client:jar:compile - -org.glassfish.jersey.core -jersey-client -2.36 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.core:jersey-client:jar:compile - - - - - -org.glassfish.jersey.core:jersey-common:jar:compile - -org.glassfish.jersey.core -jersey-common -2.36 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.core:jersey-common:jar:compile - - - - - -org.glassfish.jersey.core:jersey-server:jar:compile - -org.glassfish.jersey.core -jersey-server -2.36 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.core:jersey-server:jar:compile - - - - - -org.glassfish.jersey.containers:jersey-container-servlet:jar:compile - -org.glassfish.jersey.containers -jersey-container-servlet -2.36 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.containers:jersey-container-servlet:jar:compile - - - - - -org.glassfish.jersey.containers:jersey-container-servlet-core:jar:compile - -org.glassfish.jersey.containers -jersey-container-servlet-core -2.36 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.containers:jersey-container-servlet-core:jar:compile - - - - - -org.glassfish.jersey.inject:jersey-hk2:jar:compile - -org.glassfish.jersey.inject -jersey-hk2 -2.36 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.glassfish.jersey.inject:jersey-hk2:jar:compile - - - - - -io.netty:netty-all:jar:compile - -io.netty -netty-all -4.1.68.Final - - - -org.apache.spark:spark-core_2.12:jar:compile->io.netty:netty-all:jar:compile - - - - - -io.dropwizard.metrics:metrics-core:jar:compile - -io.dropwizard.metrics -metrics-core -4.1.1 - - - -org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-core:jar:compile - - - - - -io.dropwizard.metrics:metrics-jvm:jar:compile - -io.dropwizard.metrics -metrics-jvm -4.1.1 - - - -org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-jvm:jar:compile - - - - - -io.dropwizard.metrics:metrics-json:jar:compile - -io.dropwizard.metrics -metrics-json -4.1.1 - - - -org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-json:jar:compile - - - - - -io.dropwizard.metrics:metrics-graphite:jar:compile - -io.dropwizard.metrics -metrics-graphite -4.1.1 - - - -org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-graphite:jar:compile - - - - - -io.dropwizard.metrics:metrics-jmx:jar:compile - -io.dropwizard.metrics -metrics-jmx -4.1.1 - - - -org.apache.spark:spark-core_2.12:jar:compile->io.dropwizard.metrics:metrics-jmx:jar:compile - - - - - -org.apache.ivy:ivy:jar:compile - -org.apache.ivy -ivy -2.4.0 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.ivy:ivy:jar:compile - - - - - -oro:oro:jar:compile - -oro -oro -2.0.8 - - - -org.apache.spark:spark-core_2.12:jar:compile->oro:oro:jar:compile - - - - - -net.razorvine:pyrolite:jar:compile - -net.razorvine -pyrolite -4.30 - - - -org.apache.spark:spark-core_2.12:jar:compile->net.razorvine:pyrolite:jar:compile - - - - - -net.sf.py4j:py4j:jar:compile - -net.sf.py4j -py4j -0.10.9 - - - -org.apache.spark:spark-core_2.12:jar:compile->net.sf.py4j:py4j:jar:compile - - - - - -org.apache.spark:spark-tags_2.12:jar:compile - -org.apache.spark -spark-tags_2.12 -3.0.3 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.spark:spark-tags_2.12:jar:compile - - - - - -org.apache.commons:commons-crypto:jar:compile - -org.apache.commons -commons-crypto -1.1.0 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.apache.commons:commons-crypto:jar:compile - - - - - -org.spark-project.spark:unused:jar:compile - -org.spark-project.spark -unused -1.0.0 - - - -org.apache.spark:spark-core_2.12:jar:compile->org.spark-project.spark:unused:jar:compile - - - - - -com.esotericsoftware:kryo-shaded:jar:compile - -com.esotericsoftware -kryo-shaded -4.0.2 - - - -com.esotericsoftware:minlog:jar:compile - -com.esotericsoftware -minlog -1.3.0 - - - -com.esotericsoftware:kryo-shaded:jar:compile->com.esotericsoftware:minlog:jar:compile - - - - - -com.twitter:chill_2.12:jar:compile->com.esotericsoftware:kryo-shaded:jar:compile - - - - - -com.twitter:chill_2.12:jar:compile->com.twitter:chill-java:jar:compile - - - - - -org.apache.hadoop:hadoop-client:jar:compile->org.apache.hadoop:hadoop-common:jar:compile - - - - - -org.apache.hadoop:hadoop-client:jar:compile->org.apache.hadoop:hadoop-mapreduce-client-core:jar:compile - - - - - -org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:compile - -org.apache.hadoop -hadoop-mapreduce-client-jobclient -3.3.3 - - - -org.apache.hadoop:hadoop-client:jar:compile->org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:compile - - - - - -org.apache.hadoop:hadoop-mapreduce-client-common:jar:provided - -org.apache.hadoop -hadoop-mapreduce-client-common -3.3.3 -(provided) - - - -org.apache.hadoop:hadoop-mapreduce-client-jobclient:jar:compile->org.apache.hadoop:hadoop-mapreduce-client-common:jar:provided - - - - - -org.fusesource.leveldbjni:leveldbjni-all:jar:compile - -org.fusesource.leveldbjni -leveldbjni-all -1.8 - - - -org.apache.spark:spark-kvstore_2.12:jar:compile->org.fusesource.leveldbjni:leveldbjni-all:jar:compile - - - - - -org.roaringbitmap:shims:jar:compile - -org.roaringbitmap -shims -0.7.45 - - - -org.roaringbitmap:RoaringBitmap:jar:compile->org.roaringbitmap:shims:jar:compile - - - - - -org.json4s:json4s-core_2.12:jar:compile - -org.json4s -json4s-core_2.12 -3.6.6 - - - -org.json4s:json4s-ast_2.12:jar:compile - -org.json4s -json4s-ast_2.12 -3.6.6 - - - -org.json4s:json4s-core_2.12:jar:compile->org.json4s:json4s-ast_2.12:jar:compile - - - - - -org.json4s:json4s-scalap_2.12:jar:compile - -org.json4s -json4s-scalap_2.12 -3.6.6 - - - -org.json4s:json4s-core_2.12:jar:compile->org.json4s:json4s-scalap_2.12:jar:compile - - - - - -org.json4s:json4s-jackson_2.12:jar:compile->org.json4s:json4s-core_2.12:jar:compile - - - - - -jakarta.ws.rs:jakarta.ws.rs-api:jar:compile - -jakarta.ws.rs -jakarta.ws.rs-api -2.1.6 - - - -org.glassfish.jersey.core:jersey-client:jar:compile->jakarta.ws.rs:jakarta.ws.rs-api:jar:compile - - - - - -org.glassfish.hk2.external:jakarta.inject:jar:compile - -org.glassfish.hk2.external -jakarta.inject -2.6.1 - - - -org.glassfish.jersey.core:jersey-client:jar:compile->org.glassfish.hk2.external:jakarta.inject:jar:compile - - - - - -org.glassfish.jersey.core:jersey-client:jar:compile->org.glassfish.jersey.core:jersey-common:jar:compile - - - - - -jakarta.annotation:jakarta.annotation-api:jar:compile - -jakarta.annotation -jakarta.annotation-api -1.3.5 - - - -org.glassfish.jersey.core:jersey-common:jar:compile->jakarta.annotation:jakarta.annotation-api:jar:compile - - - - - -org.glassfish.hk2:osgi-resource-locator:jar:compile - -org.glassfish.hk2 -osgi-resource-locator -1.0.3 - - - -org.glassfish.jersey.core:jersey-common:jar:compile->org.glassfish.hk2:osgi-resource-locator:jar:compile - - - - - -org.glassfish.jersey.core:jersey-server:jar:compile->org.glassfish.jersey.core:jersey-client:jar:compile - - - - - -org.glassfish.jersey.core:jersey-server:jar:compile->org.glassfish.jersey.core:jersey-common:jar:compile - - - - - -jakarta.validation:jakarta.validation-api:jar:compile - -jakarta.validation -jakarta.validation-api -2.0.2 - - - -org.glassfish.jersey.core:jersey-server:jar:compile->jakarta.validation:jakarta.validation-api:jar:compile - - - - - -org.glassfish.jersey.containers:jersey-container-servlet:jar:compile->org.glassfish.jersey.containers:jersey-container-servlet-core:jar:compile - - - - - -org.glassfish.hk2:hk2-locator:jar:compile - -org.glassfish.hk2 -hk2-locator -2.6.1 - - - -org.glassfish.hk2.external:aopalliance-repackaged:jar:compile - -org.glassfish.hk2.external -aopalliance-repackaged -2.6.1 - - - -org.glassfish.hk2:hk2-locator:jar:compile->org.glassfish.hk2.external:aopalliance-repackaged:jar:compile - - - - - -org.glassfish.hk2:hk2-api:jar:compile - -org.glassfish.hk2 -hk2-api -2.6.1 - - - -org.glassfish.hk2:hk2-locator:jar:compile->org.glassfish.hk2:hk2-api:jar:compile - - - - - -org.glassfish.hk2:hk2-utils:jar:compile - -org.glassfish.hk2 -hk2-utils -2.6.1 - - - -org.glassfish.hk2:hk2-locator:jar:compile->org.glassfish.hk2:hk2-utils:jar:compile - - - - - -org.glassfish.hk2:hk2-api:jar:compile->org.glassfish.hk2.external:aopalliance-repackaged:jar:compile - - - - - -org.glassfish.hk2:hk2-api:jar:compile->org.glassfish.hk2:hk2-utils:jar:compile - - - - - -org.glassfish.jersey.inject:jersey-hk2:jar:compile->org.javassist:javassist:jar:compile - - - - - -org.glassfish.jersey.inject:jersey-hk2:jar:compile->org.glassfish.jersey.core:jersey-common:jar:compile - - - - - -org.glassfish.jersey.inject:jersey-hk2:jar:compile->org.glassfish.hk2:hk2-locator:jar:compile - - - - - -com.univocity:univocity-parsers:jar:compile - -com.univocity -univocity-parsers -2.9.0 - - - -org.apache.spark:spark-sql_2.12:jar:compile->com.univocity:univocity-parsers:jar:compile - - - - - -org.apache.spark:spark-sketch_2.12:jar:compile - -org.apache.spark -spark-sketch_2.12 -3.0.3 - - - -org.apache.spark:spark-sql_2.12:jar:compile->org.apache.spark:spark-sketch_2.12:jar:compile - - - - - -org.apache.orc:orc-core:jar:compile - -org.apache.orc -orc-core -1.5.10 - - - -org.apache.spark:spark-sql_2.12:jar:compile->org.apache.orc:orc-core:jar:compile - - - - - -org.apache.orc:orc-mapreduce:jar:compile - -org.apache.orc -orc-mapreduce -1.5.10 - - - -org.apache.spark:spark-sql_2.12:jar:compile->org.apache.orc:orc-mapreduce:jar:compile - - - - - -org.apache.hive:hive-storage-api:jar:compile - -org.apache.hive -hive-storage-api -2.7.1 - - - -org.apache.spark:spark-sql_2.12:jar:compile->org.apache.hive:hive-storage-api:jar:compile - - - - - -org.apache.parquet:parquet-column:jar:compile - -org.apache.parquet -parquet-column -1.10.1 - - - -org.apache.spark:spark-sql_2.12:jar:compile->org.apache.parquet:parquet-column:jar:compile - - - - - -org.apache.parquet:parquet-hadoop:jar:compile - -org.apache.parquet -parquet-hadoop -1.10.1 - - - -org.apache.spark:spark-sql_2.12:jar:compile->org.apache.parquet:parquet-hadoop:jar:compile - - - - - -org.apache.orc:orc-core:jar:compile->com.google.protobuf:protobuf-java:jar:compile - - - - - -org.apache.orc:orc-core:jar:compile->commons-lang:commons-lang:jar:compile - - - - - -org.apache.orc:orc-shims:jar:compile - -org.apache.orc -orc-shims -1.5.10 - - - -org.apache.orc:orc-core:jar:compile->org.apache.orc:orc-shims:jar:compile - - - - - -io.airlift:aircompressor:jar:compile - -io.airlift -aircompressor -0.10 - - - -org.apache.orc:orc-core:jar:compile->io.airlift:aircompressor:jar:compile - - - - - -org.threeten:threeten-extra:jar:compile - -org.threeten -threeten-extra -1.5.0 - - - -org.apache.orc:orc-core:jar:compile->org.threeten:threeten-extra:jar:compile - - - - - -org.apache.parquet:parquet-common:jar:compile - -org.apache.parquet -parquet-common -1.10.1 - - - -org.apache.parquet:parquet-column:jar:compile->org.apache.parquet:parquet-common:jar:compile - - - - - -org.apache.parquet:parquet-encoding:jar:compile - -org.apache.parquet -parquet-encoding -1.10.1 - - - -org.apache.parquet:parquet-column:jar:compile->org.apache.parquet:parquet-encoding:jar:compile - - - - - -org.apache.parquet:parquet-format:jar:compile - -org.apache.parquet -parquet-format -2.4.0 - - - -org.apache.parquet:parquet-hadoop:jar:compile->org.apache.parquet:parquet-format:jar:compile - - - - - -org.apache.parquet:parquet-jackson:jar:compile - -org.apache.parquet -parquet-jackson -1.10.1 - - - -org.apache.parquet:parquet-hadoop:jar:compile->org.apache.parquet:parquet-jackson:jar:compile - - - - - -org.scalanlp:breeze_2.12:jar:compile - -org.scalanlp -breeze_2.12 -1.0 - - - -org.scalanlp:breeze-macros_2.12:jar:compile - -org.scalanlp -breeze-macros_2.12 -1.0 - - - -org.scalanlp:breeze_2.12:jar:compile->org.scalanlp:breeze-macros_2.12:jar:compile - - - - - -net.sf.opencsv:opencsv:jar:compile - -net.sf.opencsv -opencsv -2.3 - - - -org.scalanlp:breeze_2.12:jar:compile->net.sf.opencsv:opencsv:jar:compile - - - - - -com.github.wendykierp:JTransforms:jar:compile - -com.github.wendykierp -JTransforms -3.1 - - - -org.scalanlp:breeze_2.12:jar:compile->com.github.wendykierp:JTransforms:jar:compile - - - - - -org.typelevel:spire_2.12:jar:compile - -org.typelevel -spire_2.12 -0.17.0-M1 - - - -org.scalanlp:breeze_2.12:jar:compile->org.typelevel:spire_2.12:jar:compile - - - - - -org.scala-lang.modules:scala-collection-compat_2.12:jar:compile - -org.scala-lang.modules -scala-collection-compat_2.12 -2.1.1 - - - -org.scalanlp:breeze_2.12:jar:compile->org.scala-lang.modules:scala-collection-compat_2.12:jar:compile - - - - - -pl.edu.icm:JLargeArrays:jar:compile - -pl.edu.icm -JLargeArrays -1.5 - - - -com.github.wendykierp:JTransforms:jar:compile->pl.edu.icm:JLargeArrays:jar:compile - - - - - -org.typelevel:spire-macros_2.12:jar:compile - -org.typelevel -spire-macros_2.12 -0.17.0-M1 - - - -org.typelevel:spire_2.12:jar:compile->org.typelevel:spire-macros_2.12:jar:compile - - - - - -org.typelevel:spire-platform_2.12:jar:compile - -org.typelevel -spire-platform_2.12 -0.17.0-M1 - - - -org.typelevel:spire_2.12:jar:compile->org.typelevel:spire-platform_2.12:jar:compile - - - - - -org.typelevel:spire-util_2.12:jar:compile - -org.typelevel -spire-util_2.12 -0.17.0-M1 - - - -org.typelevel:spire_2.12:jar:compile->org.typelevel:spire-util_2.12:jar:compile - - - - - -org.typelevel:machinist_2.12:jar:compile - -org.typelevel -machinist_2.12 -0.6.8 - - - -org.typelevel:spire_2.12:jar:compile->org.typelevel:machinist_2.12:jar:compile - - - - - -org.typelevel:algebra_2.12:jar:compile - -org.typelevel -algebra_2.12 -2.0.0-M2 - - - -org.typelevel:spire_2.12:jar:compile->org.typelevel:algebra_2.12:jar:compile - - - - - -org.typelevel:cats-kernel_2.12:jar:compile - -org.typelevel -cats-kernel_2.12 -2.0.0-M4 - - - -org.typelevel:algebra_2.12:jar:compile->org.typelevel:cats-kernel_2.12:jar:compile - - - - - -org.apache.spark:spark-mllib-local_2.12:jar:compile - -org.apache.spark -spark-mllib-local_2.12 -3.0.3 - - - -org.apache.spark:spark-mllib-local_2.12:jar:compile->org.scalanlp:breeze_2.12:jar:compile - - - - - -org.apache.spark:spark-graphx_2.12:jar:compile->org.apache.spark:spark-mllib-local_2.12:jar:compile - - - - - -com.github.fommil.netlib:core:jar:compile - -com.github.fommil.netlib -core -1.1.2 - - - -org.apache.spark:spark-graphx_2.12:jar:compile->com.github.fommil.netlib:core:jar:compile - - - - - -net.sourceforge.f2j:arpack_combined_all:jar:compile - -net.sourceforge.f2j -arpack_combined_all -0.1 - - - -org.apache.spark:spark-graphx_2.12:jar:compile->net.sourceforge.f2j:arpack_combined_all:jar:compile - - - - - -org.apache.spark:spark-catalyst_2.12:jar:compile->commons-codec:commons-codec:jar:compile - - - - - -org.scala-lang.modules:scala-parser-combinators_2.12:jar:compile - -org.scala-lang.modules -scala-parser-combinators_2.12 -1.0.4 - - - -org.apache.spark:spark-catalyst_2.12:jar:compile->org.scala-lang.modules:scala-parser-combinators_2.12:jar:compile - - - - - -org.codehaus.janino:janino:jar:compile - -org.codehaus.janino -janino -3.0.16 - - - -org.apache.spark:spark-catalyst_2.12:jar:compile->org.codehaus.janino:janino:jar:compile - - - - - -org.codehaus.janino:commons-compiler:jar:compile - -org.codehaus.janino -commons-compiler -3.0.16 - - - -org.apache.spark:spark-catalyst_2.12:jar:compile->org.codehaus.janino:commons-compiler:jar:compile - - - - - -org.antlr:antlr4-runtime:jar:compile - -org.antlr -antlr4-runtime -4.7.1 - - - -org.apache.spark:spark-catalyst_2.12:jar:compile->org.antlr:antlr4-runtime:jar:compile - - - - - -org.apache.arrow:arrow-vector:jar:compile - -org.apache.arrow -arrow-vector -0.15.1 - - - -org.apache.spark:spark-catalyst_2.12:jar:compile->org.apache.arrow:arrow-vector:jar:compile - - - - - -org.apache.arrow:arrow-format:jar:compile - -org.apache.arrow -arrow-format -0.15.1 - - - -org.apache.arrow:arrow-vector:jar:compile->org.apache.arrow:arrow-format:jar:compile - - - - - -org.apache.arrow:arrow-memory:jar:compile - -org.apache.arrow -arrow-memory -0.15.1 - - - -org.apache.arrow:arrow-vector:jar:compile->org.apache.arrow:arrow-memory:jar:compile - - - - - -com.google.flatbuffers:flatbuffers-java:jar:compile - -com.google.flatbuffers -flatbuffers-java -1.9.0 - - - -org.apache.arrow:arrow-vector:jar:compile->com.google.flatbuffers:flatbuffers-java:jar:compile - - - - - -uk.gov.gchq.gaffer:map-store:jar:compile - -uk.gov.gchq.gaffer -map-store -2.0.0 - - - -uk.gov.gchq.gaffer:map-store:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile - - - - - -uk.gov.gchq.gaffer:map-store:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile - - - - - -uk.gov.gchq.gaffer:accumulo-store:jar:compile - -uk.gov.gchq.gaffer -accumulo-store -2.0.0 - - - -uk.gov.gchq.gaffer:accumulo-store:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile - - - - - -uk.gov.gchq.gaffer:accumulo-store:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile - - - - - -uk.gov.gchq.gaffer:accumulo-store:jar:compile->uk.gov.gchq.gaffer:hdfs-library:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile - -org.apache.accumulo -accumulo-core -2.0.1 - - - -uk.gov.gchq.gaffer:accumulo-store:jar:compile->org.apache.accumulo:accumulo-core:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.commons:commons-lang3:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->commons-io:commons-io:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.commons:commons-collections4:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->commons-logging:commons-logging:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.commons:commons-math3:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.commons:commons-configuration2:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->com.google.protobuf:protobuf-java:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->com.google.code.gson:gson:jar:compile - - - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.zookeeper:zookeeper:jar:compile - - - - - -com.beust:jcommander:jar:compile - -com.beust -jcommander -1.72 - - - -org.apache.accumulo:accumulo-core:jar:compile->com.beust:jcommander:jar:compile - - - - - -com.github.ben-manes.caffeine:caffeine:jar:compile - -com.github.ben-manes.caffeine -caffeine -2.7.0 - - - -org.apache.accumulo:accumulo-core:jar:compile->com.github.ben-manes.caffeine:caffeine:jar:compile - - - - - -org.apache.accumulo:accumulo-start:jar:compile - -org.apache.accumulo -accumulo-start -2.0.1 - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.accumulo:accumulo-start:jar:compile - - - - - -org.apache.hadoop:hadoop-client-api:jar:compile - -org.apache.hadoop -hadoop-client-api -3.1.1 - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.hadoop:hadoop-client-api:jar:compile - - - - - -org.apache.htrace:htrace-core:jar:compile - -org.apache.htrace -htrace-core -3.2.0-incubating - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.htrace:htrace-core:jar:compile - - - - - -org.apache.thrift:libthrift:jar:compile - -org.apache.thrift -libthrift -0.12.0 - - - -org.apache.accumulo:accumulo-core:jar:compile->org.apache.thrift:libthrift:jar:compile - - - - - -org.apache.accumulo:accumulo-start:jar:compile->commons-beanutils:commons-beanutils:jar:compile - - - - - -org.apache.commons:commons-vfs2:jar:compile - -org.apache.commons -commons-vfs2 -2.3 - - - -org.apache.accumulo:accumulo-start:jar:compile->org.apache.commons:commons-vfs2:jar:compile - - - - - -org.apache.thrift:libthrift:jar:compile->org.apache.httpcomponents:httpcore:jar:compile - - - - - -uk.gov.gchq.gaffer:spark-accumulo-library:jar:compile - -uk.gov.gchq.gaffer -spark-accumulo-library -2.0.0 - - - -uk.gov.gchq.gaffer:spark-accumulo-library:jar:compile->uk.gov.gchq.gaffer:spark-library:jar:compile - - - - - -uk.gov.gchq.gaffer:spark-accumulo-library:jar:compile->uk.gov.gchq.gaffer:accumulo-store:jar:compile - - - - - -io.netty:netty:jar:compile - -io.netty -netty -3.10.6.Final - - - -uk.gov.gchq.gaffer:spark-accumulo-library:jar:compile->io.netty:netty:jar:compile - - - - - -uk.gov.gchq.gaffer:bitmap-library:jar:compile - -uk.gov.gchq.gaffer -bitmap-library -2.0.0 - - - -uk.gov.gchq.gaffer:bitmap-library:jar:compile->uk.gov.gchq.gaffer:serialisation:jar:compile - - - - - -uk.gov.gchq.gaffer:bitmap-library:jar:compile->org.roaringbitmap:RoaringBitmap:jar:compile - - - - - -uk.gov.gchq.gaffer:hazelcast-cache-service:jar:compile - -uk.gov.gchq.gaffer -hazelcast-cache-service -2.0.0 - - - -uk.gov.gchq.gaffer:hazelcast-cache-service:jar:compile->uk.gov.gchq.gaffer:cache:jar:compile - - - - - -com.hazelcast:hazelcast:jar:compile - -com.hazelcast -hazelcast -5.3.0 - - - -uk.gov.gchq.gaffer:hazelcast-cache-service:jar:compile->com.hazelcast:hazelcast:jar:compile - - - - - -uk.gov.gchq.gaffer:flink-library:jar:compile - -uk.gov.gchq.gaffer -flink-library -2.0.0 - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->uk.gov.gchq.gaffer:store:jar:compile - - - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->jakarta.xml.bind:jakarta.xml.bind-api:jar:compile - - - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->org.scala-lang:scala-library:jar:compile - - - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->org.scala-lang.modules:scala-parser-combinators_2.12:jar:compile - - - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->org.objenesis:objenesis:jar:compile - - - - - -org.apache.flink:flink-java:jar:compile - -org.apache.flink -flink-java -1.7.2 - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->org.apache.flink:flink-java:jar:compile - - - - - -org.apache.flink:flink-streaming-java_2.12:jar:compile - -org.apache.flink -flink-streaming-java_2.12 -1.7.2 - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->org.apache.flink:flink-streaming-java_2.12:jar:compile - - - - - -org.apache.flink:flink-clients_2.12:jar:compile - -org.apache.flink -flink-clients_2.12 -1.7.2 - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->org.apache.flink:flink-clients_2.12:jar:compile - - - - - -org.apache.flink:flink-connector-kafka-0.10_2.12:jar:compile - -org.apache.flink -flink-connector-kafka-0.10_2.12 -1.7.2 - - - -uk.gov.gchq.gaffer:flink-library:jar:compile->org.apache.flink:flink-connector-kafka-0.10_2.12:jar:compile - - - - - -org.apache.flink:flink-core:jar:compile - -org.apache.flink -flink-core -1.7.2 - - - -org.apache.flink:flink-core:jar:compile->org.apache.commons:commons-compress:jar:compile - - - - - -org.apache.flink:flink-core:jar:compile->commons-collections:commons-collections:jar:compile - - - - - -org.apache.flink:flink-annotations:jar:compile - -org.apache.flink -flink-annotations -1.7.2 - - - -org.apache.flink:flink-core:jar:compile->org.apache.flink:flink-annotations:jar:compile - - - - - -org.apache.flink:flink-metrics-core:jar:compile - -org.apache.flink -flink-metrics-core -1.7.2 - - - -org.apache.flink:flink-core:jar:compile->org.apache.flink:flink-metrics-core:jar:compile - - - - - -com.esotericsoftware.kryo:kryo:jar:compile - -com.esotericsoftware.kryo -kryo -2.24.0 - - - -org.apache.flink:flink-core:jar:compile->com.esotericsoftware.kryo:kryo:jar:compile - - - - - -com.esotericsoftware.minlog:minlog:jar:compile - -com.esotericsoftware.minlog -minlog -1.2 - - - -com.esotericsoftware.kryo:kryo:jar:compile->com.esotericsoftware.minlog:minlog:jar:compile - - - - - -org.apache.flink:flink-java:jar:compile->org.apache.commons:commons-lang3:jar:compile - - - - - -org.apache.flink:flink-java:jar:compile->com.google.code.findbugs:jsr305:jar:compile - - - - - -org.apache.flink:flink-java:jar:compile->org.apache.commons:commons-math3:jar:compile - - - - - -org.apache.flink:flink-java:jar:compile->org.apache.flink:flink-core:jar:compile - - - - - -org.apache.flink:flink-shaded-asm:jar:compile - -org.apache.flink -flink-shaded-asm -5.0.4-5.0 - - - -org.apache.flink:flink-java:jar:compile->org.apache.flink:flink-shaded-asm:jar:compile - - - - - -org.apache.flink:force-shading:jar:compile - -org.apache.flink -force-shading -1.7.2 - - - -org.apache.flink:flink-java:jar:compile->org.apache.flink:force-shading:jar:compile - - - - - -org.apache.flink:flink-runtime_2.12:jar:compile - -org.apache.flink -flink-runtime_2.12 -1.7.2 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->commons-io:commons-io:jar:compile - - - - - -org.apache.flink:flink-runtime_2.12:jar:compile->com.twitter:chill_2.12:jar:compile - - - - - -org.apache.flink:flink-queryable-state-client-java_2.12:jar:compile - -org.apache.flink -flink-queryable-state-client-java_2.12 -1.7.2 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->org.apache.flink:flink-queryable-state-client-java_2.12:jar:compile - - - - - -org.apache.flink:flink-hadoop-fs:jar:compile - -org.apache.flink -flink-hadoop-fs -1.7.2 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->org.apache.flink:flink-hadoop-fs:jar:compile - - - - - -org.apache.flink:flink-shaded-netty:jar:compile - -org.apache.flink -flink-shaded-netty -4.1.24.Final-5.0 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->org.apache.flink:flink-shaded-netty:jar:compile - - - - - -org.apache.flink:flink-shaded-jackson:jar:compile - -org.apache.flink -flink-shaded-jackson -2.7.9-5.0 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->org.apache.flink:flink-shaded-jackson:jar:compile - - - - - -com.typesafe.akka:akka-actor_2.12:jar:compile - -com.typesafe.akka -akka-actor_2.12 -2.4.20 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->com.typesafe.akka:akka-actor_2.12:jar:compile - - - - - -com.typesafe.akka:akka-stream_2.12:jar:compile - -com.typesafe.akka -akka-stream_2.12 -2.4.20 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->com.typesafe.akka:akka-stream_2.12:jar:compile - - - - - -com.typesafe.akka:akka-protobuf_2.12:jar:compile - -com.typesafe.akka -akka-protobuf_2.12 -2.4.20 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->com.typesafe.akka:akka-protobuf_2.12:jar:compile - - - - - -com.typesafe.akka:akka-slf4j_2.12:jar:compile - -com.typesafe.akka -akka-slf4j_2.12 -2.4.20 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->com.typesafe.akka:akka-slf4j_2.12:jar:compile - - - - - -org.clapper:grizzled-slf4j_2.12:jar:compile - -org.clapper -grizzled-slf4j_2.12 -1.3.2 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->org.clapper:grizzled-slf4j_2.12:jar:compile - - - - - -com.github.scopt:scopt_2.12:jar:compile - -com.github.scopt -scopt_2.12 -3.5.0 - - - -org.apache.flink:flink-runtime_2.12:jar:compile->com.github.scopt:scopt_2.12:jar:compile - - - - - -com.typesafe:config:jar:compile - -com.typesafe -config -1.3.0 - - - -com.typesafe.akka:akka-actor_2.12:jar:compile->com.typesafe:config:jar:compile - - - - - -org.scala-lang.modules:scala-java8-compat_2.12:jar:compile - -org.scala-lang.modules -scala-java8-compat_2.12 -0.8.0 - - - -com.typesafe.akka:akka-actor_2.12:jar:compile->org.scala-lang.modules:scala-java8-compat_2.12:jar:compile - - - - - -org.reactivestreams:reactive-streams:jar:compile - -org.reactivestreams -reactive-streams -1.0.0 - - - -com.typesafe.akka:akka-stream_2.12:jar:compile->org.reactivestreams:reactive-streams:jar:compile - - - - - -com.typesafe:ssl-config-core_2.12:jar:compile - -com.typesafe -ssl-config-core_2.12 -0.2.1 - - - -com.typesafe.akka:akka-stream_2.12:jar:compile->com.typesafe:ssl-config-core_2.12:jar:compile - - - - - -org.apache.flink:flink-streaming-java_2.12:jar:compile->org.apache.flink:flink-runtime_2.12:jar:compile - - - - - -org.apache.flink:flink-shaded-guava:jar:compile - -org.apache.flink -flink-shaded-guava -18.0-5.0 - - - -org.apache.flink:flink-streaming-java_2.12:jar:compile->org.apache.flink:flink-shaded-guava:jar:compile - - - - - -org.apache.flink:flink-clients_2.12:jar:compile->commons-cli:commons-cli:jar:compile - - - - - -org.apache.flink:flink-optimizer_2.12:jar:compile - -org.apache.flink -flink-optimizer_2.12 -1.7.2 - - - -org.apache.flink:flink-clients_2.12:jar:compile->org.apache.flink:flink-optimizer_2.12:jar:compile - - - - - -org.apache.flink:flink-connector-kafka-0.9_2.12:jar:compile - -org.apache.flink -flink-connector-kafka-0.9_2.12 -1.7.2 - - - -org.apache.flink:flink-connector-kafka-base_2.12:jar:compile - -org.apache.flink -flink-connector-kafka-base_2.12 -1.7.2 - - - -org.apache.flink:flink-connector-kafka-0.9_2.12:jar:compile->org.apache.flink:flink-connector-kafka-base_2.12:jar:compile - - - - - -org.apache.flink:flink-connector-kafka-0.10_2.12:jar:compile->org.apache.flink:flink-connector-kafka-0.9_2.12:jar:compile - - - - - -uk.gov.gchq.gaffer:time-library:jar:compile - -uk.gov.gchq.gaffer -time-library -2.0.0 - - - -uk.gov.gchq.gaffer:time-library:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile - - - - - -uk.gov.gchq.gaffer:time-library:jar:compile->uk.gov.gchq.gaffer:bitmap-library:jar:compile - - - - - -uk.gov.gchq.gaffer:common-rest:jar:compile - -uk.gov.gchq.gaffer -common-rest -2.0.0 - - - -uk.gov.gchq.gaffer:common-rest:jar:compile->uk.gov.gchq.gaffer:map-store:jar:compile - - - - - -javax.annotation:javax.annotation-api:jar:compile - -javax.annotation -javax.annotation-api -1.3.2 - - - -uk.gov.gchq.gaffer:common-rest:jar:compile->javax.annotation:javax.annotation-api:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:war:compile - -uk.gov.gchq.gaffer -core-rest -2.0.0 -.war - - - -uk.gov.gchq.gaffer:core-rest:war:compile->com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:war:compile->com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.core:jersey-server:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.containers:jersey-container-servlet:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.inject:jersey-hk2:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:war:compile->uk.gov.gchq.gaffer:common-rest:jar:compile - - - - - -javax.servlet:javax.servlet-api:jar:provided - -javax.servlet -javax.servlet-api -3.1.0 -(provided) - - - -uk.gov.gchq.gaffer:core-rest:war:compile->javax.servlet:javax.servlet-api:jar:provided - - - - - -io.swagger:swagger-jaxrs:jar:compile - -io.swagger -swagger-jaxrs -1.6.6 - - - -uk.gov.gchq.gaffer:core-rest:war:compile->io.swagger:swagger-jaxrs:jar:compile - - - - - -io.swagger:swagger-annotations:jar:compile - -io.swagger -swagger-annotations -1.6.6 - - - -uk.gov.gchq.gaffer:core-rest:war:compile->io.swagger:swagger-annotations:jar:compile - - - - - -org.glassfish.jersey.media:jersey-media-multipart:jar:compile - -org.glassfish.jersey.media -jersey-media-multipart -2.36 - - - -uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.media:jersey-media-multipart:jar:compile - - - - - -org.glassfish.jersey.media:jersey-media-json-jackson:jar:compile - -org.glassfish.jersey.media -jersey-media-json-jackson -2.36 - - - -uk.gov.gchq.gaffer:core-rest:war:compile->org.glassfish.jersey.media:jersey-media-json-jackson:jar:compile - - - - - -com.fasterxml.jackson.datatype:jackson-datatype-json-org:jar:compile - -com.fasterxml.jackson.datatype -jackson-datatype-json-org -2.13.5 - - - -uk.gov.gchq.gaffer:core-rest:war:compile->com.fasterxml.jackson.datatype:jackson-datatype-json-org:jar:compile - - - - - -com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:compile - -com.fasterxml.jackson.dataformat -jackson-dataformat-yaml -2.13.2 - - - -org.yaml:snakeyaml:jar:compile - -org.yaml -snakeyaml -1.30 - - - -com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:compile->org.yaml:snakeyaml:jar:compile - - - - - -io.swagger:swagger-core:jar:compile - -io.swagger -swagger-core -1.6.6 - - - -io.swagger:swagger-core:jar:compile->com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:compile - - - - - -io.swagger:swagger-models:jar:compile - -io.swagger -swagger-models -1.6.6 - - - -io.swagger:swagger-core:jar:compile->io.swagger:swagger-models:jar:compile - - - - - -javax.validation:validation-api:jar:compile - -javax.validation -validation-api -1.1.0.Final - - - -io.swagger:swagger-core:jar:compile->javax.validation:validation-api:jar:compile - - - - - -io.swagger:swagger-jaxrs:jar:compile->org.reflections:reflections:jar:compile - - - - - -io.swagger:swagger-jaxrs:jar:compile->io.swagger:swagger-core:jar:compile - - - - - -org.jvnet.mimepull:mimepull:jar:compile - -org.jvnet.mimepull -mimepull -1.9.13 - - - -org.glassfish.jersey.media:jersey-media-multipart:jar:compile->org.jvnet.mimepull:mimepull:jar:compile - - - - - -org.glassfish.jersey.ext:jersey-entity-filtering:jar:compile - -org.glassfish.jersey.ext -jersey-entity-filtering -2.36 - - - -org.glassfish.jersey.media:jersey-media-json-jackson:jar:compile->org.glassfish.jersey.ext:jersey-entity-filtering:jar:compile - - - - - -com.fasterxml.jackson.datatype:jackson-datatype-json-org:jar:compile->org.json:json:jar:compile - - - - - -uk.gov.gchq.gaffer:proxy-store:jar:compile - -uk.gov.gchq.gaffer -proxy-store -2.0.0 - - - -uk.gov.gchq.gaffer:proxy-store:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile - - - - - -uk.gov.gchq.gaffer:proxy-store:jar:compile->org.glassfish.jersey.core:jersey-client:jar:compile - - - - - -uk.gov.gchq.gaffer:federated-store:jar:compile - -uk.gov.gchq.gaffer -federated-store -2.0.0 - - - -uk.gov.gchq.gaffer:federated-store:jar:compile->uk.gov.gchq.gaffer:map-store:jar:compile - - - - - -uk.gov.gchq.gaffer:federated-store:jar:compile->uk.gov.gchq.gaffer:accumulo-store:jar:compile - - - - - -uk.gov.gchq.gaffer:federated-store:jar:compile->io.netty:netty:jar:compile - - - - - -uk.gov.gchq.gaffer:accumulo-rest:war:compile - -uk.gov.gchq.gaffer -accumulo-rest -2.0.0 -.war - - - -uk.gov.gchq.gaffer:accumulo-rest:war:compile->uk.gov.gchq.gaffer:accumulo-store:jar:compile - - - - - -uk.gov.gchq.gaffer:accumulo-rest:war:compile->io.netty:netty:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile - -uk.gov.gchq.gaffer -core-rest -2.0.0 -classes - - - -uk.gov.gchq.gaffer:accumulo-rest:war:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.core:jersey-server:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.containers:jersey-container-servlet:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.inject:jersey-hk2:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->uk.gov.gchq.gaffer:common-rest:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->io.swagger:swagger-jaxrs:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->io.swagger:swagger-annotations:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.media:jersey-media-multipart:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->org.glassfish.jersey.media:jersey-media-json-jackson:jar:compile - - - - - -uk.gov.gchq.gaffer:core-rest:jar:classes:compile->com.fasterxml.jackson.datatype:jackson-datatype-json-org:jar:compile - - - - - -uk.gov.gchq.gaffer:map-rest:war:compile - -uk.gov.gchq.gaffer -map-rest -2.0.0 -.war - - - -uk.gov.gchq.gaffer:map-rest:war:compile->uk.gov.gchq.gaffer:map-store:jar:compile - - - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile - -uk.gov.gchq.gaffer -spring-rest -2.0.0 - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:jcs-cache-service:jar:compile - - - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->org.glassfish.jersey.inject:jersey-hk2:jar:compile - - - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:time-library:jar:compile - - - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:common-rest:jar:compile - - - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:proxy-store:jar:compile - - - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->uk.gov.gchq.gaffer:federated-store:jar:compile - - - - - -org.springframework.boot:spring-boot-starter-web:jar:compile - -org.springframework.boot -spring-boot-starter-web -2.5.12 - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->org.springframework.boot:spring-boot-starter-web:jar:compile - - - - - -org.springdoc:springdoc-openapi-ui:jar:compile - -org.springdoc -springdoc-openapi-ui -1.6.9 - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->org.springdoc:springdoc-openapi-ui:jar:compile - - - - - -org.springframework:spring-core:jar:compile - -org.springframework -spring-core -5.3.18 - - - -uk.gov.gchq.gaffer:spring-rest:jar:compile->org.springframework:spring-core:jar:compile - - - - - -org.springframework.boot:spring-boot-starter:jar:compile - -org.springframework.boot -spring-boot-starter -2.5.12 - - - -org.springframework.boot:spring-boot-starter:jar:compile->jakarta.annotation:jakarta.annotation-api:jar:compile - - - - - -org.springframework.boot:spring-boot-starter:jar:compile->org.yaml:snakeyaml:jar:compile - - - - - -org.springframework.boot:spring-boot:jar:compile - -org.springframework.boot -spring-boot -2.5.12 - - - -org.springframework.boot:spring-boot-starter:jar:compile->org.springframework.boot:spring-boot:jar:compile - - - - - -org.springframework.boot:spring-boot-autoconfigure:jar:compile - -org.springframework.boot -spring-boot-autoconfigure -2.5.12 - - - -org.springframework.boot:spring-boot-starter:jar:compile->org.springframework.boot:spring-boot-autoconfigure:jar:compile - - - - - -org.springframework.boot:spring-boot-starter-logging:jar:compile - -org.springframework.boot -spring-boot-starter-logging -2.5.12 - - - -org.springframework.boot:spring-boot-starter:jar:compile->org.springframework.boot:spring-boot-starter-logging:jar:compile - - - - - -org.springframework.boot:spring-boot-starter-logging:jar:compile->org.slf4j:jul-to-slf4j:jar:compile - - - - - -org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework.boot:spring-boot-starter:jar:compile - - - - - -org.springframework.boot:spring-boot-starter-json:jar:compile - -org.springframework.boot -spring-boot-starter-json -2.5.12 - - - -org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework.boot:spring-boot-starter-json:jar:compile - - - - - -org.springframework.boot:spring-boot-starter-tomcat:jar:compile - -org.springframework.boot -spring-boot-starter-tomcat -2.5.12 - - - -org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework.boot:spring-boot-starter-tomcat:jar:compile - - - - - -org.springframework:spring-web:jar:compile - -org.springframework -spring-web -5.3.18 - - - -org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework:spring-web:jar:compile - - - - - -org.springframework:spring-webmvc:jar:compile - -org.springframework -spring-webmvc -5.3.18 - - - -org.springframework.boot:spring-boot-starter-web:jar:compile->org.springframework:spring-webmvc:jar:compile - - - - - -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:compile - -com.fasterxml.jackson.datatype -jackson-datatype-jdk8 -2.12.6 - - - -org.springframework.boot:spring-boot-starter-json:jar:compile->com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:compile - - - - - -com.fasterxml.jackson.module:jackson-module-parameter-names:jar:compile - -com.fasterxml.jackson.module -jackson-module-parameter-names -2.12.6 - - - -org.springframework.boot:spring-boot-starter-json:jar:compile->com.fasterxml.jackson.module:jackson-module-parameter-names:jar:compile - - - - - -org.apache.tomcat.embed:tomcat-embed-core:jar:compile - -org.apache.tomcat.embed -tomcat-embed-core -9.0.60 - - - -org.springframework.boot:spring-boot-starter-tomcat:jar:compile->org.apache.tomcat.embed:tomcat-embed-core:jar:compile - - - - - -org.apache.tomcat.embed:tomcat-embed-el:jar:compile - -org.apache.tomcat.embed -tomcat-embed-el -9.0.60 - - - -org.springframework.boot:spring-boot-starter-tomcat:jar:compile->org.apache.tomcat.embed:tomcat-embed-el:jar:compile - - - - - -org.apache.tomcat.embed:tomcat-embed-websocket:jar:compile - -org.apache.tomcat.embed -tomcat-embed-websocket -9.0.60 - - - -org.springframework.boot:spring-boot-starter-tomcat:jar:compile->org.apache.tomcat.embed:tomcat-embed-websocket:jar:compile - - - - - -org.springframework:spring-beans:jar:compile - -org.springframework -spring-beans -5.3.18 - - - -org.springframework:spring-web:jar:compile->org.springframework:spring-beans:jar:compile - - - - - -org.springframework:spring-aop:jar:compile - -org.springframework -spring-aop -5.3.18 - - - -org.springframework:spring-webmvc:jar:compile->org.springframework:spring-aop:jar:compile - - - - - -org.springframework:spring-context:jar:compile - -org.springframework -spring-context -5.3.18 - - - -org.springframework:spring-webmvc:jar:compile->org.springframework:spring-context:jar:compile - - - - - -org.springframework:spring-expression:jar:compile - -org.springframework -spring-expression -5.3.18 - - - -org.springframework:spring-webmvc:jar:compile->org.springframework:spring-expression:jar:compile - - - - - -io.swagger.core.v3:swagger-core:jar:compile - -io.swagger.core.v3 -swagger-core -2.2.0 - - - -io.swagger.core.v3:swagger-core:jar:compile->jakarta.validation:jakarta.validation-api:jar:compile - - - - - -io.swagger.core.v3:swagger-core:jar:compile->com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:compile - - - - - -io.swagger.core.v3:swagger-annotations:jar:compile - -io.swagger.core.v3 -swagger-annotations -2.2.0 - - - -io.swagger.core.v3:swagger-core:jar:compile->io.swagger.core.v3:swagger-annotations:jar:compile - - - - - -io.swagger.core.v3:swagger-models:jar:compile - -io.swagger.core.v3 -swagger-models -2.2.0 - - - -io.swagger.core.v3:swagger-core:jar:compile->io.swagger.core.v3:swagger-models:jar:compile - - - - - -org.springdoc:springdoc-openapi-common:jar:compile - -org.springdoc -springdoc-openapi-common -1.6.9 - - - -org.springdoc:springdoc-openapi-common:jar:compile->io.swagger.core.v3:swagger-core:jar:compile - - - - - -org.springdoc:springdoc-openapi-webmvc-core:jar:compile - -org.springdoc -springdoc-openapi-webmvc-core -1.6.9 - - - -org.springdoc:springdoc-openapi-webmvc-core:jar:compile->org.springdoc:springdoc-openapi-common:jar:compile - - - - - -org.springdoc:springdoc-openapi-ui:jar:compile->org.springdoc:springdoc-openapi-webmvc-core:jar:compile - - - - - -org.webjars:swagger-ui:jar:compile - -org.webjars -swagger-ui -4.11.1 - - - -org.springdoc:springdoc-openapi-ui:jar:compile->org.webjars:swagger-ui:jar:compile - - - - - -org.webjars:webjars-locator-core:jar:compile - -org.webjars -webjars-locator-core -0.46 - - - -org.springdoc:springdoc-openapi-ui:jar:compile->org.webjars:webjars-locator-core:jar:compile - - - - - -io.github.classgraph:classgraph:jar:compile - -io.github.classgraph -classgraph -4.8.147 - - - -org.springdoc:springdoc-openapi-ui:jar:compile->io.github.classgraph:classgraph:jar:compile - - - - - -org.springframework:spring-jcl:jar:compile - -org.springframework -spring-jcl -5.3.18 - - - -org.springframework:spring-core:jar:compile->org.springframework:spring-jcl:jar:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-model:jar:compile - -uk.gov.gchq.gaffer -road-traffic-model -2.0.0 - - - -uk.gov.gchq.gaffer:road-traffic-model:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-generators:jar:compile - -uk.gov.gchq.gaffer -road-traffic-generators -2.0.0 - - - -uk.gov.gchq.gaffer:road-traffic-generators:jar:compile->uk.gov.gchq.gaffer:graph:jar:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-generators:jar:compile->uk.gov.gchq.gaffer:road-traffic-model:jar:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:war:compile - -uk.gov.gchq.gaffer -road-traffic-rest -2.0.0 -.war - - - -uk.gov.gchq.gaffer:road-traffic-rest:war:compile->uk.gov.gchq.gaffer:jcs-cache-service:jar:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:war:compile->javax.servlet:javax.servlet-api:jar:provided - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:war:compile->uk.gov.gchq.gaffer:proxy-store:jar:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:war:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:war:compile->uk.gov.gchq.gaffer:road-traffic-generators:jar:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-demo:war:compile - -uk.gov.gchq.gaffer -road-traffic-demo -2.0.0 -.war - - - -uk.gov.gchq.gaffer:road-traffic-demo:war:compile->uk.gov.gchq.gaffer:road-traffic-rest:war:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile - -uk.gov.gchq.gaffer -road-traffic-rest -2.0.0 -classes - - - -uk.gov.gchq.gaffer:road-traffic-demo:war:compile->uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile->uk.gov.gchq.gaffer:proxy-store:jar:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile - - - - - -uk.gov.gchq.gaffer:road-traffic-rest:jar:classes:compile->uk.gov.gchq.gaffer:road-traffic-generators:jar:compile - - - - - -uk.gov.gchq.gaffer:basic-model:jar:compile - -uk.gov.gchq.gaffer -basic-model -2.0.0 - - - -uk.gov.gchq.gaffer:basic-model:jar:compile->uk.gov.gchq.gaffer:sketches-library:jar:compile - - - - - -uk.gov.gchq.gaffer:basic-rest:war:compile - -uk.gov.gchq.gaffer -basic-rest -2.0.0 -.war - - - -uk.gov.gchq.gaffer:basic-rest:war:compile->uk.gov.gchq.gaffer:jcs-cache-service:jar:compile - - - - - -uk.gov.gchq.gaffer:basic-rest:war:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile - - - - - -uk.gov.gchq.gaffer:federated-demo:war:compile - -uk.gov.gchq.gaffer -federated-demo -2.0.0 -.war - - - -uk.gov.gchq.gaffer:federated-demo:war:compile->uk.gov.gchq.gaffer:jcs-cache-service:jar:compile - - - - - -uk.gov.gchq.gaffer:federated-demo:war:compile->uk.gov.gchq.gaffer:federated-store:jar:compile - - - - - -uk.gov.gchq.gaffer:federated-demo:war:compile->uk.gov.gchq.gaffer:core-rest:jar:classes:compile - - - - - diff --git a/docs/dev/managing-dependencies/managing-dependencies.md b/docs/dev/managing-dependencies/managing-dependencies.md deleted file mode 100644 index 8c4f605d52..0000000000 --- a/docs/dev/managing-dependencies/managing-dependencies.md +++ /dev/null @@ -1,105 +0,0 @@ -# Managing dependencies in Gaffer - -Gaffer is a large project with a lot of dependency and transitive dependency management required. This page covers some ways to get information about and more easily visualise and understand the dependencies for given Maven modules (or even the whole project). - -For more information on how Maven handles dependencies, [see this Maven guide](https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html). - - -## Get a simple list of direct dependencies - -This outputs the direct (non-transitive) dependencies for all Maven modules in the project. For specific modules append `-pl :module-name` to the command. -``` -mvn dependency:list -DexcludeTransitive=true -Dsort=true -``` -If `-DexcludeTransitive=true` is not used it will also print transitive dependencies, but without showing where they come from. The tree option (explained below) is better for getting this kind of information. - -For more info, [see the Maven dependency plugin documentation for the 'list' goal](https://maven.apache.org/plugins/maven-dependency-plugin/list-mojo.html). - -### Output to File - -The user property `-DoutputFile=deps.txt` can be used to place output for each module into the root directory **of that module**. It is also possible to aggregate all results into a single file using `mvn dependency:list -B -DappendOutput=true -DoutputFile=$(pwd)/mvn-dep-list.txt`. - -However, any files created contain terminal escape characters for adding colour (even when using `-B` - this is likely a bug). Another option is to use `-B` to turn off colour, piping the output through `sort --unique` (which also aggregates the dependencies) and appending that to a file - instead of using the plugin's file output option. - - -## Get a tree showing Dependencies - -This produces a tree showing the **resolved** dependencies for all Maven modules in the project. For specific modules append `-pl :module-name` to the command (as before). -``` -mvn dependency:tree -``` - -To include **all** dependencies (resolved, transitive and duplicate) in the tree and to see information on versions which have been managed (via `dependencyManagement`), append `-Dverbose`. - -This verbose flag is very useful for finding out which dependencies are affected by version management and version conflicts. - -Here's a simplified tree output with the verbose output explained (expand the :material-plus-circle-outline: symbols): -```c -uk.gov.gchq.gaffer:core:pom:2.0.0 -+- org.slf4j:slf4j-api:jar:1.7.36:compile -+- org.slf4j:slf4j-reload4j:jar:1.7.36:compile -| +- (org.slf4j:slf4j-api:jar:1.7.36:compile - omitted for duplicate) // (1)! -| \- ch.qos.reload4j:reload4j:jar:1.2.18.3:compile (version managed from 1.2.19) // (2)! -+- org.junit.jupiter:junit-jupiter:jar:5.9.0:test -| +- org.junit.jupiter:junit-jupiter-api:jar:5.9.0:test -\- org.mockito:mockito-junit-jupiter:jar:4.6.1:test - +- org.mockito:mockito-core:jar:4.6.1:test - \- (org.junit.jupiter:junit-jupiter-api:jar:5.8.2:test - omitted for conflict with 5.9.0) // (3)! -``` - -1. This dependency was omitted because it was already resolved (found) earlier. If a dependency with the same version is seen more than once, subsequent instances are ignored as **duplicates**. -2. This dependency has had its version changed (managed) from the version defined in the parent transitive dependency's POM to the version for this dependency given in the `dependencyManagement` section of our POM. The "version managed from" in the brackets is the version which would have been used if it hadn't been overridden by specifying a managed version. -3. This dependency was omitted because it was already resolved (found) earlier. If a dependency with different versions is seen more than once, subsequent instances are ignored as **conflicts**. In this case, the conflicting version is a transitive dependency and the resolved dependency is a direct dependency (declared in the module POM) which takes priority. - -This verbose output is also useful for discovering which dependencies need to have exclusions added when using [Maven dependency exclusions](https://maven.apache.org/guides/introduction/introduction-to-optional-and-excludes-dependencies.html#dependency-exclusions). As an example, replacing Log4j with Reload4j can be done be excluding all Log4j dependencies and adding Reload4j as a project dependency (from a Java perspective this works because both use the same package name). This requires an exclusion section to be added for every dependency which includes Log4j as a transitive dependency. - -Without the verbose output, only a single resolved Log4j dependency is shown. In a project with 5 dependencies with Log4j as a sub-dependency, if only the resolved dependency were to be excluded, then another would become the resolved dependency. With this approach, the tree would need to be printed repeatedly until all these dependencies had been seen and excluded. With the verbose output these dependencies are all shown initially. - -For more info, [see the Maven dependency plugin documentation for the 'tree' goal](https://maven.apache.org/plugins/maven-dependency-plugin/tree-mojo.html). - -### Output to File - -Unlike `dependency:list`, the `tree` plugin goal does output to file correctly. This puts all tree output into a single file called `mvn-dep-tree.txt` in the current directory: -``` -mvn dependency:tree -DoutputFile=$(pwd)/mvn-dep-tree.txt -DappendOutput=true -``` - -## Plotting graphs of dependencies - -### Using the built-in Maven plugin - -In addition to text output, this can also produce `.dot` files which can be used to create dependency graph visualisations (in combination with [Graphviz](https://en.wikipedia.org/wiki/Graphviz)). The resulting files are simple and lack positioning information. They only work well if the number of dependencies is low (e.g. `-Dverbose=true` not used). - -This visualisation was produced using the plugin and [Graphviz `sfdp`](https://graphviz.org/docs/layouts/sfdp/). - -```bash -mvn dependency:tree -DoutputType=dot -DoutputFile=$(pwd)/core.dot -pl :core # Create the .dot file -sfdp -x -Goverlap=false -Gsplines=true -Tsvg core.dot -o core.svg # Create the SVG image -``` - -It has too much empty space to be very useful. - -![Image title](core.svg) - -### Using the Depgraph Maven plugin - -A much better tool for creating `.dot` files is the Depgraph Maven plugin. This creates graphs with optimised node positioning, better formatting and more options than the built-in plugin. - -The main options available are `showClassifiers`, `showDuplicates`, `showConflicts`, `showVersions`, `showGroupIds` & `showTypes`. All of these options are disabled by default, but have been enabled in the Gaffer POM. - -It's important to note that this plugin does not have an equivalent to the verbose mode of `dependency:tree`, and it does not have a way to shown that a dependency's version has been changed using dependency management. - -For more info on the plugin, see the [Depgraph GitHub](https://github.com/ferstl/depgraph-maven-plugin) and [plugin docs](https://ferstl.github.io/depgraph-maven-plugin/graph-mojo.html). - -An example is shown below. The `dot` tool is also [part of Graphviz](https://graphviz.org/docs/layouts/dot/). - -```bash -mvn depgraph:graph -pl :map-store -Dscope=compile -DoutputDirectory=$(pwd) -dot -Tsvg dependency-graph.dot -o filename.svg -``` - -The black dotted lines show duplicates and the red ones show conflicts which were ignored during dependency resolution. - -![Image title](map-store.svg) - -It's also possible to create an aggregated graph of all dependencies in a project using `depgraph:aggregate` (which doesn't support showing duplicates or conflicts). For Gaffer this creates [a very large image](gaffer-complete.svg). \ No newline at end of file diff --git a/docs/dev/managing-dependencies/map-store.svg b/docs/dev/managing-dependencies/map-store.svg deleted file mode 100644 index b5f433e2d5..0000000000 --- a/docs/dev/managing-dependencies/map-store.svg +++ /dev/null @@ -1,1104 +0,0 @@ - - - - - - -map-store - - - -uk.gov.gchq.gaffer:store:jar - -uk.gov.gchq.gaffer -store -2.0.0 - - - -uk.gov.gchq.gaffer:data:jar - -uk.gov.gchq.gaffer -data -2.0.0 - - - -uk.gov.gchq.gaffer:store:jar->uk.gov.gchq.gaffer:data:jar - - - - - -uk.gov.gchq.gaffer:operation:jar - -uk.gov.gchq.gaffer -operation -2.0.0 - - - -uk.gov.gchq.gaffer:store:jar->uk.gov.gchq.gaffer:operation:jar - - - - - -com.google.guava:guava:jar - -com.google.guava -guava -30.1.1-jre - - - -uk.gov.gchq.gaffer:store:jar->com.google.guava:guava:jar - - - - - -org.slf4j:slf4j-api:jar - -org.slf4j -slf4j-api -1.7.36 - - - -uk.gov.gchq.gaffer:store:jar->org.slf4j:slf4j-api:jar - - - - - -org.slf4j:slf4j-reload4j:jar - -org.slf4j -slf4j-reload4j -1.7.36 - - - -uk.gov.gchq.gaffer:store:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:data:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:data:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:data:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:exception:jar - -uk.gov.gchq.gaffer -exception -2.0.0 - - - -uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.gaffer:exception:jar - - - - - -uk.gov.gchq.gaffer:serialisation:jar - -uk.gov.gchq.gaffer -serialisation -2.0.0 - - - -uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.gaffer:serialisation:jar - - - - - -uk.gov.gchq.gaffer:access:jar - -uk.gov.gchq.gaffer -access -2.0.0 - - - -uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.gaffer:access:jar - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8 - -uk.gov.gchq.koryphe -core -2.5.2 -jdk8 - - - -uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.koryphe:core:jar:jdk8 - - - - - -uk.gov.gchq.gaffer:type:jar - -uk.gov.gchq.gaffer -type -2.0.0 - - - -uk.gov.gchq.gaffer:data:jar->uk.gov.gchq.gaffer:type:jar - - - - - -org.apache.commons:commons-collections4:jar - -org.apache.commons -commons-collections4 -4.4 - - - -uk.gov.gchq.gaffer:data:jar->org.apache.commons:commons-collections4:jar - - - - - -uk.gov.gchq.gaffer:operation:jar->uk.gov.gchq.gaffer:data:jar - - - - - -uk.gov.gchq.gaffer:operation:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:operation:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:operation:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:cache:jar - -uk.gov.gchq.gaffer -cache -2.0.0 - - - -uk.gov.gchq.gaffer:operation:jar->uk.gov.gchq.gaffer:cache:jar - - - - - -uk.gov.gchq.gaffer:operation:jar->uk.gov.gchq.gaffer:access:jar - - - - - -com.google.code.findbugs:jsr305:jar - -com.google.code.findbugs -jsr305 -3.0.2 - - - -com.google.guava:guava:jar->com.google.code.findbugs:jsr305:jar - - - - - -com.google.guava:failureaccess:jar - -com.google.guava -failureaccess -1.0.1 - - - -com.google.guava:guava:jar->com.google.guava:failureaccess:jar - - - - - -com.google.guava:listenablefuture:jar - -com.google.guava -listenablefuture -9999.0-empty-to-avoid-conflict-with-guava - - - -com.google.guava:guava:jar->com.google.guava:listenablefuture:jar - - - - - -org.checkerframework:checker-qual:jar - -org.checkerframework -checker-qual -3.8.0 - - - -com.google.guava:guava:jar->org.checkerframework:checker-qual:jar - - - - - -com.google.errorprone:error_prone_annotations:jar - -com.google.errorprone -error_prone_annotations -2.5.1 - - - -com.google.guava:guava:jar->com.google.errorprone:error_prone_annotations:jar - - - - - -com.google.j2objc:j2objc-annotations:jar - -com.google.j2objc -j2objc-annotations -1.3 - - - -com.google.guava:guava:jar->com.google.j2objc:j2objc-annotations:jar - - - - - -org.slf4j:slf4j-reload4j:jar->org.slf4j:slf4j-api:jar - - - - - -ch.qos.reload4j:reload4j:jar - -ch.qos.reload4j -reload4j -1.2.18.3 - - - -org.slf4j:slf4j-reload4j:jar->ch.qos.reload4j:reload4j:jar - - - - - -uk.gov.gchq.gaffer:map-store:jar - -uk.gov.gchq.gaffer -map-store -2.0.0 - - - -uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:store:jar - - - - - -uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:data:jar - - - - - -uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:operation:jar - - - - - -uk.gov.gchq.gaffer:map-store:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:map-store:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:map-store:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:common-util:jar - -uk.gov.gchq.gaffer -common-util -2.0.0 - - - -uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:common-util:jar - - - - - -uk.gov.gchq.gaffer:graph:jar - -uk.gov.gchq.gaffer -graph -2.0.0 - - - -uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:graph:jar - - - - - -uk.gov.gchq.gaffer:sketches-library:jar - -uk.gov.gchq.gaffer -sketches-library -2.0.0 - - - -uk.gov.gchq.gaffer:map-store:jar->uk.gov.gchq.gaffer:sketches-library:jar - - - - - -uk.gov.gchq.gaffer:cache:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:cache:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:cache:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:cache:jar->uk.gov.gchq.gaffer:exception:jar - - - - - -uk.gov.gchq.gaffer:cache:jar->uk.gov.gchq.gaffer:serialisation:jar - - - - - -uk.gov.gchq.gaffer:exception:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:exception:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:exception:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:exception:jar->uk.gov.gchq.gaffer:common-util:jar - - - - - -org.apache.commons:commons-lang3:jar - -org.apache.commons -commons-lang3 -3.12.0 - - - -uk.gov.gchq.gaffer:exception:jar->org.apache.commons:commons-lang3:jar - - - - - -com.fasterxml.jackson.core:jackson-databind:jar - -com.fasterxml.jackson.core -jackson-databind -2.13.5 - - - -uk.gov.gchq.gaffer:exception:jar->com.fasterxml.jackson.core:jackson-databind:jar - - - - - -com.fasterxml.jackson.core:jackson-core:jar - -com.fasterxml.jackson.core -jackson-core -2.13.5 - - - -uk.gov.gchq.gaffer:exception:jar->com.fasterxml.jackson.core:jackson-core:jar - - - - - -uk.gov.gchq.gaffer:serialisation:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:serialisation:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:serialisation:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:serialisation:jar->uk.gov.gchq.gaffer:exception:jar - - - - - -uk.gov.gchq.gaffer:serialisation:jar->uk.gov.gchq.gaffer:common-util:jar - - - - - -uk.gov.gchq.gaffer:serialisation:jar->com.fasterxml.jackson.core:jackson-databind:jar - - - - - -org.apache.avro:avro:jar - -org.apache.avro -avro -1.8.2 - - - -uk.gov.gchq.gaffer:serialisation:jar->org.apache.avro:avro:jar - - - - - -uk.gov.gchq.gaffer:serialisation:jar->com.fasterxml.jackson.core:jackson-core:jar - - - - - -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar - -com.fasterxml.jackson.datatype -jackson-datatype-jsr310 -2.13.5 - - - -uk.gov.gchq.gaffer:serialisation:jar->com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar - - - - - -uk.gov.gchq.gaffer:access:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:access:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:access:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:access:jar->uk.gov.gchq.gaffer:common-util:jar - - - - - -uk.gov.gchq.gaffer:common-util:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:common-util:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:common-util:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:common-util:jar->uk.gov.gchq.koryphe:core:jar:jdk8 - - - - - -uk.gov.gchq.gaffer:common-util:jar->org.apache.commons:commons-lang3:jar - - - - - -com.fasterxml.jackson.core:jackson-annotations:jar - -com.fasterxml.jackson.core -jackson-annotations -2.13.5 - - - -uk.gov.gchq.gaffer:common-util:jar->com.fasterxml.jackson.core:jackson-annotations:jar - - - - - -uk.gov.gchq.gaffer:common-util:jar->com.fasterxml.jackson.core:jackson-databind:jar - - - - - -org.reflections:reflections:jar - -org.reflections -reflections -0.9.12 - - - -uk.gov.gchq.gaffer:common-util:jar->org.reflections:reflections:jar - - - - - -com.github.spotbugs:spotbugs-annotations:jar - -com.github.spotbugs -spotbugs-annotations -4.7.3 - - - -com.github.spotbugs:spotbugs-annotations:jar->com.google.code.findbugs:jsr305:jar - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8->com.google.guava:guava:jar - - -29.0-jre - - - -uk.gov.gchq.koryphe:core:jar:jdk8->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8->com.github.spotbugs:spotbugs-annotations:jar - - - - - -io.github.lukehutch:fast-classpath-scanner:jar - -io.github.lukehutch -fast-classpath-scanner -2.10.0 - - - -uk.gov.gchq.koryphe:core:jar:jdk8->io.github.lukehutch:fast-classpath-scanner:jar - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8->org.apache.commons:commons-lang3:jar - - - - - -commons-io:commons-io:jar - -commons-io -commons-io -2.11.0 - - - -uk.gov.gchq.koryphe:core:jar:jdk8->commons-io:commons-io:jar - - - - - -org.apache.commons:commons-csv:jar - -org.apache.commons -commons-csv -1.9.0 - - - -uk.gov.gchq.koryphe:core:jar:jdk8->org.apache.commons:commons-csv:jar - - - - - -commons-codec:commons-codec:jar - -commons-codec -commons-codec -1.15 - - - -uk.gov.gchq.koryphe:core:jar:jdk8->commons-codec:commons-codec:jar - - - - - -uk.gov.gchq.koryphe:core:jar:jdk8->com.fasterxml.jackson.core:jackson-annotations:jar - - -2.13.4 - - - -uk.gov.gchq.koryphe:core:jar:jdk8->com.fasterxml.jackson.core:jackson-databind:jar - - - - - -org.json:json:jar - -org.json -json -20211205 - - - -uk.gov.gchq.koryphe:core:jar:jdk8->org.json:json:jar - - - - - -com.fasterxml.jackson.core:jackson-databind:jar->com.fasterxml.jackson.core:jackson-annotations:jar - - - - - -com.fasterxml.jackson.core:jackson-databind:jar->com.fasterxml.jackson.core:jackson-core:jar - - - - - -org.apache.avro:avro:jar->org.slf4j:slf4j-api:jar - - -1.7.7 - - - -org.codehaus.jackson:jackson-core-asl:jar - -org.codehaus.jackson -jackson-core-asl -1.9.13 - - - -org.apache.avro:avro:jar->org.codehaus.jackson:jackson-core-asl:jar - - - - - -org.codehaus.jackson:jackson-mapper-asl:jar - -org.codehaus.jackson -jackson-mapper-asl -1.9.13 - - - -org.apache.avro:avro:jar->org.codehaus.jackson:jackson-mapper-asl:jar - - - - - -com.thoughtworks.paranamer:paranamer:jar - -com.thoughtworks.paranamer -paranamer -2.8 - - - -org.apache.avro:avro:jar->com.thoughtworks.paranamer:paranamer:jar - - - - - -org.xerial.snappy:snappy-java:jar - -org.xerial.snappy -snappy-java -1.1.1.3 - - - -org.apache.avro:avro:jar->org.xerial.snappy:snappy-java:jar - - - - - -org.apache.commons:commons-compress:jar - -org.apache.commons -commons-compress -1.8.1 - - - -org.apache.avro:avro:jar->org.apache.commons:commons-compress:jar - - - - - -org.tukaani:xz:jar - -org.tukaani -xz -1.5 - - - -org.apache.avro:avro:jar->org.tukaani:xz:jar - - - - - -org.codehaus.jackson:jackson-mapper-asl:jar->org.codehaus.jackson:jackson-core-asl:jar - - - - - -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar->com.fasterxml.jackson.core:jackson-annotations:jar - - - - - -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar->com.fasterxml.jackson.core:jackson-databind:jar - - - - - -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar->com.fasterxml.jackson.core:jackson-core:jar - - - - - -uk.gov.gchq.gaffer:type:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:type:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:type:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:type:jar->uk.gov.gchq.gaffer:serialisation:jar - - - - - -uk.gov.gchq.gaffer:type:jar->uk.gov.gchq.koryphe:core:jar:jdk8 - - - - - -org.javassist:javassist:jar - -org.javassist -javassist -3.29.2-GA - - - -org.reflections:reflections:jar->org.javassist:javassist:jar - - - - - -uk.gov.gchq.gaffer:graph:jar->uk.gov.gchq.gaffer:store:jar - - - - - -uk.gov.gchq.gaffer:graph:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:graph:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:graph:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:sketches-library:jar->uk.gov.gchq.gaffer:data:jar - - - - - -uk.gov.gchq.gaffer:sketches-library:jar->com.google.guava:guava:jar - - - - - -uk.gov.gchq.gaffer:sketches-library:jar->org.slf4j:slf4j-api:jar - - - - - -uk.gov.gchq.gaffer:sketches-library:jar->org.slf4j:slf4j-reload4j:jar - - - - - -uk.gov.gchq.gaffer:sketches-library:jar->uk.gov.gchq.gaffer:serialisation:jar - - - - - -uk.gov.gchq.gaffer:sketches-library:jar->uk.gov.gchq.koryphe:core:jar:jdk8 - - - - - -com.clearspring.analytics:stream:jar - -com.clearspring.analytics -stream -2.7.0 - - - -uk.gov.gchq.gaffer:sketches-library:jar->com.clearspring.analytics:stream:jar - - - - - -com.yahoo.datasketches:sketches-core:jar - -com.yahoo.datasketches -sketches-core -0.12.0 - - - -uk.gov.gchq.gaffer:sketches-library:jar->com.yahoo.datasketches:sketches-core:jar - - - - - -it.unimi.dsi:fastutil:jar - -it.unimi.dsi -fastutil -6.5.7 - - - -com.clearspring.analytics:stream:jar->it.unimi.dsi:fastutil:jar - - - - - -com.yahoo.datasketches:memory:jar - -com.yahoo.datasketches -memory -0.12.0 - - - -com.yahoo.datasketches:memory:jar->org.slf4j:slf4j-api:jar - - -1.7.25 - - - -com.yahoo.datasketches:sketches-core:jar->com.yahoo.datasketches:memory:jar - - - - - diff --git a/docs/dev/remote-coding-environments.md b/docs/dev/remote-coding-environments.md deleted file mode 100644 index c1786964e2..0000000000 --- a/docs/dev/remote-coding-environments.md +++ /dev/null @@ -1,32 +0,0 @@ -# Remote Coding Environments For Gaffer - -Gaffer is now configured for remote coding environemnts such as GitHub -Codespaces and Gitpod. This addition allows for an easier and faster way to -contribute to Gaffer with no manual setup or need to download dependencies to a -local machine. - -## GitHub Codespaces -To use GitHub Codespaces simply open the Gaffer repository, click the "Code" -button drop down, and then the option labeled "Create codespace on develop". - -This will launch a Codespaces environment with all the configuration needed to -contribute to Gaffer. See the [GitHub documentation for more information on -Codespaces] (https://github.com/features/codespaces). - -## Gitpod -To use Gitpod you can simply prefix any GitHub URL with **`gitpod.io/#`** and -follow the steps from there. You can also install the -[extension](https://www.gitpod.io/docs/configure/user-settings/browser-extension) -on your web browser. This adds a button to GitHub that does the prefixing for -you. - -Our custom GitPod configuration removes your Git commit email so you -will need to [re-configure your Git commit -email](https://www.gitpod.io/docs/configure/authentication/github). You can also -[configure your Git commit email to be a private GitHub email or a custom email -too.](https://www.gitpod.io/docs/configure/authentication#how-to-use-a-private-github-email-or-custom-email-for-git-commits). -Once done your environment will be all set to contribute to the Gaffer -repository. - -See the [Gitpod documentation for more -information.](https://www.gitpod.io/docs/introduction). \ No newline at end of file diff --git a/docs/dev/rest-api-sketches.md b/docs/dev/rest-api-sketches.md deleted file mode 100644 index 5b6ba6055f..0000000000 --- a/docs/dev/rest-api-sketches.md +++ /dev/null @@ -1,236 +0,0 @@ -# Using Sketches with the REST API - -This page explains some nuances and special steps required when using classes from the Sketches library with the REST API. If you just want to know how to use the sketches libraries to -use cardinality, see the [cardinality](../getting-started/guide/cardinality.md) docs page. - -## Sketches Library - -To learn more about the Sketches library see [advanced properties](../../reference/properties-guide/advanced) reference page. -The sketches library is included by default with the Map and Accumulo stores. This is because the `sketches-library` is a dependency in each of -the respective store modules' poms. As well as this, the serialisation is handled by the fact the -[SketchesJsonModules](https://github.com/gchq/Gaffer/blob/gaffer2-2.0.0/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/serialisation/json/SketchesJsonModules.java) -is returned by the `getJsonSerialiserModules` method in both the -[Map](https://github.com/gchq/Gaffer/blob/gaffer2-2.0.0/store-implementation/map-store/src/main/java/uk/gov/gchq/gaffer/mapstore/MapStoreProperties.java#L127) -and [Accumulo](https://github.com/gchq/Gaffer/blob/gaffer2-2.0.0/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloProperties.java#L468) -property classes. The modules are then loaded by the [JSONSerialiser](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/jsonserialisation/JSONSerialiser.html) -and used during the deserialisation of the REST JSON queries. - -## HyperLogLog sketches - -Gaffer currently supports the [Datasketches HllSketch](https://github.com/apache/datasketches-java/blob/master/src/main/java/org/apache/datasketches/hll/HllSketch.java) and [Clearspring HyperLogLogPlus](https://github.com/addthis/stream-lib/blob/master/src/main/java/com/clearspring/analytics/stream/cardinality/HyperLogLogPlus.java) algorithms. The Clearspring HyperLogLogPlus has been **deprecated in Gaffer** and we recommend the Datasketches HllSketch to users for the reasons described in the [advanced properties guide](../reference/properties-guide/advanced.md#hyperloglogplus). - -The `HllSketch` and `HyperLogLogPlus` sketches can be used to store an approximation of -cardinality of an element. The JSON of the query is converted to Java -objects during deserialisation using the `JSONSerialiser`. During the -deserialisation, the sketch's JSON representation is converted to a Java -object using the `ObjectMapper` module which uses the relevant deserialiser ( -[HyperLogLogPlusJsonDeserialiser](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/clearspring/cardinality/serialisation/json/HyperLogLogPlusJsonDeserialiser.java) or [HllSketchJsonDeserialiser](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/datasketches/cardinality/serialisation/json/HllSketchJsonDeserialiser.java)). - -## Creating cardinality values over JSON - -When adding or updating a cardinality object over the rest api, you specify the vertex values to add to the sketch. -This is done by either using the `offers` field with `HyperLogLogPlus`, or the `values` field with `HllSketch`. -The HyperLogLog object is then instantiated and updated with -the values. The object can then be serialised and stored in the datastore. -The vertex object is serialised using the `toString` representation of the object. -???+ note - As the algorithms use the `toString` method, any user defined type - introduced must override the `toString` method returning meaningful string - value representing the object rather than the default class instance - identifier. User defined types can be introduced by either adding further - [types](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/types/package-summary.html) - to Gaffer or by adding a jar with the extra type(s) to the Gaffer - classpath on startup. - -Depending on whether you are using `HyperLogLogPlus` or `HllSketch`, either the -[`HyperLogLogPlusWithOffers`](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/clearspring/cardinality/serialisation/json/HyperLogLogPlusWithOffers.java) or the -[`HllSketchWithValues`](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/datasketches/cardinality/serialisation/json/HllSketchWithValues.java) -respectively is responsible for the JSON deserialisation. -The helper classes wrap the underlying sketch and includes the following annotation on -the `offers`/`values` field: - -```java -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "class") -``` - -This signals to the Jackson `ObjectMapper` that it needs to look for the -`class` field in each object and translate to the correct object type. - -### Primitive data types over JSON -Primitive types are converted to the correct format by Jackson -`ObjectMapper` automatically. Here are some examples of the values: - -=== "String" - `"values": ["valueA", "value2",...]` - -=== "Long" - `"values": [1, 2,...]` - -=== "Double" - `"values": [1.1, 2.2,...]` - -### Non-primitive data types over JSON -In order to convert non-primitive vertex values (like `TypeSubTypeValue`) to Java objects, the JSON values need to contain the special field **class** -containing the class name of the object. The `deserialiser` uses this `class` -field when deserialising using the [JSONSerialiser](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/jsonserialisation/JSONSerialiser.html) -`deserialise` method. - -Here are the Gaffer user defined types: - -=== "FreqMap" - ```json - "values": [ - { - "class": "uk.gov.gchq.gaffer.types.FreqMap", - "test": 1 - }, - ... - ] - ``` - -=== "CustomMap" - ```json - "values": [ - { - "class": "uk.gov.gchq.gaffer.types.CustomMap", - "keySerialiser": { - "class": "uk.gov.gchq.gaffer.serialisation.implementation.BooleanSerialiser" - }, - "valueSerialiser": { - "class": "uk.gov.gchq.gaffer.serialisation.implementation.BooleanSerialiser" - }, - "jsonStorage": [] - }, - ... - ] - ``` - -=== "TypeValue" - ```json - "values": [ - { - "class" : "uk.gov.gchq.gaffer.types.TypeValue", - "type" : "type", - "value" : "value" - }, - ... - ] - ``` - -=== "TypeSubTypeValue" - ```json - "values": [ - { - "class" : "uk.gov.gchq.gaffer.types.TypeSubTypeValue", - "type" : "type", - "subType" : "subType", - "value" : "value" - }, - ... - ] - ``` - -???+ note - The subclass fields must also have the `class` field set (for - example, the `keySerialiser` in the `CustomMap` type) if not a standard Java Object - so that the Jackson `ObjectMapper` knows how to convert the correct values - to Java objects. - -#### Composing using Java - -If you are composing the `HllSketch` with values using Java, before -converting to JSON and sending via REST, you need ensure that the `values` -objects are translated to JSON with the correct `class` field added. -To make sure of this, you could add the `sketches-library` JAR and use the -[HllSketchWithValues](https://github.com/gchq/Gaffer/blob/develop/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/datasketches/cardinality/serialisation/json/HllSketchWithValues.java) -object to construct your query (or the equivalent for HyperLogLogPlus). -This way you know that all the objects have the -correct field added. You can then convert the `HllSketchWithValues` to -JSON using the -[JSONSerialiser](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/jsonserialisation/JSONSerialiser.html) -`serialisation` method: -```java -final HllSketchWithValues hllSketchWithValues = JSONSerialiser.deserialise(treeNode.toString(), HllSketchWithValues.class); -``` -If you want to create your own class instead, rather than using -`HllSketchWithValues`, ensure -that the `values` list has the correct annotation so the `class` is added on -conversion using by the Jackson `ObjectMapper`: - -```java -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "class") -private List values = new ArrayList<>(); -``` - -#### Composing using Python - -An example of using Python to add a `HyperLogLogPlus` property with a `TypeSubTypeValue` offer: -```python -g.AddElements( - input=[ - g.Entity( - vertex="A", - group="cardinality", - properties={ - "hllp": g.hyper_log_log_plus([ - { - "class" : "uk.gov.gchq.gaffer.types.TypeSubTypeValue", - "type" : "t", - "subType" : "st", - "value" : "B" - } - ]) - } - ) - ] - ) -``` - -An example of using Python to add a `HllSketch` property with a `TypeSubTypeValue` offer: -```python -g.AddElements( - input=[ - g.Entity( - vertex="A", - group="cardinality", - properties={ - "hllSketch": g.hll_sketch([ - { - "class" : "uk.gov.gchq.gaffer.types.TypeSubTypeValue", - "type" : "t", - "subType" : "st", - "value" : "B" - } - ]) - } - ) - ] - ) -``` - -### Adding user defined vertex types into offers - -To add a user defined type you must ensure that: - -- the type is on the Gaffer classpath -- the type must override the `toString` method -- the type contains the correct annotations if you are converting from Java to - JSON before sending via REST - -The following user defined type example features the annotation required as -well as the `@Override` of the `toString` method: - -```java -@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, property = "class") -public class ExampleType implements Comparable, Serializable { - - private String value...; - - // getters and setters - - @Override - public String toString() { - return ...; - } -} -``` diff --git a/docs/dev/ways-of-working.md b/docs/dev/ways-of-working.md deleted file mode 100644 index 86a03cce7f..0000000000 --- a/docs/dev/ways-of-working.md +++ /dev/null @@ -1,91 +0,0 @@ -# Ways of Working - -## Git branching model -We have adopted the GitFlow Branching Model in order to support both Gaffer v1 and v2: ![GitFlow Branching Model](https://nvie.com/img/git-model@2x.png) - -## Issues -Where possible a pull request should correlate to a single GitHub issue. An issue should relate to a single functional or non-functional change - changes to alter/improve other pieces of functionality should be addressed in a separate issue in order to keep reviews atomic. -The reasoning behind code changes should be documented in the GitHub issue. -All resolved issues should be included in the next GitHub milestone, this enables releases to be linked to the included issues. -If a code change requires users of Gaffer to make changes in order for them to adopt it then the issue should be labelled 'migration-required' and a comment should be added similar to: - -``` -### Migration Steps - -[Description of what needs to be done to adopt the code change with examples] -``` - -### Workflow -* Assign yourself to the issue -* Create a new branch off **develop** using pattern: `gh-[issue number]-[issue-title]` -* Commit your changes using descriptive commit titles -* Check and push your changes -* Create a pull request (PR) to merge your branch into **develop**, prefixing the PR title with "Gh-[issue number]: " -* If you named the branch and PR correctly, the PR should have "Resolve #[issue-number]" automatically added to the description after it is made. If it doesn't, then please add the issue it will resolve as a "Linked issue" -* If there is a significant change, please follow the same process to document the change in [gaffer-doc](https://github.com/gchq/gaffer-doc) -* The pull request will be reviewed and following any changes and approval your branch will be squashed and merged into **develop** -* Delete the branch -* The issue will be closed automatically - -## Pull Requests -Pull requests will undergo a review by a Gaffer committer to check the code changes are compliant with our coding style. This is a community so please be respectful of other members - offer encouragement, support and suggestions. - -As described in our git branching model - please raise pull requests to merge your changes in our **develop** branch. - -When pull requests are accepted, the reviewer should squash and merge them. This is because it keeps the **develop** branch clean and populated with only merge commits, rather than intermediate ones. As well as this, it makes everyone's job reviewing pull requests easier as any insecure and unreviewed intermediate commits are not included into the **develop** branch. - -Please agree to the [GCHQ OSS Contributor License Agreement](https://github.com/GovernmentCommunicationsHeadquarters/Gaffer/wiki/GCHQ-OSS-Contributor-License-Agreement-V1.0) before submitting a pull request. Signing the CLA is enforced by the cla-assistant. - -## Documentation -As mentioned before, any significant changes in a PR should be accompanied with an addition to Gaffer's documentation: [gaffer-doc](https://github.com/gchq/gaffer-doc). -Smaller changes should be self documented in the tests. With this approach, any large feature or change has user friendly documentation, whereas technical or implementation details are documented for developers by the tests. - -## Coding style -### Java -Please ensure your coding style is consistent with the rest of the Gaffer project and the [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html). Your changes should pass the checkstyle and spotless plugins that are part of the continuous integration pipeline and check for code formatting and licenses. Before you push your changes you can check the checkstyle plugin passes with `mvn checkstyle:check` and check the spotless plugin passes with `mvn spotless:check`. - -### Python -Please ensure your coding style is consistent with the rest of the Gaffer project and the [PEP 8 Style Guide](https://peps.python.org/pep-0008/). However, there are a few exceptions to the standards set by PEP8: -* Module level imports at the top of the file - this will not be enforced but is recommended where it does not cause issues with the code generated by Fishbowl. -* Max line length of 79 characters - the max line length that will be enforced in this project has been increased to 100 characters. - -Before you create a PR for your changes you can use [autopep8](https://github.com/hhatto/autopep8) to check and fix any styling issues. The following can be run which will take into account the rule exceptions mentioned above. -`autopep8 --exit-code -r -i -a -a --max-line-length 100 --ignore E402 .` - - -### Javadoc -Ensure your java code has sufficient javadocs explaining what the section of code does and the intended use of it. Javadocs should be used in addition to clean readable code. - -In particular: -* All public classes (not required for test classes unless an explanation of the testing is required) -* public methods (not required if the functionality is obvious from the method name) -* public constants (not required if the constant is obvious from the name) - -### Tests -* All new code should be unit tested. Where this is not possible the code should be invoked and the functionality should be tested in an integration test. In a small number of cases this will not be possible - instead steps to verify the code should be thoroughly documented. -* Tests should cover edge cases and exception cases as well as normal expected behavior. -* Keep each test decoupled and don't rely on tests running in a given order - don't save state between tests. -* For a given code change, aim to improve the code coverage. -* Unit test classes should test a single class and be named [testClass]Test. -* Integration test classes should be named [functionalityUnderTest]IT. -* Tests should be readable and self documenting. -* Each test should focus on testing one small piece of functionality invoked from a single method call. -* Tests should use JUnit 5 and assertJ. -* We suggest the following pattern: - - ```java - @Test - public void should[DoSomething|ReturnSomething] { - // Given - [Setup your test here] - - // When - [Invoke the test method] - - // Then - [assertThat the method did what was expected] - } - ``` - -## Gaffer 2 -During the Gaffer 2 development process there was a **v2-alpha** branch, which acted as the **develop** branch for changes staged for Gaffer 2. This branch is no longer in use and can be ignored. diff --git a/docs/gaffer2.0/accumulo-kerberos.md b/docs/gaffer2.0/accumulo-kerberos.md deleted file mode 100644 index a3e3cbb414..0000000000 --- a/docs/gaffer2.0/accumulo-kerberos.md +++ /dev/null @@ -1,93 +0,0 @@ -# Accumulo Kerberos Support - -This page contains information on Kerberos Authentication support for Gaffer's Accumulo Store. This functionality was introduced in version `2.0.0-alpha-0.3.1` of Gaffer. - -## Using the Accumulo Store with Kerberos - -### Prerequisites -To use Gaffer's Accumulo Store with Kerberos authentication: - -- The Accumulo cluster to connect with must be correctly configured to use Kerberos. -- A principal for the system/host Gaffer will be running on must be created in the Key Distribution Center (KDC) database. -- The Gaffer principal should use the standard [`primary/instance@realm`](https://web.mit.edu/kerberos/krb5-1.5/krb5-1.5.4/doc/krb5-user/What-is-a-Kerberos-Principal_003f.html) format. Using principals without an _instance_ qualification has not been tested. -- A keytab for the Gaffer principal must be created and transferred to the Gaffer host. -- The Gaffer principal must have been added as an Accumulo user with suitable permissions granted. -- Kerberos client utilities should be installed on the host and `krb5.conf` must be correctly configured. -- An Accumulo client configuration should be available on the host and contain the correct options to enable Kerberos. -- The Gaffer store.properties should state that Kerberos is to be used, specify the principal name and the keytab path. - -The sections below cover some of these points in more detail. - -### Accumulo user for Gaffer - -When Kerberos is used with Accumulo, [any client with a principal can connect without requiring an Accumulo user to have been created previously](https://accumulo.apache.org/docs/2.x/security/kerberos#kerberosauthenticator). This works by creating an Accumulo user automatically when a new client connects. These users are not granted any permissions. - -Users can still be created manually via the Accumulo shell, with Gaffer's full principal (with all components) given as the username. Permissions to create and read tables can then be granted to this user. If this isn't done, Accumulo will create the user automatically when Gaffer first connects. In this case Gaffer will fail to start as the required permissions will not have been granted - they can then be granted via the shell and Gaffer restarted. - -### Accumulo Client configuration -Depending on the version of Accumulo used, an [`accumulo-client.properties` (2.x)](https://accumulo.apache.org/docs/2.x/security/kerberos#configuration) or [`client.conf` (1.x)](https://accumulo.apache.org/1.10/accumulo_user_manual.html#_configuration_3) must be populated as described in the respective version of the Accumulo documentation. The only value which needs to be altered is the Kerberos server primary. This should reflect the primary part of the principals used by the Accumulo cluster. - -The location of this config file can be specified using the `ACCUMULO_CLIENT_CONF_PATH` environment variable. If this is not set, then [default paths will be checked](https://accumulo.apache.org/docs/2.x/apidocs/org/apache/accumulo/core/client/ClientConfiguration.html#loadDefault()). - -Other than this file, Accumulo libraries and configuration files do not need to be installed on the Gaffer host. - -### Gaffer `store.properties` configuration -In addition to the usual [Accumulo Store settings](../reference/stores-guide/accumulo.md#properties-file), these extra options must be specified for Kerberos: -``` -accumulo.kerberos.enable=true -accumulo.kerberos.principal=gaffer/host.domain@REALM.NAME -accumulo.kerberos.keytab=/gaffer/config/gaffer.keytab -``` -The `accumulo.username` and `accumulo.password` values do not need to be set and are ignored when `accumulo.kerberos.enable` is true. - -The `kinit` Kerberos command does not need to be used, although it might be useful for ensuring the client principal works correctly. All Kerberos ticket management, renewal and re-login is handled automatically. - -### Specifying a different `krb5.conf` -If the `krb5.conf` in the default system location is not suitable, or if it's stored in a non-standard location, then -custom a custom `krb5.conf` location can be specified when starting Gaffer by setting the system property value `java.security.krb5.conf`. The simplest way to do this is by using the option flag `-Djava.security.krb5.conf=/my/path/to/krb5.conf` when launching the Gaffer JAR. - -## Federation Considerations -Due to the way Kerberos is implemented in Accumulo, it is not possible for Gaffer to use multiple principals at the same time. For the `FederatedStore`, this prevents adding graphs which are on different Accumulo clusters, if those clusters require different principals. In practice this is unlikely to be a problem, as different Accumulo clusters would only need separate client principals if they were on separate Kerberos Realms or using different KDCs. - -This only impacts Accumulo clusters which require Kerberos. It doesn't impact on adding graphs which are stored in clusters using basic authentication and not Kerberos. Nor does it affect adding graphs from a Kerberos cluster and also adding graphs from a non Kerberos cluster in the same `FederatedStore`. - -If this limitation is a problem, it can be worked around by running additional Gaffer instances and connecting to them using a `ProxyStore` in the `FederatedStore`, rather than connecting directly using an `AccumuloStore`. - -## HDFS Considerations -When using the [`AddElementsFromHdfs`](https://gchq.github.io/gaffer-doc/v1docs/getting-started/operations/addelementsfromhdfs.html) operation Gaffer acts as a HDFS client. When Kerberos is used ([Hadoop Secure Mode](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html)), HDFS clients must have [native libraries](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/NativeLibraries.html) installed and configured correctly; else Hadoop will raise a Runtime Exception stating that "Secure IO is not possible without native code extensions". - -The HDFS client also requires the Hadoop configuration files `core-site.xml` and `hdfs-site.xml` to both be present and configured as below. The location of these files can be specified using the `HADOOP_CONF_DIR` environment variable. - -```xml - - - hadoop.security.authentication - kerberos - - - hadoop.security.authorization - true - -``` - -In particular, `hdfs-site.xml` requires the `yarn.resourcemanager.principal` property to be set to the HDFS client principal - should be the same one as in the Gaffer Store properties. If this is missing Hadoop will fail to connect and raise an IO Exception with "Can't get Master Kerberos principal for use as renewer". - -```xml - - - yarn.resourcemanager.principal - primary/instance@realm - -``` - -Note that the `core-site.xml` and `hdfs-site.xml` files are **only** required if `AddElementsFromHdfs` is going to be used. For Accumulo connections the Hadoop properties (from `core-site.xml`) used for enabling Kerberos are set automatically in Gaffer's connection code. - -## Spark Accumulo Library -The [Spark Accumulo Library](https://github.com/gchq/Gaffer/tree/master/library/spark) has not yet been updated to support Kerberos. This prevents [Spark Operations](https://gchq.github.io/gaffer-doc/v1docs/getting-started/spark-operations/contents.html) from being used with an `AccumuloStore` which has Kerberos authentication enabled. It is on the backlog for support to be added in future. - -## Troubleshooting -Kerberos is not easy to configure and familiarity with Kerberos concepts is recommended. There are [some useful links to introductory information in the Accumulo Kerberos docs](https://accumulo.apache.org/docs/2.x/security/kerberos#overview). - -Improperly configured DNS will cause problems with Kerberos. Ensure all hostnames used in Principals resolved correctly, include reverse lookup. Due to how the system's hostname is used by the Hadoop Kerberos libraries, a mismatch between the configured hostname and the hostname resolved by a reverse lookup can prevent authentication from working correctly. - -Various environment variables can be set for debugging Kerberos, [see the Hadoop docs for more information](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html#Troubleshooting). These variables are applicable to Accumulo ([see docs](https://accumulo.apache.org/docs/2.x/security/kerberos#debugging)) because its Kerberos implementation uses Hadoop libraries. The Gaffer logging level (set in `log4.xml`) should be increased to at least `INFO` when using these environment variables. diff --git a/docs/gaffer2.0/accumulo-migration.md b/docs/gaffer2.0/accumulo-migration.md deleted file mode 100644 index 15a406e68b..0000000000 --- a/docs/gaffer2.0/accumulo-migration.md +++ /dev/null @@ -1,27 +0,0 @@ -# Accumulo Migration - -This page contains information on changes to the Accumulo/Hadoop versions supported by Gaffer and how to continue using the previously supported versions. - -## Accumulo 2 & Hadoop 3 become default versions - -From the `2.0.0-alpha-0.3` release of Gaffer, the default version of Accumulo has been upgraded to [Accumulo 2.0.1](https://accumulo.apache.org/release/accumulo-2.0.1/). Hadoop has also been upgraded to the latest version (currently 3.3.3). This is because Hadoop 2.x is not compatible with Accumulo 2. - -## Retained support for Accumulo 1 & Hadoop 2 - -Support for certain versions of Accumulo 1 and Hadoop 2 (specifically 1.9.3 & 2.6.5) has been retained and can be enabled by using a Maven profile when building from source (see below). This facilitates testing with these versions and creates shaded JARs (e.g. spring-rest exec, accumulo-store iterators) with the appropriate versions of supporting libraries. As described in the source docs, other versions of Accumulo 1.x and Hadoop 2.x might also work. - -### Availability of Gaffer artifacts supporting Accumulo 1 - -The shaded JARs differ based on the versions of the bundled libraries and only the default version (Accumulo 2.0.1) is published to the Maven Central repository. The 'legacy' version must be built locally. - -### Building Gaffer with the 'legacy' profile - -To build Gaffer using Accumulo 1.9.3 and Hadoop 2.6.5, the 'legacy' Maven profile needs to be used. This is enabled by supplying `-Dlegacy=true` as an extra argument at the command line when running Maven. For example, `mvn clean install -Pcoverage -Dlegacy=true` will perform a full build/test of Gaffer with this profile enabled. Java 11 cannot be used with this profile because only Hadoop 3.3.0 and higher support it. - -With the 'legacy' Maven profile active, the filenames of all shaded JARs produced are appended with `-legacy`. This is to differentiate them from the default shaded JARs which contain different libraries and different library versions. A default Gaffer Accumulo REST API JAR will not work with an Accumulo 1 cluster, and the 'legacy' version will not work with Accumulo 2 because the bundled libraries are specific to the version of Accumulo. - -## Migrating from Accumulo 1 to 2 - -See [the Accumulo documentation](https://accumulo.apache.org/docs/2.x/administration/upgrading) for guidance on upgrading from Accumulo 1 to 2. Of particular significance is the [deprecation of the dynamic reloading classpath directory functionality](https://accumulo.apache.org/release/accumulo-2.0.0/#removed-default-dynamic-reloading-classpath-directory-libext) in Accumulo 2. This affects where and how the Gaffer iterators JAR can be installed. See the Accumulo store documentation for these installation details. - -Otherwise, no Accumulo specific Gaffer configuration needs to be changed and migrating from Accumulo 1 to 2 should be as simple as swapping the Gaffer dependency versions/JARs, although this has not been actively tested. diff --git a/docs/gaffer2.0/changelist.md b/docs/gaffer2.0/changelist.md deleted file mode 100644 index dbcec62b14..0000000000 --- a/docs/gaffer2.0/changelist.md +++ /dev/null @@ -1,130 +0,0 @@ -# Gaffer 2 Changelist - -Below is a summary of changes that have been made in Gaffer version 2. - -### Accumulo 2 Support -The Accumulo store now supports Accumulo 2 and Hadoop 3 by default, with support for Accumulo 1 and Hadoop 2 retained. See the [Accumulo Migration page](accumulo-migration.md) for more information about this change. - -### Federated Store Improvements -The Federated Operation was added to greatly improve flexibility of using a Federated Store. -!!! danger "Breaking change" - To migrate, please see the [Federated Store Changes page](federation-changes.md). - -### Cache Improvements and fixes -All "caches" within Gaffer received a lot of bug fixes which should make them significantly more stable and consistent over time. This should improve usability of FederatedStores, NamedOperations and NamedViews. -!!! danger "Breaking change" - The cache will need to be reloaded, as the new internal cache interface has changed. To do this, export all of the contents of your cache, upgrade, then re-add everything manually. - -### Removal of Deprecated code -All of Gaffer 1's deprecated code has been removed. -!!! danger "Breaking change" - To migrate, please see the [deprecations](deprecations.md) page. - -### Dependency Upgrades -Dependencies have been updated, where possible to the latest version, removing vulnerabilities. -!!! danger "Breaking change" - You will need to migrate your dependencies to be compatible with Gaffer 2's new dependency versions. Please see the [dependencies](dependencies.md) page for full details. - -### Federated and Proxy store fixes -A lot of bugs have been fixed that should facilitate FederatedStores with ProxyStores in them. -!!! danger "Breaking change" - The unique store trait `DYNAMIC_SCHEMA` has been removed from Gaffer. Simply removing it from custom FederatedStore implementations should be an adequate fix. - -### Removal of CloseableIterable -The `CloseableIterable` class has been removed so Operations like `GetAllElements` now return an `Iterable` instead, but the result still implements `Closeable`. -!!! danger "Breaking change" - Everywhere `CloseableIterable` was used in client code should be replaced with an `Iterable`: - ```java - final CloseableIterable results = graph.execute(new GetAllElements(), USER); - ``` - ```java - final Iterable results = graph.execute(new GetAllElements(), USER); - ``` - -### Removal of HBase and Parquet stores -The HBase and Parquet stores have been removed from Gaffer in version 2. We made posts for both the [HBase](https://github.com/gchq/Gaffer/issues/2367) and [Parquet](https://github.com/gchq/Gaffer/discussions/2557) stores to understand the levels of usage. It was then decided to remove both stores as this would make introducing various improvements easier in the long term. HBase and Parquet remain available in Gaffer version 1. In the future, they could be reimplemented for Gaffer 2, though we do not plan to currently. -!!! danger "Breaking change" - We would recommend instead using an Accumulo Store. If you would like these store implementations in Gaffer 2, or any other potential store for that matter, please [make an issue](https://github.com/gchq/Gaffer/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.md&title=) on GitHub. - -### Gaffer now builds with Java 8 and Java 11 -There is now a maven profile that will swap dependency versions so you can build Gaffer with Java 11. The code has also been updated to build with both Java versions. - -### Accumulo Kerberos Authentication Support -The Accumulo store now supports authenticating to Accumulo and HDFS using Kerberos, in addition to username/password. For more information, see the [Kerberos support page](accumulo-kerberos.md). - -### CSV Import and Export -Basic support for importing and exporting [CSVs](../getting-started/guide/csv.md) has been added. - -### All operations can now be used within NamedOperations -Previously, `GetElementsBetweenSets` could not be used within a NamedOperation as it used `inputB`. `GetElementsBetweenSets` and `inputB` have both been deprecated and instead you should use `GetElementsBetweenSetsPairs`. -??? example - Old operation now deprecated: - ```json - { - "class": "GetElementsBetweenSets", - "input": [ - { - "class": "EntitySeed", - "vertex": "firstInput" - } - ], - "inputB": [ - { - "class": "EntitySeed", - "vertex": "secondInput" - } - ] - } - ``` - New operation that will work within NamedOperations: - ```json - { - "class": "GetElementsBetweenSetsPairs", - "input": { - "class": "Pair", - "first": { - "ArrayList" : [ - { - "class": "EntitySeed", - "vertex": "firstInput" - } - ] - }, - "second": { - "ArrayList" : [ - { - "class": "EntitySeed", - "vertex": "secondInput" - } - ] - } - } - } - ``` - -### Ability to set OperationDeclarations during AddGraph -This will mean subgraphs added to FederatedStores can have additional operation handlers set when they are added. You can directly provide the OperationsDeclarations json to the store properties with `gaffer.store.operation.declarations.json`. -??? example - ``` json - { - "class": "AddGraph", - "graphId": "myGraph", - "schema": {}, // (1)! - "storeProperties": { - "gaffer.store.class": "MapStore", - "gaffer.store.operation.declarations.json": { - "operations": [ - { - "operation": "ImportFromLocalFile", // (2)! - "handler": { - "class": "ImportFromLocalFileHandler" - } - } - ] - } - } - } - ``` - - 1. Schema left empty for brevity - 2. This example operation enables file import. Read more in the [CSV](../getting-started/guide/csv.md) docs. diff --git a/docs/gaffer2.0/dependencies.md b/docs/gaffer2.0/dependencies.md deleted file mode 100644 index 99d7b66d60..0000000000 --- a/docs/gaffer2.0/dependencies.md +++ /dev/null @@ -1,35 +0,0 @@ -# Dependency Upgrades - -This page lists the dependencies that have been upgraded as part of Gaffer 2. - - - Assertj: 3.20.2 -> 3.23.1 - - Avro: 1.7.7 -> 1.8.2 - - Commons-codec: 1.6 -> 1.15 - - Commons-csv: 1.4 -> 1.10.0 - - Commons-io: 2.7 -> 2.11.0 - - Commons-jcs-core: 2.1 -> 2.2.1 - - Commons-lang: 3.3.2 -> 3.12.0 - - Commons-logging: 1.1.3 -> 1.2 - - Commons-math3: 3.4.1 -> 3.6.1 - - Commons-math: 2.1 -> 2.2 - - Curator: 2.6.0 -> 2.13.0 - - Flink: 1.4.1 -> 1.7.2 - - Graphframes: 0.4.0 -> 0.8.1 - - Guava: 13.0.1 -> 30.1.1 - - Hadoop: 2.6.5 -> 3.3.3 - - Hazelcast: 3.8 -> 5.3.0 - - Jackson: 2.6.5 -> 2.13.5 - - Javassist: 3.19.0-GA -> 3.28.0-GA - - Jersey: 2.25 -> 2.36 - - Jersey: 2.25 -> 2.36 - - Junit5: 5.6.0 -> 5.9.0 - - Kafka: 0.10.0.0 -> 0.10.2.2 - - Koryphe: 1.14.0 -> 2.1.0 - - Log4j: 1.2.17 -> Reload4j: 1.2.18.3 - - Mockito: 3.3.3 -> 4.6.1 - - Paranamer: 2.6 -> 2.8 - - Reflections: 0.9.10 -> 0.9.12 - - Slf4j: 1.7.25 -> 1.7.36 - - Spark 2.3.2 -> 3.0.3 - - Spring API Swagger: 2.6.0 -> 3.0.0 - - Spring Boot: 1.3.2 -> 2.5.12 diff --git a/docs/gaffer2.0/deprecations.md b/docs/gaffer2.0/deprecations.md deleted file mode 100644 index 0319e5506f..0000000000 --- a/docs/gaffer2.0/deprecations.md +++ /dev/null @@ -1,563 +0,0 @@ -# Deprecations - -This page describes deprecated code which has been removed in Gaffer 2 and how to migrate to better equivalents. Each heading for a section below refers to a classname from `uk.gov.gchq.gaffer` where there have been changes or where that class has been removed entirely. The section headings link to the code on GitHub for that class (as of the Gaffer 1.21.1 release). - -Deprecations impacting the serialisers used in schemas are listed first, followed by [changes to Seed Matching and changes to Traits](#changes-to-seed-matching-and-traits). Other deprecations are then [listed in alphabetical order](#all-other-deprecations). - -## Serialisers - -### Migrating away from deprecated Serialisers -Various deprecated serialisers have been removed completely (details below). If any of these are being used in an existing schema, a new graph and schema will need to be created (see below for replacement serialisers to use) and data from existing graphs migrated. Data will need to be migrated (export and reimport) from graphs using deprecated serialisers before upgrading to Gaffer v2. - -It is essential to migrate data stored using deprecated serialisers. Simply replacing these serialisers is not enough because this **will prevent existing data from being read** and potentially put the backing store into a **corrupted state**. - -### Preservation of ordering -When using an ordered store (such as Accumulo), all serialisers used on vertices must preserve order. As such, `compactRaw` serialisers (which do not preserve order) cannot be used on vertices in ordered stores. - -However, when preserving order is not required, such as for properties, `CompactRaw` serialisers are the most effective solution and should always be used. Using an ordered serialiser on a property would reduce performance without providing any benefit. [See the schemas documentation for more detail](https://gchq.github.io/gaffer-doc/v1docs/getting-started/developer-guide/schemas.html#serialisers). - -### Removed Serialisers - -#### [`serialisation.implementation.raw.RawDateSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawDateSerialiser.java) and [`serialisation.DateSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/DateSerialiser.java) -Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedDateSerialiser` instead - note that this will preserve order. Neither of these replacement serialisers implement `.deserialiseString(String)`, instead use `new Date(Long.parseLong(String))` in place of this. - -#### [`serialisation.implementation.raw.RawDoubleSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawDoubleSerialiser.java) and [`serialisation.DoubleSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/DoubleSerialiser.java) -Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedDoubleSerialiser` instead - note that this will preserve order. Neither of these replacement serialisers implement `.deserialiseString(String)`, instead use `Double.parseDouble(String)` in place of this. - -#### [`serialisation.implementation.raw.RawFloatSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawFloatSerialiser.java) and [`serialisation.FloatSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/FloatSerialiser.java) -Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedFloatSerialiser` instead - note that this will preserve order. - -#### [`serialisation.IntegerSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/IntegerSerialiser.java) and [`serialisation.implementation.raw.RawIntegerSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawIntegerSerialiser.java) -Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedIntegerSerialiser` instead if you need order preserved (e.g. vertex types). If object ordering definitely does **not** need to be preserved (e.g. only property types), `uk.gov.gchq.gaffer.serialisation.implementation.raw.CompactRawIntegerSerialiser` should be used instead. Neither of these replacement serialisers implement `.deserialiseString(String)`, instead use `Integer.parseInt(String)` in place of this. - -#### [`serialisation.LongSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/LongSerialiser.java) and [`serialisation.implementation.raw.RawLongSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/implementation/raw/RawLongSerialiser.java) -Use `uk.gov.gchq.gaffer.serialisation.implementation.ordered.OrderedLongSerialiser` instead if you need order preserved (e.g. vertex types). If object ordering definitely does **not** need to be preserved (e.g. only property types), `uk.gov.gchq.gaffer.serialisation.implementation.raw.CompactRawLongSerialiser` could also be used instead. - -### Changed Serialisers -#### [`serialisation.ToBytesSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/ToBytesSerialiser.java) and [`serialisation.ToBytesViaStringDeserialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/serialisation/ToBytesViaStringDeserialiser.java) -In both serialisers, the method `deserialise(byte[])` has been marked as deprecated. It cannot be deleted as it is needed to implement the Serialiser interface. It is recommended for speed/performance to use the other implementation with an offset and a length: `deserialise(byte[], int, int)`. - -## Removal of Seed Matching - -### [`operation.SeedMatching`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/SeedMatching.java) -SeedMatching has been removed from Gaffer. This was previously used in [get](../reference/operations-guide/get.md) operations, like [`GetElements`](../reference/operations-guide/get.md#getelements), to select whether you wanted your results to contain only Elements that are the same type as the seed, or both Edges and Entities. For more info, see the Gaffer 1.X docs page on [SeedMatching](https://gchq.github.io/gaffer-doc/v1docs/getting-started/user-guide/filtering.html#seedmatching). As described in the Gaffer 1.X docs, `SeedMatching` can be replaced with a `View`. The default behaviour in Gaffer is the same as if you used `seed_matching="RELATED"`, so **if this is the case, there is no migration required**. However, if you used `seed_matching="EQUAL"`, you will need to migrate to a `View`. -??? example "SeedMatching migration with EdgeSeeds" - - Where SeedMatching was used to only get back Edges from EdgeSeeds - - === "Java" - - ``` java - final GetElements getEdgesWithSeedMatching = new GetElements.Builder() - .input(new EdgeSeed("source", "dest", true)) - .seedMatching(SeedMatching.SeedMatchingType.EQUAL) - .build(); - ``` - - === "JSON" - - ``` json - { - "class" : "GetElements", - "input" : [ { - "class" : "EdgeSeed", - "source" : "source", - "destination" : "dest", - "matchedVertex" : "SOURCE", - "directedType" : "DIRECTED" - } ], - "seedMatching" : "EQUAL" - } - ``` - - === "Python" - - ``` python - g.GetElements( - input=[ - g.EdgeSeed( - source="source", - destination="dest", - directed_type="DIRECTED", - matched_vertex="SOURCE" - ) - ], - seed_matching="EQUAL" - ) - ``` - - You should instead specify that in a View - - === "Java" - - ``` java - final GetElements getEdgesWithoutSeedMatching = new GetElements.Builder() - .input(new EdgeSeed("source", "dest", true)) - .view(new View.Builder() - .edge("relevantEdgeGroup") - .build()) - .build(); - ``` - - === "JSON" - - ``` json - { - "class" : "GetElements", - "input" : [ { - "class" : "EdgeSeed", - "source" : "source", - "destination" : "dest", - "matchedVertex" : "SOURCE", - "directedType" : "DIRECTED" - } ], - "view" : { - "edges" : { - "relevantEdgeGroup" : { } - } - } - } - ``` - - === "Python" - - ``` python - g.GetElements( - input=[ - g.EdgeSeed( - source="source", - destination="dest", - directed_type="DIRECTED", - matched_vertex="SOURCE" - ) - ], - view=g.View( - edges=[g.ElementDefinition(group="relevantEdgeGroup")] - ) - ) - ``` - -??? example "SeedMatching migration with EntitySeeds" - - Where SeedMatching was used to only get back Entities from EntitySeeds - - === "Java" - - ``` java - final GetElements getEntitiesWithSeedMatching = new GetElements.Builder() - .input(new EntitySeed("vertex")) - .seedMatching(SeedMatching.SeedMatchingType.EQUAL) - .build(); - ``` - - === "JSON" - - ``` json - { - "class" : "GetElements", - "input" : [ { - "class" : "EntitySeed", - "vertex" : "vertex" - } ], - "seedMatching" : "EQUAL" - } - ``` - - === "Python" - - ``` python - g.GetElements( - input=[ - g.EntitySeed( - vertex="vertex" - ) - ], - seed_matching="EQUAL" - ) - ``` - - You should instead specify that in a View - - === "Java" - - ``` java - final GetElements getEntitiesWithoutSeedMatching = new GetElements.Builder() - .input(new EntitySeed("vertex")) - .view(new View.Builder() - .entity("relevantEntityGroup") - .build()) - .build(); - ``` - - === "JSON" - - ``` json - { - "class" : "GetElements", - "input" : [ { - "class" : "EntitySeed", - "vertex" : "vertex" - } ], - "view" : { - "entities" : { - "relevantEntityGroup" : { } - } - } - } - ``` - - === "Python" - - ``` python - g.GetElements( - input=[ - g.EntitySeed( - vertex="vertex" - ) - ], - view=g.View( - entities=[g.ElementDefinition(group="relevantEntityGroup")] - ) - ) - ``` - -??? example "SeedMatching migration with EdgeSeeds and EntitySeeds" - - Where SeedMatching was used to only get back only Edges from the provided EdgeSeeds and only Entities from the provided EntitySeeds - - === "Java" - - ``` java - final GetElements getBothWithSeedMatching = new GetElements.Builder() - .input(new EntitySeed("vertex"), new EdgeSeed("source", "dest", true)) - .seedMatching(SeedMatching.SeedMatchingType.EQUAL) - .build(); - ``` - - === "JSON" - - ``` json - { - "class" : "GetElements", - "input" : [ { - "class" : "EntitySeed", - "vertex" : "vertex" - }, { - "class" : "EdgeSeed", - "source" : "source", - "destination" : "dest", - "matchedVertex" : "SOURCE", - "directedType" : "DIRECTED" - } ], - "seedMatching" : "EQUAL" - } - ``` - - === "Python" - - ``` python - g.GetElements( - input=[ - g.EntitySeed( - vertex="vertex" - ), - g.EdgeSeed( - source="source", - destination="dest", - directed_type="DIRECTED", - matched_vertex="SOURCE" - ) - ], - seed_matching="EQUAL" - ) - ``` - - You will instead need to perform multiple Operations and combine the results. To perform the above operation, you would have to combine both previous examples. - - -## Changes to Store Traits - -### [`store.Store`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/Store.java) -- The method `getTraits()` has been removed. Use `Store.execute(Operation, Context)` with the `GetTraits` operation instead. -- The method `hasTrait(StoreTrait)` has been removed. Use `Store.execute(Operation, Context)` with the `HasTrait` operation instead. - -### [`federatedstore.FederatedGraphStorage`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedGraphStorage.java) -- The method `getTraits(GetTraits, Context)` has been removed. Use `Store.execute(Operation, Context)` with the `GetTraits` operation instead. - -### [`federatedstore.FederatedStore`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedStore.java) -- The methods `getTraits()` and `getTraits(GetTraits, Context)` have been removed. Use `Store.execute(Operation, Context)` with the `GetTraits` operation instead. - -## Changes to Schemas - -Deprecated methods in `store.schema.TypeDefinition` and `store.schema.Schema` have been removed ([see below sections](#storeschematypedefinition)). - -### Elements Schema - -Specifying a property name to use as a time stamp is now set under `config`. - -Previously set at the top level: -``` -{ - "entities": { - ... - }, - "edges": { - ... - }, - "timestampProperty": "timestamp" -} -``` - -Now set in `config`: -``` -{ - "entities": { - ... - }, - "edges": { - ... - }, - "config": { - "timestampProperty": "timestamp" - } -} -``` - -### Types Schema - -Specifying a serialiser class using deprecated `serialiserClass` is no longer possible. Instead, use `class` in `serialiser`. Deprecated `vertexSerialiserClass` has also been removed. - -Old, deprecated, and now removed approach: -``` -{ - "types": { - "example.map": { - "description": "Map type description", - "class": "java.util.LinkedHashMap", - "serialiserClass": "uk.gov.gchq.gaffer.serialisation.implementation.MapSerialiser" - } - } -} -``` - -Now set in `serialiser`: -``` -{ - "types": { - "example.map": { - "description": "Map type description", - "class": "java.util.LinkedHashMap", - "serialiser": { - "class": "uk.gov.gchq.gaffer.serialisation.implementation.MapSerialiser" - } - } - } -} -``` - -## All other Deprecations - -### [`accumulostore.AccumuloProperties`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloProperties.java) -- The `TABLE` setting/variable plus the methods `getTable()` and `setTable(String)` have been removed. For `getTable()`, [uk.gov.gchq.gaffer.accumulostore.getTableName()](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/accumulostore/AccumuloStore.html#getTableName--) could be used instead. -- A `graphId` should be supplied instead of setting `TABLE` directly. - -### [`accumulostore.MockAccumuloStore`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/MockAccumuloStore.java) -- This class has been removed. -- For in memory graphs, use `uk.gov.gchq.gaffer.mapstore.MapStore` instead. -- For Accumulo specific store tests, use `uk.gov.gchq.gaffer.accumulostore.MiniAccumuloStore` instead. - -### [`commonutil.TestTypes`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/common-util/src/test/java/uk/gov/gchq/gaffer/commonutil/TestTypes.java) -- This class has been removed. -- Use the equivalent `TestTypes` class in the store module `uk.gov.gchq.gaffer.store.TestTypes` instead. - -### [`commonutil.CommonConstants`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/CommonConstants.java) -- This class has been removed as it was redundant. -- For the `UTF-8` constant use `StandardCharsets.UTF_8.name()` from built in Java libraries. -- The above also applies to the `ISO_8859_1` constant from this class. -- This also allows for more robust error handing, [this commit is an example of a change implementing this](https://github.com/gchq/Gaffer/commit/3999890f3aebadbc5384695123af89a4d9053333#diff-faa371afa5e6d548ff73af92f1f148870424e7858e3fef43b3d25bc5b740c013). - -### [`data.elementdefinition.view.NamedViewDetail`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/data/src/main/java/uk/gov/gchq/gaffer/data/elementdefinition/view/NamedViewDetail.java) -- The method `hasWriteAccess(String userId, Set opAuths, String adminAuth)` has been removed. -- Use `hasWriteAccess(User user, String adminAuth)` instead. - -### [`data.elementdefinition.view.ViewElementDefinition`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/data/src/main/java/uk/gov/gchq/gaffer/data/elementdefinition/view/ViewElementDefinition.java) -- The method `setAggregator(ElementAggregator aggregator)` has been removed. -- A `ViewElementDefinition` should be constructed using the builder `uk.gov.gchq.gaffer.data.elementdefinition.view.ViewElementDefinition.Builder` instead. - -??? example - - Where `setAggregator` was used previously - ``` java - final ViewElementDefinition elementDef = new ViewElementDefinition(); - elementDef.setAggregator(myElementAggregator); - ``` - - You should now use the Builder - ``` java - final ViewElementDefinition elementDef = new ViewElementDefinition.Builder() - .aggregator(myElementAggregator) - .build(); - ``` - -### [`federatedstore.FederatedAccess`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedAccess.java) -- The method `isAddingUser(User)` has been removed. -- Use `hasReadAccess(User user, String adminAuth)`/`hasWriteAccess(User user, String adminAuth)` instead. - -### [`federatedstore.FederatedGraphStorage`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedGraphStorage.java) -- The methods `getAllIdsAsAdmin()`, `getAllGraphAndAccessAsAdmin(List)` and `changeGraphAccessAsAdmin(String, FederatedAccess)` have all been removed. -- The method `remove(String graphId)` has been removed. The following can be used instead: - - `remove(String graphId, User user)` - - `remove(String graphId, User user, String adminAuth)` - - `remove(String graphId, Predicate>> entryPredicateForGraphRemoval)` - -### [`federatedstore.FederatedStore`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/FederatedStore.java) -- The method `updateOperationForGraph(Operation, Graph)` has been removed. Use `FederatedStoreUtil.updateOperationForGraph(Operation, Graph)` instead. -- The method `addGraphs(Set graphAuths, String addingUserId, GraphSerialisable... graphs)` has been removed. The following can be used instead: - - `addGraphs(Set graphAuths, String addingUserId, boolean isPublic, GraphSerialisable... graphs)` - - `addGraphs(Set graphAuths, String addingUserId, boolean isPublic, boolean disabledByDefault, GraphSerialisable... graphs)` - - `addGraphs(Set graphAuths, String addingUserId, boolean isPublic, boolean disabledByDefault, AccessPredicate readAccessPredicate, AccessPredicate writeAccessPredicate, GraphSerialisable... graphs)` - - `addGraphs(FederatedAccess access, GraphSerialisable... graphs)` - -### [`federatedstore.operation.RemoveGraph`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/store-implementation/federated-store/src/main/java/uk/gov/gchq/gaffer/federatedstore/operation/RemoveGraph.java) -- The method `Builder.setGraphId(String graphId)` has been removed. -- Use `Builder.graphId(String graphId)` which has identical behaviour instead. - -??? example - - Where `Builder.setGraphId` was used previously - ``` java - final RemoveGraph removeGraphOp = new RemoveGraph.Builder() - .setGraphId("myGraph") - .build(); - ``` - - You should now use `Builder.graphId` - ``` java - final RemoveGraph removeGraphOp = new RemoveGraph.Builder() - .graphId("myGraph") - .build(); - ``` - -### [`graph.Graph`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/graph/src/main/java/uk/gov/gchq/gaffer/graph/Graph.java) -- The methods `Builder.graphId`, `Builder.library`, `Builder.view`, `Builder.addHook`, `Builder.addHooks` have all been removed in all forms. -- Instead of using these methods, use `.config()` to set the `graphConfig`. - -??? example - - Where the graph config was added using the `Graph.Builder` before - ``` java - final Graph myGraph = new Graph.Builder() - .graphId("myGraph") - .library(myGraphLibrary) - .view(myView) - .addHook(customHook) - .addSchema(mySchema) - .storeProperties(storeProperties) - .build(); - ``` - - You should now use the `GraphConfig.Builder` - ``` java - final Graph myGraph = new Graph.Builder() - .config(new GraphConfig.Builder() - .graphId("myGraph") - .library(myGraphLibrary) - .view(myView) - .addHook(customHook) - .build()) - .addSchema(mySchema) // (1)! - .storeProperties(storeProperties) // (2)! - .build(); - ``` - - 1. Schemas are not part of the GraphConfig - 2. StoreProperties are not part of the GraphConfig - -### [`hdfs.operation.MapReduce`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/library/hdfs-library/src/main/java/uk/gov/gchq/gaffer/hdfs/operation/MapReduce.java) -- The methods `getNumReduceTasks()` and `setNumReduceTasks(Integer)` have been removed. -- Gaffer’s operations that inherit `MapReduce` did not make use of `numReduceTasks`, either setting it to a constant number in the `JobFactory` or using Accumulo to automatically set the number (recommended for performance) and using min/max to keep it within a range. Therefore, `numReduceTasks`, `getNumReduceTasks` and `setNumReduceTasks` have been removed from this interface. - -### [`hdfs.operation.AddElementsFromHdfs`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/library/hdfs-library/src/main/java/uk/gov/gchq/gaffer/hdfs/operation/AddElementsFromHdfs.java) -- The methods `getNumReduceTasks()` and `setNumReduceTasks(Integer)` have been removed. -- The number of reduce tasks should not be set. By default the number of reduce tasks should match the number of tablets. Use minimum and maximum reduce tasks to specify boundaries for the number of reduce tasks. - -### [`hdfs.operation.SampleDataForSplitPoints`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/library/hdfs-library/src/main/java/uk/gov/gchq/gaffer/hdfs/operation/SampleDataForSplitPoints.java) -- The methods `getNumReduceTasks()` and `setNumReduceTasks(Integer)` have been removed. -- These methods were not required as `NumReduceTasks` was always set to 1 in any case. - -### [`jobtracker.JobDetail`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/jobtracker/JobDetail.java) -- The constructors which took `userId` as a `String` have been removed. -- Instead, a `User` (`uk.gov.gchq.gaffer.user.User`) should be used in its place. See the [Builder for User](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/user/User.Builder.html). -- `getUserId` and `setUserId` have also been removed. For getting the `UserId`, `getUser().getUserId()` can be used instead. See the [Javadoc for User](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/user/User.html#getUserId--). - -### [`jsonserialisation.JSONSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/serialisation/src/main/java/uk/gov/gchq/gaffer/jsonserialisation/JSONSerialiser.java) -- The method `update(String jsonSerialiserClass, String jsonSerialiserModules)` has been removed. -- Use `update(String jsonSerialiserClass, String jsonSerialiserModules, Boolean strictJson)` instead. Passing `strictJson` as `null` will result in the same behaviour. - -### [`operation.Operation`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/Operation.java) -- The method `asOperationChain(Operation operation)` has been removed. -- Use `OperationChain.wrap` with the `Operation` instead. - -### [`operation.impl.GetWalks`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/GetWalks.java) -- The method `Builder.operation` has been removed. -- Use the vararg method `Builder.addOperations` instead. - -### [`operation.impl.SplitStore`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/SplitStore.java) -- This class has been removed. -- It is replaced by `SplitStoreFromFile` which is identical except in name. - -### [`operation.impl.join.methods.JoinFunction`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/join/methods/JoinFunction.java) -- The method `join(Iterable keys, String keyName, String matchingValuesName, Match match, Boolean flatten)` which was not implemented has been removed. - -### [`rest.SystemProperty`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/rest-api/common-rest/src/main/java/uk/gov/gchq/gaffer/rest/SystemProperty.java) -- `GRAPH_ID`, `GRAPH_HOOKS_PATH`, `GRAPH_LIBRARY_PATH` and `GRAPH_LIBRARY_CONFIG` have been removed. -- These config options have been removed in favour of providing a `graphConfig` JSON and using `GRAPH_CONFIG_PATH` instead. - -### [`rest.service.v2.example.ExamplesFactory`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/rest-api/core-rest/src/main/java/uk/gov/gchq/gaffer/rest/service/v2/example/ExamplesFactory.java) -- This class has been removed. -- It is replaced by `uk.gov.gchq.gaffer.rest.factory.ExamplesFactory`, which can be used instead. - -### [`store.StoreProperties`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/StoreProperties.java) -- StoreProperties ID (`gaffer.store.id`) and related methods (`getId()`, `setId(String)`) have been removed. -- The ID of the store properties is instead directly set in the `GraphLibrary` when adding the `StoreProperties` with `GraphLibrary.add(String graphId, String schemaId, Schema schema, String propertiesId, StoreProperties properties)`. -- See the [Javadoc for GraphLibrary](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/library/GraphLibrary.html) for more detail. -- If you aren't using a `GraphLibrary`, this change shouldn't affect you as store properties ID is only used in `GraphLibrary`. - -### [`store.Context`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/Context.java) -- The private constructor `Context(User user, Map config, String jobId)` has been removed; along with the `jobId(String)` method. -- Use `Context(User user, Map config)` instead. This does not support supplying the Job ID, this will be set automatically. To get the Job ID use `.getJobId()`. - -### [`store.schema.TypeDefinition`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/schema/TypeDefinition.java) -- The method `getSerialiserClass()` has been removed. Instead, use `getSerialiser()` with `.getClass()` and related methods. -- The method `setSerialiserClass(String)` has been removed. Instead, set the Serialiser directly using `setSerialiser(Serialiser)`. - -### [`store.schema.Schema`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/schema/Schema.java) -- Schema ID (`gaffer.store.id`) and related methods have been removed. The ID is now defined in `GraphLibrary` when adding the schema. -- `timestampProperty` and related methods have been removed. Instead, this is specified by setting `"config": {"timestampProperty": "timestamp"}` (where `"timestamp"` is the property name to use as a time stamp) in the Schema. See [this example schema](https://github.com/gchq/Gaffer/blob/gaffer2-2.0.0-alpha-0.1/core/store/src/test/resources/schema-nested/elements.json) for more info. -- The method `getVertexSerialiserClass()` has been removed. It can be replaced by calling `vertexSerialiser.getClass()` and converting the result as appropriate, e.g. `getVertexSerialiserClass()` used `SimpleClassNameIdResolver.getSimpleClassName(vertexSerialiser.getClass())`. - -### [`store.library.GraphLibrary`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/library/GraphLibrary.java) -- The method `addSchema(Schema schema)` has been removed. Use `addSchema(String id, Schema schema)` instead. -- The method `addProperties(StoreProperties properties)` has been removed. Use `addProperties(String id, StoreProperties properties)` instead. -- Both of these now require the ID to be supplied. - -### [`store.operation.OperationChainValidator`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/OperationChainValidator.java) -- The method `validateViews(Operation op, ValidationResult validationResult, Schema schemaNotUsed, Store store)` has been removed. Use `validateViews(Operation op, User user, Store store, ValidationResult validationResult)` instead, passing `user` as `null` will result in the same behaviour. -- The method `validateComparables(Operation op, ValidationResult validationResult, Schema schemaNotUsed, Store store)` has been removed. Use `validateComparables(Operation op, User user, Store store, ValidationResult validationResult)` instead, passing `user` as `null` will result in the same behaviour. - -### [`store.operation.handler.named.cache.NamedViewCache`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/store/src/main/java/uk/gov/gchq/gaffer/store/operation/handler/named/cache/NamedViewCache.java) -- The method `deleteNamedView(String name)` has been removed. Use `deleteNamedView(String name, User user)` instead, passing `user` as `null` will result in the same behaviour. -- The method `getNamedView(String name)` has been removed. Use `getNamedView(String name, User user)` instead. -- The method `getAllNamedViews()` has been removed. Use `getAllNamedViews(User user)` instead. - -### [`types.IntegerFreqMap`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/type/src/main/java/uk/gov/gchq/gaffer/types/IntegerFreqMap.java) -- This class has been removed. -- Use `uk.gov.gchq.gaffer.types.FreqMap` instead, this is identical except for using Long rather than Integer. - -#### [`types.function.IntegerFreqMapAggregator`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/type/src/main/java/uk/gov/gchq/gaffer/types/function/IntegerFreqMapAggregator.java) -- This class has been removed. -- Use `uk.gov.gchq.gaffer.types.function.FreqMapAggregator` instead. - -#### [`serialisation.IntegerFreqMapSerialiser`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/type/src/main/java/uk/gov/gchq/gaffer/serialisation/IntegerFreqMapSerialiser.java) -- This class has been removed. -- Use `uk.gov.gchq.gaffer.serialisation.FreqMapSerialiser` instead. diff --git a/docs/gaffer2.0/federation-changes.md b/docs/gaffer2.0/federation-changes.md deleted file mode 100644 index 0da482e74a..0000000000 --- a/docs/gaffer2.0/federation-changes.md +++ /dev/null @@ -1,240 +0,0 @@ -# Federated Store Changes - -This page contains information on the changes to Gaffer's Federated Store. This functionality was introduced in version `2.0.0-alpha-0.4` of Gaffer. -The main changes were the addition of the [Federated Operation](#the-federated-operation), and a change to how results are [merged](#default-results-merging) by default. - -## The Federated Operation - -The `FederatedOperationChain` was removed and replaced with a new Operation, the `FederatedOperation`. This was added to improve the control you have over how operations are federated. -The Federated Operation has 3 key parameters: `operation`, `graphIds` and `mergeFunction`: -``` json -{ - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "uk.gov.gchq.gaffer.operation.impl.get.GetAllElements" - }, - "graphIds": [ "graphA", "graphB" ], - "mergeFunction": { - "class": "uk.gov.gchq.gaffer.federatedstore.util.ConcatenateMergeFunction" - } -} -``` - -### Required parameter: operation - -This is the Operation you wish to be federated to the subgraphs. This can be a single Operation or an OperationChain. If you use an OperationChain, then the whole chain will be sent to the subgraphs. - -### Optional parameter: graphIds - -This is a list of graph IDs which you want to send the operation to. - -If the user does not specify `graphIds` in the Operation, then the `storeConfiguredGraphIds` for that store will be used. If the admin has not configured the `storeConfiguredGraphIds` then all graphIds will be used. - -For information on sending different operations in one chain to different subgraphs, see [below](#breaking-change-removal-of-federatedoperationchain). - -### Optional parameter: mergeFunction - -The `mergeFunction` parameter is the Function you want to use when merging the results from the subgraphs. - -If the user does not specify a `mergeFunction` then it will be selected from the `storeConfiguredMergeFunctions` for that store. If the admin has not configured the `storeConfiguredMergeFunctions`, it will contain pre-populated `mergeFunctions`. Lastly, if a suitable `mergeFunction` is not found then a default `ConcatenateMergeFunction` is used. - -For example, when GetElements is used as the operation inside a FederatedOperation and the user hasn't specified a `mergeFunction`, the pre-populated `ApplyViewToElementsFunction` will be selected from `storeConfiguredMergeFunctions`, unless the admin configured it to use something else. - - -## Migrating to a FederatedOperation - -Previously, graphIds were selected in queries with the now deprecated option: `gaffer.federatedstore.operation.graphIds`. This is being supported while users migrate to using a FederatedOperation. - -### Sending an Operation to specific stores - -As mentioned, the `gaffer.federatedstore.operation.graphIds` option is still being supported so if you have an Operation using that option, it will continue to work. -Despite the option still being supported, we recommend you migrate to using a FederatedOperation. - -The `gaffer.federatedstore.operation.graphIds` option does not work an OperationChain. Previously, if you wanted to send an entire OperationChain to specific graphs, then you had to use a FederatedOperationChain. This has been replaced by a FederatedOperation with an OperationChain as the payload. For migration, see [below](#breaking-change-removal-of-federatedoperationchain). - -#### Deprecated graphIds option on a single Operation -```json -{ - "class": "uk.gov.gchq.gaffer.operation.impl.get.GetAllElements", - "options": { - "gaffer.federatedstore.operation.graphIds": "graphA" - } -} -``` - -#### New FederatedOperation graphIds on a single Operation -``` json -{ - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "uk.gov.gchq.gaffer.operation.impl.get.GetAllElements" - }, - "graphIds": [ "graphA" ] -} -``` - -#### Deprecated graphIds option inside an OperationChain -```json -{ - "class": "uk.gov.gchq.gaffer.operation.OperationChain", - "operations": [ - { - "class": "ExampleOperation1", - "options": { - "gaffer.federatedstore.operation.graphIds": "graphA" - } - }, - { - "class": "ExampleOperation2", - "options": { - "gaffer.federatedstore.operation.graphIds": "graphB" - } - } - ] -} -``` - -#### New FederatedOperation graphIds inside an OperationChain -```json -{ - "class": "uk.gov.gchq.gaffer.operation.OperationChain", - "operations": [ - { - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "ExampleOperation1" - }, - "graphIds": [ "graphA" ] - }, - { - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "ExampleOperation2" - }, - "graphIds": [ "graphB" ] - } - ] -} -``` - -## Breaking change: Removal of FederatedOperationChain - -The FederatedOperationChain has been removed, and where you would have used it before you should instead use a FederatedOperation with an OperationChain inside. - -This is useful if you have an OperationChain and want to send different parts of the chain to different subgraphs. - -#### Individually sending a sequence of Operations to a subgraph -You could send a sequence of operations within one chain to the same subgraph using `graphIds`, however, this is not always efficient: -```json -{ - "class": "uk.gov.gchq.gaffer.operation.OperationChain", - "operations": [ - { - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "ExampleOperation1" - }, - "graphIds": [ "graphA" ] - }, - { - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "ExampleOperation2" - }, - "graphIds": [ "graphA" ] - }, - { - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "ExampleOperation3" - }, - "graphIds": [ "graphB" ] - } - ] -} -``` - -#### Removed FederatedOperationChain sending a sequence of operations to a subgraph -It is more efficient to group together sequences of Operations that will go to the same subgraph. -This used to be done with a FederatedOperationChain: -```json -{ - "class": "uk.gov.gchq.gaffer.operation.OperationChain", - "operations": [ - { - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperationChain", - "operations": { - [ - "class": "ExampleOperation1", - "class": "ExampleOperation2" - ] - }, - "options": { - "gaffer.federatedstore.operation.graphIds": "graphA" - } - }, - { - "class": "ExampleOperation3", - "options": { - "gaffer.federatedstore.operation.graphIds": "graphB" - } - } - ] -} -``` - - -#### New FederatedOperation sending a sequence of operations to a subgraph -Now you should instead wrap an OperationChain inside a FederatedOperation: -```json -{ - "class": "uk.gov.gchq.gaffer.operation.OperationChain", - "operations": [ - { - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "uk.gov.gchq.gaffer.operation.OperationChain", - "operations": { - [ - "class": "ExampleOperation1", - "class": "ExampleOperation2" - ] - } - }, - "graphIds": [ "graphA" ] - }, - { - "class": "uk.gov.gchq.gaffer.federatedstore.operation.FederatedOperation", - "operation": { - "class": "ExampleOperation3" - }, - "graphIds": [ "graphB" ] - } - ] -} -``` - -## Default results merging - -As described above, FederatedStores now have `storeConfiguredMergeFunctions` that dictate how the FederatedStore will merge results from different subgraphs dependent on the Operation. - -In places, these new defaults do differ from previous behaviour, hence results will too. This can be overriden on a per Operation basis using the `mergeFunction` parameter described above, or a per store basis by overriding `storeConfiguredMergeFunctions`. -The previous behaviour was that all Operation results were concatenated together, this is now a mergeFunction within Gaffer called `ConcatenateMergeFunction`. Therefore, if you wanted a FederatedOperation to use this old behaviour, you can set the `mergeFunction` to `ConcatenateMergeFunction` (as shown [above](#the-federated-operation)). - -### New Merge function examples - -By default, `GetElements` results will be merged with `ApplyViewToElementsFunction`. This uses the View from the operation and applies it to all of the results, meaning the results are now re-aggregated and re-filtered using the Schema, locally in the FederatedStore. This makes the results look like they came from one graph, rather than getting back a list of Elements from different subgraphs. - -By default, `GetTraits` results will be merged with `CollectionIntersect`. This returns the intersection of common store traits from the subgraphs. This behaviour is the same, but now it can be overriden. - -By default, `GetSchema` results will be merged with `MergeSchema`. This returns an aggregated schema from the subgraphs, unless there is a conflict. This behaviour is the same, but now it can be overriden. For example, you may wish to use the `ConcatenateMergeFunction` if there is a schema conflict. - -### Default storeConfiguredMergeFunctions - -| Operation | Merge function | -|-------------------|-----------------------------| -| GetElements | ApplyViewToElementsFunction | -| GetAllElements | ApplyViewToElementsFunction | -| GetSchema | MergeSchema | -| GetTraits | CollectionIntersect | -| others | ConcatenateMergeFunction | diff --git a/docs/gaffer2.0/log4j.md b/docs/gaffer2.0/log4j.md deleted file mode 100644 index 333d509b0d..0000000000 --- a/docs/gaffer2.0/log4j.md +++ /dev/null @@ -1,30 +0,0 @@ -# Log4j in Gaffer - -This page contains information on how logging is done in Gaffer and on previous use of Log4j in Gaffer. - -## Log4j Version - -Log4j version 1 (1.2.17), was used by Gaffer versions 1.21 and below. From Gaffer 1.22, Log4j was replaced with Reload4j. **The newer version of Log4j, Log4j2 - which is susceptible to the major Log4Shell attack, has never been used by Gaffer or its dependencies.** - -## How Logging is done - -Gaffer uses SLF4J ([Simple Logging Facade for Java](https://www.slf4j.org/)) for all logging. This is a framework/abstraction layer which allows for different loggers to be used ([known as bindings](https://www.slf4j.org/manual.html#swapping)). The binding used by Gaffer is `org.slf4j:slf4j-reload4j:jar:1.7.36`. - -## Impact of Log4j removal on projects incorporating Gaffer - -Gaffer now uses Reload4j via SLF4J. This may impact projects which are using Gaffer if they are using Log4j directly or through a transitive dependency. To help avoid dependency conflicts, we have configured `maven-enforcer-plugin` to block use of Log4j with Gaffer. If you are using Gaffer in your project and your build fails because of this plugin, you will need to add a dependency exclusion to any dependencies which depend transitively on Log4j. These can be found by using the Maven dependency tree (ideally in verbose mode). - -## Dependencies of Gaffer using Log4j 1.2.17 - -Some major Gaffer dependencies (listed below) use Log4j internally (either directly or through SLF4J). From Gaffer version 1.22 these transitive dependencies are excluded and replaced with Reload4j, such that Log4j does not appear on the classpath at all. - -- GCHQ Koryphe 1.14.0 - Uses SLF4J with Log4j. -- Apache HBase 1.3.0 - Multiple artefacts used from the group `org.apache.hbase`. All depend directly on Log4j. -- Apache Hadoop 2.6.5 - Multiple artefacts used from the group `org.apache.hadoop`. All depend directly on Log4j. -- Apache Accumulo 1.9.3 - Multiple artefacts used from the group `org.apache.accumulo`. All depend directly on Log4j. -- Apache Kafka 0.10.0.0 - Artefact depends indirectly on Log4j through a sub dependency (`com.101tec:zkclient`). -- Apache Spark 2.3.2 - Artefact depends directly on Log4j. - -## Log4j Vulnerabilities - -Current vulnerabilities in Log4j 1.12.17 relate to the JDBC, SMTP and JMS appenders, the JMS Sink and the Socket Server. Gaffer never used any of this. In its default configuration, we don't believe Gaffer is vulnerable to any of these problems. If the Log4j configuration is altered, changes could be made which may cause Gaffer to be vulnerable to one or more of the above vulnerabilities. Standard security processes to prevent unauthorised access and modification of configuration files should preclude this possibility. diff --git a/docs/getting-started/advanced-guide/aggregation.md b/docs/getting-started/advanced-guide/aggregation.md deleted file mode 100644 index b57d1d9935..0000000000 --- a/docs/getting-started/advanced-guide/aggregation.md +++ /dev/null @@ -1,7 +0,0 @@ -# Aggregation Guide - -!!! info "Work in Progress" - - This page is under construction. - - Proposed content: *Guide to cover the aggregation feature in Gaffer*. diff --git a/docs/getting-started/advanced-guide/cardinality.md b/docs/getting-started/advanced-guide/cardinality.md deleted file mode 100644 index 9aa2f0bcd8..0000000000 --- a/docs/getting-started/advanced-guide/cardinality.md +++ /dev/null @@ -1,316 +0,0 @@ -# Cardinality - -This page describes what cardinality is, how to add it to your Gaffer graph and how to use it in -Gaffer queries. - -## What is Cardinality? - -In Gaffer, cardinality represents the number of unique vertices that are connected to a given -vertex. - -``` mermaid -graph TD - 1(1, cardinality=4) --> 2 - 1 --> 3 - 1 --> 4 - 1 --> 5 - 3(3, cardinality=2) --> 5 - 3 --> 5 - 3 --> 5 - 3 --> 5 -``` - -For very large graphs, updating this number accurately would be very costly in compute. This is -because for each new Edge that is added, we would have to check both connected Entities to see if -they are already connected to the other Entity, and this could be costly for Entities with a high -cardinality. Instead, Gaffer uses approximate cardinality making use of a [HyperLogLog -Sketch](https://datasketches.apache.org/docs/HLL/HLL.html), which estimates the cardinality with -relatively low error. In Gaffer, where you see the term "cardinality" used, it is referring to this -approximate cardinality backed by a sketch. - -## How to add cardinality to your graph - -You will need to add a property to your schema that will represent the approximate cardinality of an -Entity. This property is usually added to a specific Entity group that exists solely to represent -the Cardinality of a given vertex value. An example of the schema changes can be seen in the -[advanced properties guide](../../reference/properties-guide/advanced.md#hllsketch). If you are -using an Accumulo or Map store as your data store, this should be all that is needed. However, if -you are using a custom store, or a custom rest API, some additional config is needed. - -!!! tip - It is often useful keep track of cardinality per edge group. This is usually done with an edge - group property which is group in the `groupBy`. - - ``` json - { - "entities": { - "cardinality": { - "vertex": "vertex.string", - "properties": { - "approxCardinality": "hll", - "edgeGroup": "set" - }, - "groupBy": [ - "edgeGroup" - ] - } - } - } - ``` - -??? info "Additional config" - If you are using a custom data store, you will need to make sure the `SketchesJsonModules` is - added to your store properties. This can be easily done by changing the `store.properties` file, - as shown below. Alternatively, it can be hardcoded into the store, like in the - [AccumuloProperties](https://github.com/gchq/Gaffer/blob/a91ce4cd1e04dd0a2dfdf9633b513768fccd3507/store-implementation/accumulo-store/src/main/java/uk/gov/gchq/gaffer/accumulostore/AccumuloProperties.java#L468). - - ```properties - gaffer.serialiser.json.modules=uk.gov.gchq.gaffer.sketches.serialisation.json.SketchesJsonModules - ``` - - If you are using a custom data store, or you not using the standard spring-rest Gaffer rest API, - then you will also need to ensure that the `sketches-library` dependency is added to your - `pom.xml` for the store and/or rest API. - - ```xml - - uk.gov.gchq.gaffer - sketches-library - ${gaffer.version} - - ``` - -## How to add data with cardinality in your schema - -There are two main methods of adding cardinality elements in Gaffer. The first is to do it manually -when you add your edges, the second is to use a generator that can do it for you. - -### Manually adding cardinality entities - -Adding a cardinality entity between vertex 'A' and 'B' using `AddElements`. For more examples of -different property types, see the sketches [dev -docs](../../dev/rest-api-sketches.md#primitive-data-types-over-json) page. - -=== "Java" - - ``` java - final HllSketch hll = new HllSketch(10); //(1)! - hll.update("B"); //(2)! - new AddElements.Builder() - .input(new Entity.Builder() - .group("cardinality") - .vertex("A") - .property("approxCardinality", hll) //(3)! - .build()) - .build(); - ``` - - 1. Create the HllSketch object and define the precision values. By default, logK = 10. - 2. Update the sketch with the connected entity's vertex value, this adds it to the HllSketch's - bit hash. - 3. When adding the cardinality entity, set the property. - -=== "JSON" - - ``` json - { - "class": "AddElements", - "input": [{ - "class": "Entity", - "vertex": "A", - "group": "cardinality", - "properties": { - "approxCardinality": { - "org.apache.datasketches.hll.HllSketch": { - "values": ["B"] //(1)! - } - } - } - }] - } - ``` - - 1. You can directly add the values in the json and the deserialiser and aggregator will ensure - it is properly added to the object. - -=== "Python" - - ```python - g.AddElements( - input=[ - g.Entity( - vertex="A", - group="cardinality", - properties={ - "hll": g.hll_sketch(["B"]) #(1)! - } - ) - ] - ) - ``` - - 1. The `g.hll_sketch` helper function lets you directly add the values. The deserialiser and - aggregator will ensure it is properly added to the object. - -### Automatically adding cardinality entities - -Rather than using the `AddElements` operation to manually add cardinality entities, you can add them -automatically when you add edges to the graph using the -[`HllSketchEntityGenerator`](https://github.com/gchq/Gaffer/blob/a91ce4cd1e04dd0a2dfdf9633b513768fccd3507/library/sketches-library/src/main/java/uk/gov/gchq/gaffer/sketches/datasketches/cardinality/HllSketchEntityGenerator.java). -This will take an edge and return the same edge, as well as two cardinality entities, each -representing the additional cardinality link between the two vertices in the edge. - -=== "Java" - - ```java - new OperationChain.Builder() - .first(new GenerateElements.Builder() - .input(new Edge("edgeGroup1", "A", "B", true)) //(1)! - .generator(new HllSketchEntityGenerator() //(2)! - .cardinalityPropertyName("approxCardinality") //(3)! - .group("cardinality") //(4)! - .edgeGroupProperty("edgeGroup") //(5)! - .propertiesToCopy(...) //(6)! - ) - .build()) - .then(new AddElements()) //(7)! - .build(); - ``` - - 1. The input of edges is added to the OperationChain. Here we make one Edge between "A" and "B", - with group = "edgeGroup". - 2. The input is streamed into the `HllSketchEntityGenerator`, which will return the edge as well - as the two cardinality entities. - 3. The name of the property where we should store the cardinality. - 4. The group which the cardinality entity should be put into. - 5. If you track cardinality per edge group using a set property (as described - [above](#how-to-add-cardinality-to-your-graph)), which property should track this. - 6. Any properties from the edge to copy onto the cardinality entity. - 7. The results are streamed into `AddElements` to be added to the graph. - -=== "JSON" - - ```json - { - "class": "OperationChain", - "operations": [ - { - "class": "GenerateElements", - "input": [{ - "class": "Edge", - "group": "edgeGroup1", - "source": "A", - "destination": "B", - "directed": true - }], - "elementGenerator": { - "class": "HllSketchEntityGenerator", - "cardinalityPropertyName": "approxCardinality", - "edgeGroupProperty": "edgeGroup", - "group": "cardinality" - } - }, - { - "class": "AddElements" - } - ] - } - ``` - -=== "Python" - - ```python - g.OperationChain([ - g.GenerateElements( - input=[g.Edge("edgeGroup1", "A", "B", True)], - element_generator=g.HllSketchEntityGenerator( - cardinality_property_name="approxCardinality", - group="cardinality", - edge_group_property="edgeGroup" - ) - ), - g.AddElements() - ]) - ``` - -## How to get cardinality back using Gaffer queries - -Depending on how you query Gaffer, approximate cardinality will be displayed in results in different ways. - -=== "Java" - - ```java - final GetElements query = new GetElements.Builder() - .input(new EntitySeed("A")) - .build(); - - final Element element; - try (final CloseableIterable elements = graph.execute(query, user)) { - element = elements.iterator().next(); - } - - final HllSketch hllSketch = (HllSketch) element.getProperty("approxCardinality"); - final double approxCardinality = hllSketch.getEstimate(); - final String cardinalityEstimate = "Entity A has approximate cardinality " + approxCardinality; - ``` - - Result: - - ```txt - Entity A has approximate cardinality 1.0 - ``` - -=== "JSON" - - ```json - { - "class": "GetElements", - "input": [ - { - "class": "EntitySeed", - "vertex": "A" - } - ] - } - ``` - - Result: - - ```json - [{ - "class": "Entity", - "group": "cardinality", - "vertex": "A", - "properties": { - "approxCardinality": { - "org.apache.datasketches.hll.HllSketch": { - "bytes": "AgEHCgMIAQBejtgF", "cardinality": 1.0 - } - } - } - }] - ``` - -=== "Python" - - ```python - g.GetElements(g.EntitySeed("A")) - ``` - - Result: - - ```python - [{ - 'class': 'uk.gov.gchq.gaffer.data.element.Entity', - 'group': 'BasicEntity', - 'vertex': 'A', - 'properties': { - 'approxCardinality': { - 'org.apache.datasketches.hll.HllSketch': { - 'bytes': 'AgEHCgMIAQBejtgF', - 'cardinality': 1.0 - } - } - } - }] - ``` diff --git a/docs/getting-started/advanced-guide/import-export/csv.md b/docs/getting-started/advanced-guide/import-export/csv.md deleted file mode 100644 index df29b75f8d..0000000000 --- a/docs/getting-started/advanced-guide/import-export/csv.md +++ /dev/null @@ -1,201 +0,0 @@ -# CSV Import and Export - -!!! info "Work in Progress" - - This page is under construction. - - Proposed content: *Guide to cover the available import and export options for CSV*. - -Gaffer supports both importing from and exporting to csv. - -If you configure your Gaffer graph to support the `ImportFromLocalFile` and `ExportToLocalFile` -operations, then it can do this from/to a local file. - -??? tip "Enabling these operations on your Gaffer graph" - - To enable these operations on your Gaffer graph, you would need to add the following to your - `operationsDeclarations.json`. - - ```json - { - "operations": [ - { - "operation": "uk.gov.gchq.gaffer.operation.impl.export.localfile.ImportFromLocalFile", - "handler": { - "class": "uk.gov.gchq.gaffer.store.operation.handler.export.localfile.ImportFromLocalFileHandler" - } - }, - { - "operation": "uk.gov.gchq.gaffer.operation.impl.export.localfile.ExportToLocalFile", - "handler": { - "class": "uk.gov.gchq.gaffer.store.operation.handler.export.localfile.ExportToLocalFileHandler" - } - } - ] - } - ``` - -## CSV Import - -Importing is done with an OperationChain in multiple parts. - -```json -{ - "class": "OperationChain", - "operations": [ - { - "class": "ImportFromLocalFile", //(1)! - "filePath": "mydata.csv" - }, - { - "class": "GenerateElements", //(2)! - "elementGenerator": { - "class": "Neo4jCsvElementGenerator" - } - }, - { - "class": "AddElements" //(3)! - } - ] -} -``` - -1. The `ImportFromLocalFile` operation reads each line from the file `mydata.csv` and will stream - each string into the next parts of the chain. -2. The `GenerateElements` operation will transform each line of the file into a Gaffer Element. You - will need to provide an element generator that is suitable for the file you have provided. The - two CsvElementGenerators provided in core Gaffer are [`Neo4jElementGenerator`](#neo4j-format) and - [`NeptuneCsvElementGenerator`](#neptune-format). -3. Finally, the stream of Gaffer Elements are added with an `AddElements` operation. - -## CSV Export - -Exporting to csv is done with a similar OperationChain. - -```json -{ - "class": "OperationChain", - "operations": [ - { - "class": "GetAllElements" //(1)! - }, - { - "class": "ToCsv", //(2)! - "csvGenerator": "Neo4jCsvGenerator" - }, - { - "class": "ExportToLocalFile", //(3)! - "filePath": "output.csv" - } - ] -} -``` - -1. Firstly, you need to get the Elements which you want to export, in this example we simply - `GetAllElements`. -2. The `ToCsv` operation is then used to turn each Element into a csv formatted string. You must - supply a `CsvGenerator` to do this. You can build a custom [`CsvGenerator`](#custom-formats), or - use a supplied one. The two `CsvGenerators` provided in core Gaffer are - [`Neo4jCsvGenerator`](#neo4j-format) and [`NeptuneCsvGenerator`](#neptune-format). -3. Then the `ExportToLocalFile` operation is used to save this string output into a local file. - -## Formats - -### Custom formats - -You can customise CsvGenerator to create a custom export format in a ToCsv operation. -For example, the following operation. - -```json -{ - "class": "ToCsv", - "csvGenerator": { - "class": "CsvGenerator", - "fields": ["prop1", "SOURCE", "DESTINATION", "prop2", "GROUP"], - "constants": ["constant1", "constant2"] - } -} -``` - -Would produce csv rows that look like: - -=== "Table" - | prop1Value | sourceValue | destinationValue | prop2 | groupValue | constant1 | constant2 | - | ---------- | ----------- | ---------------- | ----- | ---------- | --------- | --------- | - -=== "CSV" - ```csv - prop1Value,sourceValue,destinationValue,prop2,groupValue,constant1,constant2 - ``` - -Currently, custom import formats are not supported. Instead you should use one of the two -[OpenCypher formats](#opencypher-formats). - -### OpenCypher Formats - -Core Gaffer has some generators provided that can import from and export to OpenCypher csvs. These -will work with other graph databases like Neo4j and Neptune. - -Please note that when using these, Gaffer might change your property name headers. All instances of -`-` are replaced with `_`, and invalid characters are stripped as outlined in -[PropertiesUtil](https://github.com/gchq/Gaffer/blob/f16de7c3eccfe7a800cad1d7eea5fbae4cf01d44/core/common-util/src/main/java/uk/gov/gchq/gaffer/commonutil/PropertiesUtil.java#L26). - -As shown [later](#neo4j-format) in the examples, OpenCypher formats let you dictate property types -in the header, like `propertyName:type`. Below is a table that shows which Gaffer transform function -is used to deserialise each [OpenCypher data -type](https://docs.aws.amazon.com/neptune/latest/userguide/bulk-load-tutorial-format-opencypher.html#bulk-load-tutorial-format-opencypher-data-types) -during import. - -| Gaffer Transform Function | OpenCypher Data Types | -| ------------------------- | --------------------------------------------------------------------- | -| `ToString` | `String` `Char` `Duration` `Point` `Date` `LocalDate` `LocalDateTime` | -| `ToBoolean` | `Bool` `Boolean` | -| `ToInteger` | `Int` `Short` `Byte` | -| `ToLong` | `Long` | -| `ToFloat` | `Float` | -| `ToDouble` | `Double` | -| `ParseTime` | `DateTime` | - -## Neo4j Generators - -You can import CSV from Neo4j using the `Neo4jCsvElementGenerator` and export using the -`Neo4jCsvGenerator`. The format used is defined -[here](https://neo4j.com/labs/apoc/4.4/export/csv/#export-database-csv). - -???+ note "Example" - === "Table" - | _id | name | age | lang | _labels | _start | _end | _type | weight | - |-----|-------|-----|------|----------|--------|------|---------|--------| - | v1 | marko | 29 | | Person | | | | | - | v2 | lop | | java | Software | | | | | - | e1 | | | | | v1 | v2 | Created | 0.4 | - - === "CSV" - ```csv - _id,name:String,age:Int,lang:String,_labels,_start,_end,_type,weight:Float - v1,marko,29,,Person,,,, - v2,lop,,java,Software,,,, - e1,,,,,v1,v2,Created,0.4 - ``` - -## Neptune Generators - -You can import csv from Neptune using the `NeptuneCsvElementGenerator` and export using the -`NeptuneCsvGenerator`. The format used is defined -[here](https://docs.aws.amazon.com/neptune/latest/userguide/bulk-load-tutorial-format-opencypher.html). - -???+ note "Example" - === "Table" - | :ID | name | age | lang | :LABEL | :START_ID | :END_ID | :TYPE | weight | - |-----|-------|-----|------|----------|-----------|---------|---------|--------| - | v1 | marko | 29 | | Person | | | | | - | v2 | lop | | java | Software | | | | | - | e1 | | | | | v1 | v2 | Created | 0.4 | - - === "CSV" - ```csv - :ID,name:String,age:Int,lang:String,:LABEL,:START_ID,:END_ID,:TYPE,weight:Double - v1,marko,29,,person,,,, - v2,lop,,java,software,,,, - e1,,,,,v1,v2,created,0.4 - ``` diff --git a/docs/getting-started/basics.md b/docs/getting-started/basics.md deleted file mode 100644 index 6b400c8216..0000000000 --- a/docs/getting-started/basics.md +++ /dev/null @@ -1,216 +0,0 @@ -# Gaffer Basics - -## What is Gaffer? - -Gaffer is a graph database framework, it acts similarly to an interface providing a graph data -structure on top of a chosen storage technology to enable storage of large graphs and traversal of -it's nodes and edges. In a nutshell Gaffer allows you to take data, convert it into a graph, store it -in a database and then run queries and analytics on it. - -The high level interactions of loading data and querying are demonstrated in the diagrams below. - -```mermaid -flowchart TD - subgraph Graph Query - E((User)) --> F - G{{Schema}} --> F - F([Query]) ---> H(Gaffer) - J(key-value store) <--> H - H --> K([Result]) - end - subgraph Data Input - A(Data)-->B{{Schema}} - B --> C(Gaffer) - C --> D(key-value store) - end -``` - -## Gaffer Schemas - -One of the main differences between Gaffer and other graph database tools are its schemas. In Gaffer -JSON based schemas need to be written upfront for it to understand how to load and treat the data in -the graph. These schemas define all aspects of the nodes and edges in the graph, and can even be -used to automatically do basic analysis or aggregation on queries and ingested data. - -You can kind of think of the schema as sort of a filter or validator for the incoming data. A -given bit of data must conform with part of the schema or it will simply be ignored as it doesn't -fit the structure of the graph. - -### Elements Schema - -All distinct bits of data in a Gaffer graph (e.g. nodes and edges) are referred to as 'elements'. -The structure and properties of these graph elements are defined in the elements schema. The general -format of an element schema are two lists; one of the `"edges"` and the other of the `"entities"` -like the following: - -!!! example "Basic elements syntax" - - ```json - { - "edges": { - "Edge": { - "source": "type", - "destination": "type", - "directed": "true", - "properties": { - "property": "type" - } - } - }, - "entities": { - "Node": { - "description": "A Node", - "vertex": "type", - "properties": { - "property": "type" - } - } - } - } - ``` - -As you can see there are a few fields for both the example `"Edge"` and `"Node"`, many of these -require a type as their value (discussed in the [next section](#types-schema)) which are -essentially handlers or object types for the value associated with the field. - -For an `edge` the following fields are required: - -- `source` - A user defined type for the source node the edge originated from. - -- `directed` - Boolean true or false to define if the edge is directed or not. When an Edge is - undirected in Gaffer, it is treated as if the relationship was bidirectional and the vertices of - the edge do not have an authoritative source and destination. - - !!! note "" - The type here, `"true"` or `"false"` needs to be defined in the types schema using a class - that evaluates to it. This is demonstrated in the [example - deployment](./example-deployment/writing-the-schema.md) document. - -- `destination` - A user defined type for the destination node the edge goes to. - -For an `entity` only one field is required: - -- `vertex` - A user defined type for the node/vertex. - -!!! note "" - The example includes some of the common optional fields too such as a `"properties"` list and - `"description"`. - -### Types Schema - -Following on from the elements schema, the other necessary schema needed for a Gaffer deployment is -the types schema. The types schema allows user defined types for all elements in the graph. It can -also demonstrate the power of Gaffer as it allows for custom functions classes to be used on the -types; however, this can make it quite complex to write a full schema for a graph. - -!!! example "Example types syntax" - - ```json - { - "types": { - "type.string": { - "description": "A basic type to hold the string value of an element", - "class": "java.lang.String" - }, - "type.int": { - "description": "A basic type to hold the int value of an element", - "class": "java.lang.Integer" - } - } - } - ``` - -## The API - -This section covers the currently available APIs that can be used to interact with a Gaffer graph. - -### Swagger Rest API - -Most of the interaction with a deployed Gaffer graph covered in this documentation will be through -the rest API. When deployed, the rest API will be available at a configurable address, accessing -this address via a browser brings up a [Swagger UI](https://swagger.io/) with various GET and POST -requests predefined for you to start using. - -Most of the GET requests simply retrieve information about the graph which can be useful to ensure -your config files have been loaded correctly and the graph is operating normally. The POST requests -allow you to interact with the graph by loading data and running queries. - -The main POST request end point you will use is `/graph/operations/execute`, this will ingest raw -JSON to carry out operations on the Gaffer graph. Gaffer provides many pre built operations that are -available to use and can be chained together for more complex use cases. However be aware, that -operation chains are usually highly specific to the data and results you wish to extract from the -graph so please refer to the reference guide on [Gaffer -operations](../reference/operations-guide/operations.md) for more detail on this. - -!!! example "Example operation chain using rest API" - - The following operation chain gets all the elements in the graph then will count them and - return the total. - - ```json - { - "class": "OperationChain", - "operations": [ - { - "class": "GetAllElements" - }, - { - "class": "Count" - } - ] - } - ``` - -### Python API - -Along side the rest API there also exists a Python API. Commonly referred to as `gafferpy` this API -enables similar querying capabilities as the rest API but from Python code. Fundamentally it wraps -the rest API to use the same JSON under the hood this means you should be able to access any -features or end points available in the main rest API. - -To get started with `gafferpy` simply import the module and connect to an existing graph. - -```python -from gafferpy import gaffer -from gafferpy import gaffer_connector -g_connector = gaffer_connector.GafferConnector("http://localhost:8080/rest/latest") -``` - -Then once connected you can access and run the same endpoints and operations as you would via the -usual rest API. - -!!! example "Example operation chain using `gafferpy`" - - The following operation chain gets all the elements in the graph then will count them and - store the total in the variable `count`. - - ```python - count = g_connector.execute_operation_chain( - operation_chain = gaffer.OperationChain( - operations=[ - gaffer.GetAllElements(), - gaffer.Count() - ] - ) - ) - ``` - -### Java API - -As Gaffer is written in Java there is native support to allow use of all its public classes. Using -Gaffer via the Java interface does differ from the rest API and `gafferpy` but is fully featured -with extensive [Javadocs](https://gchq.github.io/Gaffer/overview-summary.html). - -!!! example "Example operation chain using Java" - - The following operation chain gets all the elements in the graph then will count them and - store the result in a `Long`. - - ```java - OperationChain countAllElements = new OperationChain.Builder() - .first(new GetAllElements()) - .then(new Count<>()) - .build(); - - Long result = graph.execute(countAllElements, user); - ``` diff --git a/docs/getting-started/example-deployment/project-setup.md b/docs/getting-started/example-deployment/project-setup.md deleted file mode 100644 index 1eb687b6c4..0000000000 --- a/docs/getting-started/example-deployment/project-setup.md +++ /dev/null @@ -1,231 +0,0 @@ -# Example Deployment - -This guide will run through the start up and deployment of a basic Gaffer instance. It will cover -how to write a basic Gaffer Schema from scratch along with using the pre-made containers to run the -Gaffer rest API and Accumulo based data store. - -!!! warning - Please be aware that the example is only intended to demonstrate the core Gaffer concepts it is - not a production example. Various additional configuration to Accumulo and the HDFS set up would - be required to make this production ready and likely be specific to your infrastructure. - -## The Example Graph - -For this basic example we will attempt to recreate the graph in the following diagram consisting of -two nodes (vertexes) with one directed edge between them. - -```mermaid -graph LR - A(["Person - - name: marko - age: 29"]) - -- - "Created - weight: 0.4" - --> - B(["Software - - name: lop - lang: java"]) -``` - -This data describes one individual and a single piece of software that has been created by that -individual. The data will be loaded into the graph from a CSV file that follows the Neo4j export -syntax, this demonstrates how Gaffer can be used and how it can interact, model and query data from -other popular graph databases. Even with this basic graph we should be able to start building -queries to ask questions like "Who created the software called 'lop'?" and "How much did 'marko' -contribute to the software called 'lop'?" etc. - -To go with the diagram above the following CSV file (both raw and rendered are provided) represents -the graph in [Neo4j syntax](https://neo4j.com/labs/apoc/4.4/export/csv/#export-database-csv). - -!!! note "" - Please note that Gaffer often requires additional information about the data such as, - `:String` on the column headers to help with typing of the values. This is demonstrated below - in the raw file. There's more detail on this in the [OpenCypher documentation](../advanced-guide/import-export/csv.md#opencypher-formats). - -=== "Table" - | _id | name | age | lang | _labels | _start | _end | _type | weight | - |-----|-------|-----|------|----------|--------|------|---------|--------| - | v1 | marko | 29 | | Person | | | | | - | v2 | lop | | java | Software | | | | | - | e1 | | | | | v1 | v2 | Created | 0.4 | - -=== "CSV" - ```csv - _id,name:String,age:Int,lang:String,_labels,_start,_end,_type,weight:Float - v1,marko,29,,Person,,,, - v2,lop,,java,Software,,,, - e1,,,,,v1,v2,Created,0.4 - ``` - -## Project Setup - -First you must set up the files and directories you will need for the instance. As it stands there -are a couple of different ways to run a Gaffer project this example will use a logical structure -that suites a stand alone deployment consisting of the following file structure: - -!!! example "Example Gaffer project structure" - - !!! tip "" - Click the plus symbols for a brief description of each file - - ```yaml - ├── config - │ ├── accumulo - │ │ ├── accumulo-client.properties - │ │ ├── accumulo-env.sh - │ │ ├── accumulo.properties - │ │ ├── core-site.xml - │ │ └── log4j.properties - │ ├── gaffer - │ │ ├── application.properties #(1)! - │ │ ├── data #(2)! - │ │ │ ├── neo4jExport.csv - │ │ ├── graph - │ │ │ └── graphConfig.json #(3)! - │ │ ├── schema - │ │ │ ├── elements.json #(4)! - │ │ │ └── types.json #(5)! - │ │ └── store - │ │ ├── operationsDeclarations.json #(6)! - │ │ └── store.properties #(7)! - │ └── hdfs - │ ├── core-site.xml - │ ├── hdfs-site.xml - │ └── log4j.properties - └── docker-compose.yaml #(8)! - ``` - - 1. Properties file that generally sets the file locations of other Gaffer - configs e.g. schemas (note these are the absolute paths inside the - container). - 2. Any data files, e.g. CSV, to be made available to the Gaffer container. - 3. The main graph config file to set various properties of the overall graph. - 4. This file holds the schema outlining the elements in the graph, e.g. the - nodes (aka entities) and edges. - 5. This file defines the different data types in the graph and how they are - serialised to Java classes. - 6. Config file for additional Gaffer operations and set the class to handle - them on the store. - 7. The General store properties, sets up what store to use and any additional - configuration. - 8. This file controls which containers will be started up and the configuration - of them to ensure correct ports and files are available. - -All the files in the `config/accumulo/` and `config/hdfs/` directories will be copied directly from -the two locations in the Gaffer docker repo, -[here](https://github.com/gchq/gaffer-docker/tree/develop/docker/accumulo/conf-2.0.1) and -[here](https://github.com/gchq/gaffer-docker/tree/develop/docker/hdfs/conf). The configuration of -these are out of scope of this example but are covered in other sections of the documentation. The -main focus of this guide will be on the configuration files under the `config/gaffer/` directory. - -## Configuration Files - -There's a full [break down of Gaffer schema files on the next page](./writing-the-schema.md), this -section will instead cover the smaller additional config files that go along side the main Gaffer -schema to tweak other aspects of the graph. The location of these files will need to be volume -mounted into the container for them to be included in the deployment which is covered in more detail -[later in the guide](./running-the-deployment.md). - -!!! note - Many of these files have defaults already in the standard `gaffer-rest` container image, but its - useful to still include them in the project to allow easy configuration. - -### Application Properties - -This is probably the simplest configuration file in the Gaffer deployment. In general it borrows a -concept from [Spring Boot](https://docs.spring.io/spring-boot/docs/current/reference/html/application-properties.html) -to allow changing the context root and any properties related to Gaffer. In the example that follows -we use it to set the file location properties of where the other config files are (inside the -container). - -```properties title="application.properties" -gaffer.schemas=/gaffer/schema -gaffer.storeProperties=/gaffer/store/store.properties -gaffer.graph.config=/gaffer/graph/graphConfig.json -``` - -### Graph Configuration - -The graph configuration file is a JSON file that configures few bits of the Gaffer graph. Primarily -it is used to set the name and description along with any additional hooks to run before an operation -chain e.g. to impose limits on max results etc. For the example as, it is a very basic graph we just -set the name and short description. - -```json title="graphConfig.json" -{ - "graphId": "ExampleGraph", - "description": "An example graph" -} -``` - -### Store Properties - -The store properties file is used to configure how Gaffer will store its data. There are a few -different stores available for Gaffer, these are explained in more detail in the [reference -documentation](../../reference/stores-guide/stores.md), but by default you must provide a store -class and a store properties class. For this example we are using an Accumulo store as it is -recommended for efficient storage and retrieval of large data volumes. It's set up requires a few -custom properties which are outlined in the following file. - -```properties title="store.properties" -gaffer.store.class=uk.gov.gchq.gaffer.accumulostore.AccumuloStore -gaffer.store.properties.class=uk.gov.gchq.gaffer.accumulostore.AccumuloProperties - -# Accumulo specific config -accumulo.instance=accumulo -accumulo.zookeepers=zookeeper -accumulo.user=root -accumulo.password=secret - -# General store config -gaffer.cache.service.class=uk.gov.gchq.gaffer.cache.impl.HashMapCacheService -gaffer.store.job.tracker.enabled=true -gaffer.store.operation.declarations=/gaffer/store/operationsDeclarations.json -``` - -### Operations Declarations - -The operation declarations file is a way of enabling additional operations in Gaffer. By default -there are some built in operations already available (the rest API has a get all operations request -to see a list), but its likely you might want to enable others or add your own custom ones. As the -example will load its data from a local CSV file we can activate a couple of additional operations -using the following file. - -```json title="operationsDeclarations.json" -{ - "operations": [ - { - "operation": "uk.gov.gchq.gaffer.operation.impl.export.localfile.ImportFromLocalFile", - "handler": { - "class": "uk.gov.gchq.gaffer.store.operation.handler.export.localfile.ImportFromLocalFileHandler" - } - }, - { - "operation": "uk.gov.gchq.gaffer.operation.impl.export.localfile.ExportToLocalFile", - "handler": { - "class": "uk.gov.gchq.gaffer.store.operation.handler.export.localfile.ExportToLocalFileHandler" - } - } - ] -} -``` - -The two additional operations already exist in Gaffer (in the code base: -[ImportFromLocalFile](https://github.com/gchq/Gaffer/blob/develop/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/export/localfile/ImportFromLocalFile.java) -and -[ExportToLocalFile](https://github.com/gchq/Gaffer/blob/develop/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/impl/export/localfile/ExportToLocalFile.java)), -what this file is doing is essentially activating them and setting the handler class for them. The -`ImportFromLocalFile` usage is demonstrated in the [using the API](./using-the-api.md) section to -load some data. - -This operation allows us to pass a local CSV file (in the container) which will be read line by line -and get a stream of the line strings. This is very useful when we start using Operation Chains as -we can pass this stream of data as the input to the next operation in the chain similar to shell -pipes. - -!!! note - The location of the file needs to be set via the store properties file using the - `gaffer.store.operation.declarations` property (see [previous section](#store-properties)). diff --git a/docs/getting-started/example-deployment/running-the-deployment.md b/docs/getting-started/example-deployment/running-the-deployment.md deleted file mode 100644 index 10d492d455..0000000000 --- a/docs/getting-started/example-deployment/running-the-deployment.md +++ /dev/null @@ -1,219 +0,0 @@ -# Running the Deployment - -To run the containers there are a couple of options but primarily the two main ways are with either -Docker Compose or Kubernetes (via Helm), the [gaffer-docker](https://github.com/gchq/gaffer-docker) -repository has some examples of how to run both. - -## Docker Compose - -For this example we will use a slightly modified version of the docker compose config file used in -the repository. - -??? example "docker-compose.yaml" - ```yaml - version: "3.7" - - services: - - zookeeper: - image: zookeeper:${ZOOKEEPER_VERSION} - healthcheck: - test: echo ruok | nc 127.0.0.1 2181 | grep imok - interval: 30s - timeout: 5s - retries: 3 - container_name: zookeeper - hostname: zookeeper - environment: - - ZOO_SERVERS=server.1=zookeeper:2888:3888;2181 - - ZOO_4LW_COMMANDS_WHITELIST=* - volumes: - - /data - - /datalog - - hdfs-namenode: - image: gchq/hdfs:${HADOOP_VERSION} - depends_on: - zookeeper: - condition: service_healthy - healthcheck: - test: curl -f http://localhost:9870 || exit 1 - interval: 30s - timeout: 10s - retries: 3 - command: namenode - container_name: hdfs-namenode - hostname: hdfs-namenode - environment: - - HADOOP_CONF_DIR=${HADOOP_CONF_DIR} - ports: - - 9870:9870 - volumes: - - ./configs/hdfs:${HADOOP_CONF_DIR}:ro - - /var/log/hadoop - - /data1 - - /data2 - - hdfs-datanode: - image: gchq/hdfs:${HADOOP_VERSION} - depends_on: - hdfs-namenode: - condition: service_healthy - command: datanode - container_name: hdfs-datanode - hostname: hdfs-datanode - environment: - - HADOOP_CONF_DIR=${HADOOP_CONF_DIR} - volumes: - - ./configs/hdfs:${HADOOP_CONF_DIR}:ro - - /var/log/hadoop - - /data1 - - /data2 - - accumulo-master: - image: gchq/gaffer:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} - depends_on: - hdfs-namenode: - condition: service_healthy - healthcheck: - test: cat /proc/net/tcp | grep 270F - interval: 30s - timeout: 5s - retries: 3 - start_period: 10s - build: - context: . - args: - GAFFER_VERSION: ${GAFFER_VERSION} - BASE_IMAGE_NAME: gchq/accumulo - BASE_IMAGE_TAG: ${ACCUMULO_VERSION} - command: master - container_name: accumulo-master - hostname: accumulo-master - environment: - - ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} - - HADOOP_USER_NAME=hadoop - volumes: - - ./configs/accumulo:${ACCUMULO_CONF_DIR}:ro - - /var/log/accumulo - - accumulo-tserver: - image: gchq/gaffer:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} - depends_on: - accumulo-master: - condition: service_healthy - healthcheck: - test: cat /proc/net/tcp | grep 270D - interval: 30s - timeout: 5s - retries: 3 - command: tserver - container_name: accumulo-tserver - hostname: accumulo-tserver - environment: - - ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} - - HADOOP_USER_NAME=hadoop - volumes: - - ./configs/accumulo:${ACCUMULO_CONF_DIR}:ro - - /var/log/accumulo - - accumulo-monitor: - image: gchq/gaffer:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} - depends_on: - accumulo-master: - condition: service_healthy - command: monitor - container_name: accumulo-monitor - hostname: accumulo-monitor - environment: - - ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} - - HADOOP_USER_NAME=hadoop - ports: - - 9995:9995 - volumes: - - ./configs/accumulo:${ACCUMULO_CONF_DIR}:ro - - /var/log/accumulo - - accumulo-gc: - image: gchq/gaffer:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} - depends_on: - accumulo-master: - condition: service_healthy - command: gc - container_name: accumulo-gc - hostname: accumulo-gc - environment: - - ACCUMULO_CONF_DIR=${ACCUMULO_CONF_DIR} - - HADOOP_USER_NAME=hadoop - volumes: - - ./configs/accumulo:${ACCUMULO_CONF_DIR}:ro - - /var/log/accumulo - - gaffer-rest: - image: gchq/gaffer-rest:${GAFFER_VERSION}-accumulo-${ACCUMULO_VERSION} - depends_on: - accumulo-tserver: - condition: service_healthy - ports: - - 8080:8080 - volumes: - - ./configs/gaffer/application.properties:/gaffer/config/application.properties:ro - - ./configs/gaffer/data:/gaffer/data:ro - - ./configs/gaffer/graph:/gaffer/graph:ro - - ./configs/gaffer/schema:/gaffer/schema:ro - - ./configs/gaffer/store:/gaffer/store:ro - ``` - -If you are not familiar with docker or docker compose there are plenty of -[resources](https://docs.docker.com/compose/) online to get up to speed but essentially, it is a -config file that will run up multiple containers with various bits of configuration which is a lot -easier than typing out multiple docker commands! - -Some key bits you may want to configure in the file are the shared volumes (under the `volumes` -section). The locations in the example file assume you use the project structure from the -[example setup](./project-setup.md) but if you change any of the locations then they will need -updating. - -To run up the cluster its as easy as running the following from the root of the project. - -```bash -docker compose up -``` - -The above configuration will start the following containers: - -- Zookeeper -- HDFS - - Datanode - - Namenode -- Accumulo - - Monitor - - GC - - tServer - - Master -- Gaffer Rest API - -The web UIs for nodes in cluster can then be accessed at the following addresses: - -- Access the HDFS NameNode web UI at: -- Access the Accumulo Monitor UI at: -- Access the Gaffer REST API at: - -## Environment Variables - -As you can probably see the example is using a few environment variables in the -`docker-compose.yaml` file, these set things such as the container versions and a couple of file -locations. The use of these variables are recommended as it can make it easier to update container -versions and other aspects of the containers. - -A basic set of these environment variables are shown below which can be saved in a `.env` file and -sourced before running the containers. - -```bash -ZOOKEEPER_VERSION="3.7.1" -GAFFER_VERSION="2.0.0" -ACCUMULO_VERSION="2.0.1" -HADOOP_VERSION="3.3.3" -ACCUMULO_CONF_DIR="/etc/accumulo/conf" -HADOOP_CONF_DIR="/etc/hadoop/conf" -``` diff --git a/docs/getting-started/example-deployment/using-the-api.md b/docs/getting-started/example-deployment/using-the-api.md deleted file mode 100644 index d592ff307c..0000000000 --- a/docs/getting-started/example-deployment/using-the-api.md +++ /dev/null @@ -1,191 +0,0 @@ -# Using the API - -As covered in the [Gaffer basics](../basics.md) the main POST request used in the API is -`/graph/operations/execute`. This part of the guide will cover the general usage of this part of the -API and walk through some general operations you might want to use to load data or query. - -!!! note - The [Gaffer operations reference guide](../../reference/operations-guide/operations.md) - has more detail on available operations. - -## Loading Data - -Gaffer supports various methods of loading data and depending on your use case you can even bypass -it all together to load directly into Accumulo. - -This example will focus on using the rest API to add the graph elements. In production this method -would not be recommended for large volumes of data. However, it is fine for smaller data sets and -generally can be done in a few stages outlined in the following diagram. - -```mermaid -flowchart LR - A(Raw Data) --> B(GenerateElements) - B --> C(AddElements) -``` - -The first stage is taking the raw input data and converting it into Gaffer elements via an element -generator class. Gaffer includes a few built in -[generators](../../reference/operations-guide/generate.md) but you can use a custom class or -pre-process the data before passing to Gaffer so that you're able to use a default generator. Once -the data has been converted to elements it needs to be added into the graph. To load elements there -is a standard `AddElements` operation which takes raw elements JSON as input and adds them into the -graph. - -!!! info - This is where the schema is used here to validate the elements are correct and conform before - adding. - -Using the example again we will demonstrate how we could write an operation chain to load the data -from the neo4j formatted CSV file. - -```json -{ - "class": "OperationChain", - "operations": [ - { - "class": "ImportFromLocalFile", - "filePath": "/gaffer/data/neo4jExport.csv" - }, - { - "class": "GenerateElements", - "elementGenerator": { - "class": "Neo4jCsvElementGenerator" - } - }, - { - "class": "AddElements" - } - ] -} -``` - -The operation chain above essentially mirrors the stages in the previous diagram. In the example -chain we first ingest the data via the `ImportFromLocalFile` class (an additional operation we added -via the [`operationsDeclarations.json`](./additional-config.md#operations-declarations)), which -streams the data from the CSV file into the next `GenerateElements` operation. - -For the generator we have selected the built in `Neo4jCsvElementGenerator` class, this is already -set up to be able to parse a correctly formatted neo4j exported CSV into Gaffer elements via the -schema. If you are curious as to what the output of each operation is you can try run a subset of -this chain to see how the data changes on each one, the output should be returned back to you in the -server response section of the Swagger API. - -## Querying Data - -Once data is loaded in the graph its now possible to start querying the data to gain insight and -perform analytics. Querying in Gaffer can get fairly complex but generally simple queries are made -up of two parts; a `Get` Operation and a `View`. - -Starting with the `Get` operation, say we want to get all nodes and edges based on their ID. To do -this we can use the `GetElements` operation and set the `Seed` to the entity (e.g. node) or edge -where we want to start the search. To demonstrate this on the example graph we can attempt to get -all entities and edges associated with the `Person` node with ID `v1`. - -The result from this query should return the node associated with the `v1` id along with any edges -on this node, which in this case is just one - -=== "Input Query" - ```json - { - "class": "GetElements", - "input": [ - { - "class": "EntitySeed", - "vertex": "v1" - } - ] - } - ``` - -=== "Example Result" - - ```json - [ - { - "class": "uk.gov.gchq.gaffer.data.element.Entity", - "group": "Person", - "vertex": "v1", - "properties": { - "name": "marko", - "age": 29 - } - }, - { - "class": "uk.gov.gchq.gaffer.data.element.Edge", - "group": "Created", - "source": "v1", - "destination": "v2", - "directed": true, - "matchedVertex": "SOURCE", - "properties": { - "weight": { - "java.lang.Float": 0.4 - } - } - } - ] - ``` - -### Filtering Data - -The final bit of querying this guide will go into is how to apply a `View` to a returned set of -elements. A `View` in Gaffer allows you to filter, aggregate, transform and just generally -manipulate the results. In general a `View` has the following possible use cases: - -- **Filtering** - General filtering on elements based on predicates. Filtering can be applied - pre-aggregation, post aggregation and post transformation. - -- **Aggregation** - This is to control how similar elements are aggregated together. You can provide - a subset of the schema `groupBy` properties and override the aggregation functions. - -- **Transformation** - Transformations can be applied by providing Functions to transform properties - and vertex values. This is a powerful feature, you can override the existing values or you can - transform and save the new value into a new transient property. - -- **Property Removal** - The relevant properties you want to be returned can be controlled. You can - use either `properties` or `excludeProperties` to define the list of properties to be included - or excluded. - -Taking the example from the previous section we will demonstrate general filtering on a query. As -before, the query returns the node `v1` and any edges associated with it. We will now filter it to -include only edges where the weight is over a certain value. In this scenario it is analogous to -asking, *"get all the `Created` edges on node `v1` that have a `weight` greater than 0.3"*. - -=== "Filter Query" - - ```json - { - "class": "GetElements", - "input": [ - { - "class": "EntitySeed", - "vertex": "v1" - } - ], - "view": { - "edges": { - "Created": { - "preAggregationFilterFunctions": [ - { - "selection": [ - "weight" - ], - "predicate": { - "class": "IsMoreThan", - "orEqualTo": false, - "value": { - "Float": 0.3 - } - } - } - ] - } - } - } - } - ``` - -!!! tip - As you can see filtering is based around predicates which are similar to if else statements in - traditional programming. For a full list of available predicates refer to the reference - [documentation](../../reference/predicates-guide/predicates.md). diff --git a/docs/getting-started/example-deployment/writing-the-schema.md b/docs/getting-started/example-deployment/writing-the-schema.md deleted file mode 100644 index 0be48ba33c..0000000000 --- a/docs/getting-started/example-deployment/writing-the-schema.md +++ /dev/null @@ -1,264 +0,0 @@ -# Writing the Schema - -In Gaffer JSON based schemas need to be written upfront to model and understand how to load and -treat the data in the graph. These schemas define all aspects of the nodes and edges in the graph, -and can even be used to automatically do basic analysis or aggregation on queries and ingested data. - -For reference, this guide will use the same CSV data set from the [project setup](./project-setup.md#the-example-graph) page. - -=== "Table" - | _id | name | age | lang | _labels | _start | _end | _type | weight | - |-----|-------|-----|------|----------|--------|------|---------|--------| - | v1 | marko | 29 | | Person | | | | | - | v2 | lop | | java | Software | | | | | - | e1 | | | | | v1 | v2 | Created | 0.4 | - -=== "CSV" - ```csv - _id,name:String,age:Int,lang:String,_labels,_start,_end,_type,weight:Float - v1,marko,29,,Person,,,, - v2,lop,,java,Software,,,, - e1,,,,,v1,v2,Created,0.4 - ``` - -## Elements Schema - -In Gaffer an element refers to any object in the graph, i.e. your nodes (vertexes) and edges. To set -up a graph we need to tell Gaffer what objects are in the graph and the properties they have. The -standard way to do this is a JSON config file in the schema directory. The filename can just be -called something like `elements.json`, the name is not special as all files under the `schema` -directory will be merged into a master schema but its still recommended to use an appropriate name. - -As covered in the [Gaffer basics](../basics.md), to write a schema you can see that there are some -required fields, but largely a schema is highly specific to your input data. - -Starting with the `entities` from the example, we can see there will be two distinct types of nodes -in the graph; one representing a `Person` and another for `Software`. These can be added into the -schema to give something like the the following: - -!!! info "" - - The types here such as `id.person.string` are covered in the [next section](#types-schema). - -```json -{ - "entities": { - "Person": { - "description": "Entity representing a person vertex", - "vertex": "id.person.string" - }, - "Software": { - "description": "Entity representing a software vertex", - "vertex": "id.software.string" - } - } -} -``` - -From the basic schema you can see that we have added two entity types for the graph. For now, each -`entity` just contains a short description and a type associated to the `vertex` key. The type here -is just a place holder but it has been named appropriately as it's assumed that we will just use the -string representation of the node's id (this will be defined in the `types.json` later in the -guide). - -Expanding on the basic schema we will now add the `edges` to the graph. As the example graph is -small we only need to add one edge - the `Created` edge. This is a directed edge that connects a -`Person` to a `Software` and can be defined as the following. - -```json -{ - "edges": { - "Created": { - "source": "id.person.string", - "destination": "id.software.string", - "directed": "true" - } - }, - "entities": { - "Person": { - "description": "Entity representing a person vertex", - "vertex": "id.person.string" - }, - "Software": { - "description": "Entity representing a software vertex", - "vertex": "id.software.string" - } - } -} -``` - -As discussed in the [basics guide](../basics.md), edges have some mandatory fields. Starting with -the `source` and `destination` fields, these must match the types associated with the vertex field -in the relevant entities. From the example, we can see that the source of a `Created` edge is a -`Person` so we will use the placeholder type we set as the `vertex` field which is -`id.person.string`. Similarly the destination is a `Software` node so we will use its placeholder of -`id.software.string`. - -We must also set whether an edge is directed or not, in this case it is as only a person can create -software not the other way around. To set this we will use the `true` type, but note that this is a -placeholder and must still be defined in the types.json. - -Continuing with the example, the nodes and edges also have some properties associated with each such -as name, age etc. These can also be added to the schema using a properties map to result in the -extended schema below. - -```json -{ - "edges": { - "Created": { - "source": "id.person.string", - "destination": "id.software.string", - "directed": "true", - "aggregate": "false", - "properties": { - "weight": "property.float" - } - } - }, - "entities": { - "Person": { - "description": "Entity representing a person vertex", - "vertex": "id.person.string", - "aggregate": "false", - "properties": { - "name": "property.string", - "age": "property.integer" - } - }, - "Software": { - "description": "Entity representing a software vertex", - "vertex": "id.software.string", - "aggregate": "false", - "properties": { - "name": "property.string", - "lang": "property.string" - } - } - } -} -``` - -!!! note - Take note of the `"aggregate": "false"` setting, this skips any ingest aggregation as it is not - required and out of scope of this example. All entity property types must have an aggregation - function in Gaffer unless this option is added. Aggregation is fairly advanced topic in Gaffer - but very powerful it is covered in more depth later in the documentation. - -## Types Schema - -The other schema that now needs to be written is the types schema. As you have seen in the elements -schema there are some place holder types added as the values for many of the keys. These types work -similarly to if you have ever programmed in a strongly typed language, they are essentially the -wrapper for the value to encapsulate it. - -Now starting with the types for the nodes/vertexes, we used two placeholder types, one for the -`Person` entity and one for the `Software` entity. From the example CSV you can see there is a `_id` -column that uses a string identifier that is used for the ID of the node (this will also be used by -the `edge` to identify the source and destination). We will define a type for each node ID using the -standard java `String` class to encapsulate it, this leads to a basic `type.json` like the -following. - -```json -{ - "types": { - "id.person.string": { - "description": "A basic type to hold the string id of a person entity", - "class": "java.lang.String" - }, - "id.software.string": { - "description": "A basic type to hold the string id of a person entity", - "class": "java.lang.String" - } - } -} -``` - -The next set of types that need defining are, the ones used for the properties that are attached to -the nodes/entities. Again we need to take a look back at what our input data looks like, in the CSV -file we can see there are three different types that are used for the properties which are analogous -to a `String`, an `Integer` and a `Float`. - -!!! tip - Of course technically, all of these properties could be encapsulated in a string but, assigning - a relevant type allows some additional type specific features when doing things like grouping - and aggregation as it would in traditional programming. - -If we make a type for each of the possible properties using the standard Java classes we end up with -the following. - -```json -{ - "types": { - "id.person.string": { - "description": "A basic type to hold the string id of a person entity", - "class": "java.lang.String" - }, - "id.software.string": { - "description": "A basic type to hold the string id of a person entity", - "class": "java.lang.String" - }, - "property.string": { - "description": "A type to hold string properties of entities", - "class": "java.lang.String" - }, - "property.integer": { - "description": "A basic type to hold integer properties of entities", - "class": "java.lang.Integer" - }, - "property.float": { - "description": "A basic type to hold float properties of entities", - "class": "java.lang.Float" - } - } -} -``` - -The final thing that we need to add to the schema is a type for the `true` Boolean value that's used -by the directed field of the edge element. This leaves us with the complete list of types for this -example. - -```json -{ - "types": { - "id.person.string": { - "description": "A basic type to hold the string id of a person entity", - "class": "java.lang.String" - }, - "id.software.string": { - "description": "A basic type to hold the string id of a person entity", - "class": "java.lang.String" - }, - "property.string": { - "description": "A type to hold string properties of entities", - "class": "java.lang.String" - }, - "property.integer": { - "description": "A basic type to hold integer properties of entities", - "class": "java.lang.Integer" - }, - "property.float": { - "description": "A basic type to hold float properties of entities", - "class": "java.lang.Float" - }, - "true": { - "description": "A simple boolean that must always be true.", - "class": "java.lang.Boolean", - "validateFunctions": [ - { - "class": "uk.gov.gchq.koryphe.impl.predicate.IsTrue" - } - ] - } - } -} -``` - -As you can see the Boolean value also demonstrates the validation feature which allows for -validation of any values using the type. In this example it verifies its true but you could also -check it exists, see if its less than another value etc. or even run your own custom validator -class. - -!!! tip - The Koryphe module provides lots of default functions that can be used to validate and aggregate - data, see the [predicate reference guide](../../reference/predicates-guide/koryphe-predicates.md) - for more information. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md deleted file mode 100644 index b2a7a665cb..0000000000 --- a/docs/getting-started/quickstart.md +++ /dev/null @@ -1,69 +0,0 @@ -# Quickstart - -The quickest way to get up and running with Gaffer is through its container images. To start up a -simple map store based instance with some default schemas simply pull and run the `gaffer-rest` -image. - -```bash -docker pull gchq/gaffer-rest:2.0.0 -``` - -```bash -docker run -p 8080:8080 gchq/gaffer-rest:2.0.0 -``` - -The Swagger rest API should be available at [http://127.0.0.1:8080/rest](http://127.0.0.1:8080/rest) -to try out. - -Be aware that, as the image uses the map store backend by default, all graph data will be saved in -memory so killing the container will mean you will lose any data added to the graph. Take a look at -the [possible deployments](#possible-deployments) section for a basic look at the different storage -options Gaffer supports. - -If you wish to add custom schema to try out you can mount these into the container at start up to -configure the graph. By default the `gaffer-rest` image looks under `/gaffer/schema` meaning you can -mount over this directory with a directory containing your custom schema. - -```bash -docker run -p 8080:8080 -v /path/to/your/schema:/gaffer/schema gchq/gaffer-rest:2.0.0 -``` - -!!! info - - The [example deployment](example-deployment/project-setup.md) section provides a full walkthrough - of a basic Gaffer deployment, and covers how to write a simple Gaffer Schema to create a custom - graph with an Accumulo based data store. - -## Possible Deployments - -As Gaffer essentially works as a framework to structure and save data into a data store, the storage -option is one of the largest considerations when deploying a new graph. A few technologies are -supported by Gaffer; however, some are more widely used than others, the main types you might want -to use are: - -- **Accumulo Store** - The main recommended data store for Gaffer implemented by [Apache - Accumulo](https://accumulo.apache.org/). -- **Map Store** - In memory JVM store, useful for quick prototyping. -- **Proxy Store** - This provides a way to hook into an existing Gaffer store, when used all - operations are delegated to the chosen Gaffer Rest API. -- **Federated Store** - Similar to a proxy store however, this will forward all requests to a - collection of sub graphs but merge the responses so they appear as one graph. - -Once the storage option has been chosen, the deployment can be setup and started using one or more -of the available Gaffer container images. - -!!! info - - Please see the `gaffer-docker` documentation for more information on available images and - deployments. - -To change the storage backend for Gaffer the `store.properties` file can be configured with the -chosen type, please refer to the [reference guide](../reference/stores-guide/stores.md) for more -information on store properties. - -!!! example "Example `store.properties` for MapStore" - - ```properties - gaffer.store.class=uk.gov.gchq.gaffer.mapstore.SingleUseMapStore - gaffer.store.properties.class=uk.gov.gchq.gaffer.mapstore.MapStoreProperties - ``` diff --git a/mkdocs.yml b/mkdocs.yml index 44d477cc3d..536ec73a9b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -45,59 +45,6 @@ plugins: nav: - Home: 'index.md' - - 'Getting Started': - - 'Gaffer Basics': 'getting-started/basics.md' - - 'Quickstart': 'getting-started/quickstart.md' - - 'Example Deployment': - - 'Project Setup': 'getting-started/example-deployment/project-setup.md' - - 'Writing the Schema': 'getting-started/example-deployment/writing-the-schema.md' - - 'Running the Deployment': 'getting-started/example-deployment/running-the-deployment.md' - - 'Using the API': 'getting-started/example-deployment/using-the-api.md' - - 'Advanced User Guide': - - 'Aggregation': 'getting-started/advanced-guide/aggregation.md' - - 'Import and Export': - - 'Using CSV': 'getting-started/advanced-guide/import-export/csv.md' - - 'Cardinality': 'getting-started/advanced-guide/cardinality.md' - - 'Gaffer 2.0': - - Changelist: 'gaffer2.0/changelist.md' - - Deprecations: 'gaffer2.0/deprecations.md' - - Dependencies: 'gaffer2.0/dependencies.md' - - 'Accumulo Migration': 'gaffer2.0/accumulo-migration.md' - - 'Federation Changes': 'gaffer2.0/federation-changes.md' - - 'Accumulo Kerberos': 'gaffer2.0/accumulo-kerberos.md' - - 'Log4j in Gaffer': 'gaffer2.0/log4j.md' - - 'Developer Docs': - - 'Development': 'dev/development.md' - - 'Remote Coding Environments': 'dev/remote-coding-environments.md' - - 'Managing Dependencies': 'dev/managing-dependencies/managing-dependencies.md' - - 'Ways of Working': 'dev/ways-of-working.md' - - 'Sketches custom deserialisation': 'dev/rest-api-sketches.md' - - 'Docker': 'dev/docker.md' - - Kubernetes: - - 'Kubernetes Guide': 'dev/kubernetes-guide/kubernetes.md' - - 'Deploy an Empty Graph': 'dev/kubernetes-guide/deploy-empty-graph.md' - - 'Add your schema': 'dev/kubernetes-guide/deploy-schema.md' - - 'Change the Graph Id and Description': 'dev/kubernetes-guide/change-graph-metadata.md' - - 'Adding your own libraries and functions': 'dev/kubernetes-guide/add-libraries.md' - - 'Changing Accumulo Passwords': 'dev/kubernetes-guide/change-accumulo-passwords.md' - - Components: - - 'Components/Maven Modules': 'dev/components/components.md' - - 'Cache': 'dev/components/cache.md' - - 'Data': 'dev/components/data.md' - - 'Graph': 'dev/components/graph.md' - - 'Store': 'dev/components/store.md' - - 'Operation': 'dev/components/operation.md' - - 'Serialisation': 'dev/components/serialisation.md' - - 'Integration Tests': 'dev/components/integration-test.md' - - 'Core REST': 'dev/components/core-rest.md' - - 'Spring REST': 'dev/components/spring-rest.md' - - 'Accumulo Store': 'dev/components/accumulo-store.md' - - Libraries: - - dev/components/libraries/bitmap.md - - dev/components/libraries/flink.md - - dev/components/libraries/sketches.md - - dev/components/libraries/spark.md - - dev/components/libraries/time.md - User Guide: - 'Gaffer Basics': - 'What is Gaffer?': 'user-guide/gaffer-basics/what-is-gaffer.md' From efb9b92171cfaa3cc3a8d2bbf3c3f1fcea734c3a Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:58:35 +0000 Subject: [PATCH 12/20] Corrected all doc links --- .github/workflows/build-docs.yml | 2 +- .../gaffer-stores/accumulo-store.md | 4 ++-- .../gaffer-stores/store-guide.md | 12 ++++++------ docs/change-notes/changelist/v2-changes.md | 14 +++++++------- .../migrating-from-v1-to-v2/accumulo-kerberos.md | 2 +- .../migrating-from-v1-to-v2/deprecations.md | 2 +- .../example-deployment/project-setup.md | 4 ++-- .../example-deployment/using-the-api.md | 6 +++--- .../example-deployment/writing-the-schema.md | 4 ++-- .../project-structure/components/accumulo-store.md | 4 ++-- .../project-structure/components/cache.md | 4 ++-- .../project-structure/components/data.md | 2 +- .../project-structure/components/graph.md | 2 +- .../components/libraries/flink.md | 2 +- .../components/libraries/sketches.md | 2 +- .../components/libraries/spark.md | 2 +- .../project-structure/components/operation.md | 2 +- docs/development-guide/rest-api-sketches.md | 2 +- docs/reference/operations-guide/misc.md | 2 +- .../gaffer-basics/what-is-cardinality.md | 4 ++-- docs/user-guide/getting-started/api.md | 2 +- docs/user-guide/getting-started/schema.md | 2 +- 22 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index c36b7bb135..aead51201a 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -30,7 +30,7 @@ jobs: run: pip install -r requirements.txt - name: Build using MkDocs - run: mkdocs build + run: mkdocs build -s - name: Check Links uses: gaurav-nelson/github-action-markdown-link-check@v1 diff --git a/docs/administration-guide/gaffer-stores/accumulo-store.md b/docs/administration-guide/gaffer-stores/accumulo-store.md index 026319c2ba..7628ffd46f 100644 --- a/docs/administration-guide/gaffer-stores/accumulo-store.md +++ b/docs/administration-guide/gaffer-stores/accumulo-store.md @@ -129,7 +129,7 @@ Note that here `elements` could be a never-ending stream of `Element`s and the a To ingest data via bulk import, a MapReduce job is used to convert your data into files of Accumulo key-value pairs that are pre-sorted to match the distribution of data in Accumulo. Once these files are created, Accumulo moves them from their current location in HDFS to the correct directory within Accumulo's data directory. The data in them is then available for query immediately. -Gaffer provides code to make this as simple as possible. The `AddElementsFromHdfs` operation is used to bulk import data. See [AddElementsFromHdfs](../operations-guide/hdfs.md#addelementsfromhdfs) for examples. +Gaffer provides code to make this as simple as possible. The `AddElementsFromHdfs` operation is used to bulk import data. See [AddElementsFromHdfs](../../reference/operations-guide/hdfs.md#addelementsfromhdfs) for examples. ## Visibility @@ -185,7 +185,7 @@ In Gaffer's `AccumuloStore` a key-package contains all the logic for: A key-package is an implementation of the `AccumuloKeyPackage` interface. Gaffer provides two implementations: `ByteEntityKeyPackage` and `ClassicKeyPackage`. These names are essentially meaningless. The "classic" in `ClassicKeyPackage` refers to the fact that it is similar to the implementation in the first version of Gaffer (known as "Gaffer1"). -Both key-packages should provide good performance for most use-cases. There will be slight differences in performance between the two for different types of query. The `ByteEntityKeyPackage` will be slightly faster if the query specifies that only out-going or in-coming edges are required. The `ClassicKeyPackage` will be faster when querying for all edges involving a pair of vertices. See the Key-Packages part of the [Accumulo Store Implementation page](../../dev/components/accumulo-store.md) for more information about these key-packages. +Both key-packages should provide good performance for most use-cases. There will be slight differences in performance between the two for different types of query. The `ByteEntityKeyPackage` will be slightly faster if the query specifies that only out-going or in-coming edges are required. The `ClassicKeyPackage` will be faster when querying for all edges involving a pair of vertices. See the Key-Packages part of the [Accumulo Store Implementation page](../../development-guide/project-structure/components/accumulo-store.md) for more information about these key-packages. ## Advanced properties diff --git a/docs/administration-guide/gaffer-stores/store-guide.md b/docs/administration-guide/gaffer-stores/store-guide.md index a3e1ff0e73..af99b2aa22 100644 --- a/docs/administration-guide/gaffer-stores/store-guide.md +++ b/docs/administration-guide/gaffer-stores/store-guide.md @@ -1,13 +1,13 @@ # Stores Guide -A Gaffer Store represents the backing database responsible for storing (or facilitating access to) a graph. Ordinarily a Store provides backing for a single graph. Stores which provide access to other stores can support multiple graphs. So far only the [Federated Store](federated.md) supports this. +A Gaffer Store represents the backing database responsible for storing (or facilitating access to) a graph. Ordinarily a Store provides backing for a single graph. Stores which provide access to other stores can support multiple graphs. So far only the [Federated Store](federated-store.md) supports this. Gaffer currently supplies the following store implementations: -- [Map Store](map.md) - Simple in-memory store -- [Accumulo Store](accumulo.md) - [Apache Accumulo](https://accumulo.apache.org/) backed store -- [Proxy Store](proxy.md) - Delegates/forwards queries to another Gaffer REST -- [Federated Store](federated.md) - Federates queries across multiple graphs +- [Map Store](map-store.md) - Simple in-memory store +- [Accumulo Store](accumulo-store.md) - [Apache Accumulo](https://accumulo.apache.org/) backed store +- [Proxy Store](proxy-store.md) - Delegates/forwards queries to another Gaffer REST +- [Federated Store](federated-store.md) - Federates queries across multiple graphs ## Caches @@ -19,7 +19,7 @@ Gaffer comes with three cache implementations: The `HashMap` cache is not persistent. If using the Hazelcast instance of the Cache service be aware that once the last node shuts down, all data will be lost. This is due to the data being held in memory in a distributed system. -For information on implementing caches, see [the cache developer docs page](../../dev/components/cache.md). +For information on implementing caches, see [the cache developer docs page](../../development-guide/project-structure/components/cache.md). ### Cache configuration diff --git a/docs/change-notes/changelist/v2-changes.md b/docs/change-notes/changelist/v2-changes.md index dbcec62b14..9de5bb1ef6 100644 --- a/docs/change-notes/changelist/v2-changes.md +++ b/docs/change-notes/changelist/v2-changes.md @@ -3,12 +3,12 @@ Below is a summary of changes that have been made in Gaffer version 2. ### Accumulo 2 Support -The Accumulo store now supports Accumulo 2 and Hadoop 3 by default, with support for Accumulo 1 and Hadoop 2 retained. See the [Accumulo Migration page](accumulo-migration.md) for more information about this change. +The Accumulo store now supports Accumulo 2 and Hadoop 3 by default, with support for Accumulo 1 and Hadoop 2 retained. See the [Accumulo Migration page](../migrating-from-v1-to-v2/accumulo-migration.md) for more information about this change. ### Federated Store Improvements The Federated Operation was added to greatly improve flexibility of using a Federated Store. !!! danger "Breaking change" - To migrate, please see the [Federated Store Changes page](federation-changes.md). + To migrate, please see the [Federated Store Changes page](../migrating-from-v1-to-v2/federation-changes.md). ### Cache Improvements and fixes All "caches" within Gaffer received a lot of bug fixes which should make them significantly more stable and consistent over time. This should improve usability of FederatedStores, NamedOperations and NamedViews. @@ -18,12 +18,12 @@ All "caches" within Gaffer received a lot of bug fixes which should make them si ### Removal of Deprecated code All of Gaffer 1's deprecated code has been removed. !!! danger "Breaking change" - To migrate, please see the [deprecations](deprecations.md) page. + To migrate, please see the [deprecations](../migrating-from-v1-to-v2/deprecations.md) page. ### Dependency Upgrades Dependencies have been updated, where possible to the latest version, removing vulnerabilities. !!! danger "Breaking change" - You will need to migrate your dependencies to be compatible with Gaffer 2's new dependency versions. Please see the [dependencies](dependencies.md) page for full details. + You will need to migrate your dependencies to be compatible with Gaffer 2's new dependency versions. Please see the [dependencies](../migrating-from-v1-to-v2/dependencies.md) page for full details. ### Federated and Proxy store fixes A lot of bugs have been fixed that should facilitate FederatedStores with ProxyStores in them. @@ -50,10 +50,10 @@ The HBase and Parquet stores have been removed from Gaffer in version 2. We made There is now a maven profile that will swap dependency versions so you can build Gaffer with Java 11. The code has also been updated to build with both Java versions. ### Accumulo Kerberos Authentication Support -The Accumulo store now supports authenticating to Accumulo and HDFS using Kerberos, in addition to username/password. For more information, see the [Kerberos support page](accumulo-kerberos.md). +The Accumulo store now supports authenticating to Accumulo and HDFS using Kerberos, in addition to username/password. For more information, see the [Kerberos support page](../migrating-from-v1-to-v2/accumulo-kerberos.md). ### CSV Import and Export -Basic support for importing and exporting [CSVs](../getting-started/guide/csv.md) has been added. +Basic support for importing and exporting [CSVs](../../user-guide/query/api-querying/import-export-data.md) has been added. ### All operations can now be used within NamedOperations Previously, `GetElementsBetweenSets` could not be used within a NamedOperation as it used `inputB`. `GetElementsBetweenSets` and `inputB` have both been deprecated and instead you should use `GetElementsBetweenSetsPairs`. @@ -127,4 +127,4 @@ This will mean subgraphs added to FederatedStores can have additional operation ``` 1. Schema left empty for brevity - 2. This example operation enables file import. Read more in the [CSV](../getting-started/guide/csv.md) docs. + 2. This example operation enables file import. Read more in the [CSV](../../user-guide/query/api-querying/import-export-data.md) docs. diff --git a/docs/change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md b/docs/change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md index a3e3cbb414..331a68a6be 100644 --- a/docs/change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md +++ b/docs/change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md @@ -32,7 +32,7 @@ The location of this config file can be specified using the `ACCUMULO_CLIENT_CON Other than this file, Accumulo libraries and configuration files do not need to be installed on the Gaffer host. ### Gaffer `store.properties` configuration -In addition to the usual [Accumulo Store settings](../reference/stores-guide/accumulo.md#properties-file), these extra options must be specified for Kerberos: +In addition to the usual [Accumulo Store settings](../../administration-guide/gaffer-stores/accumulo-store.md#properties-file), these extra options must be specified for Kerberos: ``` accumulo.kerberos.enable=true accumulo.kerberos.principal=gaffer/host.domain@REALM.NAME diff --git a/docs/change-notes/migrating-from-v1-to-v2/deprecations.md b/docs/change-notes/migrating-from-v1-to-v2/deprecations.md index 0319e5506f..0466650c35 100644 --- a/docs/change-notes/migrating-from-v1-to-v2/deprecations.md +++ b/docs/change-notes/migrating-from-v1-to-v2/deprecations.md @@ -40,7 +40,7 @@ In both serialisers, the method `deserialise(byte[])` has been marked as depreca ## Removal of Seed Matching ### [`operation.SeedMatching`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/SeedMatching.java) -SeedMatching has been removed from Gaffer. This was previously used in [get](../reference/operations-guide/get.md) operations, like [`GetElements`](../reference/operations-guide/get.md#getelements), to select whether you wanted your results to contain only Elements that are the same type as the seed, or both Edges and Entities. For more info, see the Gaffer 1.X docs page on [SeedMatching](https://gchq.github.io/gaffer-doc/v1docs/getting-started/user-guide/filtering.html#seedmatching). As described in the Gaffer 1.X docs, `SeedMatching` can be replaced with a `View`. The default behaviour in Gaffer is the same as if you used `seed_matching="RELATED"`, so **if this is the case, there is no migration required**. However, if you used `seed_matching="EQUAL"`, you will need to migrate to a `View`. +SeedMatching has been removed from Gaffer. This was previously used in [get](../../reference/operations-guide/get.md) operations, like [`GetElements`](../../reference/operations-guide/get.md#getelements),, to select whether you wanted your results to contain only Elements that are the same type as the seed, or both Edges and Entities. For more info, see the Gaffer 1.X docs page on [SeedMatching](https://gchq.github.io/gaffer-doc/v1docs/getting-started/user-guide/filtering.html#seedmatching). As described in the Gaffer 1.X docs, `SeedMatching` can be replaced with a `View`. The default behaviour in Gaffer is the same as if you used `seed_matching="RELATED"`, so **if this is the case, there is no migration required**. However, if you used `seed_matching="EQUAL"`, you will need to migrate to a `View`. ??? example "SeedMatching migration with EdgeSeeds" Where SeedMatching was used to only get back Edges from EdgeSeeds diff --git a/docs/development-guide/example-deployment/project-setup.md b/docs/development-guide/example-deployment/project-setup.md index 1eb687b6c4..6cff6bab24 100644 --- a/docs/development-guide/example-deployment/project-setup.md +++ b/docs/development-guide/example-deployment/project-setup.md @@ -43,7 +43,7 @@ the graph in [Neo4j syntax](https://neo4j.com/labs/apoc/4.4/export/csv/#export-d !!! note "" Please note that Gaffer often requires additional information about the data such as, `:String` on the column headers to help with typing of the values. This is demonstrated below - in the raw file. There's more detail on this in the [OpenCypher documentation](../advanced-guide/import-export/csv.md#opencypher-formats). + in the raw file. There's more detail on this in the [OpenCypher documentation](../../user-guide/query/api-querying/import-export-data.md#opencypher-formats). === "Table" | _id | name | age | lang | _labels | _start | _end | _type | weight | @@ -165,7 +165,7 @@ set the name and short description. The store properties file is used to configure how Gaffer will store its data. There are a few different stores available for Gaffer, these are explained in more detail in the [reference -documentation](../../reference/stores-guide/stores.md), but by default you must provide a store +documentation](../../administration-guide/gaffer-stores/store-guide.md), but by default you must provide a store class and a store properties class. For this example we are using an Accumulo store as it is recommended for efficient storage and retrieval of large data volumes. It's set up requires a few custom properties which are outlined in the following file. diff --git a/docs/development-guide/example-deployment/using-the-api.md b/docs/development-guide/example-deployment/using-the-api.md index d592ff307c..3088d0260e 100644 --- a/docs/development-guide/example-deployment/using-the-api.md +++ b/docs/development-guide/example-deployment/using-the-api.md @@ -1,6 +1,6 @@ # Using the API -As covered in the [Gaffer basics](../basics.md) the main POST request used in the API is +As covered in the [Getting Started API page](../../user-guide/getting-started/api.md) the main POST request used in the API is `/graph/operations/execute`. This part of the guide will cover the general usage of this part of the API and walk through some general operations you might want to use to load data or query. @@ -61,7 +61,7 @@ from the neo4j formatted CSV file. The operation chain above essentially mirrors the stages in the previous diagram. In the example chain we first ingest the data via the `ImportFromLocalFile` class (an additional operation we added -via the [`operationsDeclarations.json`](./additional-config.md#operations-declarations)), which +via the `operationsDeclarations.json`, which streams the data from the CSV file into the next `GenerateElements` operation. For the generator we have selected the built in `Neo4jCsvElementGenerator` class, this is already @@ -188,4 +188,4 @@ asking, *"get all the `Created` edges on node `v1` that have a `weight` greater !!! tip As you can see filtering is based around predicates which are similar to if else statements in traditional programming. For a full list of available predicates refer to the reference - [documentation](../../reference/predicates-guide/predicates.md). + [documentation](../../reference//predicates-guide/predicates.md) \ No newline at end of file diff --git a/docs/development-guide/example-deployment/writing-the-schema.md b/docs/development-guide/example-deployment/writing-the-schema.md index 0be48ba33c..982d67920c 100644 --- a/docs/development-guide/example-deployment/writing-the-schema.md +++ b/docs/development-guide/example-deployment/writing-the-schema.md @@ -29,7 +29,7 @@ standard way to do this is a JSON config file in the schema directory. The filen called something like `elements.json`, the name is not special as all files under the `schema` directory will be merged into a master schema but its still recommended to use an appropriate name. -As covered in the [Gaffer basics](../basics.md), to write a schema you can see that there are some +As covered in the [Getting Started Schema page](../../user-guide/getting-started/schema.md), to write a schema you can see that there are some required fields, but largely a schema is highly specific to your input data. Starting with the `entities` from the example, we can see there will be two distinct types of nodes @@ -87,7 +87,7 @@ small we only need to add one edge - the `Created` edge. This is a directed edge } ``` -As discussed in the [basics guide](../basics.md), edges have some mandatory fields. Starting with +As discussed in the [user schema guide](../../user-guide/getting-started/schema.md), edges have some mandatory fields. Starting with the `source` and `destination` fields, these must match the types associated with the vertex field in the relevant entities. From the example, we can see that the source of a `Created` edge is a `Person` so we will use the placeholder type we set as the `vertex` field which is diff --git a/docs/development-guide/project-structure/components/accumulo-store.md b/docs/development-guide/project-structure/components/accumulo-store.md index 4fb633a477..fd0617e16f 100644 --- a/docs/development-guide/project-structure/components/accumulo-store.md +++ b/docs/development-guide/project-structure/components/accumulo-store.md @@ -2,7 +2,7 @@ The [accumulo-store module](https://github.com/gchq/Gaffer/tree/master/core/store) is an implementation of the Store API which uses Apache Accumulo. -This page contains brief details on the internal implementation of the `AccumuloStore`. For information on configuring and using this store, see [the Accumulo Store reference page](../../reference/stores-guide/accumulo.md). +This page contains brief details on the internal implementation of the `AccumuloStore`. For information on configuring and using this store, see [the Accumulo Store reference page](../../../administration-guide/gaffer-stores/accumulo-store.md). ## Introduction @@ -21,7 +21,7 @@ The core of the functionality is implemented in the key-packages, the iterators ## Key-packages -As noted in the [Key-packages section of the Accumulo Store reference](../../reference/stores-guide/accumulo.md#key-packages), key-packages are responsible for converting `Element`s to and from key-value pairs, for creating ranges of keys containing all data relevant to a particular query, and for configuring the Iterators. Gaffer provides two key-packages: `ByteEntityKeyPackage` and `ClassicKeyPackage`. Advanced users are able to create their own key-packages if they wish - see [options for future key-packages](#options-for-future-key-packages) for some ideas. +As noted in the [Key-packages section of the Accumulo Store reference](../../../administration-guide/gaffer-stores/accumulo-store.md#key-packages), key-packages are responsible for converting `Element`s to and from key-value pairs, for creating ranges of keys containing all data relevant to a particular query, and for configuring the Iterators. Gaffer provides two key-packages: `ByteEntityKeyPackage` and `ClassicKeyPackage`. Advanced users are able to create their own key-packages if they wish - see [options for future key-packages](#options-for-future-key-packages) for some ideas. Before these key-packages are described, we review the main design goals: diff --git a/docs/development-guide/project-structure/components/cache.md b/docs/development-guide/project-structure/components/cache.md index da049572d4..dcf9c1e261 100644 --- a/docs/development-guide/project-structure/components/cache.md +++ b/docs/development-guide/project-structure/components/cache.md @@ -7,7 +7,7 @@ CacheServiceLoader.getService(); ``` By default, there is no service loaded so if you're using a component that makes use of the `CacheServiceLoader`, be sure to specify the service class in the `store.properties` file. -See the [Stores reference guide](../../reference/stores-guide/stores.md#cache-configuration) for configuration info. +See the [Stores reference guide](../../../administration-guide/gaffer-stores/store-guide.md#cache-configuration) for configuration info. If using an external cache service (anything found in the cache library) be sure to include the library as a dependency: ```xml @@ -21,4 +21,4 @@ If using an external cache service (anything found in the cache library) be sure When run in a servlet context, the `CacheServiceLoader` should be shutdown gracefully by the `ServletLifecycleListener` found in the REST package. Do not trust the shutdown hook in a servlet context. If running outside a servlet environment, you can either call shutdown on the cache service manually or use the shutdown hook upon initialisation of the cache service loader. -For information on Gaffer caches and cache configuration, see [the cache section of the Stores Guide](../../reference/stores-guide/stores.md#caches). +For information on Gaffer caches and cache configuration, see [the cache section of the Stores Guide](../../../administration-guide/gaffer-stores/store-guide.md#caches). diff --git a/docs/development-guide/project-structure/components/data.md b/docs/development-guide/project-structure/components/data.md index 368be8eeff..44e4f28f10 100644 --- a/docs/development-guide/project-structure/components/data.md +++ b/docs/development-guide/project-structure/components/data.md @@ -8,7 +8,7 @@ It also contains the logic for processing these `Element`s - `ElementAggregator` Gaffer makes use of Java 8's Function and Predicate interfaces to aggregate, transform and filter data. To allow these Function and Predicate classes to process tuples we make use of the [Koryphe](https://github.com/gchq/koryphe/tree/master) library. Koryphe allows us to wrap the Gaffer Elements in a tuple and pass it any Function or Predicate. -You can use any of our implementations ([see reference pages](../../reference/intro.md)) or write your own. +You can use any of our implementations ([see reference pages](../../../reference/intro.md)) or write your own. All the following classes will act on one or more Element identifiers (vertex/source/destination/directed) or properties. If you implement the Java 8 interfaces directly, you would need to add the following `JsonType` annotation to your class: diff --git a/docs/development-guide/project-structure/components/graph.md b/docs/development-guide/project-structure/components/graph.md index ffff6b9d3e..aabfbe2733 100644 --- a/docs/development-guide/project-structure/components/graph.md +++ b/docs/development-guide/project-structure/components/graph.md @@ -13,7 +13,7 @@ But, essentially a graph requires just 3 things: some store properties, a schema See the [Graph Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/graph/package-summary.html) for further documentation. ## Store Properties -The store properties tells the graph the type of store to connect to along with any required connection details. See the [Stores](../../reference/stores-guide/stores.md) reference page for more information on the different Stores for Gaffer. +The store properties tells the graph the type of store to connect to along with any required connection details. See the [Stores](../../../administration-guide/gaffer-stores/store-guide.md) reference page for more information on the different Stores for Gaffer. ## Schema The schema is passed to the store to instruct the store how to store and process the data. See [Schemas](https://gchq.github.io/gaffer-doc/v1docs/getting-started/developer-guide/schemas.html) for more information - v1 docs, to be updated and added to this documentation in future. diff --git a/docs/development-guide/project-structure/components/libraries/flink.md b/docs/development-guide/project-structure/components/libraries/flink.md index 8207bded0d..2be65a513e 100644 --- a/docs/development-guide/project-structure/components/libraries/flink.md +++ b/docs/development-guide/project-structure/components/libraries/flink.md @@ -11,7 +11,7 @@ In order to make use of the flink libraries you will need to include this librar ``` -For information on registering and using flink operations, see the [Flink Operations guide](../../../reference/operations-guide/flink.md). +For information on registering and using flink operations, see the [Flink Operations guide](../../../../reference/operations-guide/flink.md). ## I am getting errors when running the Flink operations on a cluster This could be to do with the way the Gaffer Store class is serialised and distributed around the cluster. To distribute the job, Flink requires all of the components of the job to be Serializable. diff --git a/docs/development-guide/project-structure/components/libraries/sketches.md b/docs/development-guide/project-structure/components/libraries/sketches.md index 4be4defa4b..f337d3ac01 100644 --- a/docs/development-guide/project-structure/components/libraries/sketches.md +++ b/docs/development-guide/project-structure/components/libraries/sketches.md @@ -11,4 +11,4 @@ In order to make use of the sketches libraries you will need to include this lib ``` -For information on configuring and using sketches, see the [Cardinality guide](../../../getting-started/guide/cardinality.md#how-to-add-cardinality-to-your-graph) (for configuring `SketchesJsonModules` expand "Additional config"). +For information on configuring and using sketches, see the [Cardinality guide](../../../../user-guide/gaffer-basics/what-is-cardinality.md) (for configuring `SketchesJsonModules` expand "Additional config"). diff --git a/docs/development-guide/project-structure/components/libraries/spark.md b/docs/development-guide/project-structure/components/libraries/spark.md index 6fbbbaf9e0..327e3e9763 100644 --- a/docs/development-guide/project-structure/components/libraries/spark.md +++ b/docs/development-guide/project-structure/components/libraries/spark.md @@ -20,4 +20,4 @@ To use spark with Accumulo you will need to include this dependency: ``` -For information on registering and using spark operations, see the [Spark Operations guide](../../../reference/operations-guide/spark.md). +For information on registering and using spark operations, see the [Spark Operations guide](../../../../reference/operations-guide/spark.md). diff --git a/docs/development-guide/project-structure/components/operation.md b/docs/development-guide/project-structure/components/operation.md index 1047a8b927..b3d41a8aaf 100644 --- a/docs/development-guide/project-structure/components/operation.md +++ b/docs/development-guide/project-structure/components/operation.md @@ -72,7 +72,7 @@ public static class Builder extends Operation.BaseBuilder ### Operation Scores For use with a `ScoreOperationChain`, some `Operation`s may require a custom way of calculating an associated score, therefore an implementation of the `ScoreResolver` interface may be required. -There is a `DefaultScoreResolver` to which the custom implementation should delegate, in a manner specific to the new Operation. For more info, see [ScoreOperationChain](/docs/reference/stores-guide/stores.md#scoreoperationchain) and [ScoreOperationChainExample](../../reference/operations-guide/misc.md#scoreoperationchain). +There is a `DefaultScoreResolver` to which the custom implementation should delegate, in a manner specific to the new Operation. For more info, see [ScoreOperationChain](/docs/reference/stores-guide/stores.md#scoreoperationchain) and [ScoreOperationChainExample](../../../reference/operations-guide/misc.md#scoreoperationchain). ### Documentation diff --git a/docs/development-guide/rest-api-sketches.md b/docs/development-guide/rest-api-sketches.md index 5b6ba6055f..0a468f97b0 100644 --- a/docs/development-guide/rest-api-sketches.md +++ b/docs/development-guide/rest-api-sketches.md @@ -1,7 +1,7 @@ # Using Sketches with the REST API This page explains some nuances and special steps required when using classes from the Sketches library with the REST API. If you just want to know how to use the sketches libraries to -use cardinality, see the [cardinality](../getting-started/guide/cardinality.md) docs page. +use cardinality, see the [cardinality](../user-guide/gaffer-basics/what-is-cardinality.md) docs page. ## Sketches Library diff --git a/docs/reference/operations-guide/misc.md b/docs/reference/operations-guide/misc.md index ade67db695..e82b2741fc 100644 --- a/docs/reference/operations-guide/misc.md +++ b/docs/reference/operations-guide/misc.md @@ -49,7 +49,7 @@ Gets data from an endpoint. [Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/ Determines a "score" for an OperationChain. This is used to determine whether a particular user has the required permissions to execute a given OperationChain. [Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/operation/impl/ScoreOperationChain.html) -This operation requires Store configuration to be set before it can be used. See the [Store Guide](../stores-guide/stores.md#ScoreOperationChain) for how to do this. +This operation requires Store configuration to be set before it can be used. See the [Store Guide](../../administration-guide/gaffer-stores/store-guide.md) for how to do this. ### Example ScoreOperationChain diff --git a/docs/user-guide/gaffer-basics/what-is-cardinality.md b/docs/user-guide/gaffer-basics/what-is-cardinality.md index 9aa2f0bcd8..b9dff18ebe 100644 --- a/docs/user-guide/gaffer-basics/what-is-cardinality.md +++ b/docs/user-guide/gaffer-basics/what-is-cardinality.md @@ -88,8 +88,8 @@ when you add your edges, the second is to use a generator that can do it for you ### Manually adding cardinality entities Adding a cardinality entity between vertex 'A' and 'B' using `AddElements`. For more examples of -different property types, see the sketches [dev -docs](../../dev/rest-api-sketches.md#primitive-data-types-over-json) page. +different property types, see the sketches on the [developer +guide](../../development-guide/rest-api-sketches#primitive-data-types-over-json) page. === "Java" diff --git a/docs/user-guide/getting-started/api.md b/docs/user-guide/getting-started/api.md index e0e605c40a..6231a41131 100644 --- a/docs/user-guide/getting-started/api.md +++ b/docs/user-guide/getting-started/api.md @@ -18,7 +18,7 @@ JSON to carry out operations on the Gaffer graph. Gaffer provides many pre built available to use and can be chained together for more complex use cases. However be aware, that operation chains are usually highly specific to the data and results you wish to extract from the graph so please refer to the reference guide on [Gaffer -operations](../reference/operations-guide/operations.md) for more detail on this. +operations](../../reference/operations-guide/operations.md) !!! example "Example operation chain using rest API" diff --git a/docs/user-guide/getting-started/schema.md b/docs/user-guide/getting-started/schema.md index 430e9f1284..963aca06e2 100644 --- a/docs/user-guide/getting-started/schema.md +++ b/docs/user-guide/getting-started/schema.md @@ -57,7 +57,7 @@ For an `edge` the following fields are required: !!! note "" The type here, `"true"` or `"false"` needs to be defined in the types schema using a class that evaluates to it. This is demonstrated in the [example - deployment](./example-deployment/writing-the-schema.md) document. + deployment](../../development-guide/example-deployment/writing-the-schema.md) document. - `destination` - A user defined type for the destination node the edge goes to. From b617f4681c6bf87f52d09764116240f85928e0cd Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Thu, 7 Sep 2023 15:46:26 +0000 Subject: [PATCH 13/20] Added Endpoints guide --- docs/reference/endpoints-guide/endpoints.md | 0 mkdocs.yml | 5 +++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 docs/reference/endpoints-guide/endpoints.md diff --git a/docs/reference/endpoints-guide/endpoints.md b/docs/reference/endpoints-guide/endpoints.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/mkdocs.yml b/mkdocs.yml index 536ec73a9b..9fbfbfe5cf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -136,7 +136,7 @@ nav: - 'Federation Changes': 'change-notes/migrating-from-v1-to-v2/federation-changes.md' - 'Accumulo Kerberos': 'change-notes/migrating-from-v1-to-v2/accumulo-kerberos.md' - 'Log4j in Gaffer': 'change-notes/migrating-from-v1-to-v2/log4j.md' - - 'Reference': + - Reference: - 'Reference Guide': 'reference/intro.md' - 'Glossary': 'reference/glossary.md' - 'Javadoc': 'reference/javadoc.md' @@ -172,7 +172,8 @@ nav: - 'Accumulo Operations': 'reference/operations-guide/accumulo.md' - 'Spark Operations': 'reference/operations-guide/spark.md' - 'Misc Operations': 'reference/operations-guide/misc.md' - + - Endpoints: + - 'Endpoints Guide': 'reference/endpoints-guide/endpoints.md' theme: name: material logo: assets/icon.svg From 89e5d44a283ea4b76a53d243bf288adb38953afd Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Thu, 7 Sep 2023 15:57:01 +0000 Subject: [PATCH 14/20] Corrected site contents in index.md --- docs/index.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/index.md b/docs/index.md index d1a7adf7c7..e09948e246 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,12 +16,9 @@ Gaffer [2.0.0](https://github.com/gchq/Gaffer/releases/tag/gaffer2-2.0.0) was re Site Contents ------------- -- [Gaffer 2.0](gaffer2.0/changelist/) - Information on the Gaffer 2.0 release, including changelogs, deprecation and migration details. -- [Getting Started](getting-started/quickstart/) - Guides for using and deploying Gaffer. -- [Developer Info](dev/development/) - Information about developing Gaffer itself. - [User Guide](user-guide/gaffer-basics/what-is-gaffer/) - Information about how to use Gaffer. - [Development Guide](development-guide/getting-started/) - Information about developing Gaffer itself. -- [Administration GUide](adiministration-guide/how-to-run-gaffer/gaffer-docker/) - Information about creating a gaffer instance. +- [Administration Guide](administration-guide/where-to-run-gaffer/gaffer-docker/) - Information about creating a gaffer instance. - [Change Notes](change-notes/changelist/v2-changes/) - Information on the Gaffer 2.0 release, including changelogs, deprecation and migration details. - [Reference](reference/intro/) - Documentation for Gaffer Operations, Predicates, Functions, Binary Operators and Properties. From b711ef8da557675c9ad69aa61330c976cda34aef Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:28:15 +0000 Subject: [PATCH 15/20] Added Introduction pages and removed getting started pages --- docs/administration-guide/introduction.md | 0 .../{getting-started.md => introduction.md} | 0 docs/index.md | 6 ++--- docs/user-guide/{getting-started => }/api.md | 0 docs/user-guide/introduction.md | 0 .../{getting-started => }/schema.md | 0 mkdocs.yml | 26 +++++++++---------- 7 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 docs/administration-guide/introduction.md rename docs/development-guide/{getting-started.md => introduction.md} (100%) rename docs/user-guide/{getting-started => }/api.md (100%) create mode 100644 docs/user-guide/introduction.md rename docs/user-guide/{getting-started => }/schema.md (100%) diff --git a/docs/administration-guide/introduction.md b/docs/administration-guide/introduction.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/development-guide/getting-started.md b/docs/development-guide/introduction.md similarity index 100% rename from docs/development-guide/getting-started.md rename to docs/development-guide/introduction.md diff --git a/docs/index.md b/docs/index.md index e09948e246..ee0395572c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,9 +16,9 @@ Gaffer [2.0.0](https://github.com/gchq/Gaffer/releases/tag/gaffer2-2.0.0) was re Site Contents ------------- -- [User Guide](user-guide/gaffer-basics/what-is-gaffer/) - Information about how to use Gaffer. -- [Development Guide](development-guide/getting-started/) - Information about developing Gaffer itself. -- [Administration Guide](administration-guide/where-to-run-gaffer/gaffer-docker/) - Information about creating a gaffer instance. +- [User Guide](user-guide/introduction/) - Information about how to use Gaffer. +- [Development Guide](development-guide/introduction/) - Information about developing Gaffer itself. +- [Administration Guide](administration-guide/introduction/) - Information about creating a gaffer instance. - [Change Notes](change-notes/changelist/v2-changes/) - Information on the Gaffer 2.0 release, including changelogs, deprecation and migration details. - [Reference](reference/intro/) - Documentation for Gaffer Operations, Predicates, Functions, Binary Operators and Properties. diff --git a/docs/user-guide/getting-started/api.md b/docs/user-guide/api.md similarity index 100% rename from docs/user-guide/getting-started/api.md rename to docs/user-guide/api.md diff --git a/docs/user-guide/introduction.md b/docs/user-guide/introduction.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/user-guide/getting-started/schema.md b/docs/user-guide/schema.md similarity index 100% rename from docs/user-guide/getting-started/schema.md rename to docs/user-guide/schema.md diff --git a/mkdocs.yml b/mkdocs.yml index 9fbfbfe5cf..92b2aaa34b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -46,6 +46,7 @@ plugins: nav: - Home: 'index.md' - User Guide: + - 'Introduction': 'user-guide/introduction.md' - 'Gaffer Basics': - 'What is Gaffer?': 'user-guide/gaffer-basics/what-is-gaffer.md' - 'What is a Graph?': 'user-guide/gaffer-basics/what-is-a-graph.md' @@ -53,9 +54,8 @@ nav: - 'What is Python?': 'user-guide/gaffer-basics/what-is-python.md' - 'What is Cardinality?': 'user-guide/gaffer-basics/what-is-cardinality.md' - 'What is Aggregation?': 'user-guide/gaffer-basics/what-is-aggregation.md' - - 'Getting Started': - - 'API': 'user-guide/getting-started/api.md' - - 'Schema': 'user-guide/getting-started/schema.md' + - 'API': 'user-guide/api.md' + - 'Schema': 'user-guide/schema.md' - 'Query': - 'Python Extension': - 'Python Extension Guide': 'user-guide/query/python-extension/python-extension.md' @@ -68,7 +68,7 @@ nav: - 'Gremlin Guide': 'user-guide/query/gremlin/gremlin.md' - 'Examples': 'user-guide/query/gremlin/examples.md' - Development Guide: - - 'Getting Started': 'development-guide/getting-started.md' + - 'Introduction': 'development-guide/introduction.md' - 'Example Deployment': - 'Project Setup': 'development-guide/example-deployment/project-setup.md' - 'Writing the Schema': 'development-guide/example-deployment/writing-the-schema.md' @@ -100,15 +100,15 @@ nav: - Maven Dependencies: - 'Spring': 'development-guide/project-structure/maven-dependencies/spring.md' - Administration Guide: - - 'Getting Started': - - 'Docker': 'administration-guide/where-to-run-gaffer/gaffer-docker.md' - - 'Kubernetes': - - 'Kubernetes Guide': 'administration-guide/where-to-run-gaffer/kubernetes-guide/kubernetes-guide.md' - - 'Deploy an Empty Graph': 'administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-empty-graph.md' - - 'Add your Schema': 'administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-schema.md' - - 'Change the Graph ID and Description': 'administration-guide/where-to-run-gaffer/kubernetes-guide/change-graph-metadata.md' - - 'Adding your Own Libraries and Functions': 'administration-guide/where-to-run-gaffer/kubernetes-guide/add-libraries.md' - - 'Changing Accumulo Passwords': 'administration-guide/where-to-run-gaffer/kubernetes-guide/change-accumulo-passwords.md' + - 'Introduction': 'administration-guide/introduction.md' + - 'Docker': 'administration-guide/where-to-run-gaffer/gaffer-docker.md' + - 'Kubernetes': + - 'Kubernetes Guide': 'administration-guide/where-to-run-gaffer/kubernetes-guide/kubernetes-guide.md' + - 'Deploy an Empty Graph': 'administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-empty-graph.md' + - 'Add your Schema': 'administration-guide/where-to-run-gaffer/kubernetes-guide/deploy-schema.md' + - 'Change the Graph ID and Description': 'administration-guide/where-to-run-gaffer/kubernetes-guide/change-graph-metadata.md' + - 'Adding your Own Libraries and Functions': 'administration-guide/where-to-run-gaffer/kubernetes-guide/add-libraries.md' + - 'Changing Accumulo Passwords': 'administration-guide/where-to-run-gaffer/kubernetes-guide/change-accumulo-passwords.md' - 'Store Setup': - 'Store Guide': 'administration-guide/gaffer-stores/store-guide.md' - 'Accumulo Store': 'administration-guide/gaffer-stores/accumulo-store.md' From 95f4f79290d4676c0d7f1834ce0ecb641346de5a Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:34:09 +0000 Subject: [PATCH 16/20] Fixed links --- docs/development-guide/example-deployment/using-the-api.md | 2 +- .../example-deployment/writing-the-schema.md | 4 ++-- docs/user-guide/api.md | 2 +- docs/user-guide/schema.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/development-guide/example-deployment/using-the-api.md b/docs/development-guide/example-deployment/using-the-api.md index 3088d0260e..d0bea0c2e9 100644 --- a/docs/development-guide/example-deployment/using-the-api.md +++ b/docs/development-guide/example-deployment/using-the-api.md @@ -1,6 +1,6 @@ # Using the API -As covered in the [Getting Started API page](../../user-guide/getting-started/api.md) the main POST request used in the API is +As covered in the [Getting Started API page](../../user-guide/api.md) the main POST request used in the API is `/graph/operations/execute`. This part of the guide will cover the general usage of this part of the API and walk through some general operations you might want to use to load data or query. diff --git a/docs/development-guide/example-deployment/writing-the-schema.md b/docs/development-guide/example-deployment/writing-the-schema.md index 982d67920c..64134ffeb1 100644 --- a/docs/development-guide/example-deployment/writing-the-schema.md +++ b/docs/development-guide/example-deployment/writing-the-schema.md @@ -29,7 +29,7 @@ standard way to do this is a JSON config file in the schema directory. The filen called something like `elements.json`, the name is not special as all files under the `schema` directory will be merged into a master schema but its still recommended to use an appropriate name. -As covered in the [Getting Started Schema page](../../user-guide/getting-started/schema.md), to write a schema you can see that there are some +As covered in the [Getting Started Schema page](../../user-guide/schema.md), to write a schema you can see that there are some required fields, but largely a schema is highly specific to your input data. Starting with the `entities` from the example, we can see there will be two distinct types of nodes @@ -87,7 +87,7 @@ small we only need to add one edge - the `Created` edge. This is a directed edge } ``` -As discussed in the [user schema guide](../../user-guide/getting-started/schema.md), edges have some mandatory fields. Starting with +As discussed in the [user schema guide](../../user-guide/schema.md), edges have some mandatory fields. Starting with the `source` and `destination` fields, these must match the types associated with the vertex field in the relevant entities. From the example, we can see that the source of a `Created` edge is a `Person` so we will use the placeholder type we set as the `vertex` field which is diff --git a/docs/user-guide/api.md b/docs/user-guide/api.md index 6231a41131..51f96c0e29 100644 --- a/docs/user-guide/api.md +++ b/docs/user-guide/api.md @@ -18,7 +18,7 @@ JSON to carry out operations on the Gaffer graph. Gaffer provides many pre built available to use and can be chained together for more complex use cases. However be aware, that operation chains are usually highly specific to the data and results you wish to extract from the graph so please refer to the reference guide on [Gaffer -operations](../../reference/operations-guide/operations.md) +operations](../reference/operations-guide/operations.md) !!! example "Example operation chain using rest API" diff --git a/docs/user-guide/schema.md b/docs/user-guide/schema.md index 963aca06e2..1307f6c735 100644 --- a/docs/user-guide/schema.md +++ b/docs/user-guide/schema.md @@ -57,7 +57,7 @@ For an `edge` the following fields are required: !!! note "" The type here, `"true"` or `"false"` needs to be defined in the types schema using a class that evaluates to it. This is demonstrated in the [example - deployment](../../development-guide/example-deployment/writing-the-schema.md) document. + deployment](../development-guide/example-deployment/writing-the-schema.md) document. - `destination` - A user defined type for the destination node the edge goes to. From 0dcd5688e755b82f9be0d596852ec07c0988a24d Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Wed, 13 Sep 2023 09:38:19 +0000 Subject: [PATCH 17/20] Addressed broken links comments --- docs/change-notes/migrating-from-v1-to-v2/deprecations.md | 2 +- docs/development-guide/example-deployment/using-the-api.md | 2 +- docs/reference/operations-guide/misc.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/change-notes/migrating-from-v1-to-v2/deprecations.md b/docs/change-notes/migrating-from-v1-to-v2/deprecations.md index 0466650c35..3a03134a63 100644 --- a/docs/change-notes/migrating-from-v1-to-v2/deprecations.md +++ b/docs/change-notes/migrating-from-v1-to-v2/deprecations.md @@ -40,7 +40,7 @@ In both serialisers, the method `deserialise(byte[])` has been marked as depreca ## Removal of Seed Matching ### [`operation.SeedMatching`](https://github.com/gchq/Gaffer/blob/gaffer2-1.21.1/core/operation/src/main/java/uk/gov/gchq/gaffer/operation/SeedMatching.java) -SeedMatching has been removed from Gaffer. This was previously used in [get](../../reference/operations-guide/get.md) operations, like [`GetElements`](../../reference/operations-guide/get.md#getelements),, to select whether you wanted your results to contain only Elements that are the same type as the seed, or both Edges and Entities. For more info, see the Gaffer 1.X docs page on [SeedMatching](https://gchq.github.io/gaffer-doc/v1docs/getting-started/user-guide/filtering.html#seedmatching). As described in the Gaffer 1.X docs, `SeedMatching` can be replaced with a `View`. The default behaviour in Gaffer is the same as if you used `seed_matching="RELATED"`, so **if this is the case, there is no migration required**. However, if you used `seed_matching="EQUAL"`, you will need to migrate to a `View`. +SeedMatching has been removed from Gaffer. This was previously used in [get](../../reference/operations-guide/get.md) operations, like [`GetElements`](../../reference/operations-guide/get.md#getelements), to select whether you wanted your results to contain only Elements that are the same type as the seed, or both Edges and Entities. For more info, see the Gaffer 1.X docs page on [SeedMatching](https://gchq.github.io/gaffer-doc/v1docs/getting-started/user-guide/filtering.html#seedmatching). As described in the Gaffer 1.X docs, `SeedMatching` can be replaced with a `View`. The default behaviour in Gaffer is the same as if you used `seed_matching="RELATED"`, so **if this is the case, there is no migration required**. However, if you used `seed_matching="EQUAL"`, you will need to migrate to a `View`. ??? example "SeedMatching migration with EdgeSeeds" Where SeedMatching was used to only get back Edges from EdgeSeeds diff --git a/docs/development-guide/example-deployment/using-the-api.md b/docs/development-guide/example-deployment/using-the-api.md index d0bea0c2e9..dbd793ae03 100644 --- a/docs/development-guide/example-deployment/using-the-api.md +++ b/docs/development-guide/example-deployment/using-the-api.md @@ -188,4 +188,4 @@ asking, *"get all the `Created` edges on node `v1` that have a `weight` greater !!! tip As you can see filtering is based around predicates which are similar to if else statements in traditional programming. For a full list of available predicates refer to the reference - [documentation](../../reference//predicates-guide/predicates.md) \ No newline at end of file + [documentation](../../reference/predicates-guide/predicates.md). \ No newline at end of file diff --git a/docs/reference/operations-guide/misc.md b/docs/reference/operations-guide/misc.md index e82b2741fc..0a3f2d8972 100644 --- a/docs/reference/operations-guide/misc.md +++ b/docs/reference/operations-guide/misc.md @@ -49,7 +49,7 @@ Gets data from an endpoint. [Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/ Determines a "score" for an OperationChain. This is used to determine whether a particular user has the required permissions to execute a given OperationChain. [Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/operation/impl/ScoreOperationChain.html) -This operation requires Store configuration to be set before it can be used. See the [Store Guide](../../administration-guide/gaffer-stores/store-guide.md) for how to do this. +This operation requires Store configuration to be set before it can be used. See the [Store Guide](../../administration-guide/gaffer-stores/store-guide.md#scoreoperationchain) for how to do this. ### Example ScoreOperationChain From 47333b10c3b9d8157c7ac5290f325cb4a24709fc Mon Sep 17 00:00:00 2001 From: l46978 <141753189+l46978@users.noreply.github.com> Date: Wed, 13 Sep 2023 11:38:20 +0000 Subject: [PATCH 18/20] Fixed Broken Links and added bare-ck in admin guide changes --- docs/administration-guide/schema.md | 430 ++++++++++++++++++ .../example-deployment/writing-the-schema.md | 4 +- .../project-structure/components/graph.md | 4 +- docs/getting-started/basics.md | 0 docs/user-guide/gaffer-basics/schema.md | 430 ------------------ 5 files changed, 434 insertions(+), 434 deletions(-) delete mode 100644 docs/getting-started/basics.md delete mode 100644 docs/user-guide/gaffer-basics/schema.md diff --git a/docs/administration-guide/schema.md b/docs/administration-guide/schema.md index e69de29bb2..9adb120d00 100644 --- a/docs/administration-guide/schema.md +++ b/docs/administration-guide/schema.md @@ -0,0 +1,430 @@ +# Schemas +This page delves into more detail on Schemas. As seen in the other guides, there are two main components to schemas: Elements and Types. + +Schemas are often separated into two JSON files (`elements.json` & `types.json`), but it's worth remembering that your schema can be broken up further if required. + +For example if you have a Graph that is made up of 5 different data sources, you may find it easier to develop and maintain by splitting your schema into multiple parts, for example 5 Element Schemas and 1 Types schema. +Alternatively, a single overall schema file could be used for Elements and Types, as there is no requirement to have a single Types schemas. +When you construct your Gaffer graph you must provide all the Schema parts. If provided separately, these will then be merged together to form an internal schema used for the Graph. + +When using Java directly, you don't need to use JSON files to store you schemas - although this is still recommended. See the [Java API](#java-api) section for more about interacting with Schemas in Java. + +The sections below walkthrough the features of Schemas in detail and explain how to accurately model your data in Gaffer. + + +## Elements schema +The Elements schema is designed to be a high level document describing what information your Graph contains, i.e. the different kinds of edges and entities and the list of properties associated with each. +Essentially this part of the schema should just be a list of all the entities and edges in the graph. +Edges describe the relationship between a source vertex and a destination vertex. +Entities describe a vertex. Edges describe the relationship between a source vertex and a destination vertex. +We use the term "element" to mean either an edge or an entity. + +When defining an element we must provide a "group". This is a unique string identifier for each element. +Groups must be completely unique and cannot be shared between edges and entities. + +### Edges +Edges must have the following: + +- `source` - Type of object to use as the source vertex in your graph. Needs to be a type defined in the [Types schema](#types). +- `destination` - Type of object to use as the destination vertex in your graph. Can either be the same type as `source`, or a different type. +- `directed` - Tells Gaffer if the edge is directed or undirected. Needs to be a type which means true or false, see [Types](#true--false) for more info. + +When an Edge is undirected in Gaffer (`directed` is `false`), it is treated as if the relationship was bidirectional, meaning that the vertices of the edge do not have an authoritative source and destination. +Thus, the undirected edges `A -- B` and `B -- A` are equal, and will be aggregated with any other undirected edge with the same source and destination. +Gaffer will present the undirected edges vertices in natural ordering, so a client will also see the above edge presented as `A, B`. +This means when adding an undirected Edge of `A -- B`, it will be aggregated with another existing undirected Edge of `B -- A`. + +### Entities +Entities must have a `vertex` field, which is similar to the `source` and `destination` fields on an edge. +For example, modelling `London -> Paris` requires an entity definition to represent the city and an edge definition to represent the relationship. +In the previous example the entity `vertex` field and the edge `source` and `destination` fields would all be of the same type. + +### Optional Element fields +Edges and Entities can optionally have the following fields: + +- `description` - A simple string which should provide some context and explain what the element is. +- `parents` - An array of parent group names. These must relate to the same sort of element as the child, for example an edge cannot have an entity as a parent. Elements can inherit any information from multiple parent elements. Fields will be merged/overridden, so the hierarchy of parents is important. Any fields that are defined in the child element will also merge or override information taken from the parents. +- `properties` - Properties are defined by a map of key-value pairs of property names to property types. Property types are described in the Types schema. +- `groupBy` - Allows you to specify extra properties (in addition to the element group and vertices) to use for controlling when similar elements should be grouped together and summarised. By default Gaffer uses the element group and its vertices to group similar elements together when aggregating and summarising elements. +- `visibilityProperty` - Used to specify the property to use as a visibility property when using visibility properties in your graph. If sensitive elements have a visibility property then set this field to that property name. This ensures Gaffer knows to restrict access to sensitive elements. +- `timestampProperty` - Used to specify timestamp property in your graph, so Gaffer Stores know to treat that property specially. Setting this is optional and does not affect the queries available to users. This property allows Store implementations like Accumulo to optimise the way the timestamp property is persisted. For these stores using it can have a very slight performance improvement due to the lazy loading of properties. For more information [see the timestamp section of the Accumulo Store Reference](gaffer-stores/accumulo-store.md#timestamp). +- `aggregate` - Specifies if aggregation is enabled for this element group. True by default. If you would like to disable aggregation, set this to false. + +These 2 optional fields are for advanced users. They can go in the Elements Schema, however we have split them out into separate Validation and Aggregation Schema files for this page, so the logic doesn't complicate the Elements schema. + +- `validateFunctions` - An array of selections and predicates to be applied to an element. This allows you to validate based on multiple properties at once. E.g. check a timestamp property together with a time to live property to check if the element should be aged off. Individual property validation is best done as a validateFunction in the property type definition in Types schema. +- `aggregateFunctions` - An array of selections and binary operators to be applied to an element. This allows you to aggregate based on multiple properties at once. It is important to note that types of properties (groupBy, non-groupBy, visibility) cannot be aggregated in the same aggregate function. The timestamp property is treated as a non-groupBy property. Individual property aggregation is best done as a aggregateFunction in the property type definition in the Types schema. + +### Example Elements Schema +This example Elements schema is a subset of the schema used in the [Gaffer Road Traffic example graph](https://github.com/gchq/Gaffer/blob/master/example/road-traffic/README.md). + +```json +{ + "edges": { + "RoadUse": { + "description": "A directed edge representing vehicles moving from junction A to junction B.", + "source": "junction", + "destination": "junction", + "directed": "true", + "properties": { + "startDate": "date.earliest", + "endDate": "date.latest", + "count": "count.long" + }, + "groupBy": [ + "startDate", + "endDate" + ] + }, + "RoadHasJunction": { + "description": "A directed edge from each road to all the junctions on that road.", + "source": "road", + "destination": "junction", + "directed": "true" + } + }, + "entities": { + "Cardinality": { + "description": "An entity that is added to every vertex representing the connectivity of the vertex.", + "vertex": "anyVertex", + "properties": { + "edgeGroup": "set", + "hllSketch": "hllSketch", + "count": "count.long" + }, + "groupBy": [ + "edgeGroup" + ] + } + } +} +``` + +Here is the Validation Schema. It contains advanced validation, that is applied to multiple properties within an Element group. + +```json +{ + "edges": { + "RoadUse": { + "validateFunctions": [ + { + "selection": [ + "startDate", + "endDate" + ], + "predicate": { + "class": "uk.gov.gchq.koryphe.impl.predicate.IsXLessThanY" + } + } + ] + } + } +} +``` + +Here is the Aggregation Schema. It contains advanced aggregation, that is applied to multiple properties within an Element group. +The multi property aggregate function defined here overrides the relevant single property aggregate functions defined in the Types schema. + +```json +{ + "edges": { + "RoadUse": { + "aggregateFunctions": [ + { + "selection": [ + "startDate", + "endDate" + ], + "binaryOperator": { + "class": "uk.gov.gchq.gaffer.doc.dev.aggregator.ExampleTuple2BinaryOperator" + } + } + ] + } + } +} +``` + + +## Types +All types used in the Elements schema must be defined in the Types parts of the schema. These Types explain to Gaffer what types of properties to expect and how to deal with them. + +For each type you must provide the following information: + +- `class` - The Java class of the type. + +You can optionally provide the following: + +- `description` - String which should provide a description of the type. +- `validateFunctions` - Array of predicates that will be executed against every type value to validate it. To improve performance, put quicker/low cost functions first in the array. +- `aggregateFunction` - The aggregate binary operator to use to aggregate/summarise/merge property values of the same type together. +- `serialiser` - Object which contains a field class which represents the Java class of the serialiser to use, and potentially arguments depending on the serialiser. If this is not provided Gaffer will attempt to select an appropriate one for you (only available for simple Java types). + +### True & False +The `directed` field for edge elements is either "true" or "false". Currently, you need to manually define these as types in your Types schema. +The easiest way to do this is to create a type called "true", "false" and define it as being a boolean with a filter predicate to check the boolean is either true or false respectively. +There's an example of a "true" type in the example Types section below. + +### Serialisers +Gaffer will automatically choose serialisers for you for some core types. +Where possible you should let Gaffer choose for you, as it will choose the optimal serialiser for the type and your usage. +For custom types you will need to write your own serialiser. + +When manually choosing a serialiser for your schema you will need to take the following into consideration. + +For vertex serialisation and groupBy properties you must choose serialisers that are consistent. +A consistent serialiser will serialise the equal objects into exactly the same values (bytes). +For example the JavaSerialiser and FreqMapSerialiser are not consistent. + +When using an ordered store (a store that implements the ORDERED StoreTrait, such as Accumulo), you need to check whether the serialisers are ordered: + + - For vertex serialisation you must use a serialiser that is ordered. + - For groupBy properties you must use a serialiser that is ordered. + - All other properties can be serialised with ordered/unordered serialisers. + +### Example Types Schema + +Here is an example Types schema (goes with the example Elements schema above): + +```json +{ + "types": { + "junction": { + "description": "A road junction represented by a String.", + "class": "java.lang.String" + }, + "road": { + "description": "A road represented by a String.", + "class": "java.lang.String" + }, + "anyVertex": { + "description": "An String vertex - used for cardinalities", + "class": "java.lang.String" + }, + "count.long": { + "description": "A long count that must be greater than or equal to 0.", + "class": "java.lang.Long", + "validateFunctions": [ + { + "class": "uk.gov.gchq.koryphe.impl.predicate.IsMoreThan", + "orEqualTo": true, + "value": { + "java.lang.Long": 0 + } + } + ], + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Sum" + } + }, + "true": { + "description": "A simple boolean that must always be true.", + "class": "java.lang.Boolean", + "validateFunctions": [ + { + "class": "uk.gov.gchq.koryphe.impl.predicate.IsTrue" + } + ] + }, + "date.earliest": { + "description": "A Date that when aggregated together will be the earliest date.", + "class": "java.util.Date", + "validateFunctions": [ + { + "class": "uk.gov.gchq.koryphe.impl.predicate.Exists" + } + ], + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Min" + } + }, + "date.latest": { + "description": "A Date that when aggregated together will be the latest date.", + "class": "java.util.Date", + "validateFunctions": [ + { + "class": "uk.gov.gchq.koryphe.impl.predicate.Exists" + } + ], + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Max" + } + }, + "set": { + "class": "java.util.TreeSet", + "aggregateFunction": { + "class": "uk.gov.gchq.koryphe.impl.binaryoperator.CollectionConcat" + } + }, + "hllSketch": { + "class": "org.apache.datasketches.hll.HllSketch", + "aggregateFunction": { + "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.binaryoperator.HllSketchAggregator" + }, + "serialiser": { + "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.serialisation.HllSketchSerialiser" + } + } + } +} +``` + + +## Full Schema Example +Once the schema has been loaded into a graph the parent elements are merged into the children for performance reasons. This is what the full schema created from the above example schema parts looks like: + +```json +{ + "edges" : { + "RoadUse" : { + "description" : "A directed edge representing vehicles moving from junction A to junction B.", + "source" : "junction", + "destination" : "junction", + "directed" : "true", + "properties" : { + "startDate" : "date.earliest", + "endDate" : "date.latest", + "count" : "count.long" + }, + "groupBy" : [ "startDate", "endDate" ], + "aggregateFunctions" : [ { + "selection" : [ "startDate", "endDate" ], + "binaryOperator" : { + "class" : "uk.gov.gchq.gaffer.doc.dev.aggregator.ExampleTuple2BinaryOperator" + } + } ], + "validateFunctions" : [ { + "selection" : [ "startDate", "endDate" ], + "predicate" : { + "class" : "uk.gov.gchq.koryphe.impl.predicate.IsXLessThanY" + } + } ] + }, + "RoadHasJunction" : { + "description" : "A directed edge from each road to all the junctions on that road.", + "source" : "road", + "destination" : "junction", + "directed" : "true" + } + }, + "entities" : { + "Cardinality" : { + "description" : "An entity that is added to every vertex representing the connectivity of the vertex.", + "vertex" : "anyVertex", + "properties" : { + "edgeGroup" : "set", + "hllSketch" : "hllSketch", + "count" : "count.long" + }, + "groupBy" : [ "edgeGroup" ] + } + }, + "types" : { + "junction" : { + "description" : "A road junction represented by a String.", + "class" : "java.lang.String" + }, + "road" : { + "description" : "A road represented by a String.", + "class" : "java.lang.String" + }, + "anyVertex" : { + "description" : "An String vertex - used for cardinalities", + "class" : "java.lang.String" + }, + "count.long" : { + "description" : "A long count that must be greater than or equal to 0.", + "class" : "java.lang.Long", + "aggregateFunction" : { + "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.Sum" + }, + "validateFunctions" : [ { + "class" : "uk.gov.gchq.koryphe.impl.predicate.IsMoreThan", + "orEqualTo" : true, + "value" : { + "java.lang.Long" : 0 + } + } ] + }, + "true" : { + "description" : "A simple boolean that must always be true.", + "class" : "java.lang.Boolean", + "validateFunctions" : [ { + "class" : "uk.gov.gchq.koryphe.impl.predicate.IsTrue" + } ] + }, + "date.earliest" : { + "description" : "A Date that when aggregated together will be the earliest date.", + "class" : "java.util.Date", + "aggregateFunction" : { + "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.Min" + }, + "validateFunctions" : [ { + "class" : "uk.gov.gchq.koryphe.impl.predicate.Exists" + } ] + }, + "date.latest" : { + "description" : "A Date that when aggregated together will be the latest date.", + "class" : "java.util.Date", + "aggregateFunction" : { + "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.Max" + }, + "validateFunctions" : [ { + "class" : "uk.gov.gchq.koryphe.impl.predicate.Exists" + } ] + }, + "set" : { + "class" : "java.util.TreeSet", + "aggregateFunction" : { + "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.CollectionConcat" + } + }, + "hllSketch" : { + "class" : "org.apache.datasketches.hll.HllSketch", + "aggregateFunction" : { + "class" : "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.binaryoperator.HllSketchAggregator" + }, + "serialiser" : { + "class" : "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.serialisation.HllSketchSerialiser" + } + } + } +} +``` + + +## Java API + +Schemas can be loaded from a JSON file directly using the [`fromJSON()` method of the `Schema` class](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/Schema.html#fromJson(java.io.InputStream...)). This accepts `byte[]`, `InputStream` or `Path` types, for example: + +```java +Schema mySchema = Schema.fromJson(Paths.get("path/to/schema.json")); +``` + +While it's easiest to create Schema objects using the JSON approach, you can create them directly from their component classes in Java. +Constructing a Schema from scratch this way would be tedious, but you could use this to create your own solutions for Schema generation by building on these components. + +A Schema can be created using its `Builder()` method. [See Schema Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/Schema.html). + +```java +Schema mySchema = new Schema.Builder().edges(edges) + .entities(entities) + .types(types) + .build(); +``` + +The Edges, Entities and Types are supplied as Maps with String keys. See the Javadoc covering [`SchemaEdgeDefinition`](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/SchemaEdgeDefinition.html), [`SchemaEntityDefinition`](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/SchemaEntityDefinition.html) and [`TypeDefinition`](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/TypeDefinition.html) for information about their builders. + +```java +Map myEdges = new HashMap<>(); +myEdges.put("myEdge", new SchemaEdgeDefinition.Builder()... + .build()); + +Map myEntities = new HashMap<>(); +myEntities.put("myEntity", new SchemaEntityDefinition.Builder()... + .build()); + +Map myTypes = new HashMap<>(); +myTypes.put("myType", new TypeDefinition.Builder()... + .build()); +``` diff --git a/docs/development-guide/example-deployment/writing-the-schema.md b/docs/development-guide/example-deployment/writing-the-schema.md index 558d2cf8a6..19c190a82f 100644 --- a/docs/development-guide/example-deployment/writing-the-schema.md +++ b/docs/development-guide/example-deployment/writing-the-schema.md @@ -27,7 +27,7 @@ In Gaffer an element refers to any object in the graph, i.e. your nodes (vertexe up a graph we need to tell Gaffer what objects are in the graph and the properties they have. The standard way to do this is a JSON config file in the schema directory. The filename can just be called something like `elements.json`, the name is not special as all files under the `schema` -directory will be [merged into a master schema](../advanced-guide/schema.md), but we recommended +directory will be [merged into a master schema](../../administration-guide/schema.md), but we recommended using an appropriate name. As covered in the [Getting Started Schema page](../../user-guide/schema.md), to write a schema you can see that there are some @@ -147,7 +147,7 @@ extended schema below. ## Types Schema -The other schema that now needs to be written is the types [schema](../advanced-guide/schema.md). As you have seen in the elements +The other schema that now needs to be written is the types [schema](../../administration-guide/schema.md). As you have seen in the elements schema there are some placeholder types added as the values for many of the keys. These types work similarly to if you have ever programmed in a strongly typed language, they are essentially the wrapper for the value to encapsulate it. diff --git a/docs/development-guide/project-structure/components/graph.md b/docs/development-guide/project-structure/components/graph.md index 3193625861..7d13e9d9e2 100644 --- a/docs/development-guide/project-structure/components/graph.md +++ b/docs/development-guide/project-structure/components/graph.md @@ -26,14 +26,14 @@ Graph graph = new Graph.Builder() .build(); ``` -Instead of a store properties, a store can be passed in directly. This is the easiest way to configure schema-less stores (Proxy and Federated Stores) using Java. See the [Java section of the Proxy Store reference page for an example](../../reference/stores-guide/proxy.md#using-a-proxystore-from-java). Using a Proxy Store will allow for connecting to an existing remote graph through Java, without needing to use the REST API or JSON directly. +Instead of a store properties, a store can be passed in directly. This is the easiest way to configure schema-less stores (Proxy and Federated Stores) using Java. See the [Java section of the Proxy Store reference page for an example](../../../administration-guide/gaffer-stores/proxy-store.md#using-a-proxystore-from-java). Using a Proxy Store will allow for connecting to an existing remote graph through Java, without needing to use the REST API or JSON directly. ## Store Properties The store properties tells the graph the type of store to connect to along with any required connection details. See the [Stores](../../../administration-guide/gaffer-stores/store-guide.md) reference page for more information on the different Stores for Gaffer. ## Schema The schema is passed to the store to instruct the store how to store and process the data. -See [Schemas](../../getting-started/advanced-guide/schema.md) for detailed information on schemas and the [Java API section of that page](../../getting-started/advanced-guide/schema.md#java-api) for lower level info. +See [Schemas](../../../administration-guide/schema.md) for detailed information on schemas and the [Java API section of that page](../../../administration-guide/schema.md#java-api) for lower level info. ## Graph Configuration The graph configuration allows you to apply special customisations to the Graph instance. The only required field is the `graphId`. diff --git a/docs/getting-started/basics.md b/docs/getting-started/basics.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/user-guide/gaffer-basics/schema.md b/docs/user-guide/gaffer-basics/schema.md deleted file mode 100644 index d6c35e101a..0000000000 --- a/docs/user-guide/gaffer-basics/schema.md +++ /dev/null @@ -1,430 +0,0 @@ -# Schemas -This page delves into more detail on Schemas. As seen in the other guides, there are two main components to schemas: Elements and Types. - -Schemas are often separated into two JSON files (`elements.json` & `types.json`), but it's worth remembering that your schema can be broken up further if required. - -For example if you have a Graph that is made up of 5 different data sources, you may find it easier to develop and maintain by splitting your schema into multiple parts, for example 5 Element Schemas and 1 Types schema. -Alternatively, a single overall schema file could be used for Elements and Types, as there is no requirement to have a single Types schemas. -When you construct your Gaffer graph you must provide all the Schema parts. If provided separately, these will then be merged together to form an internal schema used for the Graph. - -When using Java directly, you don't need to use JSON files to store you schemas - although this is still recommended. See the [Java API](#java-api) section for more about interacting with Schemas in Java. - -The sections below walkthrough the features of Schemas in detail and explain how to accurately model your data in Gaffer. - - -## Elements schema -The Elements schema is designed to be a high level document describing what information your Graph contains, i.e. the different kinds of edges and entities and the list of properties associated with each. -Essentially this part of the schema should just be a list of all the entities and edges in the graph. -Edges describe the relationship between a source vertex and a destination vertex. -Entities describe a vertex. Edges describe the relationship between a source vertex and a destination vertex. -We use the term "element" to mean either an edge or an entity. - -When defining an element we must provide a "group". This is a unique string identifier for each element. -Groups must be completely unique and cannot be shared between edges and entities. - -### Edges -Edges must have the following: - -- `source` - Type of object to use as the source vertex in your graph. Needs to be a type defined in the [Types schema](#types). -- `destination` - Type of object to use as the destination vertex in your graph. Can either be the same type as `source`, or a different type. -- `directed` - Tells Gaffer if the edge is directed or undirected. Needs to be a type which means true or false, see [Types](#true--false) for more info. - -When an Edge is undirected in Gaffer (`directed` is `false`), it is treated as if the relationship was bidirectional, meaning that the vertices of the edge do not have an authoritative source and destination. -Thus, the undirected edges `A -- B` and `B -- A` are equal, and will be aggregated with any other undirected edge with the same source and destination. -Gaffer will present the undirected edges vertices in natural ordering, so a client will also see the above edge presented as `A, B`. -This means when adding an undirected Edge of `A -- B`, it will be aggregated with another existing undirected Edge of `B -- A`. - -### Entities -Entities must have a `vertex` field, which is similar to the `source` and `destination` fields on an edge. -For example, modelling `London -> Paris` requires an entity definition to represent the city and an edge definition to represent the relationship. -In the previous example the entity `vertex` field and the edge `source` and `destination` fields would all be of the same type. - -### Optional Element fields -Edges and Entities can optionally have the following fields: - -- `description` - A simple string which should provide some context and explain what the element is. -- `parents` - An array of parent group names. These must relate to the same sort of element as the child, for example an edge cannot have an entity as a parent. Elements can inherit any information from multiple parent elements. Fields will be merged/overridden, so the hierarchy of parents is important. Any fields that are defined in the child element will also merge or override information taken from the parents. -- `properties` - Properties are defined by a map of key-value pairs of property names to property types. Property types are described in the Types schema. -- `groupBy` - Allows you to specify extra properties (in addition to the element group and vertices) to use for controlling when similar elements should be grouped together and summarised. By default Gaffer uses the element group and its vertices to group similar elements together when aggregating and summarising elements. -- `visibilityProperty` - Used to specify the property to use as a visibility property when using visibility properties in your graph. If sensitive elements have a visibility property then set this field to that property name. This ensures Gaffer knows to restrict access to sensitive elements. -- `timestampProperty` - Used to specify timestamp property in your graph, so Gaffer Stores know to treat that property specially. Setting this is optional and does not affect the queries available to users. This property allows Store implementations like Accumulo to optimise the way the timestamp property is persisted. For these stores using it can have a very slight performance improvement due to the lazy loading of properties. For more information [see the timestamp section of the Accumulo Store Reference](../../reference/stores-guide/accumulo.md#timestamp). -- `aggregate` - Specifies if aggregation is enabled for this element group. True by default. If you would like to disable aggregation, set this to false. - -These 2 optional fields are for advanced users. They can go in the Elements Schema, however we have split them out into separate Validation and Aggregation Schema files for this page, so the logic doesn't complicate the Elements schema. - -- `validateFunctions` - An array of selections and predicates to be applied to an element. This allows you to validate based on multiple properties at once. E.g. check a timestamp property together with a time to live property to check if the element should be aged off. Individual property validation is best done as a validateFunction in the property type definition in Types schema. -- `aggregateFunctions` - An array of selections and binary operators to be applied to an element. This allows you to aggregate based on multiple properties at once. It is important to note that types of properties (groupBy, non-groupBy, visibility) cannot be aggregated in the same aggregate function. The timestamp property is treated as a non-groupBy property. Individual property aggregation is best done as a aggregateFunction in the property type definition in the Types schema. - -### Example Elements Schema -This example Elements schema is a subset of the schema used in the [Gaffer Road Traffic example graph](https://github.com/gchq/Gaffer/blob/master/example/road-traffic/README.md). - -```json -{ - "edges": { - "RoadUse": { - "description": "A directed edge representing vehicles moving from junction A to junction B.", - "source": "junction", - "destination": "junction", - "directed": "true", - "properties": { - "startDate": "date.earliest", - "endDate": "date.latest", - "count": "count.long" - }, - "groupBy": [ - "startDate", - "endDate" - ] - }, - "RoadHasJunction": { - "description": "A directed edge from each road to all the junctions on that road.", - "source": "road", - "destination": "junction", - "directed": "true" - } - }, - "entities": { - "Cardinality": { - "description": "An entity that is added to every vertex representing the connectivity of the vertex.", - "vertex": "anyVertex", - "properties": { - "edgeGroup": "set", - "hllSketch": "hllSketch", - "count": "count.long" - }, - "groupBy": [ - "edgeGroup" - ] - } - } -} -``` - -Here is the Validation Schema. It contains advanced validation, that is applied to multiple properties within an Element group. - -```json -{ - "edges": { - "RoadUse": { - "validateFunctions": [ - { - "selection": [ - "startDate", - "endDate" - ], - "predicate": { - "class": "uk.gov.gchq.koryphe.impl.predicate.IsXLessThanY" - } - } - ] - } - } -} -``` - -Here is the Aggregation Schema. It contains advanced aggregation, that is applied to multiple properties within an Element group. -The multi property aggregate function defined here overrides the relevant single property aggregate functions defined in the Types schema. - -```json -{ - "edges": { - "RoadUse": { - "aggregateFunctions": [ - { - "selection": [ - "startDate", - "endDate" - ], - "binaryOperator": { - "class": "uk.gov.gchq.gaffer.doc.dev.aggregator.ExampleTuple2BinaryOperator" - } - } - ] - } - } -} -``` - - -## Types -All types used in the Elements schema must be defined in the Types parts of the schema. These Types explain to Gaffer what types of properties to expect and how to deal with them. - -For each type you must provide the following information: - -- `class` - The Java class of the type. - -You can optionally provide the following: - -- `description` - String which should provide a description of the type. -- `validateFunctions` - Array of predicates that will be executed against every type value to validate it. To improve performance, put quicker/low cost functions first in the array. -- `aggregateFunction` - The aggregate binary operator to use to aggregate/summarise/merge property values of the same type together. -- `serialiser` - Object which contains a field class which represents the Java class of the serialiser to use, and potentially arguments depending on the serialiser. If this is not provided Gaffer will attempt to select an appropriate one for you (only available for simple Java types). - -### True & False -The `directed` field for edge elements is either "true" or "false". Currently, you need to manually define these as types in your Types schema. -The easiest way to do this is to create a type called "true", "false" and define it as being a boolean with a filter predicate to check the boolean is either true or false respectively. -There's an example of a "true" type in the example Types section below. - -### Serialisers -Gaffer will automatically choose serialisers for you for some core types. -Where possible you should let Gaffer choose for you, as it will choose the optimal serialiser for the type and your usage. -For custom types you will need to write your own serialiser. - -When manually choosing a serialiser for your schema you will need to take the following into consideration. - -For vertex serialisation and groupBy properties you must choose serialisers that are consistent. -A consistent serialiser will serialise the equal objects into exactly the same values (bytes). -For example the JavaSerialiser and FreqMapSerialiser are not consistent. - -When using an ordered store (a store that implements the ORDERED StoreTrait, such as Accumulo), you need to check whether the serialisers are ordered: - - - For vertex serialisation you must use a serialiser that is ordered. - - For groupBy properties you must use a serialiser that is ordered. - - All other properties can be serialised with ordered/unordered serialisers. - -### Example Types Schema - -Here is an example Types schema (goes with the example Elements schema above): - -```json -{ - "types": { - "junction": { - "description": "A road junction represented by a String.", - "class": "java.lang.String" - }, - "road": { - "description": "A road represented by a String.", - "class": "java.lang.String" - }, - "anyVertex": { - "description": "An String vertex - used for cardinalities", - "class": "java.lang.String" - }, - "count.long": { - "description": "A long count that must be greater than or equal to 0.", - "class": "java.lang.Long", - "validateFunctions": [ - { - "class": "uk.gov.gchq.koryphe.impl.predicate.IsMoreThan", - "orEqualTo": true, - "value": { - "java.lang.Long": 0 - } - } - ], - "aggregateFunction": { - "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Sum" - } - }, - "true": { - "description": "A simple boolean that must always be true.", - "class": "java.lang.Boolean", - "validateFunctions": [ - { - "class": "uk.gov.gchq.koryphe.impl.predicate.IsTrue" - } - ] - }, - "date.earliest": { - "description": "A Date that when aggregated together will be the earliest date.", - "class": "java.util.Date", - "validateFunctions": [ - { - "class": "uk.gov.gchq.koryphe.impl.predicate.Exists" - } - ], - "aggregateFunction": { - "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Min" - } - }, - "date.latest": { - "description": "A Date that when aggregated together will be the latest date.", - "class": "java.util.Date", - "validateFunctions": [ - { - "class": "uk.gov.gchq.koryphe.impl.predicate.Exists" - } - ], - "aggregateFunction": { - "class": "uk.gov.gchq.koryphe.impl.binaryoperator.Max" - } - }, - "set": { - "class": "java.util.TreeSet", - "aggregateFunction": { - "class": "uk.gov.gchq.koryphe.impl.binaryoperator.CollectionConcat" - } - }, - "hllSketch": { - "class": "org.apache.datasketches.hll.HllSketch", - "aggregateFunction": { - "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.binaryoperator.HllSketchAggregator" - }, - "serialiser": { - "class": "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.serialisation.HllSketchSerialiser" - } - } - } -} -``` - - -## Full Schema Example -Once the schema has been loaded into a graph the parent elements are merged into the children for performance reasons. This is what the full schema created from the above example schema parts looks like: - -```json -{ - "edges" : { - "RoadUse" : { - "description" : "A directed edge representing vehicles moving from junction A to junction B.", - "source" : "junction", - "destination" : "junction", - "directed" : "true", - "properties" : { - "startDate" : "date.earliest", - "endDate" : "date.latest", - "count" : "count.long" - }, - "groupBy" : [ "startDate", "endDate" ], - "aggregateFunctions" : [ { - "selection" : [ "startDate", "endDate" ], - "binaryOperator" : { - "class" : "uk.gov.gchq.gaffer.doc.dev.aggregator.ExampleTuple2BinaryOperator" - } - } ], - "validateFunctions" : [ { - "selection" : [ "startDate", "endDate" ], - "predicate" : { - "class" : "uk.gov.gchq.koryphe.impl.predicate.IsXLessThanY" - } - } ] - }, - "RoadHasJunction" : { - "description" : "A directed edge from each road to all the junctions on that road.", - "source" : "road", - "destination" : "junction", - "directed" : "true" - } - }, - "entities" : { - "Cardinality" : { - "description" : "An entity that is added to every vertex representing the connectivity of the vertex.", - "vertex" : "anyVertex", - "properties" : { - "edgeGroup" : "set", - "hllSketch" : "hllSketch", - "count" : "count.long" - }, - "groupBy" : [ "edgeGroup" ] - } - }, - "types" : { - "junction" : { - "description" : "A road junction represented by a String.", - "class" : "java.lang.String" - }, - "road" : { - "description" : "A road represented by a String.", - "class" : "java.lang.String" - }, - "anyVertex" : { - "description" : "An String vertex - used for cardinalities", - "class" : "java.lang.String" - }, - "count.long" : { - "description" : "A long count that must be greater than or equal to 0.", - "class" : "java.lang.Long", - "aggregateFunction" : { - "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.Sum" - }, - "validateFunctions" : [ { - "class" : "uk.gov.gchq.koryphe.impl.predicate.IsMoreThan", - "orEqualTo" : true, - "value" : { - "java.lang.Long" : 0 - } - } ] - }, - "true" : { - "description" : "A simple boolean that must always be true.", - "class" : "java.lang.Boolean", - "validateFunctions" : [ { - "class" : "uk.gov.gchq.koryphe.impl.predicate.IsTrue" - } ] - }, - "date.earliest" : { - "description" : "A Date that when aggregated together will be the earliest date.", - "class" : "java.util.Date", - "aggregateFunction" : { - "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.Min" - }, - "validateFunctions" : [ { - "class" : "uk.gov.gchq.koryphe.impl.predicate.Exists" - } ] - }, - "date.latest" : { - "description" : "A Date that when aggregated together will be the latest date.", - "class" : "java.util.Date", - "aggregateFunction" : { - "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.Max" - }, - "validateFunctions" : [ { - "class" : "uk.gov.gchq.koryphe.impl.predicate.Exists" - } ] - }, - "set" : { - "class" : "java.util.TreeSet", - "aggregateFunction" : { - "class" : "uk.gov.gchq.koryphe.impl.binaryoperator.CollectionConcat" - } - }, - "hllSketch" : { - "class" : "org.apache.datasketches.hll.HllSketch", - "aggregateFunction" : { - "class" : "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.binaryoperator.HllSketchAggregator" - }, - "serialiser" : { - "class" : "uk.gov.gchq.gaffer.sketches.datasketches.cardinality.serialisation.HllSketchSerialiser" - } - } - } -} -``` - - -## Java API - -Schemas can be loaded from a JSON file directly using the [`fromJSON()` method of the `Schema` class](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/Schema.html#fromJson(java.io.InputStream...)). This accepts `byte[]`, `InputStream` or `Path` types, for example: - -```java -Schema mySchema = Schema.fromJson(Paths.get("path/to/schema.json")); -``` - -While it's easiest to create Schema objects using the JSON approach, you can create them directly from their component classes in Java. -Constructing a Schema from scratch this way would be tedious, but you could use this to create your own solutions for Schema generation by building on these components. - -A Schema can be created using its `Builder()` method. [See Schema Javadoc](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/Schema.html). - -```java -Schema mySchema = new Schema.Builder().edges(edges) - .entities(entities) - .types(types) - .build(); -``` - -The Edges, Entities and Types are supplied as Maps with String keys. See the Javadoc covering [`SchemaEdgeDefinition`](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/SchemaEdgeDefinition.html), [`SchemaEntityDefinition`](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/SchemaEntityDefinition.html) and [`TypeDefinition`](https://gchq.github.io/Gaffer/uk/gov/gchq/gaffer/store/schema/TypeDefinition.html) for information about their builders. - -```java -Map myEdges = new HashMap<>(); -myEdges.put("myEdge", new SchemaEdgeDefinition.Builder()... - .build()); - -Map myEntities = new HashMap<>(); -myEntities.put("myEntity", new SchemaEntityDefinition.Builder()... - .build()); - -Map myTypes = new HashMap<>(); -myTypes.put("myType", new TypeDefinition.Builder()... - .build()); -``` From 988fce4d5cc999d9ff6c765be5efefea173315ae Mon Sep 17 00:00:00 2001 From: GCHQDeveloper314 <94527357+GCHQDeveloper314@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:33:47 +0100 Subject: [PATCH 19/20] Add redirect plugin to prevent breaking incoming links --- mkdocs.yml | 29 +++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 30 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 92b2aaa34b..2dafaf4551 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,6 +42,35 @@ plugins: strict: true exclude: - index.md + - redirects: + redirect_maps: + 'dev/docker.md': 'administration-guide/where-to-run-gaffer/gaffer-docker.md' + 'dev/kubernetes-guide/kubernetes.md': 'administration-guide/where-to-run-gaffer/kubernetes-guide/kubernetes-guide.md' + 'dev/ways-of-working.md': 'development-guide/ways-of-working.md' + 'dev/components/operation.md': 'development-guide/project-structure/components/operation.md' + 'dev/components/cache.md': 'development-guide/project-structure/components/cache.md' + 'dev/components/graph.md': 'development-guide/project-structure/components/graph.md' + 'dev/components/data.md': 'development-guide/project-structure/components/data.md' + 'dev/components/store.md': 'development-guide/project-structure/components/store.md' + 'dev/components/serialisation.md': 'development-guide/project-structure/components/serialisation.md' + 'dev/components/libraries/sketches.md': 'development-guide/project-structure/components/libraries/sketches.md' + 'dev/components/libraries/bitmap.md': 'development-guide/project-structure/components/libraries/bitmap.md' + 'dev/components/libraries/time.md': 'development-guide/project-structure/components/libraries/time.md' + 'dev/components/libraries/flink.md': 'development-guide/project-structure/components/libraries/flink.md' + 'dev/components/libraries/spark.md': 'development-guide/project-structure/components/libraries/spark.md' + 'dev/components/integration-test.md': 'development-guide/project-structure/components/integration-test.md' + 'dev/components/spring-rest.md': 'development-guide/project-structure/components/spring-rest.md' + 'dev/components/core-rest.md': 'development-guide/project-structure/components/core-rest.md' + 'dev/components/accumulo-store.md': 'development-guide/project-structure/components/accumulo-store.md' + 'reference/stores-guide/stores.md': 'administration-guide/gaffer-stores/store-guide.md' + 'reference/stores-guide/map.md': 'administration-guide/gaffer-stores/map-store.md' + 'reference/stores-guide/proxy.md': 'administration-guide/gaffer-stores/proxy-store.md' + 'reference/stores-guide/federated.md': 'administration-guide/gaffer-stores/federated-store.md' + 'reference/stores-guide/accumulo.md': 'administration-guide/gaffer-stores/accumulo-store.md' + 'getting-started/basics.md': 'user-guide/introduction.md' + 'getting-started/guide/cardinality.md': 'user-guide/gaffer-basics/what-is-cardinality.md' + 'getting-started/quickstart.md': 'user-guide/introduction.md' + 'getting-started/guide/guide.md': 'user-guide/introduction.md' nav: - Home: 'index.md' diff --git a/requirements.txt b/requirements.txt index 1518dc0800..6f507cc683 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ MarkupSafe==2.1.2 mergedeep==1.3.4 mike==1.1.2 mkdocs==1.4.3 +mkdocs-redirects=1.2.1 mkdocs-git-revision-date-localized-plugin==1.2.0 mkdocs-material==9.1.11 mkdocs-material-extensions==1.1.1 From 4e0928d505d56b5dbebb05358cb0aa09443e495f Mon Sep 17 00:00:00 2001 From: GCHQDeveloper314 <94527357+GCHQDeveloper314@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:36:07 +0100 Subject: [PATCH 20/20] Typo --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f507cc683..474a65f023 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ MarkupSafe==2.1.2 mergedeep==1.3.4 mike==1.1.2 mkdocs==1.4.3 -mkdocs-redirects=1.2.1 +mkdocs-redirects==1.2.1 mkdocs-git-revision-date-localized-plugin==1.2.0 mkdocs-material==9.1.11 mkdocs-material-extensions==1.1.1