From 2e9cbd1eb90209c06e77ba782de39f543f120c3e Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 28 Feb 2019 17:15:12 +0100 Subject: [PATCH 01/24] refactor (api-v1): Don't have Knora and Sipi share a filesystem (ongoing). --- .../03-apis/api-v1/adding-resources.md | 108 +++---- .../03-apis/api-v2/resource-permalinks.md | 5 +- docs/src/paradox/07-sipi/sipi-and-knora.md | 145 ++-------- sipi/scripts/convert_from_binaries.lua | 240 ---------------- sipi/scripts/convert_from_file.lua | 2 + .../e2e/v1/KnoraSipiIntegrationV1ITSpec.scala | 270 +++++++++--------- .../e2e/v2/KnoraSipiIntegrationV2ITSpec.scala | 4 +- .../other/v1/DrawingsGodsV1ITSpec.scala | 54 ++-- webapi/src/main/resources/application.conf | 1 - webapi/src/main/resources/knoraXmlImport.xsd | 3 +- .../scala/org/knora/webapi/Settings.scala | 1 - .../store/sipimessages/SipiMessages.scala | 65 ----- .../resourcemessages/ResourceMessagesV1.scala | 10 +- .../valuemessages/ValueMessagesV1.scala | 9 - .../responders/v1/ResourcesResponderV1.scala | 33 +-- .../responders/v1/ValuesResponderV1.scala | 14 +- .../webapi/routing/v1/ResourcesRouteV1.scala | 171 ++--------- .../webapi/routing/v1/ValuesRouteV1.scala | 101 +------ .../webapi/store/iiif/SipiConnector.scala | 13 - .../knora/webapi/e2e/v1/SipiV1R2RSpec.scala | 136 +-------- .../webapi/store/iiif/MockSipiConnector.scala | 31 +- 21 files changed, 296 insertions(+), 1120 deletions(-) delete mode 100644 sipi/scripts/convert_from_binaries.lua diff --git a/docs/src/paradox/03-apis/api-v1/adding-resources.md b/docs/src/paradox/03-apis/api-v1/adding-resources.md index 5140ff6d96..341c25c509 100644 --- a/docs/src/paradox/03-apis/api-v1/adding-resources.md +++ b/docs/src/paradox/03-apis/api-v1/adding-resources.md @@ -49,59 +49,11 @@ The request header's content type has to be set to `application/json`. ## Adding Resources with Image Files -Certain resource classes can have attached image files. There are two ways to -attach a file to a resource: Either by submitting directly the binaries of the file in a -an HTTP Multipart request, or by indicating the location of the file. The two cases are referred to -as non-GUI case and GUI case (see @ref:[Sipi and Knora](../../07-sipi/sipi-and-knora.md)). - -### Including the binaries (non-GUI case) - -In order to include the binaries, a HTTP Multipart request has to be -sent. One part contains the JSON (same format as described for -[Adding Resources Without Images Files](#adding-resources-without-a-digital-representation)) -and has to be named `json`. The other part contains the file's name, its binaries, and its mime type -and has to be named `file`. The following example illustrates how to -make this type of request using Python 3: - -```python -#!/usr/bin/env python3 - -import requests, json - -# a Python dictionary that will be turned into a JSON object -resourceParams = { - 'restype_id': 'http://www.knora.org/ontology/test#testType', - 'properties': { - 'http://www.knora.org/ontology/test#testtext': [ - {'richtext_value': {'utf8str': "test"}} - ], - 'http://www.knora.org/ontology/test#testnumber': [ - {'int_value': 1} - ] - }, - 'label': "test resource", - 'project_id': 'http://rdfh.ch/projects/testproject' -} - -# the name of the file to be submitted -filename = "myimage.jpg" - -# a tuple containing the file's name, its binaries and its mimetype -file = {'file': (filename, open(filename, 'rb'), "image/jpeg")} # use name "file" - -# do a POST request providing both the JSON and the binaries -r = requests.post("http://host/v1/resources", - data={'json': json.dumps(resourceParams)}, # use name "json" - files=file, - auth=('user', 'password')) -``` - -Please note that the file has to be read in binary mode (by default it -would be read in text mode). +Certain resource classes can have attached image files. To attach +attach a file to a resource, you must first submit to the file to Sipi, then +submit the file's metadata to Knora (see @ref:[Sipi and Knora](../../07-sipi/sipi-and-knora.md)). -### Indicating the location of a file (GUI case) - -This request works similarly to +The request to Knora works similarly to [Adding Resources Without Image Files](#adding-resources-without-a-digital-representation). The JSON format is described in the TypeScript interface `createResourceWithRepresentationRequest` in module `createResourceFormats`. The request header's content type has to @@ -152,18 +104,24 @@ Only system or project administrators may use the bulk import. The procedure for using this feature is as follows (see the @ref:[example below](#bulk-import-example)). -1. Make an HTTP GET request to Knora to @ref:[get XML schemas](#1-get-xml-schemas) describing - the XML to be provided for the import. -2. @ref:[Generate an XML import document](#2-generate-xml-import-document) representing the - data to be imported, following the Knora import schemas that were generated in step 1. - You will probably want to write a script to do this. Knora is not involved in this step. - If you are also importing image files, this XML document needs to - @ref:[contain the filesystem paths](#bulk-import-with-image-files) of those files. -3. @ref:[Validate your XML import document](#3-validate-xml-import-document), using an XML schema validator such as - [Apache Xerces](http://xerces.apache.org) or [Saxon](http://www.saxonica.com), or an - XML development environment such as [Oxygen](https://www.oxygenxml.com). This will - help ensure that the data you submit to Knora is correct. Knora is not involved in this step. -4. @ref:[Submit the XML import document to Knora](#4-submit-xml-import-document-to-knora). +1. Make an HTTP GET request to Knora to @ref:[get XML schemas](#1-get-xml-schemas) describing + the XML to be provided for the import. + +2. If you are importing image files, @ref:[upload files to Sipi](#2-upload-files-to-sipi). + +3. @ref:[Generate an XML import document](#3-generate-xml-import-document) representing the + data to be imported, following the Knora import schemas that were generated in step 1. + You will probably want to write a script to do this. Knora is not involved in this step. + If you are also importing image files, this XML document needs to + @ref:[contain the filenames](#bulk-import-with-image-files) that Sipi returned + for the files you uploaded in step 2. + +4. @ref:[Validate your XML import document](#4-validate-xml-import-document), using an XML schema validator such as + [Apache Xerces](http://xerces.apache.org) or [Saxon](http://www.saxonica.com), or an + XML development environment such as [Oxygen](https://www.oxygenxml.com). This will + help ensure that the data you submit to Knora is correct. Knora is not involved in this step. + +5. @ref:[Submit the XML import document to Knora](#5-submit-xml-import-document-to-knora). In this procedure, the person responsible for generating the XML import data need not be familiar with RDF or with the ontologies involved. @@ -208,7 +166,12 @@ containing three files: - `knoraXmlImport.xsd`: The standard Knora XML import schema, used by all XML imports. -#### 2. Generate XML Import Document +#### 2. Upload Files to Sipi + +See @ref:[Upload Files to Sipi](../api-v2/editing-values.md#upload-files-to-sipi) in +the Knora API v2 documentation. + +#### 3. Generate XML Import Document We now convert our existing data to XML, probably by writing a custom script. The resulting XML import document could look like this: @@ -341,7 +304,7 @@ This illustrates several aspects of XML imports: - A text value can have a `lang` attribute, whose value is an ISO 639-1 code specifying the language of the text. -#### 3. Validate XML Import Document +#### 4. Validate XML Import Document You can use an XML schema validator such as [Apache Xerces](http://xerces.apache.org) or [Saxon](http://saxon.sourceforge.net/), or an XML development environment @@ -354,7 +317,7 @@ For example, using Saxon: java -cp ./saxon9ee.jar com.saxonica.Validate -xsd:p0801-biblio.xsd -s:data.xml ``` -#### 4. Submit XML Import Document to Knora +#### 5. Submit XML Import Document to Knora To create these resources in Knora, make an HTTP post request with the XML import document as the request body. The URL must specify the (URL-encoded) IRI of the project in which @@ -411,8 +374,8 @@ contains the IRI of the target resource. To attach an image file to a resource, we must provide the element `knoraXmlImport:file` before the property elements. In this -element, we must give the absolute filesystem path to the file that -should be attached to the resource, along with its MIME type: +element, we must give the filename that Sipi returned for the file in +@ref:[2. Upload Files to Sipi](#2-upload-files-to-sipi). ```xml @@ -427,7 +390,7 @@ should be attached to the resource, along with its MIME type: a page with an image - + Chlaus 1a @@ -438,6 +401,5 @@ should be attached to the resource, along with its MIME type: ``` -During the processing of the bulk import, Knora will -communicate the location of file to Sipi, which will convert it to JPEG 2000 -for storage. +During the processing of the bulk import, Knora will ask Sipi for the rest of the +file's metadata, and store that metadata in a file value attached to the resource. diff --git a/docs/src/paradox/03-apis/api-v2/resource-permalinks.md b/docs/src/paradox/03-apis/api-v2/resource-permalinks.md index 8c5d4c0e50..f731c9c972 100644 --- a/docs/src/paradox/03-apis/api-v2/resource-permalinks.md +++ b/docs/src/paradox/03-apis/api-v2/resource-permalinks.md @@ -101,7 +101,4 @@ http://ark.dasch.swiss/ark:/72163/1/0001/0C=0L1kORryKzJAJxxRyRQY.20180604T085622 ``` Without a timestamp, a Knora ARK URL refers to the latest version of the -resource at the time when the URL is resolved. Knora currently returns only ARK URLs -without timestamps, because querying past versions of resources is not yet -implemented (@github[#1115](#1115)). When it is implemented, Knora will also return -ARK URLs with timestamps. +resource at the time when the URL is resolved. \ No newline at end of file diff --git a/docs/src/paradox/07-sipi/sipi-and-knora.md b/docs/src/paradox/07-sipi/sipi-and-knora.md index 5e644f50ab..376b23776d 100644 --- a/docs/src/paradox/07-sipi/sipi-and-knora.md +++ b/docs/src/paradox/07-sipi/sipi-and-knora.md @@ -37,19 +37,9 @@ for the client to request them from Sipi, but the whole handling of files (storing, naming, organization of the internal directory structure, format conversions, and serving) is taken care of by Sipi. -## Adding Files to Knora: Using the GUI or directly the API +## Adding Files to Knora -To create a resource with a digital representation attached to, either -the browser-based GUI (SALSAH) can be used or this can be done by -*directly* addressing the API. (Of course, also the GUI uses the API. -But the user does not need to know about it.) The same applies for -changing an existing digital representation for a resource. Subsequently, the first -case will be called the *GUI case* and the second the *non-GUI case*. - -### GUI Case - -In this case, the user may choose a file to upload using his -web-browser. The file is directly sent to Sipi (route: +The file is directly sent to Sipi (route: `create_thumbnail`) to calculate a thumbnail hosted by Sipi which then gets displayed to the user in the browser. Sipi copies the original file into a temporary directory and keeps it there (for later processing in @@ -89,83 +79,7 @@ representing the information about the file to be attached to the new resource. Along with the other parameters, it is sent to the resources responder. -See @ref:[Further Handling of the GUI and the non GUI-case in the Resources Responder](#further-handling-of-the-gui-and-the-non-gui-case-in-the-resources-responder) for -details of how the resources responder then handles the request. - -#### Change the Digital Representation of a Resource - -The request is taken care of in `ValuesRouteV1.scala`. The PUT request -is handled in path `v1/filevalue/{resIri}` which receives the resource -Iri as a part of the URL: *The submitted file will update the existing -file values of the given resource.* - -The file parameters are submitted as json and are parsed into a -`ChangeFileValueApiRequestV1`. To represent the conversion request for -the Sipi responder, a `SipiResponderConversionFileRequestV1` is created. -A `ChangeFileValueRequestV1` containing the resource Iri and the message -for Sipi is then created and sent to the values responder. - -See @ref:[Further Handling of the GUI and the non GUI-case in the Values Responder](#further-handling-of-the-gui-and-the-non-gui-case-in-the-values-responder) -for details of how the values responder then handles the request. - -### Non-GUI case - -In this case, the API receives an HTTP multipart request containing the -binary data. - -#### Create a new Resource with a Digital Representation - -The request is handled in `ResourcesRouteV1.scala`. The multipart POST -request consists of two named body parts: `json` containing the resource -parameters (properties) and `file` containing the binary data as well as -the file name and its mime type. Using Python's [request -module](http://docs.python-requests.org/en/master/user/quickstart/#post-a-multipart-encoded-file), -a request could look like this: - -```python -import requests, json - -params = {...} // resource parameters -files = {'file': (filename, open(path + filename, 'rb'), mimetype)} // filename, binary data, and mime type - -r = requests.post(knora_url + '/resources', - data={'json': json.dumps(params)}, - files=files, - headers=None) -``` - -The binary data is saved to a temporary location by Knora. The route -then creates a `SipiResponderConversionPathRequestV1` representing the -information about the file (i.e. the temporary path to the file) to be -attached to the new resource. Along with the other parameters, it is -sent to the resources responder. - -See @ref:[Further Handling of the GUI and the non GUI-case in the Resources Responder](#further-handling-of-the-gui-and-the-non-gui-case-in-the-resources-responder) for -details of how the resources responder then handles the request. - -#### Change the Digital Representation of a Resource - -The request is taken care of in `ValuesRouteV1.scala`. The multipart PUT -request is handled in path `v1/filevalue/{resIri}` which receives the -resource Iri as a part of the URL: *The submitted file will update the -existing file values of the given resource.* - -For the request, no json parameters are required. So its body just -consists of the binary data -(see @ref:[Create a new Resource with a Digital Representation](#create-a-new-resource-with-a-digital-representation)). -The values route stores the submitted -binaries as a temporary file and creates a -`SipiResponderConversionPathRequestV1`. A `ChangeFileValueRequestV1` -containing the resource Iri and the message for Sipi is then created and -sent to the values responder. - -See @ref:[Further Handling of the GUI and the non GUI-case in the Values Responder](#further-handling-of-the-gui-and-the-non-gui-case-in-the-values-responder) for details -of how the values responder then handles the request. - -### Further Handling of the GUI and the Non-GUI case in the Resources Responder - -Once a `SipiResponderConversionFileRequestV1` (GUI case) or a -`SipiResponderConversionPathRequestV1` (non-GUI case) has been created +Once a `SipiResponderConversionFileRequestV1` has been created and passed to the resources responder, the GUI and the non-GUI case can be handled in a very similar way. This is why they are both implementations of the trait `SipiResponderConversionRequestV1`. @@ -173,34 +87,18 @@ implementations of the trait `SipiResponderConversionRequestV1`. The resource responder calls the ontology responder to check if all required properties were submitted for the given resource type. Also it is checked if the given resource type may have a digital representation. -The resources responder then sends a message to Sipi responder that does -a request to the Sipi server. Depending on the type of the message -(`SipiResponderConversionFileRequestV1` or -`SipiResponderConversionPathRequestV1`), a different Sipi route is -called. In the first case (GUI case), the file is already managed by -Sipi and only the filename has to be indicated. In the latter case, Sipi -is told about the location where Knora has saved the binary data to. - -To make this handling easy for Knora, both messages have their own -implementation for creating the parameters for Sipi (declared in the -trait as `toFormData`). If Knora deals with a -`SipiResponderConversionPathRequestV1`, it has to delete the temporary -file after it has been processed by SIPI. Here, we assume that we deal -with an image. - -For both cases, Sipi returns the same answer containing the following -information: +The resources responder then sends a message to Sipi connector, which makes +a request to the Sipi server. + +Sipi's response contains the following information: - `file_type`: the type of the file that has been handled by Sipi (image | video | audio | text | binary) - - `mimetype_full` and `mimetype_thumb`: mime types of the full image - representation and the thumbnail + - `mimetype_full`: mime type of the image - `original_mimetype`: the mime type of the original file - `original_filename`: the name of the original file - - `nx_full`, `ny_full`, `nx_thumb`, and `ny_thumb`: the x and y - dimensions of both the full image and the thumbnail - - `filename_full` and `filename_full`: the names of the full image - and the thumbnail (needed to request the images from Sipi) + - `nx_full`, `ny_full`: the x and y dimensions of the image + - `filename_full`: the internal filename of the image (needed to request the image from Sipi) The `file_type` is important because representations for resources are restricted to media types: image, audio, video or a generic binary file. @@ -215,22 +113,28 @@ Depending on the given file type, Sipi responder can create the apt message (here: `StillImageFileValueV1`) to save the data to the triplestore. -### Further Handling of the GUI and the non-GUI case in the Values Responder +#### Change the Digital Representation of a Resource + +The request is taken care of in `ValuesRouteV1.scala`. The PUT request +is handled in path `v1/filevalue/{resIri}` which receives the resource +Iri as a part of the URL: *The submitted file will update the existing +file value of the given resource.* + +The file parameters are submitted as json and are parsed into a +`ChangeFileValueApiRequestV1`. To represent the conversion request for +the Sipi responder, a `SipiResponderConversionFileRequestV1` is created. +A `ChangeFileValueRequestV1` containing the resource Iri and the message +for Sipi is then created and sent to the values responder. In the values responder, `ChangeFileValueRequestV1` is passed to the method `changeFileValueV1`. Unlike ordinary value change requests, the Iris of the value objects to be updated are not known yet. Because of this, all the existing file values of the given resource Iri have to be -queried first. Also their quality levels are queried because in case of -a `StillImageFileValue`, we have to deal with a file value for the -thumbnail and another one for the full quality representation. When -these two file values are being updated, the quality levels have to be -considered for the sake of consistency (otherwise a full quality value's -`knora-base:previous-value` may point to a thumbnail file value). +queried first. With the file values being returned, we actually know about the current Iris of the value objects. Now the Sipi responder is called to handle -the file conversion request (see @ref:[Further Handling of the GUI and the non GUI-case in the Resources Responder](#further-handling-of-the-gui-and-the-non-gui-case-in-the-resources-responder)). +the file conversion request (see @ref:[Further Processing in the Resources Responder](#further-processing-in-the-resources-responder)). After that, it is checked that the `file_type` returned by Sipi responder corresponds to the property type of the existing file values. For example, if the `file_type` is an image, the property pointing to the @@ -313,7 +217,6 @@ in two cookies, is the fact that cookies can not be shared among different domains. Since Knora and Sipi are likely to be running under different domains, this solution offers the necessary flexibility. - ## Authentication of users with Sipi Whenever a file is requested, Sipi asks Knora about the current user's permissions on the given file. diff --git a/sipi/scripts/convert_from_binaries.lua b/sipi/scripts/convert_from_binaries.lua deleted file mode 100644 index 03b88ae311..0000000000 --- a/sipi/scripts/convert_from_binaries.lua +++ /dev/null @@ -1,240 +0,0 @@ --- Copyright © 2015-2019 the contributors (see Contributors.md). --- --- This file is part of Knora. --- --- Knora is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as published --- by the Free Software Foundation, either version 3 of the License, or --- (at your option) any later version. --- --- Knora is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU Affero General Public License for more details. --- --- You should have received a copy of the GNU Affero General Public --- License along with Knora. If not, see . - --- handles the Knora non GUI-case: Knora uploaded a file to sourcePath - -require "send_response" -require "get_mediatype" - -success, errmsg = server.setBuffer() -if not success then - server.log("server.setBuffer() failed: " .. errmsg, server.loglevel.LOG_ERR) - send_error(500, "buffer could not be set correctly") - return -end - -if server.post == nil then - send_error(400, PARAMETERS_INCORRECT) - - return -end - -originalFilename = server.post['originalfilename'] -originalMimetype = server.post['originalmimetype'] -sourcePath = server.post['source'] -prefix = server.post['prefix'] - --- check if all the expected params are set -if originalFilename == nil or originalMimetype == nil or sourcePath == nil or prefix == nil then - send_error(400, PARAMETERS_INCORRECT) - return -end - --- all params are set - --- check if source is readable - -success, readable = server.fs.is_readable(sourcePath) -if not success then - server.log("server.fs.is_readable() failed: " .. readable, server.loglevel.LOG_ERR) - send_error(500, "server.fs.is_readable() failed") - return -end - -if not readable then - - send_error(500, FILE_NOT_READABLE .. sourcePath) - return -end - --- check for the mimetype of the file -success, real_mimetype = server.file_mimetype(sourcePath) - -if not success then - server.log("server.file_mimetype() failed: " .. exists, server.loglevel.LOG_ERR) - send_error(500, "mimetype of file could not be determined") -end - --- handle the file depending on its media type (image, text file) -mediatype = get_mediatype(real_mimetype.mimetype) - --- in case of an unsupported mimetype, the function returns false -if not mediatype then - send_error(400, "Mimetype '" .. real_mimetype.mimetype .. "' is not supported") - return -end - --- depending on the media type, decide what to do -if mediatype == IMAGE then - - -- it is an image - - -- - -- check if project directory is available, if not, create it - -- - - projectDir = config.imgroot .. '/' .. prefix .. '/' - - success, exists = server.fs.exists(projectDir) - if not success then - server.log("server.fs.exists() failed: " .. exists, server.loglevel.LOG_ERR) - end - - if not exists then - success, errmsg = server.fs.mkdir(projectDir, 511) - if not success then - server.log("server.fs.mkdir() failed: " .. errmsg, server.loglevel.LOG_ERR) - send_error(500, "Project directory could not be created on server") - return - end - end - - success, baseName = server.uuid62() - if not success then - server.log("server.uuid62() failed: " .. baseName, server.loglevel.LOG_ERR) - send_error(500, "unique name could not be created") - return - end - - - -- - -- create full quality image (jp2) - -- - success, fullImg = SipiImage.new(sourcePath) - if not success then - server.log("SipiImage.new() failed: " .. fullImg, server.loglevel.LOG_ERR) - return - end - - success, check = fullImg:mimetype_consistency(originalMimetype, originalFilename) - if not success then - server.log("fullImg:mimetype_consistency() failed: " .. check, server.loglevel.LOG_ERR) - return - end - - -- if check returns false, the user's input is invalid - if not check then - - send_error(400, MIMETYPES_INCONSISTENCY) - - return - end - - success, fullDims = fullImg:dims() - if not success then - server.log("fullImg:dims() failed: " .. fullDIms, server.loglevel.LOG_ERR) - return - end - - fullImgName = baseName .. '.jpx' - - -- - -- create new full quality image file path with sublevels: - -- - success, newFilePath = helper.filename_hash(fullImgName); - if not success then - server.sendStatus(500) - server.log(gaga, server.loglevel.error) - return false - end - - success, errmsg = fullImg:write(projectDir .. newFilePath) - if not success then - server.log("fullImg:write() failed: " .. errmsg, server.loglevel.LOG_ERR) - return - end - - result = { - mimetype_full = "image/jp2", - filename_full = fullImgName, - nx_full = fullDims.nx, - ny_full = fullDims.ny, - original_mimetype = originalMimetype, - original_filename = originalFilename, - file_type = IMAGE - } - - send_success(result) - -elseif mediatype == TEXT then - - -- it is a text file - - -- - -- check if project directory is available, if not, create it - -- - projectFileDir = config.docroot .. '/' .. prefix .. '/' - success, exists = server.fs.exists(projectFileDir) - if not success then - server.log("server.fs.exists() failed: " .. exists, server.loglevel.LOG_ERR) - end - - if not exists then - success, errmsg = server.fs.mkdir(projectFileDir, 511) - if not success then - server.log("server.fs.mkdir() failed: " .. errmsg, server.loglevel.LOG_ERR) - send_error(500, "Project directory could not be created on server") - return - end - end - - local success, filename = server.uuid62() - if not success then - send_error(500, "Couldn't generate uuid62") - return -1 - end - - -- check file extension - if not check_file_extension(real_mimetype.mimetype, originalFilename) then - send_error(400, MIMETYPES_INCONSISTENCY) - return - end - - -- check that the submitted mimetype is the same as the real mimetype of the file - - local success, submitted_mimetype = server.parse_mimetype(originalMimetype) - - if not success then - send_error(400, "Couldn't parse mimetype: " .. originalMimetype) - return -1 - end - - if (real_mimetype.mimetype ~= submitted_mimetype.mimetype) then - send_error(400, MIMETYPES_INCONSISTENCY) - return - end - - local filePath = projectFileDir .. filename - - local success, result = server.fs.copyFile(sourcePath, filePath) - if not success then - send_error(400, "Couldn't copy file: " .. result) - return -1 - end - - result = { - mimetype = submitted_mimetype.mimetype, - charset = submitted_mimetype.charset, - file_type = TEXT, - filename = filename, - original_mimetype = originalMimetype, - original_filename = originalFilename - } - - send_success(result) - -end diff --git a/sipi/scripts/convert_from_file.lua b/sipi/scripts/convert_from_file.lua index 43823b454a..acf0654576 100644 --- a/sipi/scripts/convert_from_file.lua +++ b/sipi/scripts/convert_from_file.lua @@ -112,6 +112,8 @@ if not success then return -1 end +server.log("********* Checking consistency of mimetype " .. submitted_mimetype.mimetype .. " and filename " .. originalFilename) + success, check = fullImg:mimetype_consistency(submitted_mimetype.mimetype, originalFilename) if not success then diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala index 33eaea6e80..19bd723cb3 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala @@ -22,11 +22,14 @@ package org.knora.webapi.e2e.v1 import java.io.File import java.net.URLEncoder +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import akka.http.scaladsl.model.headers._ import akka.http.scaladsl.model.{HttpEntity, _} +import akka.http.scaladsl.unmarshalling.Unmarshal import com.typesafe.config.{Config, ConfigFactory} import org.knora.webapi._ import org.knora.webapi.messages.store.triplestoremessages.{RdfDataObject, TriplestoreJsonProtocol} +import org.knora.webapi.messages.v2.routing.authenticationmessages.{AuthenticationV2JsonProtocol, LoginResponse} import org.knora.webapi.util.MutableTestIri import org.xmlunit.builder.{DiffBuilder, Input} import org.xmlunit.diff.Diff @@ -51,14 +54,14 @@ object KnoraSipiIntegrationV1ITSpec { * End-to-End (E2E) test specification for testing Knora-Sipi integration. Sipi must be running with the config file * `sipi.knora-docker-it-config.lua`. */ -class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV1ITSpec.config) with TriplestoreJsonProtocol { +class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV1ITSpec.config) with AuthenticationV2JsonProtocol with TriplestoreJsonProtocol { override lazy val rdfDataObjects: List[RdfDataObject] = List( RdfDataObject(path = "_test_data/all_data/incunabula-data.ttl", name = "http://www.knora.org/data/0803/incunabula"), RdfDataObject(path = "_test_data/all_data/anything-data.ttl", name = "http://www.knora.org/data/0001/anything") ) - private val username = "root@example.com" + private val userEmail = "root@example.com" private val password = "test" private val pathToChlaus = "_test_data/test_route/images/Chlaus.jpg" private val pathToMarbles = "_test_data/test_route/images/marbles.tif" @@ -75,6 +78,86 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV private val pathToBEOLBulkXML = "_test_data/test_route/texts/beol/testLetter/bulk.xml" private val letterIri = new MutableTestIri + /** + * Represents a file to be uploaded to Sipi. + * + * @param path the path of the file. + * @param mimeType the MIME type of the file. + */ + case class FileToUpload(path: String, mimeType: ContentType) + + /** + * Represents an image file to be uploaded to Sipi. + * + * @param fileToUpload the file to be uploaded. + * @param width the image's width in pixels. + * @param height the image's height in pixels. + */ + case class InputFile(fileToUpload: FileToUpload, width: Int, height: Int) + + /** + * Represents the information that Sipi returns about each file that has been uploaded. + * + * @param originalFilename the original filename that was submitted to Sipi. + * @param internalFilename Sipi's internal filename for the stored temporary file. + * @param temporaryBaseIIIFUrl the base URL at which the temporary file can be accessed. + */ + case class SipiUploadResponseEntry(originalFilename: String, internalFilename: String, temporaryBaseIIIFUrl: String) + + /** + * Represents Sipi's response to a file upload request. + * + * @param uploadedFiles the information about each file that was uploaded. + */ + case class SipiUploadResponse(uploadedFiles: Seq[SipiUploadResponseEntry]) + + + object SipiUploadResponseV2JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { + implicit val sipiUploadResponseEntryFormat: RootJsonFormat[SipiUploadResponseEntry] = jsonFormat3(SipiUploadResponseEntry) + implicit val sipiUploadResponseFormat: RootJsonFormat[SipiUploadResponse] = jsonFormat1(SipiUploadResponse) + } + + import SipiUploadResponseV2JsonProtocol._ + + /** + * Uploads a file to Sipi and returns the information in Sipi's response. + * + * @param loginToken the login token to be included in the request to Sipi. + * @param filesToUpload the files to be uploaded. + * @return a [[SipiUploadResponse]] representing Sipi's response. + */ + private def uploadToSipi(loginToken: String, filesToUpload: Seq[FileToUpload]): SipiUploadResponse = { + // Make a multipart/form-data request containing the files. + + val formDataParts: Seq[Multipart.FormData.BodyPart] = filesToUpload.map { + fileToUpload => + val fileToSend = new File(fileToUpload.path) + assert(fileToSend.exists(), s"File ${fileToUpload.path} does not exist") + + Multipart.FormData.BodyPart( + "file", + HttpEntity.fromPath(fileToUpload.mimeType, fileToSend.toPath), + Map("filename" -> fileToSend.getName) + ) + } + + val sipiFormData = Multipart.FormData(formDataParts: _*) + + // Send a POST request to Sipi, asking it to convert the image to JPEG 2000 and store it in a temporary file. + val sipiRequest = Post(s"$baseSipiUrl/upload?token=$loginToken", sipiFormData) + val sipiUploadResponseJson: JsObject = getResponseJson(sipiRequest) + // println(sipiUploadResponseJson.prettyPrint) + val sipiUploadResponse: SipiUploadResponse = sipiUploadResponseJson.convertTo[SipiUploadResponse] + + // Request the temporary image from Sipi. + for (responseEntry <- sipiUploadResponse.uploadedFiles) { + val sipiGetTmpFileRequest = Get(responseEntry.temporaryBaseIIIFUrl + "/full/full/0/default.jpg") + checkResponseOK(sipiGetTmpFileRequest) + } + + sipiUploadResponse + } + /** * Adds the IRI of a XSL transformation to the given mapping. * @@ -87,7 +170,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV val mappingXML: Elem = XML.loadString(mapping) // add the XSL transformation's Iri to the mapping XML (replacing the string 'toBeDefined') - val rule = new RewriteRule() { + val rule: RewriteRule = new RewriteRule() { override def transform(node: Node): Node = { node match { @@ -153,117 +236,29 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV } "Knora and Sipi" should { + var loginToken: String = "" - "create an 'incunabula:page' with binary data" in { + "log in as a Knora user" in { + /* Correct username and correct password */ - // JSON describing the resource to be created. - val paramsPageWithBinaries = + val params = s""" |{ - | "restype_id": "http://www.knora.org/ontology/0803/incunabula#page", - | "label": "test", - | "project_id": "http://rdfh.ch/projects/0803", - | "properties": { - | "http://www.knora.org/ontology/0803/incunabula#pagenum": [ - | { - | "richtext_value": { - | "utf8str": "test_page" - | } - | } - | ], - | "http://www.knora.org/ontology/0803/incunabula#origname": [ - | { - | "richtext_value": { - | "utf8str": "test" - | } - | } - | ], - | "http://www.knora.org/ontology/0803/incunabula#partOf": [ - | { - | "link_value": "http://rdfh.ch/0803/5e77e98d2603" - | } - | ], - | "http://www.knora.org/ontology/0803/incunabula#seqnum": [ - | { - | "int_value": 999 - | } - | ] - | } + | "email": "$userEmail", + | "password": "$password" |} - """.stripMargin - - // The image to be uploaded. - val fileToSend = new File(pathToChlaus) - assert(fileToSend.exists(), s"File $pathToChlaus does not exist") - - // A multipart/form-data request containing the image and the JSON. - val formData = Multipart.FormData( - Multipart.FormData.BodyPart( - "json", - HttpEntity(ContentTypes.`application/json`, paramsPageWithBinaries) - ), - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - - // Send the multipart/form-data request to the Knora API server. - val knoraPostRequest = Post(baseApiUrl + "/v1/resources", formData) ~> addCredentials(BasicHttpCredentials(username, password)) - val knoraPostResponseJson = getResponseJson(knoraPostRequest) - - // Get the IRI of the newly created resource. - val resourceIri: String = knoraPostResponseJson.fields("res_id").asInstanceOf[JsString].value - firstPageIri.set(resourceIri) - - // Request the resource from the Knora API server. - val knoraRequestNewResource = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(resourceIri, "UTF-8")) ~> addCredentials(BasicHttpCredentials(username, password)) - val knoraNewResourceJson = getResponseJson(knoraRequestNewResource) - - // Get the URL of the image that was uploaded. - val iiifUrl = knoraNewResourceJson.fields.get("resinfo") match { - case Some(resinfo: JsObject) => - resinfo.fields.get("locdata") match { - case Some(locdata: JsObject) => - locdata.fields.get("path") match { - case Some(JsString(path)) => path - case None => throw InvalidApiJsonException("no 'path' given") - case other => throw InvalidApiJsonException("'path' could not pe parsed correctly") - } - case None => throw InvalidApiJsonException("no 'locdata' given") - - case _ => throw InvalidApiJsonException("'locdata' could not pe parsed correctly") - } - - case None => throw InvalidApiJsonException("no 'resinfo' given") - - case _ => throw InvalidApiJsonException("'resinfo' could not pe parsed correctly") - } + """.stripMargin - // Request the image from Sipi. - val sipiGetRequest = Get(iiifUrl) ~> addCredentials(BasicHttpCredentials(username, password)) - checkResponseOK(sipiGetRequest) - } + val request = Post(baseApiUrl + s"/v2/authentication", HttpEntity(ContentTypes.`application/json`, params)) + val response: HttpResponse = singleAwaitingRequest(request) + assert(response.status == StatusCodes.OK) - "change an 'incunabula:page' with binary data" in { - // The image to be uploaded. - val fileToSend = new File(pathToMarbles) - assert(fileToSend.exists(), s"File $pathToMarbles does not exist") + val lr: LoginResponse = Await.result(Unmarshal(response.entity).to[LoginResponse], 1.seconds) + loginToken = lr.token - // A multipart/form-data request containing the image. - val formData = Multipart.FormData( - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/tiff`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) + loginToken.nonEmpty should be(true) - // Send the image in a PUT request to the Knora API server. - val knoraPutRequest = Put(baseApiUrl + "/v1/filevalue/" + URLEncoder.encode(firstPageIri.get, "UTF-8"), formData) ~> addCredentials(BasicHttpCredentials(username, password)) - checkResponseOK(knoraPutRequest) + log.debug("token: {}", loginToken) } "create an 'incunabula:page' with parameters" in { @@ -281,13 +276,13 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // Send a POST request to Sipi, asking it to make a thumbnail of the image. - val sipiRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(username, password)) + val sipiRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val sipiResponseJson = getResponseJson(sipiRequest) // Request the thumbnail from Sipi. val jsonFields = sipiResponseJson.fields val previewUrl = jsonFields("preview_path").asInstanceOf[JsString].value - val sipiGetRequest = Get(previewUrl) ~> addCredentials(BasicHttpCredentials(username, password)) + val sipiGetRequest = Get(previewUrl) ~> addCredentials(BasicHttpCredentials(userEmail, password)) checkResponseOK(sipiGetRequest) val fileParams = JsObject( @@ -321,7 +316,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV """.stripMargin // Send the JSON in a POST request to the Knora API server. - val knoraPostRequest = Post(baseApiUrl + "/v1/resources", HttpEntity(ContentTypes.`application/json`, knoraParams)) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraPostRequest = Post(baseApiUrl + "/v1/resources", HttpEntity(ContentTypes.`application/json`, knoraParams)) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val knoraPostResponseJson = getResponseJson(knoraPostRequest) // Get the IRI of the newly created resource. @@ -329,7 +324,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV secondPageIri.set(resourceIri) // Request the resource from the Knora API server. - val knoraRequestNewResource = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(resourceIri, "UTF-8")) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraRequestNewResource = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(resourceIri, "UTF-8")) ~> addCredentials(BasicHttpCredentials(userEmail, password)) checkResponseOK(knoraRequestNewResource) } @@ -348,13 +343,13 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // Send a POST request to Sipi, asking it to make a thumbnail of the image. - val sipiRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(username, password)) + val sipiRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val sipiResponseJson = getResponseJson(sipiRequest) // Request the thumbnail from Sipi. val jsonFields = sipiResponseJson.fields val previewUrl = jsonFields("preview_path").asInstanceOf[JsString].value - val sipiGetRequest = Get(previewUrl) ~> addCredentials(BasicHttpCredentials(username, password)) + val sipiGetRequest = Get(previewUrl) ~> addCredentials(BasicHttpCredentials(userEmail, password)) checkResponseOK(sipiGetRequest) // JSON describing the new image to Knora. @@ -371,7 +366,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // Send the JSON in a PUT request to the Knora API server. - val knoraPutRequest = Put(baseApiUrl + "/v1/filevalue/" + URLEncoder.encode(secondPageIri.get, "UTF-8"), HttpEntity(ContentTypes.`application/json`, knoraParams.compactPrint)) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraPutRequest = Put(baseApiUrl + "/v1/filevalue/" + URLEncoder.encode(secondPageIri.get, "UTF-8"), HttpEntity(ContentTypes.`application/json`, knoraParams.compactPrint)) ~> addCredentials(BasicHttpCredentials(userEmail, password)) checkResponseOK(knoraPutRequest) } @@ -413,14 +408,19 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // Send the JSON in a POST request to the Knora API server. - val knoraPostRequest = Post(baseApiUrl + "/v1/resources", HttpEntity(ContentTypes.`application/json`, knoraParams.compactPrint)) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraPostRequest = Post(baseApiUrl + "/v1/resources", HttpEntity(ContentTypes.`application/json`, knoraParams.compactPrint)) ~> addCredentials(BasicHttpCredentials(userEmail, password)) checkResponseOK(knoraPostRequest) } - "create an 'p0803-incunabula:book' and an 'p0803-incunabula:page' with file parameters via XML import" in { - val fileToUpload = new File(pathToChlaus) - val absoluteFilePath = fileToUpload.getAbsolutePath + "create a 'p0803-incunabula:book' and a 'p0803-incunabula:page' with file parameters via XML import" in { + // Upload the image to Sipi. + val sipiUploadResponse: SipiUploadResponse = uploadToSipi( + loginToken = loginToken, + filesToUpload = Seq(FileToUpload(path = pathToChlaus, mimeType = MediaTypes.`image/tiff`)) + ) + + val uploadedFile: SipiUploadResponseEntry = sipiUploadResponse.uploadedFiles.head val knoraParams = s""" @@ -435,7 +435,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV | | | a page with an image - | + | | Chlaus | 1a | @@ -448,7 +448,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV val projectIri = URLEncoder.encode("http://rdfh.ch/projects/0803", "UTF-8") // Send the JSON in a POST request to the Knora API server. - val knoraPostRequest = Post(baseApiUrl + s"/v1/resources/xmlimport/$projectIri", HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), knoraParams)) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraPostRequest = Post(baseApiUrl + s"/v1/resources/xmlimport/$projectIri", HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), knoraParams)) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val knoraPostResponseJson: JsObject = getResponseJson(knoraPostRequest) val createdResources = knoraPostResponseJson.fields("createdResources").asInstanceOf[JsArray].elements @@ -458,23 +458,25 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV val pageResourceIri = createdResources(1).asJsObject.fields("resourceIri").asInstanceOf[JsString].value // Request the book resource from the Knora API server. - val knoraRequestNewBookResource = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(bookResourceIri, "UTF-8")) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraRequestNewBookResource = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(bookResourceIri, "UTF-8")) ~> addCredentials(BasicHttpCredentials(userEmail, password)) checkResponseOK(knoraRequestNewBookResource) // Request the page resource from the Knora API server. - val knoraRequestNewPageResource = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(pageResourceIri, "UTF-8")) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraRequestNewPageResource = Get(baseApiUrl + "/v1/resources/" + URLEncoder.encode(pageResourceIri, "UTF-8")) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val pageJson: JsObject = getResponseJson(knoraRequestNewPageResource) val locdata = pageJson.fields("resinfo").asJsObject.fields("locdata").asJsObject val origname = locdata.fields("origname").asInstanceOf[JsString].value val imageUrl = locdata.fields("path").asInstanceOf[JsString].value - assert(origname == fileToUpload.getName) + assert(origname == "Chlaus.jpg") // Request the file from Sipi. - val sipiGetRequest = Get(imageUrl) ~> addCredentials(BasicHttpCredentials(username, password)) + val sipiGetRequest = Get(imageUrl) ~> addCredentials(BasicHttpCredentials(userEmail, password)) checkResponseOK(sipiGetRequest) } - "create a TextRepresentation of type XSLTransformation and refer to it in a mapping" in { + "create a TextRepresentation of type XSLTransformation and refer to it in a mapping" ignore { + + // TODO: fix this when we can upload non-image files to Sipi (PR #1206). // create an XSL transformation val knoraParams = JsObject( @@ -502,7 +504,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // Send the JSON in a POST request to the Knora API server. - val knoraPostRequest: HttpRequest = Post(baseApiUrl + "/v1/resources", formData) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraPostRequest: HttpRequest = Post(baseApiUrl + "/v1/resources", formData) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val responseJson: JsObject = getResponseJson(knoraPostRequest) @@ -543,13 +545,14 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // send mapping xml to route - val knoraPostRequest2 = Post(baseApiUrl + "/v1/mapping", formDataMapping) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraPostRequest2 = Post(baseApiUrl + "/v1/mapping", formDataMapping) ~> addCredentials(BasicHttpCredentials(userEmail, password)) checkResponseOK(knoraPostRequest2) } - "create a sample BEOL letter" in { + "create a sample BEOL letter" ignore { + // TODO: fix this when we can upload non-image files to Sipi (PR #1206). val mapping = Source.fromFile(pathToBEOLLetterMapping).getLines.mkString @@ -576,7 +579,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // send mapping xml to route - val knoraPostRequest = Post(baseApiUrl + "/v1/mapping", formDataMapping) ~> addCredentials(BasicHttpCredentials(username, password)) + val knoraPostRequest = Post(baseApiUrl + "/v1/mapping", formDataMapping) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val mappingResponse: JsValue = getResponseJson(knoraPostRequest) @@ -584,7 +587,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV val bulkXML = Source.fromFile(pathToBEOLBulkXML).getLines.mkString - val bulkRequest = Post(baseApiUrl + "/v1/resources/xmlimport/" + URLEncoder.encode("http://rdfh.ch/projects/yTerZGyxjZVqFMNNKXCDPF", "UTF-8"), HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), bulkXML)) ~> addCredentials(BasicHttpCredentials(username, password)) + val bulkRequest = Post(baseApiUrl + "/v1/resources/xmlimport/" + URLEncoder.encode("http://rdfh.ch/projects/yTerZGyxjZVqFMNNKXCDPF", "UTF-8"), HttpEntity(ContentType(MediaTypes.`application/xml`, HttpCharsets.`UTF-8`), bulkXML)) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val bulkResponse: JsObject = getResponseJson(bulkRequest) @@ -592,7 +595,8 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV } - "create a mapping for standoff conversion to TEI referring to an XSLT and also create a Gravsearch template and an XSLT for transforming TEI header data" in { + "create a mapping for standoff conversion to TEI referring to an XSLT and also create a Gravsearch template and an XSLT for transforming TEI header data" ignore { + // TODO: fix this when we can upload non-image files to Sipi (PR #1206). // create an XSL transformation val standoffXSLTParams = JsObject( @@ -620,7 +624,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // Send the JSON in a POST request to the Knora API server. - val bodyXSLTRequest: HttpRequest = Post(baseApiUrl + "/v1/resources", formData) ~> addCredentials(BasicHttpCredentials(username, password)) + val bodyXSLTRequest: HttpRequest = Post(baseApiUrl + "/v1/resources", formData) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val bodyXSLTJson: JsObject = getResponseJson(bodyXSLTRequest) @@ -661,7 +665,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // send mapping xml to route - val mappingRequest = Post(baseApiUrl + "/v1/mapping", formDataMapping) ~> addCredentials(BasicHttpCredentials(username, password)) + val mappingRequest = Post(baseApiUrl + "/v1/mapping", formDataMapping) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val mappingJSON = getResponseJson(mappingRequest) @@ -691,7 +695,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // Send the JSON in a POST request to the Knora API server. - val gravsearchTemplateRequest: HttpRequest = Post(baseApiUrl + "/v1/resources", formDataGravsearch) ~> addCredentials(BasicHttpCredentials(username, password)) + val gravsearchTemplateRequest: HttpRequest = Post(baseApiUrl + "/v1/resources", formDataGravsearch) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val gravsearchTemplateJSON: JsObject = getResponseJson(gravsearchTemplateRequest) @@ -728,7 +732,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV ) // Send the JSON in a POST request to the Knora API server. - val headerXSLTRequest: HttpRequest = Post(baseApiUrl + "/v1/resources", formDataHeader) ~> addCredentials(BasicHttpCredentials(username, password)) + val headerXSLTRequest: HttpRequest = Post(baseApiUrl + "/v1/resources", formDataHeader) ~> addCredentials(BasicHttpCredentials(userEmail, password)) val headerXSLTJson = getResponseJson(headerXSLTRequest) diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala index 8bc887399e..d7d0d89715 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala @@ -86,12 +86,12 @@ class KnoraSipiIntegrationV2ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV */ case class SipiUploadResponse(uploadedFiles: Seq[SipiUploadResponseEntry]) - object GetImageMetadataResponseV2JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { + object SipiUploadResponseV2JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { implicit val sipiUploadResponseEntryFormat: RootJsonFormat[SipiUploadResponseEntry] = jsonFormat3(SipiUploadResponseEntry) implicit val sipiUploadResponseFormat: RootJsonFormat[SipiUploadResponse] = jsonFormat1(SipiUploadResponse) } - import GetImageMetadataResponseV2JsonProtocol._ + import SipiUploadResponseV2JsonProtocol._ /** * Represents the information that Knora returns about an image file value that was created. diff --git a/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala index 74a39038a6..4badc7ba75 100644 --- a/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala @@ -57,9 +57,39 @@ class DrawingsGodsV1ITSpec extends ITKnoraLiveSpec(DrawingsGodsV1ITSpec.config) val testPass = "test" val pathToChlaus = "_test_data/test_route/images/Chlaus.jpg" - "be able to create a resource, only find one DOAP (with combined resource class / property), and have permission to access the image" in { + // The image to be uploaded. + val fileToSend = new File(pathToChlaus) + assert(fileToSend.exists(), s"File $pathToChlaus does not exist") + + // A multipart/form-data request containing the image. + val sipiFormData = Multipart.FormData( + Multipart.FormData.BodyPart( + "file", + HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), + Map("filename" -> fileToSend.getName) + ) + ) + + // Send a POST request to Sipi, asking it to make a thumbnail of the image. + val sipiRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(drawingsOfGodsUserEmail, testPass)) + val sipiResponseJson = getResponseJson(sipiRequest) + + // Request the thumbnail from Sipi. + val jsonFields = sipiResponseJson.fields + val previewUrl = jsonFields("preview_path").asInstanceOf[JsString].value + val sipiThumbnailGetRequest = Get(previewUrl) ~> addCredentials(BasicHttpCredentials(drawingsOfGodsUserEmail, testPass)) + checkResponseOK(sipiThumbnailGetRequest) + + val fileParams = JsObject( + Map( + "originalFilename" -> jsonFields("original_filename"), + "originalMimeType" -> jsonFields("original_mimetype"), + "filename" -> jsonFields("filename") + ) + ) + val params = s""" |{ @@ -74,30 +104,14 @@ class DrawingsGodsV1ITSpec extends ITKnoraLiveSpec(DrawingsGodsV1ITSpec.config) | "http://www.knora.org/ontology/0105/drawings-gods#hasCommentAuthor":[{"hlist_value":"http://rdfh.ch/lists/0105/drawings-gods-2016-list-CommentAuthorList-child"}], | "http://www.knora.org/ontology/0105/drawings-gods#hasCodeVerso":[{"richtext_value":{"utf8str":"dayyad"}}] | }, + | "file": ${fileParams.compactPrint}, | "project_id":"http://rdfh.ch/projects/0105", | "label":"dayyad" |} """.stripMargin - // The image to be uploaded. - val fileToSend = new File(pathToChlaus) - assert(fileToSend.exists(), s"File $pathToChlaus does not exist") - - // A multipart/form-data request containing the image and the JSON. - val formData = Multipart.FormData( - Multipart.FormData.BodyPart( - "json", - HttpEntity(ContentTypes.`application/json`, params) - ), - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - // Send the multipart/form-data request to the Knora API server. - val knoraPostRequest = Post(baseApiUrl + "/v1/resources", formData) ~> addCredentials(BasicHttpCredentials(drawingsOfGodsUserEmail, testPass)) + // Send the JSON in a POST request to the Knora API server. + val knoraPostRequest = Post(baseApiUrl + "/v1/resources", HttpEntity(ContentTypes.`application/json`, params)) ~> addCredentials(BasicHttpCredentials(drawingsOfGodsUserEmail, testPass)) val knoraPostResponseJson = getResponseJson(knoraPostRequest) // Get the IRI of the newly created resource. diff --git a/webapi/src/main/resources/application.conf b/webapi/src/main/resources/application.conf index 4d84b94eee..f14868b996 100644 --- a/webapi/src/main/resources/application.conf +++ b/webapi/src/main/resources/application.conf @@ -281,7 +281,6 @@ app { file-server-path = "server" v1 { - path-conversion-route = "convert_from_binaries" file-conversion-route = "convert_from_file" } diff --git a/webapi/src/main/resources/knoraXmlImport.xsd b/webapi/src/main/resources/knoraXmlImport.xsd index 41a792b7d4..f1a18ba506 100644 --- a/webapi/src/main/resources/knoraXmlImport.xsd +++ b/webapi/src/main/resources/knoraXmlImport.xsd @@ -31,8 +31,7 @@ - - + diff --git a/webapi/src/main/scala/org/knora/webapi/Settings.scala b/webapi/src/main/scala/org/knora/webapi/Settings.scala index 333da3dc91..6f89e1d5f9 100644 --- a/webapi/src/main/scala/org/knora/webapi/Settings.scala +++ b/webapi/src/main/scala/org/knora/webapi/Settings.scala @@ -103,7 +103,6 @@ class SettingsImpl(config: Config) extends Extension { val externalSipiFileServerGetUrl: String = s"$externalSipiBaseUrl/$sipiFileServerPrefix" val internalSipiImageConversionUrlV1: String = s"$internalSipiBaseUrl" - val sipiPathConversionRouteV1: String = config.getString("app.sipi.v1.path-conversion-route") val sipiFileConversionRouteV1: String = config.getString("app.sipi.v1.file-conversion-route") val sipiFileMetadataRouteV2: String = config.getString("app.sipi.v2.file-metadata-route") diff --git a/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala b/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala index 4711d03836..9ac57dca5a 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala @@ -66,44 +66,6 @@ sealed trait SipiConversionRequestV1 extends SipiRequestV1 { } -/** - * Represents a binary file that has been temporarily stored by Knora (non GUI-case). Knora route received a multipart request - * containing binary data which it saved to a temporary location, so it can be accessed by Sipi. Knora has to delete that file afterwards. - * For further details, please read the docs: Sipi -> Interaction Between Sipi and Knora. - * - * @param originalFilename the original name of the binary file. - * @param originalMimeType the MIME type of the binary file (e.g. image/tiff). - * @param source the temporary location of the source file on disk (absolute path). - * @param userProfile the user making the request. - */ -case class SipiConversionPathRequestV1(originalFilename: String, - originalMimeType: String, - projectShortcode: String, - source: File, - userProfile: UserProfileV1) extends SipiConversionRequestV1 { - - /** - * Creates the parameters needed to call the Sipi route convert_path. - * - * Required parameters: - * - originalFilename: original name of the file to be converted. - * - originalMimeType: original mime type of the file to be converted. - * - source: path to the file to be converted (file was created by Knora). - * - * @return a Map of key-value pairs that can be turned into form data by Sipi responder. - */ - def toFormData: Map[String, String] = { - Map( - "originalFilename" -> originalFilename, - "originalMimeType" -> originalMimeType, - "source" -> source.toString, - "prefix" -> projectShortcode - ) - } - - def toJsValue: JsValue = RepresentationV1JsonProtocol.SipiConversionPathRequestV1Format.write(this) -} - /** * Represents an binary file that has been temporarily stored by Sipi (GUI-case). Knora route received a request telling it about * a file that is already managed by Sipi. The binary file data have already been sent to Sipi by the client (browser-based GUI). @@ -244,33 +206,6 @@ case class SipiConversionResponseV1(fileValueV1: FileValueV1, file_type: SipiCon */ object RepresentationV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol with NullOptions { - /** - * Converts between [[SipiConversionPathRequestV1]] objects and [[JsValue]] objects. - */ - implicit object SipiConversionPathRequestV1Format extends RootJsonFormat[SipiConversionPathRequestV1] { - /** - * Not implemented. - */ - def read(jsonVal: JsValue): SipiConversionPathRequestV1 = ??? - - /** - * Converts a [[SipiConversionPathRequestV1]] into [[JsValue]] for formatting as JSON. - * - * @param request the [[SipiConversionPathRequestV1]] to be converted. - * @return a [[JsValue]]. - */ - def write(request: SipiConversionPathRequestV1): JsValue = { - - val fields = Map( - "originalFilename" -> request.originalFilename.toJson, - "originalMimeType" -> request.originalMimeType.toJson, - "source" -> request.source.toString.toJson - ) - - JsObject(fields) - } - } - /** * Converts between [[SipiConversionFileRequestV1]] objects and [[JsValue]] objects. */ diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala index 406c4e2b00..cb67bdb092 100755 --- a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala @@ -25,7 +25,7 @@ import java.util.UUID import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{SipiConversionPathRequestV1, SipiConversionRequestV1} +import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataResponseV2, SipiConversionFileRequestV1, SipiConversionRequestV1} import org.knora.webapi.messages.v1.responder.valuemessages._ import org.knora.webapi.messages.v1.responder.{KnoraRequestV1, KnoraResponseV1} import spray.json._ @@ -62,14 +62,14 @@ case class CreateResourceApiRequestV1(restype_id: IRI, * @param label the resource's label. * @param client_id the client's unique ID for the resource. * @param properties the resource's properties. - * @param file a file on disk that should be attached to the resource. + * @param file a file in Sipi's temporary storage that should be attached to the resource. * @param creationDate the creation date that should be attached to the resource. */ case class CreateResourceFromXmlImportRequestV1(restype_id: IRI, client_id: String, label: String, properties: Map[IRI, Seq[CreateResourceValueV1]], - file: Option[ReadFileV1] = None, + file: Option[String] = None, creationDate: Option[Instant]) /** @@ -221,14 +221,14 @@ case class ResourceCreateRequestV1(resourceTypeIri: IRI, * @param clientResourceID the client's ID for the resource. * @param label the rdfs:label of the resource. * @param values the properties to add: type and value(s): a Map of propertyIris to ApiValueV1. - * @param file a file on disk that should be stored by Sipi and should be attached to the resource. + * @param file a file in Sipi's temporary storage that should be attached to the resource. * @param creationDate the creation date that should be attached to the resource. */ case class OneOfMultipleResourceCreateRequestV1(resourceTypeIri: IRI, clientResourceID: String, label: String, values: Map[IRI, Seq[CreateValueV1WithComment]], - file: Option[SipiConversionPathRequestV1] = None, + file: Option[GetImageMetadataResponseV2] = None, creationDate: Option[Instant]) /** diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala index 9a2ae9c814..9ec44c5a81 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala @@ -19,7 +19,6 @@ package org.knora.webapi.messages.v1.responder.valuemessages -import java.io.File import java.time.Instant import java.util.UUID @@ -139,14 +138,6 @@ case class CreateFileV1(originalFilename: String, } -/** - * Represents a file on disk to be added to a Knora resource in the context of a bulk import. - * - * @param file the file. - * @param mimeType the file's MIME type. - */ -case class ReadFileV1(file: File, mimeType: String) - /** * Represents a quality level of a file value to added to a Knora resource. * diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala b/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala index e1778ee799..8b0a47cb51 100755 --- a/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala @@ -1249,6 +1249,8 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo apiRequestID: UUID): Future[MultipleResourceCreateResponseV1] = { val userProfileV1 = userProfile.asUserProfileV1 + // TODO: this method has to send MoveTemporaryFileToPermanentStorageRequestV2 for each file, as ResourcesResponderV2 does. + for { // Get user's IRI and don't allow anonymous users to create resources. userIri: IRI <- Future { @@ -1409,7 +1411,17 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo resourceClassInfo = resourceClassesEntityInfoResponse.resourceClassInfoMap(resourceCreateRequest.resourceTypeIri), propertyInfoMap = propertyEntityInfoMapsPerResource(resourceCreateRequest.resourceTypeIri), values = resourceCreateRequest.values, - sipiConversionRequest = resourceCreateRequest.file, + sipiConversionRequest = resourceCreateRequest.file.map { + fileMetadata => + // TODO: this is nonsense, just to get this file to compile for now. + SipiConversionFileRequestV1( + originalFilename = fileMetadata.originalFilename, + originalMimeType = fileMetadata.originalMimeType, + projectShortcode = projectInfoResponse.project_info.shortcode, + filename = fileMetadata.originalFilename, + userProfile = userProfile.asUserProfileV1 + ) + }, clientResourceIDsToResourceClasses = clientResourceIDsToResourceClasses, userProfile = userProfile ) @@ -1629,8 +1641,6 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo case None => Future(None) // expected behaviour case Some(_: SipiConversionFileRequestV1) => throw BadRequestException(s"File params (GUI-case) are given but resource class $resourceClassIri does not allow any representation") - case Some(_: SipiConversionPathRequestV1) => - throw BadRequestException(s"A binary file was provided (non GUI-case) but resource class $resourceClassIri does not have any binary representation") } } } yield fileValues @@ -1902,7 +1912,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo apiRequestID: UUID): Future[ResourceCreateResponseV1] = { val userProfileV1 = userProfile.asUserProfileV1 - val resultFuture = for { + for { // Get user's IRI and don't allow anonymous users to create resources. userIri: IRI <- Future { @@ -1969,21 +1979,6 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo ) ) } yield result - - // If a temporary file was created, ensure that it's deleted, regardless of whether the request succeeded or failed. - resultFuture.andThen { - case _ => - sipiConversionRequest match { - case Some(conversionRequest) => - conversionRequest match { - case conversionPathRequest: SipiConversionPathRequestV1 => - // a tmp file has been created by the resources route (non GUI-case), delete it - FileUtil.deleteFileFromTmpLocation(conversionPathRequest.source, log) - case _ => () - } - case None => () - } - } } /** diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala b/webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala index 8ad024081f..2fdf815de4 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala @@ -25,7 +25,7 @@ import akka.pattern._ import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.permissionsmessages.{DefaultObjectAccessPermissionsStringForPropertyGetADM, DefaultObjectAccessPermissionsStringResponseADM, PermissionADM, PermissionType} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{SipiConstants, SipiConversionPathRequestV1, SipiConversionRequestV1, SipiConversionResponseV1} +import org.knora.webapi.messages.store.sipimessages.{SipiConstants, SipiConversionRequestV1, SipiConversionResponseV1} import org.knora.webapi.messages.store.triplestoremessages._ import org.knora.webapi.messages.v1.responder.ontologymessages.{EntityInfoGetRequestV1, EntityInfoGetResponseV1} import org.knora.webapi.messages.v1.responder.projectmessages.{ProjectInfoByIRIGetV1, ProjectInfoV1} @@ -630,7 +630,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde def makeTaskFuture(changeFileValueRequest: ChangeFileValueRequestV1): Future[ChangeFileValueResponseV1] = { // get the Iris of the current file value(s) - val resultFuture = for { + for { resourceIri <- Future(changeFileValueRequest.resourceIri) @@ -696,16 +696,6 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde } yield ChangeFileValueResponseV1( locations = Vector(changedLocation) ) - - // If a temporary file was created, ensure that it's deleted, regardless of whether the request succeeded or failed. - resultFuture.andThen { - case _ => changeFileValueRequest.file match { - case conversionPathRequest: SipiConversionPathRequestV1 => - // a tmp file has been created by the resources route (non GUI-case), delete it - FileUtil.deleteFileFromTmpLocation(conversionPathRequest.source, log) - case _ => () - } - } } for { diff --git a/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala b/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala index 2eb2f40b76..f5aff285c5 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala @@ -22,27 +22,23 @@ package org.knora.webapi.routing.v1 import java.io._ import java.nio.charset.StandardCharsets -import java.nio.file.Paths import java.time.Instant import java.util.UUID -import akka.http.scaladsl.model.Multipart.BodyPart import akka.http.scaladsl.model._ import akka.http.scaladsl.model.headers._ import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route -import akka.http.scaladsl.server.directives.FileInfo import akka.http.scaladsl.util.FastFuture import akka.pattern._ import akka.stream.ActorMaterializer -import akka.stream.scaladsl.FileIO import javax.xml.XMLConstants import javax.xml.transform.stream.StreamSource import javax.xml.validation.{Schema, SchemaFactory, Validator} import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.projectsmessages.{ProjectGetRequestADM, ProjectGetResponseADM} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{SipiConversionFileRequestV1, SipiConversionPathRequestV1} +import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataRequestV2, GetImageMetadataResponseV2, SipiConversionFileRequestV1} import org.knora.webapi.messages.v1.responder.ontologymessages._ import org.knora.webapi.messages.v1.responder.resourcemessages.ResourceV1JsonProtocol._ import org.knora.webapi.messages.v1.responder.resourcemessages._ @@ -51,15 +47,13 @@ import org.knora.webapi.routing.{Authenticator, KnoraRoute, KnoraRouteData, Rout import org.knora.webapi.util.IriConversions._ import org.knora.webapi.util.StringFormatter.XmlImportNamespaceInfoV1 import org.knora.webapi.util.standoff.StandoffTagUtilV2.TextWithStandoffTagsV2 -import org.knora.webapi.util.{DateUtilV1, FileUtil, SmartIri} +import org.knora.webapi.util.{DateUtilV1, FileUtil, MessageUtil, SmartIri} import org.knora.webapi.viewhandlers.ResourceHtmlView import org.w3c.dom.ls.{LSInput, LSResourceResolver} import org.xml.sax.SAXException -import spray.json._ import scala.collection.immutable -import scala.concurrent.duration._ -import scala.concurrent.{Future, Promise} +import scala.concurrent.Future import scala.util.{Failure, Success, Try} import scala.xml._ @@ -220,7 +214,7 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) } - def makeCreateResourceRequestMessage(apiRequest: CreateResourceApiRequestV1, multipartConversionRequest: Option[SipiConversionPathRequestV1] = None, userADM: UserADM): Future[ResourceCreateRequestV1] = { + def makeCreateResourceRequestMessage(apiRequest: CreateResourceApiRequestV1, userADM: UserADM): Future[ResourceCreateRequestV1] = { val projectIri = stringFormatter.validateAndEscapeIri(apiRequest.project_id, throw BadRequestException(s"Invalid project IRI: ${apiRequest.project_id}")) val resourceTypeIri = stringFormatter.validateAndEscapeIri(apiRequest.restype_id, throw BadRequestException(s"Invalid resource IRI: ${apiRequest.restype_id}")) val label = stringFormatter.toSparqlEncodedString(apiRequest.label, throw BadRequestException(s"Invalid label: '${apiRequest.label}'")) @@ -250,11 +244,7 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) userProfile = userADM ) - // since this function `makeCreateResourceRequestMessage` is called by the POST multipart route receiving the binaries (non GUI-case) - // and by the other POST route, either multipartConversionRequest or paramConversionRequest is set if a file should be attached to the resource, but not both. - _ = if (multipartConversionRequest.nonEmpty && paramConversionRequest.nonEmpty) throw BadRequestException("Binaries sent and file params set to route. This is illegal.") - - // make the whole Map a Future + // make the whole Map a Future valuesToBeCreated: Iterable[(IRI, Seq[CreateValueV1WithComment])] <- Future.traverse(valuesToBeCreatedWithFuture) { case (propIri: IRI, valuesFuture: Future[Seq[CreateValueV1WithComment]]) => for { @@ -266,11 +256,7 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) label = label, projectIri = projectIri, values = valuesToBeCreated.toMap, - file = if (multipartConversionRequest.nonEmpty) // either multipartConversionRequest or paramConversionRequest might be given, but never both - multipartConversionRequest // Non GUI-case - else if (paramConversionRequest.nonEmpty) - paramConversionRequest // GUI-case - else None, // no file given + file = paramConversionRequest, userProfile = userADM, apiRequestID = UUID.randomUUID ) @@ -292,21 +278,24 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) values <- valuesFuture } yield propIri -> values } + + maybeImageMetadataResponse <- resourceRequest.file match { + case Some(filename) => + // Ask Sipi about the file's metadata. + val tempFileUrl = s"${settings.internalSipiBaseUrl}/tmp/$filename" + + for { + imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequestV2(fileUrl = tempFileUrl, requestingUser = userProfile)).mapTo[GetImageMetadataResponseV2] + } yield Some(imageMetadataResponse) + + case None => FastFuture.successful(None) + } } yield OneOfMultipleResourceCreateRequestV1( resourceTypeIri = resourceRequest.restype_id, clientResourceID = resourceRequest.client_id, label = resourceRequest.label, values = valuesToBeCreated.toMap, - file = resourceRequest.file.map { - fileToRead => - SipiConversionPathRequestV1( - originalFilename = stringFormatter.toSparqlEncodedString(fileToRead.file.getName, throw BadRequestException(s"The filename is invalid: '${fileToRead.file.getName}'")), - originalMimeType = stringFormatter.toSparqlEncodedString(fileToRead.mimeType, throw BadRequestException(s"The MIME type is invalid: '${fileToRead.mimeType}'")), - projectShortcode = projectShortcode, - source = fileToRead.file, - userProfile = userProfile.asUserProfileV1 - ) - }, + file = maybeImageMetadataResponse, creationDate = resourceRequest.creationDate ) } @@ -326,11 +315,12 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) } yield projectResponse.project.shortcode resourcesToCreate: Seq[Future[OneOfMultipleResourceCreateRequestV1]] = resourceRequest.map { - createResourceRequest => createOneResourceRequestFromXmlImport( - resourceRequest = createResourceRequest, - projectShortcode = projectShortcode, - userProfile = userProfile - ) + createResourceRequest => + createOneResourceRequestFromXmlImport( + resourceRequest = createResourceRequest, + projectShortcode = projectShortcode, + userProfile = userProfile + ) } resToCreateCollection: Seq[OneOfMultipleResourceCreateRequestV1] <- Future.sequence(resourcesToCreate) @@ -709,21 +699,12 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) val childElementsAfterLabel = childElements.tail - // Get the resource's file metadata, if any. This represents a file that has already been stored by Sipi. + // Get the name of the resource's file, if any. This represents a file that in Sipi's temporary storage. // If provided, it must be the second child element of the resource element. - val file: Option[ReadFileV1] = childElementsAfterLabel.headOption match { + val file: Option[String] = childElementsAfterLabel.headOption match { case Some(secondChildElem) => if (secondChildElem.label == "file") { - val path = Paths.get(secondChildElem.attribute("path").get.text) - - if (!path.isAbsolute) { - throw BadRequestException(s"File path $path in resource '$clientIDForResource' is not absolute") - } - - Some(ReadFileV1( - file = path.toFile, - mimeType = secondChildElem.attribute("mimetype").get.text - )) + Some(secondChildElem.attribute("filename").get.text) } else { None } @@ -942,7 +923,7 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) log = log ) } ~ post { - // Create a new resource with he given type and possibly a file (GUI-case). + // Create a new resource with the given type and possibly a file. // The binary file is already managed by Sipi. // For further details, please read the docs: Sipi -> Interaction Between Sipi and Knora. entity(as[CreateResourceApiRequestV1]) { apiRequest => @@ -952,100 +933,6 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) request <- makeCreateResourceRequestMessage(apiRequest = apiRequest, userADM = userProfile) } yield request - RouteUtilV1.runJsonRouteWithFuture( - requestMessageF = requestMessageFuture, - requestContext = requestContext, - settings = settings, - responderManager = responderManager, - log = log - ) - } - } ~ post { - // Create a new resource with the given type, properties, and binary data (file) (non GUI-case). - // The binary data are contained in the request and have to be temporarily stored by Knora. - // For further details, please read the docs: Sipi -> Interaction Between Sipi and Knora. - entity(as[Multipart.FormData]) { formdata: Multipart.FormData => - requestContext => - - log.debug("/v1/resources - POST - Multipart.FormData - Route") - - type Name = String - - val JSON_PART = "json" - val FILE_PART = "file" - - val receivedFile = Promise[File] - - log.debug(s"receivedFile is completed before: ${receivedFile.isCompleted}") - - // collect all parts of the multipart as it arrives into a map - val allPartsFuture: Future[Map[Name, Any]] = formdata.parts.mapAsync[(Name, Any)](1) { - case b: BodyPart if b.name == JSON_PART => - log.debug(s"inside allPartsFuture - processing $JSON_PART") - b.toStrict(2.seconds).map(strict => (b.name, strict.entity.data.utf8String.parseJson)) - - case b: BodyPart if b.name == FILE_PART => - log.debug(s"inside allPartsFuture - processing $FILE_PART") - val filename = b.filename.getOrElse(throw BadRequestException(s"Filename is not given")) - val tmpFile = FileUtil.createTempFile(settings) - val written = b.entity.dataBytes.runWith(FileIO.toPath(tmpFile.toPath)) - written.map { written => - //println(s"written result: ${written.wasSuccessful}, ${b.filename.get}, ${tmpFile.getAbsolutePath}") - receivedFile.success(tmpFile) - (b.name, FileInfo(b.name, b.filename.get, b.entity.contentType)) - } - - case b: BodyPart if b.name.isEmpty => throw BadRequestException("part of HTTP multipart request has no name") - case b: BodyPart => throw BadRequestException(s"multipart contains invalid name: ${b.name}") - }.runFold(Map.empty[Name, Any])((map, tuple) => map + tuple) - - // this file will be deleted by Knora once it is not needed anymore - // TODO: add a script that cleans files in the tmp location that have a certain age - // TODO (in case they were not deleted by Knora which should not happen -> this has also to be implemented for Sipi for the thumbnails) - // TODO: how to check if the user has sent multiple files? - - val requestMessageFuture: Future[ResourceCreateRequestV1] = for { - - userADM <- getUserADM(requestContext) - userProfile = userADM.asUserProfileV1 - - allParts <- allPartsFuture - // get the json params and turn them into a case class - apiRequest: CreateResourceApiRequestV1 = try { - allParts.getOrElse(JSON_PART, throw BadRequestException(s"MultiPart POST request was sent without required '$JSON_PART' part!")).asInstanceOf[JsValue].convertTo[CreateResourceApiRequestV1] - } catch { - case e: DeserializationException => throw BadRequestException("JSON params structure is invalid: " + e.toString) - } - - // check if the API request contains file information: this is illegal for this route - _ = if (apiRequest.file.nonEmpty) throw BadRequestException("param 'file' is set for a post multipart request. This is not allowed.") - - sourcePath <- receivedFile.future - - // get the file info containing the original filename and content type. - fileInfo = allParts.getOrElse(FILE_PART, throw BadRequestException(s"MultiPart POST request was sent without required '$FILE_PART' part!")).asInstanceOf[FileInfo] - originalFilename = fileInfo.fileName - originalMimeType = fileInfo.contentType.toString - - projectIri = stringFormatter.validateAndEscapeIri(apiRequest.project_id, throw BadRequestException(s"Invalid project IRI: ${apiRequest.project_id}")) - - projectResponse: ProjectGetResponseADM <- (responderManager ? ProjectGetRequestADM(maybeIri = Some(projectIri), requestingUser = userADM)).mapTo[ProjectGetResponseADM] - - sipiConvertPathRequest = SipiConversionPathRequestV1( - originalFilename = stringFormatter.toSparqlEncodedString(originalFilename, throw BadRequestException(s"Original filename is invalid: '$originalFilename'")), - originalMimeType = stringFormatter.toSparqlEncodedString(originalMimeType, throw BadRequestException(s"Original MIME type is invalid: '$originalMimeType'")), - projectShortcode = projectResponse.project.shortcode, - source = sourcePath, - userProfile = userProfile - ) - - requestMessage <- makeCreateResourceRequestMessage( - apiRequest = apiRequest, - multipartConversionRequest = Some(sipiConvertPathRequest), - userADM = userADM - ) - } yield requestMessage - RouteUtilV1.runJsonRouteWithFuture( requestMessageF = requestMessageFuture, requestContext = requestContext, diff --git a/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala b/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala index e18a8625f5..e1cca9dc11 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala @@ -19,29 +19,24 @@ package org.knora.webapi.routing.v1 -import java.io.File import java.util.UUID -import akka.pattern._ -import akka.http.scaladsl.model.Multipart -import akka.http.scaladsl.model.Multipart.BodyPart import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route -import akka.http.scaladsl.server.directives.FileInfo import akka.http.scaladsl.util.FastFuture +import akka.pattern._ import akka.stream.ActorMaterializer -import akka.stream.scaladsl.FileIO import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{SipiConversionFileRequestV1, SipiConversionPathRequestV1} +import org.knora.webapi.messages.store.sipimessages.SipiConversionFileRequestV1 import org.knora.webapi.messages.v1.responder.resourcemessages.{ResourceInfoGetRequestV1, ResourceInfoResponseV1} import org.knora.webapi.messages.v1.responder.valuemessages.ApiValueV1JsonProtocol._ import org.knora.webapi.messages.v1.responder.valuemessages._ import org.knora.webapi.routing.{Authenticator, KnoraRoute, KnoraRouteData, RouteUtilV1} +import org.knora.webapi.util.DateUtilV1 import org.knora.webapi.util.standoff.StandoffTagUtilV2.TextWithStandoffTagsV2 -import org.knora.webapi.util.{DateUtilV1, FileUtil} -import scala.concurrent.{Future, Promise} +import scala.concurrent.Future /** * Provides a spray-routing function for API routes that deal with values. @@ -315,9 +310,7 @@ class ValuesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit ) } - def makeChangeFileValueRequest(resIriStr: IRI, projectShortcode: String, apiRequest: Option[ChangeFileValueApiRequestV1], multipartConversionRequest: Option[SipiConversionPathRequestV1], userADM: UserADM): ChangeFileValueRequestV1 = { - if (apiRequest.nonEmpty && multipartConversionRequest.nonEmpty) throw BadRequestException("File information is present twice, only one is allowed.") - + def makeChangeFileValueRequest(resIriStr: IRI, projectShortcode: String, apiRequest: Option[ChangeFileValueApiRequestV1], userADM: UserADM): ChangeFileValueRequestV1 = { val resourceIri = stringFormatter.validateAndEscapeIri(resIriStr, throw BadRequestException(s"Invalid resource IRI: $resIriStr")) if (apiRequest.nonEmpty) { @@ -335,14 +328,6 @@ class ValuesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit file = fileRequest, apiRequestID = UUID.randomUUID, userProfile = userADM) - } - else if (multipartConversionRequest.nonEmpty) { - // non GUI-case - ChangeFileValueRequestV1( - resourceIri = resourceIri, - file = multipartConversionRequest.get, - apiRequestID = UUID.randomUUID, - userProfile = userADM) } else { // no file information was provided throw BadRequestException("A file value change was requested but no file information was provided") @@ -487,7 +472,6 @@ class ValuesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit resIriStr = resIriStr, projectShortcode = projectShortcode, apiRequest = Some(apiRequest), - multipartConversionRequest = None, userADM = userADM ) @@ -499,81 +483,6 @@ class ValuesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit log ) } - } ~ put { - entity(as[Multipart.FormData]) { formdata => - requestContext => - - log.debug("/v1/filevalue - PUT - Multipart.FormData - Route") - - - - val FILE_PART = "file" - - type Name = String - - val receivedFile = Promise[File] - - // this file will be deleted by Knora once it is not needed anymore - // TODO: add a script that cleans files in the tmp location that have a certain age - // TODO (in case they were not deleted by Knora which should not happen -> this has also to be implemented for Sipi for the thumbnails) - // TODO: how to check if the user has sent multiple files? - - /* get the file data and save file to temporary location */ - // collect all parts of the multipart as it arrives into a map - val allPartsFuture: Future[Map[Name, Any]] = formdata.parts.mapAsync[(Name, Any)](1) { - case b: BodyPart => - if (b.name == FILE_PART) { - log.debug(s"inside allPartsFuture - processing $FILE_PART") - val filename = b.filename.getOrElse(throw BadRequestException(s"Filename is not given")) - val tmpFile = FileUtil.createTempFile(settings) - val written = b.entity.dataBytes.runWith(FileIO.toPath(tmpFile.toPath)) - written.map { written => - log.debug(s"written result: ${written.wasSuccessful}, ${b.filename.get}, ${tmpFile.getAbsolutePath}") - receivedFile.success(tmpFile) - (b.name, FileInfo(b.name, filename, b.entity.contentType)) - } - } else { - throw BadRequestException(s"Unexpected body part '${b.name}' in multipart request") - } - }.runFold(Map.empty[Name, Any])((map, tuple) => map + tuple) - - val requestMessageFuture = for { - userADM <- getUserADM(requestContext) - allParts <- allPartsFuture - sourcePath <- receivedFile.future - // get the file info containing the original filename and content type. - fileInfo = allParts.getOrElse(FILE_PART, throw BadRequestException(s"MultiPart POST request was sent without required '$FILE_PART' part!")).asInstanceOf[FileInfo] - originalFilename = fileInfo.fileName - originalMimeType = fileInfo.contentType.toString - - resourceIri = stringFormatter.validateAndEscapeIri(resIriStr, throw BadRequestException(s"Invalid resource IRI: $resIriStr")) - resourceInfoResponse <- (responderManager ? ResourceInfoGetRequestV1(resourceIri, userADM)).mapTo[ResourceInfoResponseV1] - projectShortcode = resourceInfoResponse.resource_info.getOrElse(throw NotFoundException(s"Resource not found: $resourceIri")).project_shortcode - - sipiConvertPathRequest = SipiConversionPathRequestV1( - originalFilename = stringFormatter.toSparqlEncodedString(originalFilename, throw BadRequestException(s"The original filename is invalid: '$originalFilename'")), - originalMimeType = stringFormatter.toSparqlEncodedString(originalMimeType, throw BadRequestException(s"The original MIME type is invalid: '$originalMimeType'")), - projectShortcode = projectShortcode, - source = sourcePath, - userProfile = userADM.asUserProfileV1 - ) - - } yield makeChangeFileValueRequest( - resIriStr = resIriStr, - projectShortcode = projectShortcode, - apiRequest = None, - multipartConversionRequest = Some(sipiConvertPathRequest), - userADM = userADM - ) - - RouteUtilV1.runJsonRouteWithFuture( - requestMessageFuture, - requestContext, - settings, - responderManager, - log - ) - } } } } diff --git a/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala b/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala index c2bc7f2082..07abf2c0ff 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala @@ -71,7 +71,6 @@ class SipiConnector extends Actor with ActorLogging { private val httpClient: CloseableHttpClient = HttpClients.custom.setDefaultRequestConfig(sipiRequestConfig).build override def receive: Receive = { - case convertPathRequest: SipiConversionPathRequestV1 => try2Message(sender(), convertPathV1(convertPathRequest), log) case convertFileRequest: SipiConversionFileRequestV1 => try2Message(sender(), convertFileV1(convertFileRequest), log) case getFileMetadataRequestV2: GetImageMetadataRequestV2 => try2Message(sender(), getFileMetadataV2(getFileMetadataRequestV2), log) case moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequestV2 => try2Message(sender(), moveTemporaryFileToPermanentStorageV2(moveTemporaryFileToPermanentStorageRequestV2), log) @@ -80,18 +79,6 @@ class SipiConnector extends Actor with ActorLogging { case other => handleUnexpectedMessage(sender(), other, log, this.getClass.getName) } - /** - * Convert a file that has been sent to Knora (non GUI-case). - * - * @param conversionRequest the information about the file (uploaded by Knora). - * @return a [[SipiConversionResponseV1]] representing the file values to be added to the triplestore. - */ - private def convertPathV1(conversionRequest: SipiConversionPathRequestV1): Try[SipiConversionResponseV1] = { - val url = s"${settings.internalSipiImageConversionUrlV1}/${settings.sipiPathConversionRouteV1}" - - callSipiConvertRoute(url, conversionRequest) - } - /** * Convert a file that is already managed by Sipi (GUI-case). * diff --git a/webapi/src/test/scala/org/knora/webapi/e2e/v1/SipiV1R2RSpec.scala b/webapi/src/test/scala/org/knora/webapi/e2e/v1/SipiV1R2RSpec.scala index d3fb44e5a3..c79333af9b 100644 --- a/webapi/src/test/scala/org/knora/webapi/e2e/v1/SipiV1R2RSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/e2e/v1/SipiV1R2RSpec.scala @@ -26,7 +26,6 @@ import java.nio.file.{Files, Paths} import akka.actor.{Props, _} import akka.http.scaladsl.model._ import akka.http.scaladsl.model.headers.BasicHttpCredentials -import akka.http.scaladsl.server.Route import akka.http.scaladsl.testkit.RouteTestTimeout import org.knora.webapi._ import org.knora.webapi.messages.store.triplestoremessages.RdfDataObject @@ -34,7 +33,7 @@ import org.knora.webapi.messages.v1.responder.resourcemessages.{CreateResourceAp import org.knora.webapi.messages.v1.responder.valuemessages.{ChangeFileValueApiRequestV1, CreateFileV1, CreateRichtextV1} import org.knora.webapi.routing.v1.{ResourcesRouteV1, ValuesRouteV1} import org.knora.webapi.store.SipiConnectorActorName -import org.knora.webapi.store.iiif.{MockSipiConnector, SourcePath} +import org.knora.webapi.store.iiif.MockSipiConnector /** @@ -52,9 +51,8 @@ class SipiV1R2RSpec extends R2RSpec { private val resourcesPath = new ResourcesRouteV1(routeData).knoraApiPath private val valuesPath = new ValuesRouteV1(routeData).knoraApiPath - implicit def default(implicit system: ActorSystem) = RouteTestTimeout(settings.defaultTimeout) + implicit def default(implicit system: ActorSystem): RouteTestTimeout = RouteTestTimeout(settings.defaultTimeout) - private val rootEmail = SharedTestDataV1.rootUser.userData.email.get private val incunabulaProjectAdminEmail = SharedTestDataV1.incunabulaProjectAdminUser.userData.email.get private val testPass = "test" @@ -109,71 +107,7 @@ class SipiV1R2RSpec extends R2RSpec { "The Resources Endpoint" should { - "create a resource with a digital representation doing a multipart request containing the binary data (non GUI-case)" in { - - val fileToSend = new File(RequestParams.pathToFile) - // check if the file exists - assert(fileToSend.exists(), s"File ${RequestParams.pathToFile} does not exist") - - val formData = Multipart.FormData( - Multipart.FormData.BodyPart( - "json", - HttpEntity(ContentTypes.`application/json`, RequestParams.createResourceParams.toJsValue.compactPrint) - ), - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - RequestParams.createTmpFileDir() - - Post("/v1/resources", formData) ~> addCredentials(BasicHttpCredentials(incunabulaProjectAdminEmail, testPass)) ~> resourcesPath ~> check { - - val tmpFile = SourcePath.getSourcePath() - - //println("response in test: " + responseAs[String]) - assert(!tmpFile.exists(), s"Tmp file $tmpFile was not deleted.") - assert(status == StatusCodes.OK, "Status code is not set to OK, Knora says:\n" + responseAs[String]) - } - } - - "try to create a resource sending binaries (multipart request) but fail because the mimetype is wrong" in { - - val fileToSend = new File(RequestParams.pathToFile) - // check if the file exists - assert(fileToSend.exists(), s"File ${RequestParams.pathToFile} does not exist") - - val formData = Multipart.FormData( - Multipart.FormData.BodyPart( - "json", - HttpEntity(MediaTypes.`application/json`, RequestParams.createResourceParams.toJsValue.compactPrint) - ), - // set mimetype tiff, but jpeg is expected - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/tiff`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - RequestParams.createTmpFileDir() - - Post("/v1/resources", formData) ~> addCredentials(BasicHttpCredentials(incunabulaProjectAdminEmail, testPass)) ~> Route.seal(resourcesPath) ~> check { - - val tmpFile = SourcePath.getSourcePath() - - // this test is expected to fail - - // check that the tmp file is also deleted in case the test fails - assert(!tmpFile.exists(), s"Tmp file $tmpFile was not deleted.") - //FIXME: Check for correct status code. This would then also test if the negative case is handled correctly inside Knora. - assert(status != StatusCodes.OK, "Status code is not set to OK, Knora says:\n" + responseAs[String]) - } - } - - "create a resource with a digital representation doing a params only request without binary data (GUI-case)" in { + "create a resource with a digital representation" in { val params = RequestParams.createResourceParams.copy( file = Some(CreateFileV1( @@ -192,69 +126,7 @@ class SipiV1R2RSpec extends R2RSpec { "The Values endpoint" should { - "change the file value of an existing page (submitting binaries)" in { - - val fileToSend = new File(RequestParams.pathToFile) - // check if the file exists - assert(fileToSend.exists(), s"File ${RequestParams.pathToFile} does not exist") - - val formData = Multipart.FormData( - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - RequestParams.createTmpFileDir() - - val resIri = URLEncoder.encode("http://rdfh.ch/0803/8a0b1e75", "UTF-8") - - Put("/v1/filevalue/" + resIri, formData) ~> addCredentials(BasicHttpCredentials(incunabulaProjectAdminEmail, testPass)) ~> valuesPath ~> check { - - val tmpFile = SourcePath.getSourcePath() - - assert(!tmpFile.exists(), s"Tmp file $tmpFile was not deleted.") - assert(status == StatusCodes.OK, "Status code is not set to OK, Knora says:\n" + responseAs[String]) - } - - } - - "try to change the file value of an existing page (submitting binaries) but fail because the mimetype is wrong" in { - - val fileToSend = new File(RequestParams.pathToFile) - // check if the file exists - assert(fileToSend.exists(), s"File ${RequestParams.pathToFile} does not exist") - - val formData = Multipart.FormData( - // set mimetype tiff, but jpeg is expected - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/tiff`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - RequestParams.createTmpFileDir() - - val resIri = URLEncoder.encode("http://rdfh.ch/0803/8a0b1e75", "UTF-8") - - Put("/v1/filevalue/" + resIri, formData) ~> addCredentials(BasicHttpCredentials(incunabulaProjectAdminEmail, testPass)) ~> valuesPath ~> check { - - val tmpFile = SourcePath.getSourcePath() - - // this test is expected to fail - - // check that the tmp file is also deleted in case the test fails - assert(!tmpFile.exists(), s"Tmp file $tmpFile was not deleted.") - //FIXME: Check for correct status code. This would then also test if the negative case is handled correctly inside Knora. - assert(status != StatusCodes.OK, "Status code is not set to OK, Knora says:\n" + responseAs[String]) - } - - } - - - "change the file value of an existing page (submitting params only, no binaries)" in { + "change the file value of an existing page" in { val params = ChangeFileValueApiRequestV1( file = CreateFileV1( diff --git a/webapi/src/test/scala/org/knora/webapi/store/iiif/MockSipiConnector.scala b/webapi/src/test/scala/org/knora/webapi/store/iiif/MockSipiConnector.scala index 59dc5a7e0c..d1ab1b3f7c 100644 --- a/webapi/src/test/scala/org/knora/webapi/store/iiif/MockSipiConnector.scala +++ b/webapi/src/test/scala/org/knora/webapi/store/iiif/MockSipiConnector.scala @@ -19,8 +19,6 @@ package org.knora.webapi.store.iiif -import java.io.File - import akka.actor.{Actor, ActorLogging, ActorSystem} import org.knora.webapi.messages.store.sipimessages._ import org.knora.webapi.messages.v1.responder.valuemessages.StillImageFileValueV1 @@ -31,22 +29,6 @@ import org.knora.webapi.{BadRequestException, KnoraDispatchers, Settings, SipiEx import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} -/** - * Keep track of the temporary files that was written in the route - * when submitting a multipart request - */ -object SourcePath { - private var sourcePath: File = new File("") // for init - - def setSourcePath(path: File) = { - sourcePath = path - } - - def getSourcePath() = { - sourcePath - } -} - /** * Constants for [[MockSipiConnector]]. */ @@ -68,12 +50,11 @@ class MockSipiConnector extends Actor with ActorLogging { implicit val system: ActorSystem = context.system implicit val executionContext: ExecutionContext = system.dispatchers.lookup(KnoraDispatchers.KnoraActorDispatcher) - val settings = Settings(system) + private val settings = Settings(system) def receive = { case sipiResponderConversionFileRequest: SipiConversionFileRequestV1 => future2Message(sender(), imageConversionResponse(sipiResponderConversionFileRequest), log) - case sipiResponderConversionPathRequest: SipiConversionPathRequestV1 => future2Message(sender(), imageConversionResponse(sipiResponderConversionPathRequest), log) case getFileMetadataRequestV2: GetImageMetadataRequestV2 => try2Message(sender(), getFileMetadataV2(getFileMetadataRequestV2), log) case moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequestV2 => try2Message(sender(), moveTemporaryFileToPermanentStorageV2(moveTemporaryFileToPermanentStorageRequestV2), log) case deleteTemporaryFileRequestV2: DeleteTemporaryFileRequestV2 => try2Message(sender(), deleteTemporaryFileV2(deleteTemporaryFileRequestV2), log) @@ -104,16 +85,6 @@ class MockSipiConnector extends Actor with ActorLogging { internalFilename = "full.jp2" ) - // Whenever Knora had to create a temporary file, store its path - // the calling test context can then make sure that is has actually been deleted after the test is done - // (on successful or failed conversion) - conversionRequest match { - case conversionPathRequest: SipiConversionPathRequestV1 => - // store path to tmp file - SourcePath.setSourcePath(conversionPathRequest.source) - case _ => () // params request only - } - SipiConversionResponseV1(fileValueV1, file_type = SipiConstants.FileType.IMAGE) } } From 00bdf4392d670b63fd4583fb61bae18e63725b4a Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Fri, 1 Mar 2019 12:33:12 +0100 Subject: [PATCH 02/24] feature (api-v1): Make bulk import with v2-style file uploads. --- .../resourcemessages/ResourceMessagesV1.scala | 31 +++- .../responders/v1/ResourcesResponderV1.scala | 158 +++++++++++------- .../webapi/routing/v1/ResourcesRouteV1.scala | 14 +- 3 files changed, 135 insertions(+), 68 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala index cb67bdb092..bc11f0807c 100755 --- a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala @@ -24,10 +24,12 @@ import java.util.UUID import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import org.knora.webapi._ +import org.knora.webapi.messages.admin.responder.projectsmessages.ProjectADM import org.knora.webapi.messages.admin.responder.usersmessages.UserADM import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataResponseV2, SipiConversionFileRequestV1, SipiConversionRequestV1} import org.knora.webapi.messages.v1.responder.valuemessages._ import org.knora.webapi.messages.v1.responder.{KnoraRequestV1, KnoraResponseV1} +import org.knora.webapi.messages.v2.responder.UpdateResultInProject import spray.json._ import scala.collection.breakOut @@ -228,7 +230,7 @@ case class OneOfMultipleResourceCreateRequestV1(resourceTypeIri: IRI, clientResourceID: String, label: String, values: Map[IRI, Seq[CreateValueV1WithComment]], - file: Option[GetImageMetadataResponseV2] = None, + file: Option[StillImageFileValueV1] = None, creationDate: Option[Instant]) /** @@ -251,9 +253,9 @@ case class MultipleResourceCreateRequestV1(resourcesToCreate: Seq[OneOfMultipleR * @param createdResources created resources * */ -case class MultipleResourceCreateResponseV1(createdResources: Seq[OneOfMultipleResourcesCreateResponseV1]) extends KnoraResponseV1 { +case class MultipleResourceCreateResponseV1(createdResources: Seq[OneOfMultipleResourcesCreateResponseV1], projectADM: ProjectADM) extends KnoraResponseV1 with UpdateResultInProject { - def toJsValue: JsValue = ResourceV1JsonProtocol.multipleResourceCreateResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.MultipleResourceCreateResponseV1Format.write(this) } @@ -1096,7 +1098,6 @@ object ResourceV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol } } - /** * Converts between [[ResourceInfoV1]] objects and [[JsValue]] objects. */ @@ -1141,6 +1142,27 @@ object ResourceV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol } } + /** + * Converts between [[MultipleResourceCreateResponseV1]] objects and [[JsValue]] objects. + */ + implicit object MultipleResourceCreateResponseV1Format extends JsonFormat[MultipleResourceCreateResponseV1] { + /** + * Not implemented. + */ + override def read(json: JsValue): MultipleResourceCreateResponseV1 = ??? + + /** + * Converts a [[MultipleResourceCreateResponseV1]] into a [[JsValue]] for formatting as JSON. + */ + override def write(response: MultipleResourceCreateResponseV1): JsValue = { + val fields = Map( + "createdResources" -> response.createdResources.toJson + ) + + JsObject(fields) + } + } + implicit val createResourceValueV1Format: RootJsonFormat[CreateResourceValueV1] = jsonFormat14(CreateResourceValueV1) implicit val createResourceApiRequestV1Format: RootJsonFormat[CreateResourceApiRequestV1] = jsonFormat5(CreateResourceApiRequestV1) implicit val ChangeResourceLabelApiRequestV1Format: RootJsonFormat[ChangeResourceLabelApiRequestV1] = jsonFormat1(ChangeResourceLabelApiRequestV1) @@ -1157,7 +1179,6 @@ object ResourceV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol implicit val resourceCreateValueObjectResponseV1Format: RootJsonFormat[ResourceCreateValueObjectResponseV1] = jsonFormat14(ResourceCreateValueObjectResponseV1) implicit val resourceCreateValueResponseV1Format: RootJsonFormat[ResourceCreateValueResponseV1] = jsonFormat2(ResourceCreateValueResponseV1) implicit val oneOfMultipleResourcesCreateResponseFormat: JsonFormat[OneOfMultipleResourcesCreateResponseV1] = jsonFormat3(OneOfMultipleResourcesCreateResponseV1) - implicit val multipleResourceCreateResponseV1Format: RootJsonFormat[MultipleResourceCreateResponseV1] = jsonFormat1(MultipleResourceCreateResponseV1) implicit val resourceCreateResponseV1Format: RootJsonFormat[ResourceCreateResponseV1] = jsonFormat2(ResourceCreateResponseV1) implicit val resourceDeleteResponseV1Format: RootJsonFormat[ResourceDeleteResponseV1] = jsonFormat1(ResourceDeleteResponseV1) implicit val changeResourceLabelResponseV1Format: RootJsonFormat[ChangeResourceLabelResponseV1] = jsonFormat2(ChangeResourceLabelResponseV1) diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala b/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala index 8b0a47cb51..61f7fbbd79 100755 --- a/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala @@ -26,6 +26,7 @@ import akka.http.scaladsl.util.FastFuture import akka.pattern._ import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.permissionsmessages.{DefaultObjectAccessPermissionsStringForPropertyGetADM, DefaultObjectAccessPermissionsStringForResourceClassGetADM, DefaultObjectAccessPermissionsStringResponseADM, ResourceCreateOperation} +import org.knora.webapi.messages.admin.responder.projectsmessages.{ProjectGetRequestADM, ProjectGetResponseADM} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM import org.knora.webapi.messages.store.sipimessages._ import org.knora.webapi.messages.store.triplestoremessages._ @@ -35,8 +36,10 @@ import org.knora.webapi.messages.v1.responder.resourcemessages.{MultipleResource import org.knora.webapi.messages.v1.responder.valuemessages._ import org.knora.webapi.messages.v2.responder.ontologymessages.Cardinality.KnoraCardinalityInfo import org.knora.webapi.messages.v2.responder.ontologymessages.{Cardinality, OntologyMetadataGetByIriRequestV2, OntologyMetadataV2, ReadOntologyMetadataV2} +import org.knora.webapi.messages.v2.responder.valuemessages.{FileValueV2, StillImageFileValueContentV2} import org.knora.webapi.responders.Responder.handleUnexpectedMessage import org.knora.webapi.responders.v1.GroupedProps._ +import org.knora.webapi.responders.v2.ResourceUtilV2 import org.knora.webapi.responders.{IriLocker, Responder, ResponderData} import org.knora.webapi.twirl.SparqlTemplateResourceToCreate import org.knora.webapi.util.IriConversions._ @@ -44,7 +47,7 @@ import org.knora.webapi.util._ import scala.collection.immutable import scala.concurrent.Future -import scala.util.Try +import scala.util.{Failure, Success, Try} /** * Responds to requests for information about resources, and returns responses in Knora API v1 format. @@ -1213,15 +1216,14 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo } } } - else - { - // ?firstProp is sufficient: the client requested just one property per resource that was found - ResourceSearchResultRowV1( - id = row.rowMap("resourceIri"), - value = Vector(firstProp), - rights = permissionCode - ) - } + else { + // ?firstProp is sufficient: the client requested just one property per resource that was found + ResourceSearchResultRowV1( + id = row.rowMap("resourceIri"), + value = Vector(firstProp), + rights = permissionCode + ) + } } yield searchResultRow @@ -1240,38 +1242,56 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo * @param resourcesToCreate collection of ResourceRequests . * @param projectIri IRI of the project . * @param apiRequestID the the ID of the API request. - * @param userProfile the profile of the user making the request. + * @param requestingUser the user making the request. * @return a [[MultipleResourceCreateResponseV1]] informing the client about the new resources. */ private def createMultipleNewResources(resourcesToCreate: Seq[OneOfMultipleResourceCreateRequestV1], projectIri: IRI, - userProfile: UserADM, + requestingUser: UserADM, apiRequestID: UUID): Future[MultipleResourceCreateResponseV1] = { - val userProfileV1 = userProfile.asUserProfileV1 - - // TODO: this method has to send MoveTemporaryFileToPermanentStorageRequestV2 for each file, as ResourcesResponderV2 does. + // Convert all the image metadata in the request to StillImageFileValueContentV2 instances, so we + // can use ResourceUtilV2.doSipiPostUpdate after updating the triplestore. + val stillImageFileValueContentV2s: Seq[StillImageFileValueContentV2] = resourcesToCreate.flatMap { + resourceToCreate => + resourceToCreate.file.map { + stillImageFileValueV1: StillImageFileValueV1 => + StillImageFileValueContentV2( + ontologySchema = InternalSchema, + fileValue = FileValueV2( + internalFilename = stillImageFileValueV1.internalFilename, + internalMimeType = stillImageFileValueV1.internalMimeType, + originalFilename = stillImageFileValueV1.originalFilename, + originalMimeType = stillImageFileValueV1.internalMimeType + ), + dimX = stillImageFileValueV1.dimX, + dimY = stillImageFileValueV1.dimY + ) + } + } - for { + val updateFuture: Future[MultipleResourceCreateResponseV1] = for { // Get user's IRI and don't allow anonymous users to create resources. userIri: IRI <- Future { - if (userProfile.isAnonymousUser) { + if (requestingUser.isAnonymousUser) { throw ForbiddenException("Anonymous users aren't allowed to create resources") } else { - userProfile.id + requestingUser.id } } // Get information about the project in which the resources will be created. projectInfoResponse <- { - responderManager ? ProjectInfoByIRIGetRequestV1( - projectIri, - Some(userProfileV1) + responderManager ? ProjectGetRequestADM( + maybeIri = Some(projectIri), + requestingUser = requestingUser ) - }.mapTo[ProjectInfoResponseV1] + }.mapTo[ProjectGetResponseADM] + + projectADM = projectInfoResponse.project // Ensure that the project isn't the system project or the shared ontologies project. - resourceProjectIri: IRI = projectInfoResponse.project_info.id + resourceProjectIri: IRI = projectADM.id _ = if (resourceProjectIri == OntologyConstants.KnoraBase.SystemProject || resourceProjectIri == OntologyConstants.KnoraBase.DefaultSharedOntologiesProject) { throw BadRequestException(s"Resources cannot be created in project $resourceProjectIri") @@ -1279,11 +1299,11 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo // Ensure that the resource class isn't from a non-shared ontology in another project. - namedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfoResponse.project_info) + namedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV2(projectADM) // Create random IRIs for resources, collect in Map[clientResourceID, IRI] clientResourceIDsToResourceIris: Map[String, IRI] = new ErrorHandlingMap( - toWrap = resourcesToCreate.map(resRequest => resRequest.clientResourceID -> knoraIdUtil.makeRandomResourceIri(projectInfoResponse.project_info.shortcode)).toMap, + toWrap = resourcesToCreate.map(resRequest => resRequest.clientResourceID -> knoraIdUtil.makeRandomResourceIri(projectADM.shortcode)).toMap, errorTemplateFun = { key => s"Resource $key is the target of a link, but was not provided in the request" }, errorFun = { errorMsg => throw BadRequestException(errorMsg) } ) @@ -1302,7 +1322,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo // Ensure that none of the resource classes is from a non-shared ontology in another project. resourceClassOntologyIris: Set[SmartIri] = resourceClasses.map(_.toSmartIri.getOntologyFromEntity) - readOntologyMetadataV2: ReadOntologyMetadataV2 <- (responderManager ? OntologyMetadataGetByIriRequestV2(resourceClassOntologyIris, userProfile)).mapTo[ReadOntologyMetadataV2] + readOntologyMetadataV2: ReadOntologyMetadataV2 <- (responderManager ? OntologyMetadataGetByIriRequestV2(resourceClassOntologyIris, requestingUser)).mapTo[ReadOntologyMetadataV2] _ = for (ontologyMetadata <- readOntologyMetadataV2.ontologies) { val ontologyProjectIri: IRI = ontologyMetadata.projectIri.getOrElse(throw InconsistentTriplestoreDataException(s"Ontology ${ontologyMetadata.ontologyIri} has no project")).toString @@ -1315,7 +1335,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo resourceClassesEntityInfoResponse: EntityInfoGetResponseV1 <- (responderManager ? EntityInfoGetRequestV1( resourceClassIris = resourceClasses, propertyIris = Set.empty[IRI], - userProfile = userProfile + userProfile = requestingUser )).mapTo[EntityInfoGetResponseV1] allPropertyIris: Set[IRI] = resourceClassesEntityInfoResponse.resourceClassInfoMap.flatMap { @@ -1326,7 +1346,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo propertyEntityInfoResponse: EntityInfoGetResponseV1 <- (responderManager ? EntityInfoGetRequestV1( resourceClassIris = Set.empty[IRI], propertyIris = allPropertyIris, - userProfile = userProfile + userProfile = requestingUser )).mapTo[EntityInfoGetResponseV1] propertyEntityInfoMapsPerResource: Map[IRI, Map[IRI, PropertyInfoV1]] = resourceClassesEntityInfoResponse.resourceClassInfoMap.map { @@ -1348,7 +1368,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo responderManager ? DefaultObjectAccessPermissionsStringForResourceClassGetADM( projectIri = projectIri, resourceClassIri = resourceClassIri, - targetUser = userProfile, + targetUser = requestingUser, requestingUser = KnoraSystemInstances.Users.SystemUser ) }.mapTo[DefaultObjectAccessPermissionsStringResponseADM] @@ -1368,7 +1388,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo projectIri = projectIri, resourceClassIri = resourceClassIri, propertyIri = propertyIri, - targetUser = userProfile, + targetUser = requestingUser, requestingUser = KnoraSystemInstances.Users.SystemUser) }.mapTo[DefaultObjectAccessPermissionsStringResponseADM] } yield (propertyIri, defaultObjectAccessPermissions.permissionLiteral) @@ -1411,19 +1431,10 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo resourceClassInfo = resourceClassesEntityInfoResponse.resourceClassInfoMap(resourceCreateRequest.resourceTypeIri), propertyInfoMap = propertyEntityInfoMapsPerResource(resourceCreateRequest.resourceTypeIri), values = resourceCreateRequest.values, - sipiConversionRequest = resourceCreateRequest.file.map { - fileMetadata => - // TODO: this is nonsense, just to get this file to compile for now. - SipiConversionFileRequestV1( - originalFilename = fileMetadata.originalFilename, - originalMimeType = fileMetadata.originalMimeType, - projectShortcode = projectInfoResponse.project_info.shortcode, - filename = fileMetadata.originalFilename, - userProfile = userProfile.asUserProfileV1 - ) - }, + sipiConversionRequest = None, + convertedFile = resourceCreateRequest.file, clientResourceIDsToResourceClasses = clientResourceIDsToResourceClasses, - userProfile = userProfile + userProfile = requestingUser ) // Convert each LinkToClientIDUpdateV1 into a LinkUpdateV1. @@ -1456,7 +1467,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo clientResourceIDsToResourceIris = clientResourceIDsToResourceIris, creationDate = creationDate, fileValues = fileValues, - userProfile = userProfile, + userProfile = requestingUser, apiRequestID = apiRequestID ) @@ -1496,7 +1507,28 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo label = resourceToCreate.label ) } - } yield MultipleResourceCreateResponseV1(responses) + } yield MultipleResourceCreateResponseV1(responses, projectADM) + + // Use ResourceUtilV2.doSipiPostUpdate to ask Sipi to to move temporary image files to permanent storage if the + // triplestore update was successful, or to delete the temporary files if the triplestore update failed. + val sipiPostUpdateResultFutures: Seq[Future[MultipleResourceCreateResponseV1]] = stillImageFileValueContentV2s.map { + valueContent => + ResourceUtilV2.doSipiPostUpdate( + updateFuture = updateFuture, + valueContent = valueContent, + requestingUser = requestingUser, + responderManager = responderManager, + storeManager = storeManager, + log = log + ) + } + + // If ResourceUtilV2.doSipiPostUpdate returned an error, return it to the client, otherwise return + // a MultipleResourceCreateResponseV1. + Future.sequence(sipiPostUpdateResultFutures).transformWith { + case Success(_) => updateFuture + case Failure(e) => Future.failed(e) + } } /** @@ -1508,7 +1540,8 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo * @param values values to be created for resource. If `linkTargetsAlreadyExist` is true, any links must be represented as [[LinkUpdateV1]] instances. * Otherwise, they must be represented as [[LinkToClientIDUpdateV1]] instances, so that appropriate error messages can * be generated for links to missing resources. - * @param sipiConversionRequest a file (binary representation) to be attached to the resource (GUI and non GUI-case). + * @param sipiConversionRequest a file to be converted and attached to the resource. + * @param convertedFile an already converted file to be attached to the resource. * @param clientResourceIDsToResourceClasses for each client resource ID, the IRI of the resource's class. Used only if `linkTargetsAlreadyExist` is false. * @param userProfile the profile of the user making the request. * @return a tuple (IRI, Vector[CreateValueV1WithComment]) containing the IRI of the resource and a collection of holders of [[UpdateValueV1]] and comment. @@ -1518,6 +1551,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo propertyInfoMap: Map[IRI, PropertyInfoV1], values: Map[IRI, Seq[CreateValueV1WithComment]], sipiConversionRequest: Option[SipiConversionRequestV1], + convertedFile: Option[StillImageFileValueV1], clientResourceIDsToResourceClasses: Map[String, IRI] = new ErrorHandlingMap[IRI, IRI]( toWrap = Map.empty[IRI, IRI], errorTemplateFun = { key => s"Resource $key is the target of a link, but was not provided in the request" }, @@ -1602,9 +1636,6 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo } } - // maximally one file value can be handled here - _ = if (resourceClassInfo.fileValueProperties.size > 1) throw BadRequestException(s"The given resource type $resourceClassIri requires more than on file value. This is not supported for API V1") - // Check that no required values are missing. requiredProps: Set[IRI] = resourceClassInfo.knoraResourceCardinalities.filter { case (propIri, cardinalityInfo) => cardinalityInfo.cardinality == Cardinality.MustHaveOne || cardinalityInfo.cardinality == Cardinality.MustHaveSome @@ -1619,20 +1650,26 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo // check if a file value is required by the ontology fileValues: Option[(IRI, Vector[CreateValueV1WithComment])] <- if (resourceClassInfo.fileValueProperties.nonEmpty) { - // call sipi responder - for { - sipiResponse: SipiConversionResponseV1 <- (storeManager ? sipiConversionRequest.getOrElse(throw OntologyConstraintException(s"No file (required) given for resource type $resourceClassIri"))).mapTo[SipiConversionResponseV1] + (sipiConversionRequest, convertedFile) match { + case (Some(conversionRequest), None) => + // Send a message to SipiConnector to ask Sipi to convert the image file. + for { + sipiResponse: SipiConversionResponseV1 <- (storeManager ? conversionRequest).mapTo[SipiConversionResponseV1] - // check if the file type returned by Sipi corresponds to the expected fileValue property in resourceClassInfo.fileValueProperties.head - _ = if (SipiConstants.fileType2FileValueProperty(sipiResponse.file_type) != resourceClassInfo.fileValueProperties.head) { - // TODO: remove the file from SIPI (delete request) - throw BadRequestException(s"Type of submitted file (${sipiResponse.file_type}) does not correspond to expected property type ${resourceClassInfo.fileValueProperties.head}") - } + // check if the file type returned by Sipi corresponds to the expected fileValue property in resourceClassInfo.fileValueProperties.head + _ = if (SipiConstants.fileType2FileValueProperty(sipiResponse.file_type) != resourceClassInfo.fileValueProperties.head) { + // TODO: remove the file from SIPI (delete request) + throw BadRequestException(s"Type of submitted file (${sipiResponse.file_type}) does not correspond to expected property type ${resourceClassInfo.fileValueProperties.head}") + } + } yield Some(resourceClassInfo.fileValueProperties.head -> Vector(CreateValueV1WithComment(sipiResponse.fileValueV1))) - // in case we deal with a SipiResponderConversionPathRequestV1 (non GUI-case), the tmp file created by resources route - // has already been deleted by the SipiResponder + case (None, Some(converted)) => + // The file has already been converted, just return it. + Future.successful(Some(resourceClassInfo.fileValueProperties.head -> Vector(CreateValueV1WithComment(converted)))) + + case other => throw AssertionException(s"Expected a SipiConversionRequestV1 or a StillImageFileValueV1, got $other") + } - } yield Some(resourceClassInfo.fileValueProperties.head -> Vector(CreateValueV1WithComment(sipiResponse.fileValueV1))) } else { // resource class requires no binary representation // check if there was no file sent @@ -1640,7 +1677,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo sipiConversionRequest match { case None => Future(None) // expected behaviour case Some(_: SipiConversionFileRequestV1) => - throw BadRequestException(s"File params (GUI-case) are given but resource class $resourceClassIri does not allow any representation") + throw BadRequestException(s"File params are given but resource class $resourceClassIri does not allow any representation") } } } yield fileValues @@ -1840,6 +1877,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo propertyInfoMap = propertyInfoMap, values = values, sipiConversionRequest = sipiConversionRequest, + convertedFile = None, userProfile = userProfile ) diff --git a/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala b/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala index f5aff285c5..7ce7f38378 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala @@ -279,14 +279,22 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) } yield propIri -> values } - maybeImageMetadataResponse <- resourceRequest.file match { + convertedFileV1 <- resourceRequest.file match { case Some(filename) => // Ask Sipi about the file's metadata. val tempFileUrl = s"${settings.internalSipiBaseUrl}/tmp/$filename" for { imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequestV2(fileUrl = tempFileUrl, requestingUser = userProfile)).mapTo[GetImageMetadataResponseV2] - } yield Some(imageMetadataResponse) + } yield Some(StillImageFileValueV1( + internalFilename = filename, + internalMimeType = "image/jp2", + originalFilename = imageMetadataResponse.originalFilename, + originalMimeType = Some(imageMetadataResponse.originalMimeType), + projectShortcode = projectShortcode, + dimX = imageMetadataResponse.width, + dimY = imageMetadataResponse.height + )) case None => FastFuture.successful(None) } @@ -295,7 +303,7 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) clientResourceID = resourceRequest.client_id, label = resourceRequest.label, values = valuesToBeCreated.toMap, - file = maybeImageMetadataResponse, + file = convertedFileV1, creationDate = resourceRequest.creationDate ) } From 2694c04ba066ac1cd3b94a7fce90191411ed0a5d Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Fri, 1 Mar 2019 13:25:23 +0100 Subject: [PATCH 03/24] test (api-v1): Fix integration test. --- .../e2e/v1/KnoraSipiScriptsV1ITSpec.scala | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiScriptsV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiScriptsV1ITSpec.scala index 9c4f090a2a..fdd80760b2 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiScriptsV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiScriptsV1ITSpec.scala @@ -191,40 +191,6 @@ class KnoraSipiScriptsV1ITSpec extends ITKnoraFakeSpec(KnoraSipiScriptsV1ITSpec. } - "successfully call convert_from_binaries.lua sipi script" in { - - // The image to be uploaded. - val fileToSend = new File(pathToChlaus) - assert(fileToSend.exists(), s"File $pathToChlaus does not exist") - - // A multipart/form-data request containing the image. - val sipiFormData = FormData( - Map( - "originalFilename" -> fileToSend.getName, - "originalMimeType" -> "image/jpeg", - "prefix" -> "0001", - "source" -> fileToSend.getAbsolutePath - ) - ) - - // Send a POST request to Sipi, asking it to make a thumbnail of the image. - val sipiConvertFromBinariesPostRequest = Post(baseSipiUrl + "/convert_from_binaries", sipiFormData) - val sipiConvertFromBinariesPostResponseJson = getResponseJson(sipiConvertFromBinariesPostRequest) - - val filenameFull = sipiConvertFromBinariesPostResponseJson.fields("filename_full").asInstanceOf[JsString].value - - //log.debug("sipiConvertFromBinariesPostResponseJson: {}", sipiConvertFromBinariesPostResponseJson) - - // Running with KnoraFakeService which always allows access to files. - val sipiGetImageRequest = Get(baseSipiUrl + "/0001/" + filenameFull + "/full/full/0/default.jpg") ~> addCredentials(BasicHttpCredentials(username, password)) - checkResponseOK(sipiGetImageRequest) - - // Send a GET request to Sipi, asking for the info.json of the image - val sipiGetInfoRequest = Get(baseSipiUrl + "/0001/" + filenameFull + "/info.json" ) ~> addCredentials(BasicHttpCredentials(username, password)) - val sipiGetInfoResponseJson = getResponseJson(sipiGetInfoRequest) - log.debug("sipiGetInfoResponseJson: {}", sipiGetInfoResponseJson) - } - } } From cbca42e72a38ad0a9e068ca3dc255cecb5eda980 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Fri, 1 Mar 2019 16:25:15 +0100 Subject: [PATCH 04/24] fix (sipi): Remove debugging output. --- sipi/scripts/convert_from_file.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/sipi/scripts/convert_from_file.lua b/sipi/scripts/convert_from_file.lua index acf0654576..43823b454a 100644 --- a/sipi/scripts/convert_from_file.lua +++ b/sipi/scripts/convert_from_file.lua @@ -112,8 +112,6 @@ if not success then return -1 end -server.log("********* Checking consistency of mimetype " .. submitted_mimetype.mimetype .. " and filename " .. originalFilename) - success, check = fullImg:mimetype_consistency(submitted_mimetype.mimetype, originalFilename) if not success then From 4f5fed973c4487dbd723229cbac0239790821e49 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 5 Mar 2019 15:59:36 +0100 Subject: [PATCH 05/24] feature (api-v1): Change API v1 to use the same Sipi scripts as API v2. --- sipi/scripts/convert_from_file.lua | 182 -------------- sipi/scripts/make_thumbnail.lua | 208 ---------------- sipi/scripts/upload.lua | 1 + .../org/knora/webapi/ITKnoraLiveSpec.scala | 98 +++++++- .../e2e/v1/KnoraSipiIntegrationV1ITSpec.scala | 213 ++++------------ .../e2e/v1/KnoraSipiScriptsV1ITSpec.scala | 136 +---------- .../e2e/v2/KnoraSipiIntegrationV2ITSpec.scala | 112 ++------- .../other/v1/DrawingsGodsV1ITSpec.scala | 75 +++--- .../store/sipimessages/SipiMessages.scala | 230 +----------------- .../resourcemessages/ResourceMessagesV1.scala | 56 +++-- .../valuemessages/ValueMessagesV1.scala | 45 +++- .../valuemessages/ValueMessagesV2.scala | 6 +- .../responders/v1/ResourcesResponderV1.scala | 207 ++++++++-------- .../responders/v1/ValuesResponderV1.scala | 101 ++++---- .../webapi/responders/v2/ResourceUtilV2.scala | 6 +- .../webapi/routing/v1/ResourcesRouteV1.scala | 46 ++-- .../webapi/routing/v1/ValuesRouteV1.scala | 56 ++--- .../webapi/store/iiif/SipiConnector.scala | 160 +----------- .../knora/webapi/util/StringFormatter.scala | 13 +- .../knora/webapi/e2e/v1/SipiV1R2RSpec.scala | 14 +- .../v1/ResourcesResponderV1Spec.scala | 13 +- .../responders/v1/ValuesResponderV1Spec.scala | 15 +- .../webapi/store/iiif/MockSipiConnector.scala | 49 +--- .../webapi/util/StringFormatterSpec.scala | 2 +- 24 files changed, 531 insertions(+), 1513 deletions(-) delete mode 100644 sipi/scripts/convert_from_file.lua delete mode 100644 sipi/scripts/make_thumbnail.lua diff --git a/sipi/scripts/convert_from_file.lua b/sipi/scripts/convert_from_file.lua deleted file mode 100644 index 43823b454a..0000000000 --- a/sipi/scripts/convert_from_file.lua +++ /dev/null @@ -1,182 +0,0 @@ --- Copyright © 2015-2019 the contributors (see Contributors.md). --- --- This file is part of Knora. --- --- Knora is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as published --- by the Free Software Foundation, either version 3 of the License, or --- (at your option) any later version. --- --- Knora is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU Affero General Public License for more details. --- --- You should have received a copy of the GNU Affero General Public --- License along with Knora. If not, see . - --- Knora GUI-case: Sipi has already saved the file that is supposed to be converted --- the file was saved to: config.imgroot .. '/tmp/' (route make_thumbnail) - -require "send_response" - -success, errmsg = server.setBuffer() -if not success then - server.log("server.setBuffer() failed: " .. errmsg, server.loglevel.LOG_ERR) - return -end - -if server.post == nil then - send_error(400, PARAMETERS_INCORRECT) - return -end - --- --- check if the project directory is available. it needs to be created before sipi is started, --- so that sipi can create the directory sublevels on startup. --- - -prefix = server.post['prefix'] - -if prefix == nil then - send_error(400, PARAMETERS_INCORRECT) - return -end - -projectDir = config.imgroot .. '/' .. prefix .. '/' - -local success, exists = server.fs.exists(projectDir) -if not exists then - local errorMsg = "Directory " .. projectDir .. " not found. Please make sure it exists before starting Sipi." - send_error(500, errorMsg) - server.log(errorMsg, server.loglevel.LOG_ERR) - return -1 -end - -originalFilename = server.post['originalfilename'] -originalMimetype = server.post['originalmimetype'] -filename = server.post['filename'] - --- check if all the expected params are set -if originalFilename == nil or originalMimetype == nil or filename == nil then - send_error(400, PARAMETERS_INCORRECT) - return -end - --- file with name given in param "filename" has been saved by make_thumbnail.lua beforehand -tmpDir = config.imgroot .. '/tmp/' - -local success, hashed_filename = helper.filename_hash(filename) - -if not success then - send_error(500, hashed_filename) - return -end - -sourcePath = tmpDir .. hashed_filename - --- check if source is readable -success, readable = server.fs.is_readable(sourcePath) -if not success then - server.log("Source: " .. sourcePath .. "not readable, " .. readable, server.loglevel.LOG_ERR) - return -end -if not readable then - - send_error(500, FILE_NOT_READABLE .. sourcePath) - - return -end - --- all params are set - -success, baseName = server.uuid62() -if not success then - server.log("server.uuid62() failed: " .. baseName, server.loglevel.LOG_ERR) - return -end - --- --- create full quality image (jp2) --- -success, fullImg = SipiImage.new(sourcePath) -if not success then - server.log("SipiImage.new() failed: " .. fullImg, server.loglevel.LOG_ERR) - return -end - -local success, submitted_mimetype = server.parse_mimetype(originalMimetype) - -if not success then - send_error(400, "Couldn't parse mimetype: " .. originalMimetype) - return -1 -end - -success, check = fullImg:mimetype_consistency(submitted_mimetype.mimetype, originalFilename) - -if not success then - server.log("fullImg:mimetype_consistency() failed: " .. check, server.loglevel.LOG_ERR) - return -end - --- if check returns false, the user's input is invalid -if not check then - - send_error(400, MIMETYPES_INCONSISTENCY) - - return -end - -fullImgName = baseName .. '.jpx' - --- --- create new full quality image file path with sublevels: --- -success, newFilePath = helper.filename_hash(fullImgName); -if not success then - server.sendStatus(500) - server.log(gaga, server.loglevel.LOG_ERR) - return false -end - -success, fullDims = fullImg:dims() -if not success then - server.log("fullImg:dims() failed: " .. fullDIms, server.loglevel.LOG_ERR) - return -end -fullImg:write(projectDir .. newFilePath) - --- create thumbnail (jpg) -success, thumbImg = SipiImage.new(sourcePath, { size = config.thumb_size }) -if not success then - server.log("SipiImage.new failed: " .. thumbImg, server.loglevel.LOG_ERR) - return -end - -success, thumbDims = thumbImg:dims() -if not success then - server.log("thumbImg:dims failed: " .. thumbDims, server.loglevel.LOG_ERR) - return -end - --- --- delete tmp and preview files --- -success, errmsg = server.fs.unlink(sourcePath) -if not success then - server.log("server.fs.unlink failed: " .. errmsg, server.loglevel.LOG_ERR) - return -end - -result = { - status = 0, - mimetype_full = "image/jp2", - filename_full = fullImgName, - nx_full = fullDims.nx, - ny_full = fullDims.ny, - original_mimetype = originalMimetype, - original_filename = originalFilename, - file_type = 'image' -} - -send_success(result) diff --git a/sipi/scripts/make_thumbnail.lua b/sipi/scripts/make_thumbnail.lua deleted file mode 100644 index a435e4bdd0..0000000000 --- a/sipi/scripts/make_thumbnail.lua +++ /dev/null @@ -1,208 +0,0 @@ --- Copyright © 2015-2019 the contributors (see Contributors.md). --- --- This file is part of Knora. --- --- Knora is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as published --- by the Free Software Foundation, either version 3 of the License, or --- (at your option) any later version. --- --- Knora is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU Affero General Public License for more details. --- --- You should have received a copy of the GNU Affero General Public --- License along with Knora. If not, see . - --- Knora GUI-case: create a thumbnail - -require "send_response" - -success, errormsg = server.setBuffer() -if not success then - return -1 -end - --- --- check if temporary directory is available, if not, create it. --- -local tmpDir = config.imgroot .. '/tmp/' -local success, exists = server.fs.exists(tmpDir) -if not success then - send_error(500, "Internal server error: " .. exists) - return -1 -end -if not exists then - local success, result = server.fs.mkdir(tmpDir, 511) - if not success then - local errorMsg = "Could not create tmpDir: " .. tmpDir .. " , result: " .. result - send_error(500, errorMsg) - server.log(errorMsg, server.loglevel.LOG_ERR) - return -1 - end -end - --- --- check if thumbs directory is available, if not, create it. --- -local thumbsDir = config.imgroot .. '/thumbs/' -local success, exists = server.fs.exists(thumbsDir) -if not success then - send_error(500, "Internal server error: " .. exists) - return -1 -end -if not exists then - local success, result = server.fs.mkdir(thumbsDir, 511) - if not success then - local errorMsg = "Could not create thumbsDir: " .. thumbsDir .. " , result: " .. result - send_error(500, errorMsg) - server.log(errorMsg, server.loglevel.LOG_ERR) - return -1 - end -end - --- --- check if something was uploaded --- -if server.uploads == nil then - send_error(400, "no image uploaded") - return -1 -end - -for imgindex, imgparam in pairs(server.uploads) do - - -- - -- copy the uploaded file (from config.tmpdir) to tmpDir so we have access to it in later requests - -- - - -- create tmp name - local success, tmpName = server.uuid62() - if not success then - send_error(500, "Couldn't generate uuid62!") - return -1 - end - - local success, hashed_tmpName = helper.filename_hash(tmpName) - - if not success then - send_error(500, hashed_tmpName) - return - end - - local tmpPath = tmpDir .. hashed_tmpName - - local success, result = server.copyTmpfile(imgindex, tmpPath) - if not success then - local errorMsg = "Couldn't copy uploaded file to tmp path: " .. tmpPath .. ", result: " .. result - send_error(500, errorMsg) - server.log(errorMsg, server.loglevel.LOG_ERR) - return -1 - end - - - -- - -- create a thumnail sized SipiImage - -- - local success, thumbImg = SipiImage.new(tmpPath, {size = config.thumb_size}) - if not success then - local errorMsg = "Couldn't create thumbnail for path: " .. tmpPath .. ", result: " .. tostring(thumbImg) - send_error(500, errorMsg) - server.log(errorMsg, server.loglevel.LOG_ERR) - return -1 - end - - local filename = imgparam["origname"] - local success, submitted_mimetype = server.parse_mimetype(imgparam["mimetype"]) - - if not success then - send_error(400, "Couldn't parse mimetype: " .. imgparam["mimetype"]) - return -1 - end - - local success, check = thumbImg:mimetype_consistency(submitted_mimetype.mimetype, filename) - if not success then - send_error(500, "Couldn't check mimteype consistency: " .. check) - return -1 - end - - -- - -- if check returns false, the user's input is invalid - -- - - if not check then - send_error(400, MIMETYPES_INCONSISTENCY) - return -1 - end - - -- - -- get the dimensions - -- - local success, dims = thumbImg:dims() - if not success then - send_error(500, "Couldn't get image dimensions: " .. dims) - return -1 - end - - - -- - -- write the thumbnail file - -- - thumbName = tmpName .. ".jpg" - thumbPath = thumbsDir .. thumbName - - server.log("thumbnail path: " .. thumbPath, server.loglevel.LOG_DEBUG) - - local success, result = thumbImg:write(thumbPath) - if not success then - local errorMsg = "Couldn't create thumbnail for path: " .. tostring(thumbPath) .. ", result: " .. tostring(result) - send_error(500, errorMsg) - server.log(errorMsg , server.loglevel.LOG_ERR) - return -1 - end - - -- #snip_marker - -- We need to be able to run behind a proxy and to configure this easily. - -- Allows to set SIPI_EXTERNAL_PROTOCOL environment variable and use its value. - -- - local external_protocol = os.getenv("SIPI_EXTERNAL_PROTOCOL") - if external_protocol == nil then - external_protocol = "http" - end - server.log("make_thumbnail - external_protocol: " .. external_protocol, server.loglevel.LOG_DEBUG) - - -- - -- We need to be able to run behind a proxy and to configure this easily. - -- Allows to set SIPI_EXTERNAL_HOSTNAME environment variable and use its value. - -- - local external_hostname = os.getenv("SIPI_EXTERNAL_HOSTNAME") - if external_hostname == nil then - external_hostname = config.hostname - end - server.log("make_thumbnail - external_hostname: " .. external_hostname, server.loglevel.LOG_DEBUG) - - -- - -- We need to be able to run behind a proxy and to configure this easily. - -- Allows to set SIPI_EXTERNAL_PORT environment variable and use its value. - -- - local external_port = os.getenv("SIPI_EXTERNAL_PORT") - if external_port == nil then - external_port = config.port - end - server.log("make_thumbnail - external_port: " .. external_port, server.loglevel.LOG_DEBUG) - - answer = { - nx_thumb = dims.nx, - ny_thumb = dims.ny, - mimetype_thumb = 'image/jpeg', - preview_path = external_protocol .. "://" .. external_hostname .. ":" .. external_port .."/thumbs/" .. thumbName .. "/full/full/0/default.jpg", - filename = tmpName, -- make this a IIIF URL - original_mimetype = submitted_mimetype.mimetype, - original_filename = filename, - file_type = 'IMAGE' - } - -- #snip_marker - -end - -send_success(answer) diff --git a/sipi/scripts/upload.lua b/sipi/scripts/upload.lua index ebe3f8018e..8d21e00b4e 100644 --- a/sipi/scripts/upload.lua +++ b/sipi/scripts/upload.lua @@ -109,6 +109,7 @@ for image_index, image_params in pairs(server.uploads) do end local jp2_filename = uuid62 .. '.jp2' + local protocol if server.secure then protocol = 'https://' diff --git a/webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala b/webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala index bb35d7a429..e1bb0a9a4d 100644 --- a/webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala @@ -19,23 +19,25 @@ package org.knora.webapi +import java.io.File + import akka.actor.ActorSystem import akka.event.LoggingAdapter import akka.http.scaladsl.Http import akka.http.scaladsl.client.RequestBuilding +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import akka.http.scaladsl.model._ -import akka.http.scaladsl.unmarshalling.Unmarshal import akka.stream.ActorMaterializer import com.typesafe.config.{Config, ConfigFactory} import org.knora.webapi.messages.app.appmessages.SetAllowReloadOverHTTPState import org.knora.webapi.messages.store.triplestoremessages.{RdfDataObject, TriplestoreJsonProtocol} -import org.knora.webapi.util.{StartupUtils, StringFormatter} import org.knora.webapi.util.jsonld.{JsonLDDocument, JsonLDUtil} +import org.knora.webapi.util.{StartupUtils, StringFormatter} import org.scalatest.{BeforeAndAfterAll, Matchers, Suite, WordSpecLike} import spray.json.{JsObject, _} import scala.concurrent.duration.{Duration, _} -import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.{Await, ExecutionContext} import scala.languageFeature.postfixOps object ITKnoraLiveSpec { @@ -151,4 +153,94 @@ class ITKnoraLiveSpec(_system: ActorSystem) extends Core with KnoraService with val responseBodyStr = getResponseString(request) JsonLDUtil.parseJsonLD(responseBodyStr) } + + /** + * Represents a file to be uploaded to Sipi. + * + * @param path the path of the file. + * @param mimeType the MIME type of the file. + */ + protected case class FileToUpload(path: String, mimeType: ContentType) + + /** + * Represents an image file to be uploaded to Sipi. + * + * @param fileToUpload the file to be uploaded. + * @param width the image's width in pixels. + * @param height the image's height in pixels. + */ + protected case class InputFile(fileToUpload: FileToUpload, width: Int, height: Int) + + /** + * Represents the information that Sipi returns about each file that has been uploaded. + * + * @param originalFilename the original filename that was submitted to Sipi. + * @param internalFilename Sipi's internal filename for the stored temporary file. + * @param temporaryBaseIIIFUrl the base URL at which the temporary file can be accessed. + */ + protected case class SipiUploadResponseEntry(originalFilename: String, internalFilename: String, temporaryBaseIIIFUrl: String) + + /** + * Represents Sipi's response to a file upload request. + * + * @param uploadedFiles the information about each file that was uploaded. + */ + protected case class SipiUploadResponse(uploadedFiles: Seq[SipiUploadResponseEntry]) + + object SipiUploadResponseV2JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { + implicit val sipiUploadResponseEntryFormat: RootJsonFormat[SipiUploadResponseEntry] = jsonFormat3(SipiUploadResponseEntry) + implicit val sipiUploadResponseFormat: RootJsonFormat[SipiUploadResponse] = jsonFormat1(SipiUploadResponse) + } + + import SipiUploadResponseV2JsonProtocol._ + + /** + * Represents the information that Knora returns about an image file value that was created. + * + * @param internalFilename the image's internal filename. + * @param iiifUrl the image's IIIF URL. + * @param width the image's width in pixels. + * @param height the image's height in pixels. + */ + protected case class SavedImage(internalFilename: String, iiifUrl: String, width: Int, height: Int) + + /** + * Uploads a file to Sipi and returns the information in Sipi's response. + * + * @param loginToken the login token to be included in the request to Sipi. + * @param filesToUpload the files to be uploaded. + * @return a [[SipiUploadResponse]] representing Sipi's response. + */ + protected def uploadToSipi(loginToken: String, filesToUpload: Seq[FileToUpload]): SipiUploadResponse = { + // Make a multipart/form-data request containing the files. + + val formDataParts: Seq[Multipart.FormData.BodyPart] = filesToUpload.map { + fileToUpload => + val fileToSend = new File(fileToUpload.path) + assert(fileToSend.exists(), s"File ${fileToUpload.path} does not exist") + + Multipart.FormData.BodyPart( + "file", + HttpEntity.fromPath(fileToUpload.mimeType, fileToSend.toPath), + Map("filename" -> fileToSend.getName) + ) + } + + val sipiFormData = Multipart.FormData(formDataParts: _*) + + // Send a POST request to Sipi, asking it to convert the image to JPEG 2000 and store it in a temporary file. + val sipiRequest = Post(s"$baseSipiUrl/upload?token=$loginToken", sipiFormData) + val sipiUploadResponseJson: JsObject = getResponseJson(sipiRequest) + // println(sipiUploadResponseJson.prettyPrint) + val sipiUploadResponse: SipiUploadResponse = sipiUploadResponseJson.convertTo[SipiUploadResponse] + + // Request the temporary image from Sipi. + for (responseEntry <- sipiUploadResponse.uploadedFiles) { + val sipiGetTmpFileRequest = Get(responseEntry.temporaryBaseIIIFUrl + "/full/full/0/default.jpg") + checkResponseOK(sipiGetTmpFileRequest) + } + + sipiUploadResponse + } + } diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala index e19f37c6fd..7ec9aed3a2 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala @@ -22,9 +22,8 @@ package org.knora.webapi.e2e.v1 import java.io.File import java.net.URLEncoder -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport +import akka.http.scaladsl.model._ import akka.http.scaladsl.model.headers._ -import akka.http.scaladsl.model.{HttpEntity, _} import akka.http.scaladsl.unmarshalling.Unmarshal import com.typesafe.config.{Config, ConfigFactory} import org.knora.webapi._ @@ -67,7 +66,6 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV private val pathToMarbles = "_test_data/test_route/images/marbles.tif" private val pathToXSLTransformation = "_test_data/test_route/texts/letterToHtml.xsl" private val pathToMappingWithXSLT = "_test_data/test_route/texts/mappingForLetterWithXSLTransformation.xml" - private val firstPageIri = new MutableTestIri private val secondPageIri = new MutableTestIri private val pathToBEOLBodyXSLTransformation = "_test_data/test_route/texts/beol/standoffToTEI.xsl" @@ -78,86 +76,6 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV private val pathToBEOLBulkXML = "_test_data/test_route/texts/beol/testLetter/bulk.xml" private val letterIri = new MutableTestIri - /** - * Represents a file to be uploaded to Sipi. - * - * @param path the path of the file. - * @param mimeType the MIME type of the file. - */ - case class FileToUpload(path: String, mimeType: ContentType) - - /** - * Represents an image file to be uploaded to Sipi. - * - * @param fileToUpload the file to be uploaded. - * @param width the image's width in pixels. - * @param height the image's height in pixels. - */ - case class InputFile(fileToUpload: FileToUpload, width: Int, height: Int) - - /** - * Represents the information that Sipi returns about each file that has been uploaded. - * - * @param originalFilename the original filename that was submitted to Sipi. - * @param internalFilename Sipi's internal filename for the stored temporary file. - * @param temporaryBaseIIIFUrl the base URL at which the temporary file can be accessed. - */ - case class SipiUploadResponseEntry(originalFilename: String, internalFilename: String, temporaryBaseIIIFUrl: String) - - /** - * Represents Sipi's response to a file upload request. - * - * @param uploadedFiles the information about each file that was uploaded. - */ - case class SipiUploadResponse(uploadedFiles: Seq[SipiUploadResponseEntry]) - - - object SipiUploadResponseV2JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { - implicit val sipiUploadResponseEntryFormat: RootJsonFormat[SipiUploadResponseEntry] = jsonFormat3(SipiUploadResponseEntry) - implicit val sipiUploadResponseFormat: RootJsonFormat[SipiUploadResponse] = jsonFormat1(SipiUploadResponse) - } - - import SipiUploadResponseV2JsonProtocol._ - - /** - * Uploads a file to Sipi and returns the information in Sipi's response. - * - * @param loginToken the login token to be included in the request to Sipi. - * @param filesToUpload the files to be uploaded. - * @return a [[SipiUploadResponse]] representing Sipi's response. - */ - private def uploadToSipi(loginToken: String, filesToUpload: Seq[FileToUpload]): SipiUploadResponse = { - // Make a multipart/form-data request containing the files. - - val formDataParts: Seq[Multipart.FormData.BodyPart] = filesToUpload.map { - fileToUpload => - val fileToSend = new File(fileToUpload.path) - assert(fileToSend.exists(), s"File ${fileToUpload.path} does not exist") - - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(fileToUpload.mimeType, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - } - - val sipiFormData = Multipart.FormData(formDataParts: _*) - - // Send a POST request to Sipi, asking it to convert the image to JPEG 2000 and store it in a temporary file. - val sipiRequest = Post(s"$baseSipiUrl/upload?token=$loginToken", sipiFormData) - val sipiUploadResponseJson: JsObject = getResponseJson(sipiRequest) - // println(sipiUploadResponseJson.prettyPrint) - val sipiUploadResponse: SipiUploadResponse = sipiUploadResponseJson.convertTo[SipiUploadResponse] - - // Request the temporary image from Sipi. - for (responseEntry <- sipiUploadResponse.uploadedFiles) { - val sipiGetTmpFileRequest = Get(responseEntry.temporaryBaseIIIFUrl + "/full/full/0/default.jpg") - checkResponseOK(sipiGetTmpFileRequest) - } - - sipiUploadResponse - } - /** * Adds the IRI of a XSL transformation to the given mapping. * @@ -199,7 +117,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV * Given the id originally provided by the client, gets the generated IRI from a bulk import response. * * @param bulkResponse the response from the bulk import route. - * @param clientID the client id to look for. + * @param clientID the client id to look for. * @return the Knora IRI of the resource. */ private def getResourceIriFromBulkResponse(bulkResponse: JsObject, clientID: String): String = { @@ -262,36 +180,13 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV } "create an 'incunabula:page' with parameters" in { - // The image to be uploaded. - val fileToSend = new File(pathToChlaus) - assert(fileToSend.exists(), s"File $pathToChlaus does not exist") - - // A multipart/form-data request containing the image. - val sipiFormData = Multipart.FormData( - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) + // Upload the image to Sipi. + val sipiUploadResponse: SipiUploadResponse = uploadToSipi( + loginToken = loginToken, + filesToUpload = Seq(FileToUpload(path = pathToChlaus, mimeType = MediaTypes.`image/tiff`)) ) - // Send a POST request to Sipi, asking it to make a thumbnail of the image. - val sipiRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(userEmail, password)) - val sipiResponseJson = getResponseJson(sipiRequest) - - // Request the thumbnail from Sipi. - val jsonFields = sipiResponseJson.fields - val previewUrl = jsonFields("preview_path").asInstanceOf[JsString].value - val sipiGetRequest = Get(previewUrl) ~> addCredentials(BasicHttpCredentials(userEmail, password)) - checkResponseOK(sipiGetRequest) - - val fileParams = JsObject( - Map( - "originalFilename" -> jsonFields("original_filename"), - "originalMimeType" -> jsonFields("original_mimetype"), - "filename" -> jsonFields("filename") - ) - ) + val uploadedFile: SipiUploadResponseEntry = sipiUploadResponse.uploadedFiles.head val knoraParams = s""" @@ -309,7 +204,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV | ], | "http://www.knora.org/ontology/0803/incunabula#seqnum": [{"int_value": 99999999}] | }, - | "file": ${fileParams.compactPrint}, + | "file": "${uploadedFile.internalFilename}", | "label": "test page", | "project_id": "http://rdfh.ch/projects/0803" |} @@ -329,39 +224,18 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV } "change an 'incunabula:page' with parameters" in { - // The image to be uploaded. - val fileToSend = new File(pathToMarbles) - assert(fileToSend.exists(), s"File $pathToMarbles does not exist") - - // A multipart/form-data request containing the image. - val sipiFormData = Multipart.FormData( - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/tiff`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) + // Upload the image to Sipi. + val sipiUploadResponse: SipiUploadResponse = uploadToSipi( + loginToken = loginToken, + filesToUpload = Seq(FileToUpload(path = pathToMarbles, mimeType = MediaTypes.`image/tiff`)) ) - // Send a POST request to Sipi, asking it to make a thumbnail of the image. - val sipiRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(userEmail, password)) - val sipiResponseJson = getResponseJson(sipiRequest) - - // Request the thumbnail from Sipi. - val jsonFields = sipiResponseJson.fields - val previewUrl = jsonFields("preview_path").asInstanceOf[JsString].value - val sipiGetRequest = Get(previewUrl) ~> addCredentials(BasicHttpCredentials(userEmail, password)) - checkResponseOK(sipiGetRequest) + val uploadedFile: SipiUploadResponseEntry = sipiUploadResponse.uploadedFiles.head // JSON describing the new image to Knora. val knoraParams = JsObject( Map( - "file" -> JsObject( - Map( - "originalFilename" -> jsonFields("original_filename"), - "originalMimeType" -> jsonFields("original_mimetype"), - "filename" -> jsonFields("filename") - ) - ) + "file" -> JsString(s"${uploadedFile.internalFilename}") ) ) @@ -412,7 +286,6 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV checkResponseOK(knoraPostRequest) } - "create a 'p0803-incunabula:book' and a 'p0803-incunabula:page' with file parameters via XML import" in { // Upload the image to Sipi. val sipiUploadResponse: SipiUploadResponse = uploadToSipi( @@ -759,36 +632,36 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV val xmlExpected = s""" - | - | - | - | - | Testletter - | - | - |

This is the TEI/XML representation of the resource identified by the Iri - | ${letterIri.get}.

- | - | - |

Representation of the resource's text as TEI/XML

- | - | - | - | - | - | Scheuchzer, Johann Jacob - | - | - | - | Hermann, Jacob - | - | - | - | - | + | + | + | + | + | Testletter + | + | + |

This is the TEI/XML representation of the resource identified by the Iri + | ${letterIri.get}.

+ | + | + |

Representation of the resource's text as TEI/XML

+ | + | + | + | + | + | Scheuchzer, Johann Jacob + | + | + | + | Hermann, Jacob + | + | + | + | + | |

[...] Viro Clarissimo.

Dn. Jacobo Hermanno S. S. M. C.

et Ph. M.

S. P. D.

J. J. Sch.

En quae desideras, vir Erud.e κεχαρισμένω θυμῷ Actorum Lipsiensium fragmentaGemeint sind die im Brief Hermanns von 1703.06.05 erbetenen Exemplare AE Aprilis 1703 und AE Suppl., tom. III, 1702. animi mei erga te prope[n]sissimi tenuia indicia. Dudum est, ex quo Tibi innotescere, et tuam ambire amicitiam decrevi, dudum, ex quo Ingenij Tui acumen suspexi, immo non potui quin admirarer pro eo, quod summam Demonstrationem Tuam de Iride communicare dignatus fueris summas ago grates; quamvis in hoc studij genere, non alias [siquid] μετρικώτατος, propter aliorum negotiorum continuam seriem non altos possim scandere gradus. Perge Vir Clariss. Erudito orbi propalare Ingenij Tui fructum; sed et me amare.

d. [10] Jun. 1703.Der Tag ist im Manuskript unleserlich. Da der Entwurf in Scheuchzers "Copiae epistolarum" zwischen zwei Einträgen vom 10. Juni 1703 steht, ist der Brief wohl auf den gleichen Tag zu datieren.

- | - | + | + | """.stripMargin val xmlDiff: Diff = DiffBuilder.compare(Input.fromString(letterResponseBodyXML)).withTest(Input.fromString(xmlExpected)).build() diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiScriptsV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiScriptsV1ITSpec.scala index fdd80760b2..6350cf8e04 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiScriptsV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiScriptsV1ITSpec.scala @@ -19,15 +19,10 @@ package org.knora.webapi.e2e.v1 -import java.io.File - -import akka.http.scaladsl.model.headers._ -import akka.http.scaladsl.model.{HttpEntity, _} +import akka.event.LoggingAdapter import com.typesafe.config.{Config, ConfigFactory} import org.knora.webapi.ITKnoraFakeSpec import org.knora.webapi.messages.store.triplestoremessages.TriplestoreJsonProtocol -import org.knora.webapi.util.MutableTestIri -import spray.json._ object KnoraSipiScriptsV1ITSpec { @@ -45,14 +40,7 @@ object KnoraSipiScriptsV1ITSpec { */ class KnoraSipiScriptsV1ITSpec extends ITKnoraFakeSpec(KnoraSipiScriptsV1ITSpec.config) with TriplestoreJsonProtocol { - implicit override lazy val log = akka.event.Logging(system, this.getClass) - - private val username = "root@example.com" - private val password = "test" - private val pathToChlaus = "_test_data/test_route/images/Chlaus.jpg" - private val pathToMarbles = "_test_data/test_route/images/marbles.tif" - private val firstPageIri = new MutableTestIri - private val secondPageIri = new MutableTestIri + implicit override lazy val log: LoggingAdapter = akka.event.Logging(system, this.getClass) "Calling Knora Sipi Scripts" should { @@ -71,126 +59,6 @@ class KnoraSipiScriptsV1ITSpec extends ITKnoraFakeSpec(KnoraSipiScriptsV1ITSpec. getResponseString(request) } - "successfully call make_thumbnail.lua sipi script" in { - - // The image to be uploaded. - val fileToSend = new File(pathToChlaus) - assert(fileToSend.exists(), s"File $pathToChlaus does not exist") - - // A multipart/form-data request containing the image. - val sipiFormData = Multipart.FormData( - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - // Send a POST request to Sipi, asking it to make a thumbnail of the image. - val sipiPostRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(username, password)) - val sipiPostResponseJson = getResponseJson(sipiPostRequest) - - /* sipiResponseJson will be something like this - { - "mimetype_thumb":"image/jpeg", - "original_mimetype":"image/jpeg", - "nx_thumb":93, - "preview_path":"http://localhost:1024/thumbs/CjwDMhlrctI-BG7gms08BJ4.jpg/full/full/0/default.jpg", - "filename":"CjwDMhlrctI-BG7gms08BJ4", - "file_type":"IMAGE", - "original_filename":"Chlaus.jpg", - "ny_thumb":128 - } - */ - - // get the preview_path - val previewPath = sipiPostResponseJson.fields("preview_path").asInstanceOf[JsString].value - - // get the filename - val filename = sipiPostResponseJson.fields("filename").asInstanceOf[JsString].value - - // Send a GET request to Sipi, asking for the preview image - val sipiGetRequest01 = Get(previewPath) - val sipiGetResponseJson01 = getResponseString(sipiGetRequest01) - - // Send a GET request to Sipi, asking for the info.json of the image - val sipiGetRequest02 = Get(baseSipiUrl + "/thumbs/" + filename + ".jpg/info.json" ) - val sipiGetResponseJson = getResponseJson(sipiGetRequest02) - } - - - - "successfully call convert_from_file.lua sipi script" in { - - /* This is the case where the file is already stored on the sipi server as part of make_thumbnail*/ - - // The image to be uploaded. - val fileToSend = new File(pathToChlaus) - assert(fileToSend.exists(), s"File $pathToChlaus does not exist") - - // A multipart/form-data request containing the image. - val sipiFormData = Multipart.FormData( - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - // Send a POST request to Sipi, asking it to make a thumbnail of the image. - val sipiMakeThumbnailRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) - val sipiMakeThumbnailResponseJson = getResponseJson(sipiMakeThumbnailRequest) - - - val originalFilename = sipiMakeThumbnailResponseJson.fields("original_filename").asInstanceOf[JsString].value - val originalMimeType = sipiMakeThumbnailResponseJson.fields("original_mimetype").asInstanceOf[JsString].value - val filename = sipiMakeThumbnailResponseJson.fields("filename").asInstanceOf[JsString].value - - // ToDo: Find out why this way of sending json is not working with sipi - /* - val params = - s""" - |{ - | "originalfilename": "$originalFilename", - | "originalmimetype": "$originalMimeType", - | "filename": "$filename" - |} - """.stripMargin - - val convertFromFileRequest = Post(baseSipiUrl + "/convert_from_file", HttpEntity(ContentTypes.`application/json`, params)) - val convertFromFileResponseJson = getResponseJson(convertFromFileRequest) - */ - - // A form-data request containing the payload for convert_from_file. - - val sipiFormData02 = FormData( - Map( - "originalFilename" -> originalFilename, - "originalMimeType" -> originalMimeType, - "prefix" -> "0001", - "filename" -> filename - ) - ) - - val convertFromFileRequest = Post(baseSipiUrl + "/convert_from_file", sipiFormData02) - val convertFromFileResponseJson = getResponseJson(convertFromFileRequest) - - val filenameFull = convertFromFileResponseJson.fields("filename_full").asInstanceOf[JsString].value - - // Running with KnoraFakeService which always allows access to files. - // Send a GET request to Sipi, asking for full image - // not possible as authentication is required and file needs to be known by knora to be able to authenticate the request - val sipiGetImageRequest = Get(baseSipiUrl + "/0001/" + filenameFull + "/full/full/0/default.jpg") ~> addCredentials(BasicHttpCredentials(username, password)) - checkResponseOK(sipiGetImageRequest) - - // Send a GET request to Sipi, asking for the info.json of the image - val sipiGetInfoRequest = Get(baseSipiUrl + "/0001/" + filenameFull + "/info.json" ) ~> addCredentials(BasicHttpCredentials(username, password)) - val sipiGetInfoResponseJson = getResponseJson(sipiGetInfoRequest) - log.debug("sipiGetInfoResponseJson: {}", sipiGetInfoResponseJson) - - - } - } } diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala index d7d0d89715..91f9b85116 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala @@ -1,9 +1,27 @@ +/* + * Copyright © 2015-2019 the contributors (see Contributors.md). + * + * This file is part of Knora. + * + * Knora is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Knora is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Knora. If not, see . + */ + package org.knora.webapi.e2e.v2 import java.io.File import java.net.URLEncoder -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import akka.http.scaladsl.model._ import akka.http.scaladsl.model.headers.BasicHttpCredentials import akka.http.scaladsl.unmarshalling.Unmarshal @@ -14,7 +32,6 @@ import org.knora.webapi.messages.v2.routing.authenticationmessages._ import org.knora.webapi.util.IriConversions._ import org.knora.webapi.util.jsonld.{JsonLDArray, JsonLDConstants, JsonLDDocument, JsonLDObject} import org.knora.webapi.util.{MutableTestIri, SmartIri, StringFormatter} -import spray.json._ import scala.concurrent.Await import scala.concurrent.duration._ @@ -39,8 +56,6 @@ class KnoraSipiIntegrationV2ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV private val marblesOriginalFilename = "marbles.tif" private val pathToMarbles = s"_test_data/test_route/images/$marblesOriginalFilename" - private val marblesWidth = 1419 - private val marblesHeight = 1001 private val trp88OriginalFilename = "Trp88.tiff" private val pathToTrp88 = s"_test_data/test_route/images/$trp88OriginalFilename" @@ -53,95 +68,6 @@ class KnoraSipiIntegrationV2ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV private val stillImageFileValueIri = new MutableTestIri private val aThingPictureIri = "http://rdfh.ch/0001/a-thing-picture" - /** - * Represents a file to be uploaded to Sipi. - * - * @param path the path of the file. - * @param mimeType the MIME type of the file. - */ - case class FileToUpload(path: String, mimeType: ContentType) - - /** - * Represents an image file to be uploaded to Sipi. - * - * @param fileToUpload the file to be uploaded. - * @param width the image's width in pixels. - * @param height the image's height in pixels. - */ - case class InputFile(fileToUpload: FileToUpload, width: Int, height: Int) - - /** - * Represents the information that Sipi returns about each file that has been uploaded. - * - * @param originalFilename the original filename that was submitted to Sipi. - * @param internalFilename Sipi's internal filename for the stored temporary file. - * @param temporaryBaseIIIFUrl the base URL at which the temporary file can be accessed. - */ - case class SipiUploadResponseEntry(originalFilename: String, internalFilename: String, temporaryBaseIIIFUrl: String) - - /** - * Represents Sipi's response to a file upload request. - * - * @param uploadedFiles the information about each file that was uploaded. - */ - case class SipiUploadResponse(uploadedFiles: Seq[SipiUploadResponseEntry]) - - object SipiUploadResponseV2JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { - implicit val sipiUploadResponseEntryFormat: RootJsonFormat[SipiUploadResponseEntry] = jsonFormat3(SipiUploadResponseEntry) - implicit val sipiUploadResponseFormat: RootJsonFormat[SipiUploadResponse] = jsonFormat1(SipiUploadResponse) - } - - import SipiUploadResponseV2JsonProtocol._ - - /** - * Represents the information that Knora returns about an image file value that was created. - * - * @param internalFilename the image's internal filename. - * @param iiifUrl the image's IIIF URL. - * @param width the image's width in pixels. - * @param height the image's height in pixels. - */ - case class SavedImage(internalFilename: String, iiifUrl: String, width: Int, height: Int) - - /** - * Uploads a file to Sipi and returns the information in Sipi's response. - * - * @param loginToken the login token to be included in the request to Sipi. - * @param filesToUpload the files to be uploaded. - * @return a [[SipiUploadResponse]] representing Sipi's response. - */ - private def uploadToSipi(loginToken: String, filesToUpload: Seq[FileToUpload]): SipiUploadResponse = { - // Make a multipart/form-data request containing the files. - - val formDataParts: Seq[Multipart.FormData.BodyPart] = filesToUpload.map { - fileToUpload => - val fileToSend = new File(fileToUpload.path) - assert(fileToSend.exists(), s"File ${fileToUpload.path} does not exist") - - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(fileToUpload.mimeType, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - } - - val sipiFormData = Multipart.FormData(formDataParts: _*) - - // Send a POST request to Sipi, asking it to convert the image to JPEG 2000 and store it in a temporary file. - val sipiRequest = Post(s"$baseSipiUrl/upload?token=$loginToken", sipiFormData) - val sipiUploadResponseJson: JsObject = getResponseJson(sipiRequest) - // println(sipiUploadResponseJson.prettyPrint) - val sipiUploadResponse: SipiUploadResponse = sipiUploadResponseJson.convertTo[SipiUploadResponse] - - // Request the temporary image from Sipi. - for (responseEntry <- sipiUploadResponse.uploadedFiles) { - val sipiGetTmpFileRequest = Get(responseEntry.temporaryBaseIIIFUrl + "/full/full/0/default.jpg") - checkResponseOK(sipiGetTmpFileRequest) - } - - sipiUploadResponse - } - /** * Given a JSON-LD document representing a resource, returns a JSON-LD array containing the values of the specified * property. diff --git a/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala index 4badc7ba75..e01a26474d 100644 --- a/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala @@ -19,18 +19,23 @@ package org.knora.webapi.other.v1 -import java.io.File import java.net.URLEncoder +import akka.event.LoggingAdapter import akka.http.scaladsl.model._ import akka.http.scaladsl.model.headers.BasicHttpCredentials -import com.typesafe.config.ConfigFactory +import akka.http.scaladsl.unmarshalling.Unmarshal +import com.typesafe.config.{Config, ConfigFactory} import org.knora.webapi.messages.store.triplestoremessages.{RdfDataObject, TriplestoreJsonProtocol} +import org.knora.webapi.messages.v2.routing.authenticationmessages.{AuthenticationV2JsonProtocol, LoginResponse} import org.knora.webapi.{ITKnoraLiveSpec, InvalidApiJsonException} import spray.json._ +import scala.concurrent.Await +import scala.concurrent.duration._ + object DrawingsGodsV1ITSpec { - val config = ConfigFactory.parseString( + val config: Config = ConfigFactory.parseString( """ akka.loglevel = "DEBUG" akka.stdout-loglevel = "DEBUG" @@ -40,9 +45,9 @@ object DrawingsGodsV1ITSpec { /** * End-to-End (E2E) test specification for additional testing of permissions. */ -class DrawingsGodsV1ITSpec extends ITKnoraLiveSpec(DrawingsGodsV1ITSpec.config) with TriplestoreJsonProtocol { +class DrawingsGodsV1ITSpec extends ITKnoraLiveSpec(DrawingsGodsV1ITSpec.config) with AuthenticationV2JsonProtocol with TriplestoreJsonProtocol { - implicit override lazy val log = akka.event.Logging(system, this.getClass()) + implicit override lazy val log: LoggingAdapter = akka.event.Logging(system, this.getClass) override lazy val rdfDataObjects: List[RdfDataObject] = List( RdfDataObject(path = "_test_data/other.v1.DrawingsGodsV1Spec/drawings-gods_admin-data.ttl", name = "http://www.knora.org/data/admin"), @@ -56,40 +61,40 @@ class DrawingsGodsV1ITSpec extends ITKnoraLiveSpec(DrawingsGodsV1ITSpec.config) val drawingsOfGodsUserEmail = "ddd1@unil.ch" val testPass = "test" val pathToChlaus = "_test_data/test_route/images/Chlaus.jpg" + var loginToken: String = "" - "be able to create a resource, only find one DOAP (with combined resource class / property), and have permission to access the image" in { + "log in as a Knora user" in { + /* Correct username and correct password */ - // The image to be uploaded. - val fileToSend = new File(pathToChlaus) - assert(fileToSend.exists(), s"File $pathToChlaus does not exist") - - // A multipart/form-data request containing the image. - val sipiFormData = Multipart.FormData( - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/jpeg`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) + val params = + s""" + |{ + | "email": "$drawingsOfGodsUserEmail", + | "password": "$testPass" + |} + """.stripMargin - // Send a POST request to Sipi, asking it to make a thumbnail of the image. - val sipiRequest = Post(baseSipiUrl + "/make_thumbnail", sipiFormData) ~> addCredentials(BasicHttpCredentials(drawingsOfGodsUserEmail, testPass)) - val sipiResponseJson = getResponseJson(sipiRequest) - - // Request the thumbnail from Sipi. - val jsonFields = sipiResponseJson.fields - val previewUrl = jsonFields("preview_path").asInstanceOf[JsString].value - val sipiThumbnailGetRequest = Get(previewUrl) ~> addCredentials(BasicHttpCredentials(drawingsOfGodsUserEmail, testPass)) - checkResponseOK(sipiThumbnailGetRequest) - - val fileParams = JsObject( - Map( - "originalFilename" -> jsonFields("original_filename"), - "originalMimeType" -> jsonFields("original_mimetype"), - "filename" -> jsonFields("filename") - ) + val request = Post(baseApiUrl + s"/v2/authentication", HttpEntity(ContentTypes.`application/json`, params)) + val response: HttpResponse = singleAwaitingRequest(request) + assert(response.status == StatusCodes.OK) + + val lr: LoginResponse = Await.result(Unmarshal(response.entity).to[LoginResponse], 1.seconds) + loginToken = lr.token + + loginToken.nonEmpty should be(true) + + log.debug("token: {}", loginToken) + } + + "be able to create a resource, only find one DOAP (with combined resource class / property), and have permission to access the image" in { + // Upload the image to Sipi. + val sipiUploadResponse: SipiUploadResponse = uploadToSipi( + loginToken = loginToken, + filesToUpload = Seq(FileToUpload(path = pathToChlaus, mimeType = MediaTypes.`image/tiff`)) ) + val uploadedFile: SipiUploadResponseEntry = sipiUploadResponse.uploadedFiles.head + val params = s""" |{ @@ -104,7 +109,7 @@ class DrawingsGodsV1ITSpec extends ITKnoraLiveSpec(DrawingsGodsV1ITSpec.config) | "http://www.knora.org/ontology/0105/drawings-gods#hasCommentAuthor":[{"hlist_value":"http://rdfh.ch/lists/0105/drawings-gods-2016-list-CommentAuthorList-child"}], | "http://www.knora.org/ontology/0105/drawings-gods#hasCodeVerso":[{"richtext_value":{"utf8str":"dayyad"}}] | }, - | "file": ${fileParams.compactPrint}, + | "file": "${uploadedFile.internalFilename}", | "project_id":"http://rdfh.ch/projects/0105", | "label":"dayyad" |} diff --git a/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala b/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala index 9ac57dca5a..e74f3b818a 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala @@ -19,18 +19,10 @@ package org.knora.webapi.messages.store.sipimessages -import java.io.File - import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport -import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.v1.responder.usermessages.UserProfileV1 -import org.knora.webapi.messages.v1.responder.valuemessages.FileValueV1 import spray.json._ -////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Messages V1 - /** * An abstract trait for messages that can be sent to the [[org.knora.webapi.store.iiif.IIIFManager]] */ @@ -39,211 +31,7 @@ sealed trait IIIFRequest /** * An abstract trait for messages that can be sent to [[org.knora.webapi.store.iiif.SipiConnector]]. */ -sealed trait SipiRequestV1 extends IIIFRequest - -/** - * Abstract trait to represent a conversion request to Sipi Connector. - * - * For each type of conversion request, an implementation of `toFormData` must be provided. - * - */ -sealed trait SipiConversionRequestV1 extends SipiRequestV1 { - val originalFilename: String - val originalMimeType: String - val projectShortcode: String - val userProfile: UserProfileV1 - - /** - * Creates a Map representing the parameters to be submitted to Sipi's conversion routes. - * This method must be implemented for each type of conversion request - * because different Sipi routes are called and the parameters differ. - * - * @return a Map of key-value pairs that can be turned into form data by Sipi responder. - */ - def toFormData: Map[String, String] - - def toJsValue: JsValue -} - - -/** - * Represents an binary file that has been temporarily stored by Sipi (GUI-case). Knora route received a request telling it about - * a file that is already managed by Sipi. The binary file data have already been sent to Sipi by the client (browser-based GUI). - * Knora has to tell Sipi about the name of the file to be converted. - * For further details, please read the docs: Sipi -> Interaction Between Sipi and Knora. - * - * @param originalFilename the original name of the binary file. - * @param originalMimeType the MIME type of the binary file (e.g. image/tiff). - * @param filename the name of the binary file created by SIPI. - * @param userProfile the user making the request. - */ - -case class SipiConversionFileRequestV1(originalFilename: String, - originalMimeType: String, - projectShortcode: String, - filename: String, - userProfile: UserProfileV1) extends SipiConversionRequestV1 { - - /** - * Creates the parameters needed to call the Sipi route convert_file. - * - * Required parameters: - * - originalFilename: original name of the file to be converted. - * - originalMimeType: original mime type of the file to be converted. - * - filename: name of the file to be converted (already managed by Sipi). - * - * @return a Map of key-value pairs that can be turned into form data by Sipi responder. - */ - def toFormData: Map[String, String] = { - Map( - "originalFilename" -> originalFilename, - "originalMimeType" -> originalMimeType, - "filename" -> filename, - "prefix" -> projectShortcode - ) - } - - def toJsValue: JsValue = RepresentationV1JsonProtocol.SipiConversionFileRequestV1Format.write(this) - -} - - -/** - * Represents the response received from SIPI after an image conversion request. - * - * @param nx_full x dim of the full quality representation. - * @param ny_full y dim of the full quality representation. - * @param mimetype_full mime type of the full quality representation. - * @param filename_full filename of the full quality representation. - * @param original_mimetype mime type of the original file. - * @param original_filename name of the original file. - * @param file_type type of file that has been converted (image). - */ -case class SipiImageConversionResponse(nx_full: Int, - ny_full: Int, - mimetype_full: String, - filename_full: String, - original_mimetype: String, - original_filename: String, - file_type: String) - -/** - * Represents the response received from Sipi after a text file store request. - * - * @param mimetype mime type of the text file. - * @param charset encoding of the text file. - * @param filename filename of the text file. - * @param original_mimetype original mime type of the text file (equals `mimetype`). - * @param original_filename original name of the text file. - * @param file_type type of file that has been stored (text). - */ -case class SipiTextResponse(mimetype: String, - charset: String, - filename: String, - original_mimetype: String, - original_filename: String, - file_type: String) - - -object SipiConstants { - // TODO: Shall we better use an ErrorHandlingMap here? - // map file types converted by Sipi to file value properties in Knora - val fileType2FileValueProperty: Map[FileType.Value, IRI] = Map( - FileType.TEXT -> OntologyConstants.KnoraBase.HasTextFileValue, - FileType.IMAGE -> OntologyConstants.KnoraBase.HasStillImageFileValue, - FileType.MOVIE -> OntologyConstants.KnoraBase.HasMovingImageFileValue, - FileType.AUDIO -> OntologyConstants.KnoraBase.HasAudioFileValue, - FileType.BINARY -> OntologyConstants.KnoraBase.HasDocumentFileValue - - ) - - object FileType extends Enumeration { - // the string representations correspond to Sipi's internal enum. - val IMAGE: Value = Value(0, "image") - val TEXT: Value = Value(1, "text") - val MOVIE: Value = Value(2, "movie") - val AUDIO: Value = Value(3, "audio") - val BINARY: Value = Value(4, "binary") - - val valueMap: Map[String, Value] = values.map(v => (v.toString, v)).toMap - - /** - * Given the name of a file type in this enumeration, returns the file type. If the file type is not found, throws an - * [[SipiException]]. - * - * @param filetype the name of the file type. - * @return the requested file type. - */ - def lookup(filetype: String): Value = { - valueMap.get(filetype) match { - case Some(ftype) => ftype - case None => throw SipiException(message = s"File type $filetype returned by Sipi not found in enumeration") - } - } - - } - - object StillImage { - val fullQuality = "full" - val thumbnailQuality = "thumbnail" - } - -} - -/** - * Response from [[org.knora.webapi.store.iiif.SipiConnector]] to a [[SipiConversionRequestV1]] representing a [[FileValueV1]]. - * - * @param fileValueV1 a [[FileValueV1]] - */ -case class SipiConversionResponseV1(fileValueV1: FileValueV1, file_type: SipiConstants.FileType.Value) - - -////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// JSON formatting V1 - -/** - * A spray-json protocol for generating Knora API v1 JSON providing data about representations of a resource. - */ -object RepresentationV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol with NullOptions { - - /** - * Converts between [[SipiConversionFileRequestV1]] objects and [[JsValue]] objects. - */ - implicit object SipiConversionFileRequestV1Format extends RootJsonFormat[SipiConversionFileRequestV1] { - /** - * Not implemented. - */ - def read(jsonVal: JsValue) = ??? - - /** - * Converts a [[SipiConversionFileRequestV1]] into [[JsValue]] for formatting as JSON. - * - * @param request the [[SipiConversionFileRequestV1]] to be converted. - * @return a [[JsValue]]. - */ - def write(request: SipiConversionFileRequestV1): JsValue = { - - val fields = Map( - "originalFilename" -> request.originalFilename.toJson, - "originalMimeType" -> request.originalMimeType.toJson, - "filename" -> request.filename.toJson - ) - - JsObject(fields) - } - } - - implicit val sipiImageConversionResponseFormat: RootJsonFormat[SipiImageConversionResponse] = jsonFormat7(SipiImageConversionResponse) - implicit val textStoreResponseFormat: RootJsonFormat[SipiTextResponse] = jsonFormat6(SipiTextResponse) -} - -////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// Messages V2 - -/** - * An abstract trait for messages that can be sent to [[org.knora.webapi.store.iiif.SipiConnector]]. - */ -sealed trait SipiRequestV2 extends IIIFRequest { +sealed trait SipiRequest extends IIIFRequest { def requestingUser: UserADM } @@ -253,8 +41,8 @@ sealed trait SipiRequestV2 extends IIIFRequest { * @param fileUrl the URL at which Sipi can serve the file. * @param requestingUser the user making the request. */ -case class GetImageMetadataRequestV2(fileUrl: String, - requestingUser: UserADM) extends SipiRequestV2 +case class GetImageMetadataRequest(fileUrl: String, + requestingUser: UserADM) extends SipiRequest /** @@ -281,9 +69,9 @@ object GetImageMetadataResponseV2JsonProtocol extends SprayJsonSupport with Defa * @param prefix the prefix under which the file should be stored. * @param requestingUser the user making the request. */ -case class MoveTemporaryFileToPermanentStorageRequestV2(internalFilename: String, - prefix: String, - requestingUser: UserADM) extends SipiRequestV2 +case class MoveTemporaryFileToPermanentStorageRequest(internalFilename: String, + prefix: String, + requestingUser: UserADM) extends SipiRequest /** * Asks Sipi to delete a temporary file. @@ -291,8 +79,8 @@ case class MoveTemporaryFileToPermanentStorageRequestV2(internalFilename: String * @param internalFilename the name of the file. * @param requestingUser the user making the request. */ -case class DeleteTemporaryFileRequestV2(internalFilename: String, - requestingUser: UserADM) extends SipiRequestV2 +case class DeleteTemporaryFileRequest(internalFilename: String, + requestingUser: UserADM) extends SipiRequest /** @@ -302,7 +90,7 @@ case class DeleteTemporaryFileRequestV2(internalFilename: String, * @param requestingUser the user making the request. */ case class SipiGetTextFileRequest(fileUrl: String, - requestingUser: UserADM) extends SipiRequestV2 + requestingUser: UserADM) extends SipiRequest /** * Represents a response for [[SipiGetTextFileRequest]]. diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala index bc11f0807c..774be81e2f 100755 --- a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/resourcemessages/ResourceMessagesV1.scala @@ -26,7 +26,6 @@ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.projectsmessages.ProjectADM import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataResponseV2, SipiConversionFileRequestV1, SipiConversionRequestV1} import org.knora.webapi.messages.v1.responder.valuemessages._ import org.knora.webapi.messages.v1.responder.{KnoraRequestV1, KnoraResponseV1} import org.knora.webapi.messages.v2.responder.UpdateResultInProject @@ -44,16 +43,16 @@ import scala.collection.breakOut * @param restype_id the resource type of the resource to be created. * @param label the rdfs:label of the resource. * @param properties the properties to be created as a Map of property types to property value(s). - * @param file a file to be attached to the resource (GUI-case). + * @param file the filename of a file that has been uploaded to Sipi's temporary storage. * @param project_id the IRI of the project the resources is added to. */ case class CreateResourceApiRequestV1(restype_id: IRI, label: String, properties: Map[IRI, Seq[CreateResourceValueV1]], - file: Option[CreateFileV1] = None, + file: Option[String] = None, project_id: IRI) { - def toJsValue = ResourceV1JsonProtocol.createResourceApiRequestV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.createResourceApiRequestV1Format.write(this) } @@ -202,7 +201,7 @@ case class ResourceSearchGetRequestV1(searchString: String, resourceTypeIri: Opt * @param resourceTypeIri the type of the new resource. * @param label the rdfs:label of the resource. * @param values the properties to add: type and value(s): a Map of propertyIris to ApiValueV1. - * @param file a file that should be attached to the resource. + * @param file a file that has been uploaded to Sipi's temporary storage and should be attached to the resource. * @param projectIri the IRI of the project the resources is added to. * @param userProfile the profile of the user making the request. * @param apiRequestID the ID of the API request. @@ -210,7 +209,7 @@ case class ResourceSearchGetRequestV1(searchString: String, resourceTypeIri: Opt case class ResourceCreateRequestV1(resourceTypeIri: IRI, label: String, values: Map[IRI, Seq[CreateValueV1WithComment]], - file: Option[SipiConversionRequestV1] = None, + file: Option[StillImageFileValueV1] = None, projectIri: IRI, userProfile: UserADM, apiRequestID: UUID) extends ResourcesResponderRequestV1 @@ -303,7 +302,7 @@ case class ResourceCheckClassResponseV1(isInClass: Boolean) * @param id the IRI of the resource that was marked as deleted. */ case class ResourceDeleteResponseV1(id: IRI) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.resourceDeleteResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.resourceDeleteResponseV1Format.write(this) } /** @@ -314,7 +313,7 @@ case class ResourceDeleteResponseV1(id: IRI) extends KnoraResponseV1 { */ case class ResourceInfoResponseV1(resource_info: Option[ResourceInfoV1] = None, rights: Option[Int] = None) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.resourceInfoResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.resourceInfoResponseV1Format.write(this) } /** @@ -332,7 +331,7 @@ case class ResourceFullResponseV1(resinfo: Option[ResourceInfoV1] = None, props: Option[PropsV1] = None, incoming: Seq[IncomingV1] = Nil, access: String) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.resourceFullResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.resourceFullResponseV1Format.write(this) } /** @@ -341,7 +340,7 @@ case class ResourceFullResponseV1(resinfo: Option[ResourceInfoV1] = None, * @param resource_context resources relating to this resource via `knora-base:partOf`. */ case class ResourceContextResponseV1(resource_context: ResourceContextV1) extends KnoraResponseV1 { - def toJsValue = ResourceContextV1JsonProtocol.resourceContextResponseV1Format.write(this) + def toJsValue: JsValue = ResourceContextV1JsonProtocol.resourceContextResponseV1Format.write(this) } @@ -352,7 +351,7 @@ case class ResourceContextResponseV1(resource_context: ResourceContextV1) extend */ case class ResourceRightsResponseV1(rights: Option[Int]) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.resourceRightsResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.resourceRightsResponseV1Format.write(this) } /** @@ -363,7 +362,7 @@ case class ResourceRightsResponseV1(rights: Option[Int]) extends KnoraResponseV1 */ case class ResourceSearchResponseV1(resources: Seq[ResourceSearchResultRowV1] = Vector.empty[ResourceSearchResultRowV1]) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.resourceSearchResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.resourceSearchResponseV1Format.write(this) } /** @@ -372,13 +371,14 @@ case class ResourceSearchResponseV1(resources: Seq[ResourceSearchResultRowV1] = * @param res_id the IRI ow the new resource. * @param results the values that have been attached to the resource. The key in the Map refers * to the property IRI and the Seq contains all instances of values of this type. + * @param projectADM the project in which the resource is to be created. */ case class ResourceCreateResponseV1(res_id: IRI, - results: Map[IRI, Seq[ResourceCreateValueResponseV1]] = Map.empty[IRI, Seq[ResourceCreateValueResponseV1]]) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.resourceCreateResponseV1Format.write(this) + results: Map[IRI, Seq[ResourceCreateValueResponseV1]] = Map.empty[IRI, Seq[ResourceCreateValueResponseV1]], + projectADM: ProjectADM) extends KnoraResponseV1 with UpdateResultInProject { + def toJsValue: JsValue = ResourceV1JsonProtocol.ResourceCreateResponseV1Format.write(this) } - /** * Requests the properties of a given resource. * @@ -395,7 +395,7 @@ case class PropertiesGetRequestV1(iri: IRI, userProfile: UserADM) extends Resour * @param properties the properties of the specified resource. */ case class PropertiesGetResponseV1(properties: PropsGetV1) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.propertiesGetResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.propertiesGetResponseV1Format.write(this) } /** @@ -416,7 +416,7 @@ case class ChangeResourceLabelRequestV1(resourceIri: IRI, label: String, userADM * @param label the resource's new label. */ case class ChangeResourceLabelResponseV1(res_id: IRI, label: String) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.changeResourceLabelResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.changeResourceLabelResponseV1Format.write(this) } /** @@ -437,7 +437,7 @@ case class GraphDataGetRequestV1(resourceIri: IRI, depth: Int, userADM: UserADM) * @param edges the edges that are visible in the graph. */ case class GraphDataGetResponseV1(nodes: Seq[GraphNodeV1], edges: Seq[GraphEdgeV1]) extends KnoraResponseV1 { - def toJsValue = ResourceV1JsonProtocol.graphDataGetResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.graphDataGetResponseV1Format.write(this) } /** @@ -741,7 +741,7 @@ case class ResourceSearchResultRowV1(id: IRI, value: Seq[String], rights: Option[Int] = None) { - def toJsValue = ResourceV1JsonProtocol.resourceSearchResultV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.resourceSearchResultV1Format.write(this) } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -812,7 +812,7 @@ object SalsahGuiConversions { * @param id the value object IRI of the value. */ case class ResourceCreateValueResponseV1(value: ResourceCreateValueObjectResponseV1, id: IRI) { - def toJsValue = ResourceV1JsonProtocol.resourceCreateValueResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.resourceCreateValueResponseV1Format.write(this) } /** @@ -871,7 +871,7 @@ case class ResourceCreateValueObjectResponseV1(textval: Map[LiteralValueType.Val order: Map[LiteralValueType.Value, Int]) { // TODO: do we need to add geonames here? - def toJsValue = ResourceV1JsonProtocol.resourceCreateValueObjectResponseV1Format.write(this) + def toJsValue: JsValue = ResourceV1JsonProtocol.resourceCreateValueObjectResponseV1Format.write(this) } /** @@ -1163,6 +1163,19 @@ object ResourceV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol } } + implicit object ResourceCreateResponseV1Format extends JsonFormat[ResourceCreateResponseV1] { + override def read(json: JsValue): ResourceCreateResponseV1 = ??? + + override def write(obj: ResourceCreateResponseV1): JsValue = { + val fields = Map( + "res_id" -> obj.res_id.toJson, + "results" -> obj.results.toJson + ) + + JsObject(fields) + } + } + implicit val createResourceValueV1Format: RootJsonFormat[CreateResourceValueV1] = jsonFormat14(CreateResourceValueV1) implicit val createResourceApiRequestV1Format: RootJsonFormat[CreateResourceApiRequestV1] = jsonFormat5(CreateResourceApiRequestV1) implicit val ChangeResourceLabelApiRequestV1Format: RootJsonFormat[ChangeResourceLabelApiRequestV1] = jsonFormat1(ChangeResourceLabelApiRequestV1) @@ -1179,7 +1192,6 @@ object ResourceV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol implicit val resourceCreateValueObjectResponseV1Format: RootJsonFormat[ResourceCreateValueObjectResponseV1] = jsonFormat14(ResourceCreateValueObjectResponseV1) implicit val resourceCreateValueResponseV1Format: RootJsonFormat[ResourceCreateValueResponseV1] = jsonFormat2(ResourceCreateValueResponseV1) implicit val oneOfMultipleResourcesCreateResponseFormat: JsonFormat[OneOfMultipleResourcesCreateResponseV1] = jsonFormat3(OneOfMultipleResourcesCreateResponseV1) - implicit val resourceCreateResponseV1Format: RootJsonFormat[ResourceCreateResponseV1] = jsonFormat2(ResourceCreateResponseV1) implicit val resourceDeleteResponseV1Format: RootJsonFormat[ResourceDeleteResponseV1] = jsonFormat1(ResourceDeleteResponseV1) implicit val changeResourceLabelResponseV1Format: RootJsonFormat[ChangeResourceLabelResponseV1] = jsonFormat2(ChangeResourceLabelResponseV1) implicit val graphNodeV1Format: JsonFormat[GraphNodeV1] = jsonFormat4(GraphNodeV1) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala index 9ec44c5a81..70ec58a6cd 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala @@ -24,11 +24,13 @@ import java.util.UUID import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import org.knora.webapi._ +import org.knora.webapi.messages.admin.responder.projectsmessages.ProjectADM import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.SipiConversionRequestV1 import org.knora.webapi.messages.v1.responder.resourcemessages.LocationV1 import org.knora.webapi.messages.v1.responder.{KnoraRequestV1, KnoraResponseV1} +import org.knora.webapi.messages.v2.responder.UpdateResultInProject import org.knora.webapi.messages.v2.responder.standoffmessages.MappingXMLtoStandoff +import org.knora.webapi.messages.v2.responder.valuemessages.{FileValueV2, StillImageFileValueContentV2} import org.knora.webapi.twirl.{StandoffTagAttributeV2, StandoffTagInternalReferenceAttributeV2, StandoffTagV2} import org.knora.webapi.util.standoff.StandoffTagUtilV2 import org.knora.webapi.util.{DateUtilV1, KnoraIdUtil, StringFormatter} @@ -208,11 +210,11 @@ case class ChangeValueApiRequestV1(richtext_value: Option[CreateRichtextV1] = No /** * Represents an API request payload that asks the Knora API server to change the file attached to a resource - * (i. e. to create a new version of its file values). + * (i. e. to create a new version of its file value). * - * @param file the new file to be attached to the resource (GUI-case). + * @param file the name of a file that has been uploaded to Sipi's temporary storage. */ -case class ChangeFileValueApiRequestV1(file: CreateFileV1) { +case class ChangeFileValueApiRequestV1(file: String) { def toJsValue: JsValue = ApiValueV1JsonProtocol.changeFileValueApiRequestV1Format.write(this) } @@ -487,9 +489,9 @@ case class DeleteValueResponseV1(id: IRI) extends KnoraResponseV1 { * In case of an image, two file valueshave to be changed: thumbnail and full quality. * * @param resourceIri the resource whose files value(s) should be changed. - * @param file the file to be created and added. + * @param file a file that has been uploaded to Sipi's temporary storage. */ -case class ChangeFileValueRequestV1(resourceIri: IRI, file: SipiConversionRequestV1, apiRequestID: UUID, userProfile: UserADM) extends ValuesResponderRequestV1 +case class ChangeFileValueRequestV1(resourceIri: IRI, file: StillImageFileValueV1, apiRequestID: UUID, userProfile: UserADM) extends ValuesResponderRequestV1 /** * Represents a response to a [[ChangeFileValueRequestV1]]. @@ -497,8 +499,8 @@ case class ChangeFileValueRequestV1(resourceIri: IRI, file: SipiConversionReques * * @param locations the updated file value(s). */ -case class ChangeFileValueResponseV1(locations: Vector[LocationV1]) extends KnoraResponseV1 { - def toJsValue: JsValue = ApiValueV1JsonProtocol.changeFileValueresponseV1Format.write(this) +case class ChangeFileValueResponseV1(locations: Vector[LocationV1], projectADM: ProjectADM) extends KnoraResponseV1 with UpdateResultInProject { + def toJsValue: JsValue = ApiValueV1JsonProtocol.ChangeFileValueResponseV1Format.write(this) } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1452,6 +1454,20 @@ case class StillImageFileValueV1(internalMimeType: String, case other => throw InconsistentTriplestoreDataException(s"Cannot compare a $valueTypeIri to a ${other.valueTypeIri}") } } + + def toStillImageFileValueContentV2: StillImageFileValueContentV2 = { + StillImageFileValueContentV2( + ontologySchema = InternalSchema, + fileValue = FileValueV2( + internalFilename = internalFilename, + internalMimeType = internalMimeType, + originalFilename = originalFilename, + originalMimeType = internalMimeType + ), + dimX = dimX, + dimY = dimY + ) + } } case class MovingImageFileValueV1(internalMimeType: String, @@ -1602,6 +1618,16 @@ object ApiValueV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol def write(valueV1: ApiValueV1): JsValue = valueV1.toJsValue } + implicit object ChangeFileValueResponseV1Format extends JsonFormat[ChangeFileValueResponseV1] { + override def read(json: JsValue): ChangeFileValueResponseV1 = ??? + + override def write(obj: ChangeFileValueResponseV1): JsValue = { + JsObject(Map( + "locations" -> obj.locations.toJson + )) + } + } + implicit val createFileQualityLevelFormat: RootJsonFormat[CreateFileQualityLevelV1] = jsonFormat4(CreateFileQualityLevelV1) implicit val createFileV1Format: RootJsonFormat[CreateFileV1] = jsonFormat3(CreateFileV1) implicit val valueGetResponseV1Format: RootJsonFormat[ValueGetResponseV1] = jsonFormat7(ValueGetResponseV1) @@ -1619,5 +1645,4 @@ object ApiValueV1JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol implicit val changeValueResponseV1Format: RootJsonFormat[ChangeValueResponseV1] = jsonFormat4(ChangeValueResponseV1) implicit val deleteValueResponseV1Format: RootJsonFormat[DeleteValueResponseV1] = jsonFormat1(DeleteValueResponseV1) implicit val changeFileValueApiRequestV1Format: RootJsonFormat[ChangeFileValueApiRequestV1] = jsonFormat1(ChangeFileValueApiRequestV1) - implicit val changeFileValueresponseV1Format: RootJsonFormat[ChangeFileValueResponseV1] = jsonFormat1(ChangeFileValueResponseV1) -} +} \ No newline at end of file diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala b/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala index fdbd156921..4f2461642c 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v2/responder/valuemessages/ValueMessagesV2.scala @@ -29,7 +29,7 @@ import akka.util.Timeout import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.projectsmessages.ProjectADM import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataRequestV2, GetImageMetadataResponseV2} +import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataRequest, GetImageMetadataResponseV2} import org.knora.webapi.messages.v2.responder._ import org.knora.webapi.messages.v2.responder.resourcemessages.ReadResourceV2 import org.knora.webapi.messages.v2.responder.standoffmessages.{GetMappingRequestV2, GetMappingResponseV2, MappingXMLtoStandoff, StandoffDataTypeClasses} @@ -2377,8 +2377,8 @@ object StillImageFileValueContentV2 extends ValueContentReaderV2[StillImageFileV internalFilename <- Future(jsonLDObject.requireStringWithValidation(OntologyConstants.KnoraApiV2WithValueObjects.FileValueHasFilename, stringFormatter.toSparqlEncodedString)) // Ask Sipi about the rest of the file's metadata. - tempFileUrl = s"${settings.internalSipiBaseUrl}/tmp/$internalFilename" - imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequestV2(fileUrl = tempFileUrl, requestingUser = requestingUser)).mapTo[GetImageMetadataResponseV2] + tempFileUrl = stringFormatter.makeSipiTempFileUrl(settings, internalFilename) + imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequest(fileUrl = tempFileUrl, requestingUser = requestingUser)).mapTo[GetImageMetadataResponseV2] fileValue = FileValueV2( internalFilename = internalFilename, diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala b/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala index 61f7fbbd79..cd3a919b81 100755 --- a/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v1/ResourcesResponderV1.scala @@ -26,17 +26,17 @@ import akka.http.scaladsl.util.FastFuture import akka.pattern._ import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.permissionsmessages.{DefaultObjectAccessPermissionsStringForPropertyGetADM, DefaultObjectAccessPermissionsStringForResourceClassGetADM, DefaultObjectAccessPermissionsStringResponseADM, ResourceCreateOperation} -import org.knora.webapi.messages.admin.responder.projectsmessages.{ProjectGetRequestADM, ProjectGetResponseADM} +import org.knora.webapi.messages.admin.responder.projectsmessages.{ProjectADM, ProjectGetRequestADM, ProjectGetResponseADM} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages._ import org.knora.webapi.messages.store.triplestoremessages._ import org.knora.webapi.messages.v1.responder.ontologymessages._ import org.knora.webapi.messages.v1.responder.projectmessages._ -import org.knora.webapi.messages.v1.responder.resourcemessages.{MultipleResourceCreateResponseV1, _} +import org.knora.webapi.messages.v1.responder.resourcemessages._ import org.knora.webapi.messages.v1.responder.valuemessages._ +import org.knora.webapi.messages.v2.responder.UpdateResultInProject import org.knora.webapi.messages.v2.responder.ontologymessages.Cardinality.KnoraCardinalityInfo import org.knora.webapi.messages.v2.responder.ontologymessages.{Cardinality, OntologyMetadataGetByIriRequestV2, OntologyMetadataV2, ReadOntologyMetadataV2} -import org.knora.webapi.messages.v2.responder.valuemessages.{FileValueV2, StillImageFileValueContentV2} +import org.knora.webapi.messages.v2.responder.valuemessages.StillImageFileValueContentV2 import org.knora.webapi.responders.Responder.handleUnexpectedMessage import org.knora.webapi.responders.v1.GroupedProps._ import org.knora.webapi.responders.v2.ResourceUtilV2 @@ -70,7 +70,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo case ResourceRightsGetRequestV1(resourceIri, userProfile) => getRightsResponseV1(resourceIri, userProfile) case graphDataGetRequest: GraphDataGetRequestV1 => getGraphDataResponseV1(graphDataGetRequest) case ResourceSearchGetRequestV1(searchString: String, resourceIri: Option[IRI], numberOfProps: Int, limitOfResults: Int, userProfile: UserADM) => getResourceSearchResponseV1(searchString, resourceIri, numberOfProps, limitOfResults, userProfile) - case ResourceCreateRequestV1(resourceTypeIri, label, values, convertRequest, projectIri, userProfile, apiRequestID) => createNewResource(resourceTypeIri, label, values, convertRequest, projectIri, userProfile, apiRequestID) + case ResourceCreateRequestV1(resourceTypeIri, label, values, file, projectIri, userProfile, apiRequestID) => createNewResource(resourceTypeIri, label, values, file, projectIri, userProfile, apiRequestID) case MultipleResourceCreateRequestV1(resourcesToCreate, projectIri, userProfile, apiRequestID) => createMultipleNewResources(resourcesToCreate, projectIri, userProfile, apiRequestID) case ResourceCheckClassRequestV1(resourceIri: IRI, owlClass: IRI, userProfile: UserADM) => checkResourceClass(resourceIri, owlClass, userProfile) case PropertiesGetRequestV1(resourceIri: IRI, userProfile: UserADM) => getPropertiesV1(resourceIri = resourceIri, userProfile = userProfile) @@ -1252,21 +1252,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo // Convert all the image metadata in the request to StillImageFileValueContentV2 instances, so we // can use ResourceUtilV2.doSipiPostUpdate after updating the triplestore. val stillImageFileValueContentV2s: Seq[StillImageFileValueContentV2] = resourcesToCreate.flatMap { - resourceToCreate => - resourceToCreate.file.map { - stillImageFileValueV1: StillImageFileValueV1 => - StillImageFileValueContentV2( - ontologySchema = InternalSchema, - fileValue = FileValueV2( - internalFilename = stillImageFileValueV1.internalFilename, - internalMimeType = stillImageFileValueV1.internalMimeType, - originalFilename = stillImageFileValueV1.originalFilename, - originalMimeType = stillImageFileValueV1.internalMimeType - ), - dimX = stillImageFileValueV1.dimX, - dimY = stillImageFileValueV1.dimY - ) - } + resourceToCreate => resourceToCreate.file.map(_.toStillImageFileValueContentV2) } val updateFuture: Future[MultipleResourceCreateResponseV1] = for { @@ -1431,7 +1417,6 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo resourceClassInfo = resourceClassesEntityInfoResponse.resourceClassInfoMap(resourceCreateRequest.resourceTypeIri), propertyInfoMap = propertyEntityInfoMapsPerResource(resourceCreateRequest.resourceTypeIri), values = resourceCreateRequest.values, - sipiConversionRequest = None, convertedFile = resourceCreateRequest.file, clientResourceIDsToResourceClasses = clientResourceIDsToResourceClasses, userProfile = requestingUser @@ -1509,9 +1494,26 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo } } yield MultipleResourceCreateResponseV1(responses, projectADM) - // Use ResourceUtilV2.doSipiPostUpdate to ask Sipi to to move temporary image files to permanent storage if the - // triplestore update was successful, or to delete the temporary files if the triplestore update failed. - val sipiPostUpdateResultFutures: Seq[Future[MultipleResourceCreateResponseV1]] = stillImageFileValueContentV2s.map { + doSipiPostUpdateForResources( + updateFuture = updateFuture, + fileValueContentV2s = stillImageFileValueContentV2s, + requestingUser = requestingUser + ) + } + + /** + * Asks Sipi to to move temporary image files to permanent storage if a triplestore update was successful, + * or to delete the temporary files if the triplestore update failed. + * + * @param updateFuture the future resulting from the triplestore update. + * @param fileValueContentV2s the file values that were created, if any. + * @param requestingUser the user making the request. + * @return `updateFuture`, or a failed future (if Sipi failed to move a file to permanent storage). + */ + private def doSipiPostUpdateForResources[T <: UpdateResultInProject](updateFuture: Future[T], + fileValueContentV2s: Seq[StillImageFileValueContentV2], + requestingUser: UserADM): Future[T] = { + val resultFutures: Seq[Future[T]] = fileValueContentV2s.map { valueContent => ResourceUtilV2.doSipiPostUpdate( updateFuture = updateFuture, @@ -1523,9 +1525,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo ) } - // If ResourceUtilV2.doSipiPostUpdate returned an error, return it to the client, otherwise return - // a MultipleResourceCreateResponseV1. - Future.sequence(sipiPostUpdateResultFutures).transformWith { + Future.sequence(resultFutures).transformWith { case Success(_) => updateFuture case Failure(e) => Future.failed(e) } @@ -1540,7 +1540,6 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo * @param values values to be created for resource. If `linkTargetsAlreadyExist` is true, any links must be represented as [[LinkUpdateV1]] instances. * Otherwise, they must be represented as [[LinkToClientIDUpdateV1]] instances, so that appropriate error messages can * be generated for links to missing resources. - * @param sipiConversionRequest a file to be converted and attached to the resource. * @param convertedFile an already converted file to be attached to the resource. * @param clientResourceIDsToResourceClasses for each client resource ID, the IRI of the resource's class. Used only if `linkTargetsAlreadyExist` is false. * @param userProfile the profile of the user making the request. @@ -1550,7 +1549,6 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo resourceClassInfo: ClassInfoV1, propertyInfoMap: Map[IRI, PropertyInfoV1], values: Map[IRI, Seq[CreateValueV1WithComment]], - sipiConversionRequest: Option[SipiConversionRequestV1], convertedFile: Option[StillImageFileValueV1], clientResourceIDsToResourceClasses: Map[String, IRI] = new ErrorHandlingMap[IRI, IRI]( toWrap = Map.empty[IRI, IRI], @@ -1558,12 +1556,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo errorFun = { errorMsg => throw BadRequestException(errorMsg) } ), userProfile: UserADM): Future[Option[(IRI, Vector[CreateValueV1WithComment])]] = { - val userProfileV1 = userProfile.asUserProfileV1 - for { - // Get ontology information about the resource class's cardinalities and about each property's knora-base:objectClassConstraint. - - // Check that each submitted value is consistent with the knora-base:objectClassConstraint of the property that is supposed to // point to it. propertyObjectClassConstraintChecks: Seq[Unit] <- Future.sequence { @@ -1638,7 +1631,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo // Check that no required values are missing. requiredProps: Set[IRI] = resourceClassInfo.knoraResourceCardinalities.filter { - case (propIri, cardinalityInfo) => cardinalityInfo.cardinality == Cardinality.MustHaveOne || cardinalityInfo.cardinality == Cardinality.MustHaveSome + case (_, cardinalityInfo) => cardinalityInfo.cardinality == Cardinality.MustHaveOne || cardinalityInfo.cardinality == Cardinality.MustHaveSome }.keySet -- resourceClassInfo.linkValueProperties -- resourceClassInfo.fileValueProperties // exclude link value and file value properties from checking submittedPropertyIris = values.keySet @@ -1649,39 +1642,23 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo } // check if a file value is required by the ontology - fileValues: Option[(IRI, Vector[CreateValueV1WithComment])] <- if (resourceClassInfo.fileValueProperties.nonEmpty) { - (sipiConversionRequest, convertedFile) match { - case (Some(conversionRequest), None) => - // Send a message to SipiConnector to ask Sipi to convert the image file. - for { - sipiResponse: SipiConversionResponseV1 <- (storeManager ? conversionRequest).mapTo[SipiConversionResponseV1] - - // check if the file type returned by Sipi corresponds to the expected fileValue property in resourceClassInfo.fileValueProperties.head - _ = if (SipiConstants.fileType2FileValueProperty(sipiResponse.file_type) != resourceClassInfo.fileValueProperties.head) { - // TODO: remove the file from SIPI (delete request) - throw BadRequestException(s"Type of submitted file (${sipiResponse.file_type}) does not correspond to expected property type ${resourceClassInfo.fileValueProperties.head}") - } - } yield Some(resourceClassInfo.fileValueProperties.head -> Vector(CreateValueV1WithComment(sipiResponse.fileValueV1))) - - case (None, Some(converted)) => - // The file has already been converted, just return it. - Future.successful(Some(resourceClassInfo.fileValueProperties.head -> Vector(CreateValueV1WithComment(converted)))) + fileValues: Option[(IRI, Vector[CreateValueV1WithComment])] = if (resourceClassInfo.fileValueProperties.nonEmpty) { + convertedFile match { + case Some(converted) => + // TODO: check if the file type returned by Sipi corresponds to the expected fileValue property in resourceClassInfo.fileValueProperties.head + Some(resourceClassInfo.fileValueProperties.head -> Vector(CreateValueV1WithComment(converted))) - case other => throw AssertionException(s"Expected a SipiConversionRequestV1 or a StillImageFileValueV1, got $other") + case None => throw BadRequestException(s"File required but none submitted") } } else { - // resource class requires no binary representation - // check if there was no file sent - // TODO: in all cases of an error, the tmp file has to be deleted - sipiConversionRequest match { - case None => Future(None) // expected behaviour - case Some(_: SipiConversionFileRequestV1) => - throw BadRequestException(s"File params are given but resource class $resourceClassIri does not allow any representation") + if (convertedFile.nonEmpty) { + throw BadRequestException(s"File params are given but resource class $resourceClassIri does not allow any representation") + } else { + None } } } yield fileValues - } /** @@ -1755,6 +1732,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo * @param creatorIri the creator of the resources to be created. * @param createNewResourceSparql Sparql query to create the resource . * @param generateSparqlForValuesResponse Sparql statement for creation of values of resource. + * @param projectADM the project in which the resource was created. * @param userProfile the profile of the user making the request. * @return a [[ResourceCreateResponseV1]] containing information about the created resource . */ @@ -1762,6 +1740,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo creatorIri: IRI, createNewResourceSparql: String, generateSparqlForValuesResponse: GenerateSparqlToCreateMultipleValuesResponseV1, + projectADM: ProjectADM, userProfile: UserADM): Future[ResourceCreateResponseV1] = { // Verify that the resource was created. for { @@ -1797,7 +1776,11 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo }) } - apiResponse: ResourceCreateResponseV1 = ResourceCreateResponseV1(results = resourceCreateValueResponses, res_id = resourceIri) + apiResponse: ResourceCreateResponseV1 = ResourceCreateResponseV1( + results = resourceCreateValueResponses, + res_id = resourceIri, + projectADM = projectADM + ) } yield apiResponse } @@ -1805,30 +1788,37 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo /** * Does pre-update checks, creates a resource, and verifies that it was created. * - * @param resourceIri the IRI of the resource to be created. - * @param values the values to be attached to the resource. - * @param creatorIri the creator of the resource to be created. - * @param namedGraph the named graph the resource belongs to. - * @param apiRequestID the request ID used for locking the resource. + * @param resourceClassIri the IRI of the resource class. + * @param projectADM the project in which the resource should be created. + * @param label the `rdfs:label` of the resource to be created. + * @param resourceIri the IRI of the resource to be created. + * @param values the values to be attached to the resource. + * @param file a file that has been uploaded to Sipi's temporary storage and should be attached to the resource. + * @param creatorIri the creator of the resource to be created. + * @param namedGraph the named graph the resource belongs to. + * @param requestingUser the user making the request. + * @param apiRequestID the request ID used for locking the resource. * @return a [[ResourceCreateResponseV1]] containing information about the created resource. */ def createResourceAndCheck(resourceClassIri: IRI, - projectIri: IRI, + projectADM: ProjectADM, label: String, resourceIri: IRI, values: Map[IRI, Seq[CreateValueV1WithComment]], - sipiConversionRequest: Option[SipiConversionRequestV1], + file: Option[StillImageFileValueV1], creatorIri: IRI, namedGraph: IRI, - userProfile: UserADM, + requestingUser: UserADM, apiRequestID: UUID): Future[ResourceCreateResponseV1] = { - for { + val fileValueContent: Option[StillImageFileValueContentV2] = file.map(_.toStillImageFileValueContentV2) + + val updateFuture = for { // Get ontology information about the resource class and its properties. resourceClassEntityInfoResponse: EntityInfoGetResponseV1 <- (responderManager ? EntityInfoGetRequestV1( resourceClassIris = Set(resourceClassIri), propertyIris = Set.empty[IRI], - userProfile = userProfile + userProfile = requestingUser )).mapTo[EntityInfoGetResponseV1] resourceClassInfo = resourceClassEntityInfoResponse.resourceClassInfoMap(resourceClassIri) @@ -1836,7 +1826,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo propertyEntityInfoResponse: EntityInfoGetResponseV1 <- (responderManager ? EntityInfoGetRequestV1( resourceClassIris = Set.empty[IRI], propertyIris = resourceClassInfo.knoraResourceCardinalities.keySet, - userProfile = userProfile + userProfile = requestingUser )).mapTo[EntityInfoGetResponseV1] propertyInfoMap = propertyEntityInfoResponse.propertyInfoMap @@ -1845,9 +1835,9 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo defaultResourceClassAccessPermissionsResponse: DefaultObjectAccessPermissionsStringResponseADM <- { responderManager ? DefaultObjectAccessPermissionsStringForResourceClassGetADM( - projectIri = projectIri, + projectIri = projectADM.id, resourceClassIri = resourceClassIri, - targetUser = userProfile, + targetUser = requestingUser, requestingUser = KnoraSystemInstances.Users.SystemUser ) }.mapTo[DefaultObjectAccessPermissionsStringResponseADM] @@ -1859,10 +1849,10 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo for { defaultObjectAccessPermissions <- { responderManager ? DefaultObjectAccessPermissionsStringForPropertyGetADM( - projectIri = projectIri, + projectIri = projectADM.id, resourceClassIri = resourceClassIri, propertyIri = propertyIri, - targetUser = userProfile, + targetUser = requestingUser, requestingUser = KnoraSystemInstances.Users.SystemUser) }.mapTo[DefaultObjectAccessPermissionsStringResponseADM] } yield (propertyIri, defaultObjectAccessPermissions.permissionLiteral) @@ -1876,9 +1866,8 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo resourceClassInfo = resourceClassInfo, propertyInfoMap = propertyInfoMap, values = values, - sipiConversionRequest = sipiConversionRequest, - convertedFile = None, - userProfile = userProfile + convertedFile = file, + userProfile = requestingUser ) // Everything looks OK, so we can create the resource and its values. @@ -1887,7 +1876,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo creationDate: Instant = Instant.now generateSparqlForValuesResponse: GenerateSparqlToCreateMultipleValuesResponseV1 <- generateSparqlForValuesOfNewResource( - projectIri = projectIri, + projectIri = projectADM.id, resourceIri = resourceIri, resourceClassIri = resourceClassIri, defaultPropertyAccessPermissions = defaultPropertyAccessPermissions, @@ -1895,7 +1884,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo fileValues = fileValues, clientResourceIDsToResourceIris = Map.empty[String, IRI], creationDate = creationDate, - userProfile = userProfile, + userProfile = requestingUser, apiRequestID = apiRequestID ) @@ -1912,7 +1901,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo createNewResourceSparql = generateSparqlForNewResources( resourcesToCreate = resourcesToCreate, - projectIri = projectIri, + projectIri = projectADM.id, namedGraph = namedGraph, creatorIri = creatorIri ) @@ -1925,31 +1914,36 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo creatorIri = creatorIri, createNewResourceSparql = createNewResourceSparql, generateSparqlForValuesResponse = generateSparqlForValuesResponse, - userProfile = userProfile + projectADM = projectADM, + userProfile = requestingUser ) } yield apiResponse + + doSipiPostUpdateForResources( + updateFuture = updateFuture, + fileValueContentV2s = fileValueContent.toSeq, + requestingUser = requestingUser + ) } /** * Creates a new resource and attaches the given values to it. * - * @param resourceClassIri the resource type of the resource to be created. - * @param values the values to be attached to the resource. - * @param sipiConversionRequest a file (binary representation) to be attached to the resource (GUI and non GUI-case) - * @param projectIri the project the resource belongs to. - * @param userProfile the user that is creating the resource - * @param apiRequestID the ID of this API request. + * @param resourceClassIri the resource type of the resource to be created. + * @param values the values to be attached to the resource. + * @param file a file that has been uploaded to Sipi's temporary storage and should be attached to the resource. + * @param projectIri the project the resource belongs to. + * @param userProfile the user that is creating the resource + * @param apiRequestID the ID of this API request. * @return a [[ResourceCreateResponseV1]] informing the client about the new resource. */ private def createNewResource(resourceClassIri: IRI, label: String, values: Map[IRI, Seq[CreateValueV1WithComment]], - sipiConversionRequest: Option[SipiConversionRequestV1] = None, + file: Option[StillImageFileValueV1] = None, projectIri: IRI, userProfile: UserADM, apiRequestID: UUID): Future[ResourceCreateResponseV1] = { - val userProfileV1 = userProfile.asUserProfileV1 - for { // Get user's IRI and don't allow anonymous users to create resources. @@ -1965,16 +1959,17 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo throw BadRequestException(s"Instances of knora-base:Resource cannot be created, only instances of subclasses") } - projectInfoResponse <- { - responderManager ? ProjectInfoByIRIGetRequestV1( - projectIri, - Some(userProfileV1) + // Get project info + projectResponse <- { + responderManager ? ProjectGetRequestADM( + maybeIri = Some(projectIri), + requestingUser = userProfile ) - }.mapTo[ProjectInfoResponseV1] + }.mapTo[ProjectGetResponseADM] // Ensure that the project isn't the system project or the shared ontologies project. - resourceProjectIri: IRI = projectInfoResponse.project_info.id + resourceProjectIri: IRI = projectResponse.project.id _ = if (resourceProjectIri == OntologyConstants.KnoraBase.SystemProject || resourceProjectIri == OntologyConstants.KnoraBase.DefaultSharedOntologiesProject) { throw BadRequestException(s"Resources cannot be created in project $resourceProjectIri") @@ -1991,8 +1986,8 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo throw BadRequestException(s"Cannot create a resource in project $resourceProjectIri with resource class $resourceClassIri, which is defined in a non-shared ontology in another project") } - namedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfoResponse.project_info) - resourceIri: IRI = knoraIdUtil.makeRandomResourceIri(projectInfoResponse.project_info.shortcode) + namedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV2(projectResponse.project) + resourceIri: IRI = knoraIdUtil.makeRandomResourceIri(projectResponse.project.shortcode) // Check user's PermissionProfile (part of UserADM) to see if the user has the permission to // create a new resource in the given project. @@ -2005,14 +2000,14 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo resourceIri, () => createResourceAndCheck( resourceClassIri = resourceClassIri, - projectIri = resourceProjectIri, + projectADM = projectResponse.project, label = label, resourceIri = resourceIri, values = values, - sipiConversionRequest = sipiConversionRequest, + file = file, creatorIri = userIri, namedGraph = namedGraph, - userProfile = userProfile, + requestingUser = userProfile, apiRequestID = apiRequestID ) ) @@ -2048,7 +2043,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo // Create update sparql string sparqlUpdate = queries.sparql.v1.txt.deleteResource( - dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfoResponse.project_info), + dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfoResponse.project_info), triplestore = settings.triplestoreType, resourceIri = resourceDeleteRequest.resourceIri, maybeDeleteComment = resourceDeleteRequest.deleteComment, @@ -2151,7 +2146,7 @@ class ResourcesResponderV1(responderData: ResponderData) extends Responder(respo }.mapTo[ProjectInfoResponseV1] // get the named graph the resource is contained in by the resource's project - namedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfoResponse.project_info) + namedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfoResponse.project_info) // Make a timestamp to indicate when the resource was updated. currentTime: String = Instant.now.toString diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala b/webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala index 2fdf815de4..aa5f59dfb2 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v1/ValuesResponderV1.scala @@ -24,8 +24,8 @@ import java.time.Instant import akka.pattern._ import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.permissionsmessages.{DefaultObjectAccessPermissionsStringForPropertyGetADM, DefaultObjectAccessPermissionsStringResponseADM, PermissionADM, PermissionType} +import org.knora.webapi.messages.admin.responder.projectsmessages.{ProjectADM, ProjectGetRequestADM, ProjectGetResponseADM} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{SipiConstants, SipiConversionRequestV1, SipiConversionResponseV1} import org.knora.webapi.messages.store.triplestoremessages._ import org.knora.webapi.messages.v1.responder.ontologymessages.{EntityInfoGetRequestV1, EntityInfoGetResponseV1} import org.knora.webapi.messages.v1.responder.projectmessages.{ProjectInfoByIRIGetV1, ProjectInfoV1} @@ -34,7 +34,9 @@ import org.knora.webapi.messages.v1.responder.usermessages.{UserProfileByIRIGetV import org.knora.webapi.messages.v1.responder.valuemessages._ import org.knora.webapi.messages.v2.responder.ontologymessages.Cardinality import org.knora.webapi.messages.v2.responder.standoffmessages.StandoffDataTypeClasses +import org.knora.webapi.messages.v2.responder.valuemessages.StillImageFileValueContentV2 import org.knora.webapi.responders.Responder.handleUnexpectedMessage +import org.knora.webapi.responders.v2.ResourceUtilV2 import org.knora.webapi.responders.{IriLocker, Responder, ResponderData} import org.knora.webapi.twirl.{SparqlTemplateLinkUpdate, StandoffTagIriAttributeV2, StandoffTagV2} import org.knora.webapi.util.IriConversions._ @@ -217,7 +219,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde // Everything seems OK, so create the value. unverifiedValue <- createValueV1AfterChecks( - dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfo), + dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfo), projectIri = resourceFullResponse.resinfo.get.project_id, resourceIri = createValueRequest.resourceIri, propertyIri = createValueRequest.propertyIri, @@ -611,26 +613,18 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde */ case class CurrentFileValue(property: IRI, valueObjectIri: IRI, quality: Option[Int]) - def changeFileValue(oldFileValue: CurrentFileValue, newFileValue: FileValueV1): Future[ChangeValueResponseV1] = { - changeValueV1(ChangeValueRequestV1( - valueIri = oldFileValue.valueObjectIri, - value = newFileValue, - userProfile = changeFileValueRequest.userProfile, - apiRequestID = changeFileValueRequest.apiRequestID // re-use the same id - )) - } - /** - * Preprocesses a file value change request by calling the Sipi responder to create a new file - * and calls [[changeValueV1]] to actually change the file value in Knora. + * Changes a file value in the triplestore. * * @param changeFileValueRequest a [[ChangeFileValueRequestV1]] sent by the values route. + * @param projectADM the project in which the value is being updated. * @return a [[ChangeFileValueResponseV1]] representing all the changed file values. */ - def makeTaskFuture(changeFileValueRequest: ChangeFileValueRequestV1): Future[ChangeFileValueResponseV1] = { + def makeTaskFuture(changeFileValueRequest: ChangeFileValueRequestV1, projectADM: ProjectADM): Future[ChangeFileValueResponseV1] = { + val fileValueContent: StillImageFileValueContentV2 = changeFileValueRequest.file.toStillImageFileValueContentV2 // get the Iris of the current file value(s) - for { + val triplestoreUpdateFuture = for { resourceIri <- Future(changeFileValueRequest.resourceIri) @@ -648,7 +642,6 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde // get the property Iris, file value Iris and qualities attached to the resource fileValues: Seq[CurrentFileValue] = getFileValuesResponse.results.bindings.map { row: VariableResultsRow => - CurrentFileValue( property = row.rowMap("p"), valueObjectIri = row.rowMap("fileValueIri"), @@ -659,46 +652,44 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde ) } - // the message to be sent to SipiConnector - sipiConversionRequest: SipiConversionRequestV1 = changeFileValueRequest.file + // TODO: check if the file type returned by Sipi corresponds to the already existing file value type - sipiResponse: SipiConversionResponseV1 <- (storeManager ? sipiConversionRequest).mapTo[SipiConversionResponseV1] + response: ChangeValueResponseV1 <- changeValueV1(ChangeValueRequestV1( + valueIri = fileValues.head.valueObjectIri, + value = changeFileValueRequest.file, + userProfile = changeFileValueRequest.userProfile, + apiRequestID = changeFileValueRequest.apiRequestID // re-use the same id + )) - // check if the file type returned by Sipi corresponds to the already existing file value type (e.g., hasStillImageRepresentation) - _ = if (SipiConstants.fileType2FileValueProperty(sipiResponse.file_type) != fileValues.head.property) { - // TODO: remove the file from SIPI (delete request) - throw BadRequestException(s"Type of submitted file (${sipiResponse.file_type}) does not correspond to expected property type ${fileValues.head.property}") - } - - // - // handle file types individually - // - - // create the apt case class depending on the file type returned by Sipi - changedLocation: LocationV1 <- sipiResponse.file_type match { - case SipiConstants.FileType.IMAGE => - if (fileValues.size != 1) { - throw InconsistentTriplestoreDataException(s"Expected 1 file value for $resourceIri, but ${fileValues.size} given.") - } - - val oldFileValue: CurrentFileValue = fileValues.head - val newFileValue: FileValueV1 = sipiResponse.fileValueV1 - - for { - response: ChangeValueResponseV1 <- changeFileValue(oldFileValue, newFileValue) - } yield response.value match { - case fileValueV1: FileValueV1 => valueUtilV1.fileValueV12LocationV1(fileValueV1) - case other => throw AssertionException(s"Expected Sipi to change a file value, but it changed one of these: ${other.valueTypeIri}") - } - - case otherFileType => throw NotImplementedException(s"File type $otherFileType not yet supported") + changedLocation = response.value match { + case fileValueV1: FileValueV1 => valueUtilV1.fileValueV12LocationV1(fileValueV1) + case other => throw AssertionException(s"Expected Sipi to change a file value, but it changed one of these: ${other.valueTypeIri}") } } yield ChangeFileValueResponseV1( - locations = Vector(changedLocation) + locations = Vector(changedLocation), + projectADM = projectADM + ) + + ResourceUtilV2.doSipiPostUpdate( + updateFuture = triplestoreUpdateFuture, + valueContent = fileValueContent, + requestingUser = changeFileValueRequest.userProfile, + responderManager = responderManager, + storeManager = storeManager, + log = log ) } for { + resourceInfoResponse <- (responderManager ? ResourceInfoGetRequestV1(iri = changeFileValueRequest.resourceIri, userProfile = changeFileValueRequest.userProfile)).mapTo[ResourceInfoResponseV1] + + // Get project info + projectResponse <- { + responderManager ? ProjectGetRequestADM( + maybeIri = Some(resourceInfoResponse.resource_info.get.project_id), + requestingUser = changeFileValueRequest.userProfile + ) + }.mapTo[ProjectGetResponseADM] // Do the preparations of a file value change while already holding an update lock on the resource. // This is necessary because in `makeTaskFuture` the current file value Iris for the given resource IRI have to been retrieved. @@ -708,7 +699,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde taskResult <- IriLocker.runWithIriLock( changeFileValueRequest.apiRequestID, changeFileValueRequest.resourceIri, - () => makeTaskFuture(changeFileValueRequest) + () => makeTaskFuture(changeFileValueRequest, projectResponse.project) ) } yield taskResult @@ -870,7 +861,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde // We'll need to create a new LinkValue. changeLinkValueV1AfterChecks(projectIri = currentValueQueryResult.projectIri, - dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfo), + dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfo), resourceIri = findResourceWithValueResult.resourceIri, propertyIri = propertyIri, currentLinkValueV1 = currentLinkValueQueryResult.value, @@ -973,7 +964,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde // Generate a SPARQL update. sparqlUpdate = queries.sparql.v1.txt.changeComment( - dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfo), + dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfo), triplestore = settings.triplestoreType, resourceIri = findResourceWithValueResult.resourceIri, propertyIri = findResourceWithValueResult.propertyIri, @@ -1091,7 +1082,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde ) sparqlUpdate = queries.sparql.v1.txt.deleteLink( - dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfo), + dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfo), triplestore = settings.triplestoreType, linkSourceIri = findResourceWithValueResult.resourceIri, linkUpdate = sparqlTemplateLinkUpdate, @@ -1142,7 +1133,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde } sparqlUpdate = queries.sparql.v1.txt.deleteValue( - dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfo), + dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfo), triplestore = settings.triplestoreType, resourceIri = findResourceWithValueResult.resourceIri, propertyIri = findResourceWithValueResult.propertyIri, @@ -2161,7 +2152,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde // Generate a SPARQL update string. sparqlUpdate = queries.sparql.v1.txt.changeLink( - dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfo), + dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfo), triplestore = settings.triplestoreType, linkSourceIri = resourceIri, linkUpdateForCurrentLink = sparqlTemplateLinkUpdateForCurrentLink, @@ -2307,7 +2298,7 @@ class ValuesResponderV1(responderData: ResponderData) extends Responder(responde // Generate a SPARQL update. sparqlUpdate = queries.sparql.v1.txt.addValueVersion( - dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraph(projectInfo), + dataNamedGraph = StringFormatter.getGeneralInstance.projectDataNamedGraphV1(projectInfo), triplestore = settings.triplestoreType, resourceIri = resourceIri, propertyIri = propertyIri, diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v2/ResourceUtilV2.scala b/webapi/src/main/scala/org/knora/webapi/responders/v2/ResourceUtilV2.scala index 99f2cc3415..089c08cdfb 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v2/ResourceUtilV2.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v2/ResourceUtilV2.scala @@ -25,7 +25,7 @@ import akka.pattern._ import akka.util.Timeout import org.knora.webapi.messages.admin.responder.permissionsmessages.{DefaultObjectAccessPermissionsStringForPropertyGetADM, DefaultObjectAccessPermissionsStringResponseADM} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{DeleteTemporaryFileRequestV2, MoveTemporaryFileToPermanentStorageRequestV2} +import org.knora.webapi.messages.store.sipimessages.{DeleteTemporaryFileRequest, MoveTemporaryFileToPermanentStorageRequest} import org.knora.webapi.messages.store.triplestoremessages.{SparqlAskRequest, SparqlAskResponse} import org.knora.webapi.messages.v2.responder.resourcemessages.ReadResourceV2 import org.knora.webapi.messages.v2.responder.valuemessages.{FileValueContentV2, ReadValueV2, ValueContentV2} @@ -161,7 +161,7 @@ object ResourceUtilV2 { updateFuture.transformWith { case Success(updateInProject: UpdateResultInProject) => // Yes. Ask Sipi to move the file to permanent storage. - val sipiRequest = MoveTemporaryFileToPermanentStorageRequestV2( + val sipiRequest = MoveTemporaryFileToPermanentStorageRequest( internalFilename = fileValueContent.fileValue.internalFilename, prefix = updateInProject.projectADM.shortcode, requestingUser = requestingUser @@ -172,7 +172,7 @@ object ResourceUtilV2 { case Failure(_) => // The file value update failed. Ask Sipi to delete the temporary file. - val sipiRequest = DeleteTemporaryFileRequestV2( + val sipiRequest = DeleteTemporaryFileRequest( internalFilename = fileValueContent.fileValue.internalFilename, requestingUser = requestingUser ) diff --git a/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala b/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala index 7ce7f38378..07524ac476 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala @@ -38,7 +38,7 @@ import javax.xml.validation.{Schema, SchemaFactory, Validator} import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.projectsmessages.{ProjectGetRequestADM, ProjectGetResponseADM} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataRequestV2, GetImageMetadataResponseV2, SipiConversionFileRequestV1} +import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataRequest, GetImageMetadataResponseV2} import org.knora.webapi.messages.v1.responder.ontologymessages._ import org.knora.webapi.messages.v1.responder.resourcemessages.ResourceV1JsonProtocol._ import org.knora.webapi.messages.v1.responder.resourcemessages._ @@ -224,18 +224,26 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) projectResponse: ProjectGetResponseADM <- (responderManager ? ProjectGetRequestADM(maybeIri = Some(projectIri), requestingUser = userADM)).mapTo[ProjectGetResponseADM] } yield projectResponse.project.shortcode - // for GUI-case: - // file has already been stored by Sipi. - // TODO: in the old SALSAH, the file params were sent as a property salsah:__location__ -> the GUI has to be adapated - paramConversionRequest: Option[SipiConversionFileRequestV1] = apiRequest.file match { - case Some(createFile: CreateFileV1) => Some(SipiConversionFileRequestV1( - originalFilename = stringFormatter.toSparqlEncodedString(createFile.originalFilename, throw BadRequestException(s"The original filename is invalid: '${createFile.originalFilename}'")), - originalMimeType = stringFormatter.toSparqlEncodedString(createFile.originalMimeType, throw BadRequestException(s"The original MIME type is invalid: '${createFile.originalMimeType}'")), - projectShortcode = projectShortcode, - filename = stringFormatter.toSparqlEncodedString(createFile.filename, throw BadRequestException(s"Invalid filename: '${createFile.filename}'")), - userProfile = userADM.asUserProfileV1 - )) - case None => None + file <- apiRequest.file match { + case Some(filename) => + // Ask Sipi about the file's metadata. + val tempFileUrl = stringFormatter.makeSipiTempFileUrl(settings, filename) + + for { + imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequest(fileUrl = tempFileUrl, requestingUser = userADM)).mapTo[GetImageMetadataResponseV2] + + // TODO: check that the file stored is an image. + } yield Some(StillImageFileValueV1( + internalFilename = filename, + internalMimeType = "image/jp2", + originalFilename = imageMetadataResponse.originalFilename, + originalMimeType = Some(imageMetadataResponse.originalMimeType), + projectShortcode = projectShortcode, + dimX = imageMetadataResponse.width, + dimY = imageMetadataResponse.height + )) + + case None => FastFuture.successful(None) } valuesToBeCreatedWithFuture: Map[IRI, Future[Seq[CreateValueV1WithComment]]] = valuesToCreate( @@ -256,7 +264,7 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) label = label, projectIri = projectIri, values = valuesToBeCreated.toMap, - file = paramConversionRequest, + file = file, userProfile = userADM, apiRequestID = UUID.randomUUID ) @@ -279,13 +287,15 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) } yield propIri -> values } - convertedFileV1 <- resourceRequest.file match { + convertedFile <- resourceRequest.file match { case Some(filename) => // Ask Sipi about the file's metadata. - val tempFileUrl = s"${settings.internalSipiBaseUrl}/tmp/$filename" + val tempFileUrl = stringFormatter.makeSipiTempFileUrl(settings, filename) for { - imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequestV2(fileUrl = tempFileUrl, requestingUser = userProfile)).mapTo[GetImageMetadataResponseV2] + imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequest(fileUrl = tempFileUrl, requestingUser = userProfile)).mapTo[GetImageMetadataResponseV2] + + // TODO: check that the file stored is an image. } yield Some(StillImageFileValueV1( internalFilename = filename, internalMimeType = "image/jp2", @@ -303,7 +313,7 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) clientResourceID = resourceRequest.client_id, label = resourceRequest.label, values = valuesToBeCreated.toMap, - file = convertedFileV1, + file = convertedFile, creationDate = resourceRequest.creationDate ) } diff --git a/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala b/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala index e1cca9dc11..dce33b7f91 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala @@ -28,7 +28,7 @@ import akka.pattern._ import akka.stream.ActorMaterializer import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.SipiConversionFileRequestV1 +import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataRequest, GetImageMetadataResponseV2} import org.knora.webapi.messages.v1.responder.resourcemessages.{ResourceInfoGetRequestV1, ResourceInfoResponseV1} import org.knora.webapi.messages.v1.responder.valuemessages.ApiValueV1JsonProtocol._ import org.knora.webapi.messages.v1.responder.valuemessages._ @@ -310,29 +310,28 @@ class ValuesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit ) } - def makeChangeFileValueRequest(resIriStr: IRI, projectShortcode: String, apiRequest: Option[ChangeFileValueApiRequestV1], userADM: UserADM): ChangeFileValueRequestV1 = { + def makeChangeFileValueRequest(resIriStr: IRI, projectShortcode: String, apiRequest: ChangeFileValueApiRequestV1, userADM: UserADM): Future[ChangeFileValueRequestV1] = { val resourceIri = stringFormatter.validateAndEscapeIri(resIriStr, throw BadRequestException(s"Invalid resource IRI: $resIriStr")) + val tempFileUrl = stringFormatter.makeSipiTempFileUrl(settings, apiRequest.file) - if (apiRequest.nonEmpty) { - // GUI-case - val fileRequest = SipiConversionFileRequestV1( - originalFilename = stringFormatter.toSparqlEncodedString(apiRequest.get.file.originalFilename, throw BadRequestException(s"The original filename is invalid: '${apiRequest.get.file.originalFilename}'")), - originalMimeType = stringFormatter.toSparqlEncodedString(apiRequest.get.file.originalMimeType, throw BadRequestException(s"The original MIME type is invalid: '${apiRequest.get.file.originalMimeType}'")), - projectShortcode = projectShortcode, - filename = stringFormatter.toSparqlEncodedString(apiRequest.get.file.filename, throw BadRequestException(s"Invalid filename: '${apiRequest.get.file.filename}'")), - userProfile = userADM.asUserProfileV1 - ) - - ChangeFileValueRequestV1( - resourceIri = resourceIri, - file = fileRequest, - apiRequestID = UUID.randomUUID, - userProfile = userADM) - } else { - // no file information was provided - throw BadRequestException("A file value change was requested but no file information was provided") - } + for { + imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequest(fileUrl = tempFileUrl, requestingUser = userADM)).mapTo[GetImageMetadataResponseV2] + // TODO: check that the file stored is an image. + } yield ChangeFileValueRequestV1( + resourceIri = resourceIri, + file = StillImageFileValueV1( + internalFilename = apiRequest.file, + internalMimeType = "image/jp2", + originalFilename = imageMetadataResponse.originalFilename, + originalMimeType = Some(imageMetadataResponse.originalMimeType), + projectShortcode = projectShortcode, + dimX = imageMetadataResponse.width, + dimY = imageMetadataResponse.height + ), + apiRequestID = UUID.randomUUID, + userProfile = userADM + ) } // Version history request requires 3 URL path segments: resource IRI, property IRI, and current value IRI @@ -461,19 +460,18 @@ class ValuesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit put { entity(as[ChangeFileValueApiRequestV1]) { apiRequest => requestContext => - val requestMessage = for { userADM <- getUserADM(requestContext) resourceIri = stringFormatter.validateAndEscapeIri(resIriStr, throw BadRequestException(s"Invalid resource IRI: $resIriStr")) resourceInfoResponse <- (responderManager ? ResourceInfoGetRequestV1(resourceIri, userADM)).mapTo[ResourceInfoResponseV1] projectShortcode = resourceInfoResponse.resource_info.getOrElse(throw NotFoundException(s"Resource not found: $resourceIri")).project_shortcode - - } yield makeChangeFileValueRequest( - resIriStr = resIriStr, - projectShortcode = projectShortcode, - apiRequest = Some(apiRequest), - userADM = userADM - ) + request <- makeChangeFileValueRequest( + resIriStr = resIriStr, + projectShortcode = projectShortcode, + apiRequest = apiRequest, + userADM = userADM + ) + } yield request RouteUtilV1.runJsonRouteWithFuture( requestMessage, diff --git a/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala b/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala index 07abf2c0ff..e77063995e 100644 --- a/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala +++ b/webapi/src/main/scala/org/knora/webapi/store/iiif/SipiConnector.scala @@ -32,15 +32,12 @@ import org.apache.http.util.EntityUtils import org.apache.http.{Consts, HttpHost, HttpRequest, NameValuePair} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM import org.knora.webapi.messages.store.sipimessages.GetImageMetadataResponseV2JsonProtocol._ -import org.knora.webapi.messages.store.sipimessages.RepresentationV1JsonProtocol._ -import org.knora.webapi.messages.store.sipimessages.SipiConstants.FileType import org.knora.webapi.messages.store.sipimessages._ -import org.knora.webapi.messages.v1.responder.valuemessages.{FileValueV1, StillImageFileValueV1, TextFileValueV1} import org.knora.webapi.messages.v2.responder.SuccessResponseV2 import org.knora.webapi.routing.JWTHelper import org.knora.webapi.util.ActorUtil.{handleUnexpectedMessage, try2Message} import org.knora.webapi.util.{SipiUtil, StringFormatter} -import org.knora.webapi.{BadRequestException, KnoraDispatchers, NotImplementedException, Settings, SipiException} +import org.knora.webapi.{BadRequestException, KnoraDispatchers, Settings, SipiException} import spray.json._ import scala.concurrent.ExecutionContext @@ -71,159 +68,20 @@ class SipiConnector extends Actor with ActorLogging { private val httpClient: CloseableHttpClient = HttpClients.custom.setDefaultRequestConfig(sipiRequestConfig).build override def receive: Receive = { - case convertFileRequest: SipiConversionFileRequestV1 => try2Message(sender(), convertFileV1(convertFileRequest), log) - case getFileMetadataRequestV2: GetImageMetadataRequestV2 => try2Message(sender(), getFileMetadataV2(getFileMetadataRequestV2), log) - case moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequestV2 => try2Message(sender(), moveTemporaryFileToPermanentStorageV2(moveTemporaryFileToPermanentStorageRequestV2), log) - case deleteTemporaryFileRequestV2: DeleteTemporaryFileRequestV2 => try2Message(sender(), deleteTemporaryFileV2(deleteTemporaryFileRequestV2), log) - case SipiGetTextFileRequest(fileUrl, requestingUser) => try2Message(sender(), sipiGetXsltTransformationRequestV2(fileUrl, requestingUser), log) + case getFileMetadataRequest: GetImageMetadataRequest => try2Message(sender(), getFileMetadata(getFileMetadataRequest), log) + case moveTemporaryFileToPermanentStorageRequest: MoveTemporaryFileToPermanentStorageRequest => try2Message(sender(), moveTemporaryFileToPermanentStorage(moveTemporaryFileToPermanentStorageRequest), log) + case deleteTemporaryFileRequest: DeleteTemporaryFileRequest => try2Message(sender(), deleteTemporaryFile(deleteTemporaryFileRequest), log) + case SipiGetTextFileRequest(fileUrl, requestingUser) => try2Message(sender(), sipiGetXsltTransformationRequest(fileUrl, requestingUser), log) case other => handleUnexpectedMessage(sender(), other, log, this.getClass.getName) } - /** - * Convert a file that is already managed by Sipi (GUI-case). - * - * @param conversionRequest the information about the file (managed by Sipi). - * @return a [[SipiConversionResponseV1]] representing the file values to be added to the triplestore. - */ - private def convertFileV1(conversionRequest: SipiConversionFileRequestV1): Try[SipiConversionResponseV1] = { - val url = s"${settings.internalSipiImageConversionUrlV1}/${settings.sipiFileConversionRouteV1}" - - callSipiConvertRoute(url, conversionRequest) - } - - /** - * Makes a conversion request to Sipi and creates a [[SipiConversionResponseV1]] - * containing the file values to be added to the triplestore. - * - * @param urlPath the Sipi route to be called. - * @param conversionRequest the message holding the information to make the request. - * @return a [[SipiConversionResponseV1]]. - */ - private def callSipiConvertRoute(urlPath: String, conversionRequest: SipiConversionRequestV1): Try[SipiConversionResponseV1] = { - val context: HttpClientContext = HttpClientContext.create - - val formParams = new util.ArrayList[NameValuePair]() - - for ((key, value) <- conversionRequest.toFormData) { - formParams.add(new BasicNameValuePair(key, value)) - } - - val postEntity = new UrlEncodedFormEntity(formParams, Consts.UTF_8) - val httpPost = new HttpPost(urlPath) - httpPost.setEntity(postEntity) - - val conversionResultTry: Try[String] = Try { - var maybeResponse: Option[CloseableHttpResponse] = None - - try { - maybeResponse = Some(httpClient.execute(targetHost, httpPost, context)) - - val responseEntityStr: String = Option(maybeResponse.get.getEntity) match { - case Some(responseEntity) => EntityUtils.toString(responseEntity) - case None => "" - } - - val statusCode: Int = maybeResponse.get.getStatusLine.getStatusCode - val statusCategory: Int = statusCode / 100 - - // Was the request successful? - if (statusCategory == 2) { - // Yes. - responseEntityStr - } else { - // No. Throw an appropriate exception. - val sipiErrorMsg = SipiUtil.getSipiErrorMessage(responseEntityStr) - - if (statusCategory == 4) { - throw BadRequestException(s"Sipi responded with HTTP status code $statusCode: $sipiErrorMsg") - } else { - throw SipiException(s"Sipi responded with HTTP status code $statusCode: $sipiErrorMsg") - } - } - } finally { - maybeResponse match { - case Some(response) => response.close() - case None => () - } - } - } - - // - // handle unsuccessful requests to Sipi - // - val recoveredConversionResultTry = conversionResultTry.recoverWith { - case badRequestException: BadRequestException => throw badRequestException - case sipiException: SipiException => throw sipiException - case e: Exception => throw SipiException("Failed to connect to Sipi", e, log) - } - - for { - responseAsStr: String <- recoveredConversionResultTry - - /* get json from response body */ - responseAsJson: JsValue = JsonParser(responseAsStr) - - // get file type from Sipi response - fileType: String = responseAsJson.asJsObject.fields.getOrElse("file_type", throw SipiException(message = "Sipi did not return a file type")) match { - case JsString(ftype: String) => ftype - case other => throw SipiException(message = s"Sipi returned an invalid file type: $other") - } - - // turn fileType returned by Sipi (a string) into an enum - fileTypeEnum: FileType.Value = SipiConstants.FileType.lookup(fileType) - - // create the apt case class depending on the file type returned by Sipi - fileValueV1: FileValueV1 = fileTypeEnum match { - case SipiConstants.FileType.IMAGE => - // parse response as a [[SipiImageConversionResponse]] - val imageConversionResult = try { - responseAsJson.convertTo[SipiImageConversionResponse] - } catch { - case e: DeserializationException => throw SipiException(message = "JSON response returned by Sipi is invalid, it cannot be turned into a SipiImageConversionResponse", e = e, log = log) - } - - StillImageFileValueV1( - internalMimeType = stringFormatter.toSparqlEncodedString(imageConversionResult.mimetype_full, throw BadRequestException(s"The internal MIME type returned by Sipi is invalid: '${imageConversionResult.mimetype_full}")), - originalFilename = stringFormatter.toSparqlEncodedString(imageConversionResult.original_filename, throw BadRequestException(s"The original filename returned by Sipi is invalid: '${imageConversionResult.original_filename}")), - originalMimeType = Some(stringFormatter.toSparqlEncodedString(imageConversionResult.original_mimetype, throw BadRequestException(s"The original MIME type returned by Sipi is invalid: '${imageConversionResult.original_mimetype}"))), - projectShortcode = conversionRequest.projectShortcode, - dimX = imageConversionResult.nx_full, - dimY = imageConversionResult.ny_full, - internalFilename = stringFormatter.toSparqlEncodedString(imageConversionResult.filename_full, throw BadRequestException(s"The internal filename returned by Sipi is invalid: '${imageConversionResult.filename_full}")) - ) - - case SipiConstants.FileType.TEXT => - - // parse response as a [[SipiTextResponse]] - val textStoreResult = try { - responseAsJson.convertTo[SipiTextResponse] - } catch { - case e: DeserializationException => throw SipiException(message = "JSON response returned by Sipi is invalid, it cannot be turned into a SipiTextResponse", e = e, log = log) - } - - TextFileValueV1( - internalMimeType = stringFormatter.toSparqlEncodedString(textStoreResult.mimetype, throw BadRequestException(s"The internal MIME type returned by Sipi is invalid: '${textStoreResult.mimetype}")), - internalFilename = stringFormatter.toSparqlEncodedString(textStoreResult.filename, throw BadRequestException(s"The internal filename returned by Sipi is invalid: '${textStoreResult.filename}")), - originalFilename = stringFormatter.toSparqlEncodedString(textStoreResult.original_filename, throw BadRequestException(s"The internal filename returned by Sipi is invalid: '${textStoreResult.original_filename}")), - originalMimeType = Some(stringFormatter.toSparqlEncodedString(textStoreResult.mimetype, throw BadRequestException(s"The orignal MIME type returned by Sipi is invalid: '${textStoreResult.original_mimetype}"))), - projectShortcode = conversionRequest.projectShortcode - ) - - case unknownType => throw NotImplementedException(s"Could not handle file type $unknownType") - - // TODO: add missing file types - } - - } yield SipiConversionResponseV1(fileValueV1, file_type = fileTypeEnum) - } - /** * Asks Sipi for metadata about a file. * * @param getFileMetadataRequestV2 the request. * @return a [[GetImageMetadataResponseV2]] containing the requested metadata. */ - private def getFileMetadataV2(getFileMetadataRequestV2: GetImageMetadataRequestV2): Try[GetImageMetadataResponseV2] = { + private def getFileMetadata(getFileMetadataRequestV2: GetImageMetadataRequest): Try[GetImageMetadataResponseV2] = { val knoraInfoUrl = getFileMetadataRequestV2.fileUrl + "/knora.json" val request = new HttpGet(knoraInfoUrl) @@ -239,7 +97,7 @@ class SipiConnector extends Actor with ActorLogging { * @param moveTemporaryFileToPermanentStorageRequestV2 the request. * @return a [[SuccessResponseV2]]. */ - private def moveTemporaryFileToPermanentStorageV2(moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequestV2): Try[SuccessResponseV2] = { + private def moveTemporaryFileToPermanentStorage(moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequest): Try[SuccessResponseV2] = { val token: String = JWTHelper.createToken( userIri = moveTemporaryFileToPermanentStorageRequestV2.requestingUser.id, secret = settings.jwtSecretKey, @@ -275,7 +133,7 @@ class SipiConnector extends Actor with ActorLogging { * @param deleteTemporaryFileRequestV2 the request. * @return a [[SuccessResponseV2]]. */ - private def deleteTemporaryFileV2(deleteTemporaryFileRequestV2: DeleteTemporaryFileRequestV2): Try[SuccessResponseV2] = { + private def deleteTemporaryFile(deleteTemporaryFileRequestV2: DeleteTemporaryFileRequest): Try[SuccessResponseV2] = { val token: String = JWTHelper.createToken( userIri = deleteTemporaryFileRequestV2.requestingUser.id, secret = settings.jwtSecretKey, @@ -303,7 +161,7 @@ class SipiConnector extends Actor with ActorLogging { * @param xsltFileUrl the file's URL. * @param requestingUser the user making the request. */ - private def sipiGetXsltTransformationRequestV2(xsltFileUrl: String, requestingUser: UserADM): Try[SipiGetTextFileResponse] = { + private def sipiGetXsltTransformationRequest(xsltFileUrl: String, requestingUser: UserADM): Try[SipiGetTextFileResponse] = { // ask Sipi to return the XSL transformation val request = new HttpGet(xsltFileUrl) diff --git a/webapi/src/main/scala/org/knora/webapi/util/StringFormatter.scala b/webapi/src/main/scala/org/knora/webapi/util/StringFormatter.scala index 847eb0b5e0..3449fcdd43 100644 --- a/webapi/src/main/scala/org/knora/webapi/util/StringFormatter.scala +++ b/webapi/src/main/scala/org/knora/webapi/util/StringFormatter.scala @@ -2267,7 +2267,7 @@ class StringFormatter private(val maybeSettings: Option[SettingsImpl], initForTe * @param projectInfo the project's [[ProjectInfoV1]]. * @return the IRI of the project's data named graph. */ - def projectDataNamedGraph(projectInfo: ProjectInfoV1): IRI = { + def projectDataNamedGraphV1(projectInfo: ProjectInfoV1): IRI = { OntologyConstants.NamedGraphs.DataNamedGraphStart + "/" + projectInfo.shortcode + "/" + projectInfo.shortname } @@ -2569,4 +2569,15 @@ class StringFormatter private(val maybeSettings: Option[SettingsImpl], initForTe arkUrlWithoutTimestamp } } + + /** + * Constructs a URL for accessing a file that has been uploaded to Sipi's temporary storage. + * + * @param settings the application settings. + * @param filename the filename. + * @return a URL for accessing the file. + */ + def makeSipiTempFileUrl(settings: SettingsImpl, filename: String): String = { + s"${settings.internalSipiBaseUrl}/tmp/$filename" + } } diff --git a/webapi/src/test/scala/org/knora/webapi/e2e/v1/SipiV1R2RSpec.scala b/webapi/src/test/scala/org/knora/webapi/e2e/v1/SipiV1R2RSpec.scala index c79333af9b..81112e93ca 100644 --- a/webapi/src/test/scala/org/knora/webapi/e2e/v1/SipiV1R2RSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/e2e/v1/SipiV1R2RSpec.scala @@ -108,13 +108,10 @@ class SipiV1R2RSpec extends R2RSpec { "The Resources Endpoint" should { "create a resource with a digital representation" in { + val internalFilename = "IQUO3t1AABm-FSLC0vNvVpr.jp2" val params = RequestParams.createResourceParams.copy( - file = Some(CreateFileV1( - originalFilename = "Chlaus.jpg", - originalMimeType = "image/jpeg", - filename = "./test_server/images/Chlaus.jpg" - )) + file = Some(internalFilename) ) Post("/v1/resources", HttpEntity(MediaTypes.`application/json`, params.toJsValue.compactPrint)) ~> addCredentials(BasicHttpCredentials(incunabulaProjectAdminEmail, testPass)) ~> resourcesPath ~> check { @@ -127,13 +124,10 @@ class SipiV1R2RSpec extends R2RSpec { "The Values endpoint" should { "change the file value of an existing page" in { + val internalFilename ="FSLC0vNvVpr-IQUO3t1AABm.jp2" val params = ChangeFileValueApiRequestV1( - file = CreateFileV1( - originalFilename = "Chlaus.jpg", - originalMimeType = "image/jpeg", - filename = "./test_server/images/Chlaus.jpg" - ) + file = internalFilename ) val resIri = URLEncoder.encode("http://rdfh.ch/0803/8a0b1e75", "UTF-8") diff --git a/webapi/src/test/scala/org/knora/webapi/responders/v1/ResourcesResponderV1Spec.scala b/webapi/src/test/scala/org/knora/webapi/responders/v1/ResourcesResponderV1Spec.scala index 8992837e35..ae7e1d0371 100644 --- a/webapi/src/test/scala/org/knora/webapi/responders/v1/ResourcesResponderV1Spec.scala +++ b/webapi/src/test/scala/org/knora/webapi/responders/v1/ResourcesResponderV1Spec.scala @@ -27,7 +27,6 @@ import com.typesafe.config.{Config, ConfigFactory} import org.knora.webapi.SharedOntologyTestDataADM._ import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.permissionsmessages.{ObjectAccessPermissionADM, ObjectAccessPermissionsForResourceGetADM, PermissionADM} -import org.knora.webapi.messages.store.sipimessages.SipiConversionFileRequestV1 import org.knora.webapi.messages.store.triplestoremessages._ import org.knora.webapi.messages.v1.responder.resourcemessages._ import org.knora.webapi.messages.v1.responder.valuemessages._ @@ -1196,7 +1195,7 @@ class ResourcesResponderV1Spec extends CoreSpec(ResourcesResponderV1Spec.config) val origname = TextValueSimpleV1("Blatt") val seqnum = IntegerValueV1(1) - val fileValueFull = StillImageFileValueV1( + val fileValue = StillImageFileValueV1( internalMimeType = "image/jp2", internalFilename = "gaga.jpg", originalFilename = "test.jpg", @@ -1222,7 +1221,7 @@ class ResourcesResponderV1Spec extends CoreSpec(ResourcesResponderV1Spec.config) "http://www.knora.org/ontology/0803/incunabula#partOf" -> Vector(LinkV1(book)), "http://www.knora.org/ontology/0803/incunabula#origname" -> Vector(origname), "http://www.knora.org/ontology/0803/incunabula#seqnum" -> Vector(seqnum), - OntologyConstants.KnoraBase.HasStillImageFileValue -> Vector(fileValueFull) + OntologyConstants.KnoraBase.HasStillImageFileValue -> Vector(fileValue) ) responderManager ! ResourceCreateRequestV1( @@ -1230,13 +1229,7 @@ class ResourcesResponderV1Spec extends CoreSpec(ResourcesResponderV1Spec.config) label = "Test-Page", projectIri = SharedTestDataADM.INCUNABULA_PROJECT_IRI, values = valuesToBeCreated, - file = Some(SipiConversionFileRequestV1( - originalFilename = "test.jpg", - originalMimeType = "image/jpeg", - filename = "./test_server/images/Chlaus.jpg", - projectShortcode = "0803", - userProfile = SharedTestDataADM.incunabulaProjectAdminUser.asUserProfileV1 - )), + file = Some(fileValue), userProfile = SharedTestDataADM.incunabulaProjectAdminUser, apiRequestID = UUID.randomUUID ) diff --git a/webapi/src/test/scala/org/knora/webapi/responders/v1/ValuesResponderV1Spec.scala b/webapi/src/test/scala/org/knora/webapi/responders/v1/ValuesResponderV1Spec.scala index 1db80b077c..7f3774bf36 100644 --- a/webapi/src/test/scala/org/knora/webapi/responders/v1/ValuesResponderV1Spec.scala +++ b/webapi/src/test/scala/org/knora/webapi/responders/v1/ValuesResponderV1Spec.scala @@ -27,7 +27,6 @@ import com.typesafe.config.ConfigFactory import org.knora.webapi.SharedOntologyTestDataADM._ import org.knora.webapi.SharedTestDataADM._ import org.knora.webapi._ -import org.knora.webapi.messages.store.sipimessages.SipiConversionFileRequestV1 import org.knora.webapi.messages.store.triplestoremessages._ import org.knora.webapi.messages.v1.responder.resourcemessages.{LocationV1, ResourceFullGetRequestV1, ResourceFullResponseV1} import org.knora.webapi.messages.v1.responder.valuemessages._ @@ -1568,17 +1567,19 @@ class ValuesResponderV1Spec extends CoreSpec(ValuesResponderV1Spec.config) with "add a new image file value to an incunabula:page" in { - val fileRequest = SipiConversionFileRequestV1( - originalFilename = "Chlaus.jpg", - originalMimeType = "image/jpeg", + val fileValue = StillImageFileValueV1( + internalMimeType = "image/jp2", + internalFilename = "gaga.jpg", + originalFilename = "test.jpg", + originalMimeType = Some("image/jpg"), projectShortcode = "0803", - filename = "./test_server/images/Chlaus.jpg", - userProfile = incunabulaUser.asUserProfileV1 + dimX = 1000, + dimY = 1000 ) val fileChangeRequest = ChangeFileValueRequestV1( resourceIri = "http://rdfh.ch/0803/8a0b1e75", - file = fileRequest, + file = fileValue, apiRequestID = UUID.randomUUID, userProfile = incunabulaUser) diff --git a/webapi/src/test/scala/org/knora/webapi/store/iiif/MockSipiConnector.scala b/webapi/src/test/scala/org/knora/webapi/store/iiif/MockSipiConnector.scala index d1ab1b3f7c..3ed33e4096 100644 --- a/webapi/src/test/scala/org/knora/webapi/store/iiif/MockSipiConnector.scala +++ b/webapi/src/test/scala/org/knora/webapi/store/iiif/MockSipiConnector.scala @@ -21,12 +21,11 @@ package org.knora.webapi.store.iiif import akka.actor.{Actor, ActorLogging, ActorSystem} import org.knora.webapi.messages.store.sipimessages._ -import org.knora.webapi.messages.v1.responder.valuemessages.StillImageFileValueV1 import org.knora.webapi.messages.v2.responder.SuccessResponseV2 import org.knora.webapi.util.ActorUtil._ -import org.knora.webapi.{BadRequestException, KnoraDispatchers, Settings, SipiException} +import org.knora.webapi.{KnoraDispatchers, SipiException} -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.ExecutionContext import scala.util.{Failure, Success, Try} /** @@ -50,46 +49,14 @@ class MockSipiConnector extends Actor with ActorLogging { implicit val system: ActorSystem = context.system implicit val executionContext: ExecutionContext = system.dispatchers.lookup(KnoraDispatchers.KnoraActorDispatcher) - private val settings = Settings(system) - - def receive = { - case sipiResponderConversionFileRequest: SipiConversionFileRequestV1 => future2Message(sender(), imageConversionResponse(sipiResponderConversionFileRequest), log) - case getFileMetadataRequestV2: GetImageMetadataRequestV2 => try2Message(sender(), getFileMetadataV2(getFileMetadataRequestV2), log) - case moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequestV2 => try2Message(sender(), moveTemporaryFileToPermanentStorageV2(moveTemporaryFileToPermanentStorageRequestV2), log) - case deleteTemporaryFileRequestV2: DeleteTemporaryFileRequestV2 => try2Message(sender(), deleteTemporaryFileV2(deleteTemporaryFileRequestV2), log) + case getFileMetadataRequest: GetImageMetadataRequest => try2Message(sender(), getFileMetadata(getFileMetadataRequest), log) + case moveTemporaryFileToPermanentStorageRequest: MoveTemporaryFileToPermanentStorageRequest => try2Message(sender(), moveTemporaryFileToPermanentStorage(moveTemporaryFileToPermanentStorageRequest), log) + case deleteTemporaryFileRequest: DeleteTemporaryFileRequest => try2Message(sender(), deleteTemporaryFile(deleteTemporaryFileRequest), log) case other => handleUnexpectedMessage(sender(), other, log, this.getClass.getName) } - /** - * Imitates the Sipi server by returning a [[SipiConversionResponseV1]] representing an image conversion request. - * - * @param conversionRequest the conversion request to be handled. - * @return a [[SipiConversionResponseV1]] imitating the answer from Sipi. - */ - private def imageConversionResponse(conversionRequest: SipiConversionRequestV1): Future[SipiConversionResponseV1] = { - Future { - val originalFilename = conversionRequest.originalFilename - val originalMimeType: String = conversionRequest.originalMimeType - - // we expect original mimetype to be "image/jpeg" - if (originalMimeType != "image/jpeg") throw BadRequestException("Wrong mimetype for jpg file") - - val fileValueV1 = StillImageFileValueV1( - internalMimeType = "image/jp2", - originalFilename = originalFilename, - originalMimeType = Some(originalMimeType), - projectShortcode = conversionRequest.projectShortcode, - dimX = 800, - dimY = 800, - internalFilename = "full.jp2" - ) - - SipiConversionResponseV1(fileValueV1, file_type = SipiConstants.FileType.IMAGE) - } - } - - private def getFileMetadataV2(getFileMetadataRequestV2: GetImageMetadataRequestV2): Try[GetImageMetadataResponseV2] = + private def getFileMetadata(getFileMetadataRequestV2: GetImageMetadataRequest): Try[GetImageMetadataResponseV2] = Success { GetImageMetadataResponseV2( originalFilename = "test2.tiff", @@ -99,7 +66,7 @@ class MockSipiConnector extends Actor with ActorLogging { ) } - private def moveTemporaryFileToPermanentStorageV2(moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequestV2): Try[SuccessResponseV2] = { + private def moveTemporaryFileToPermanentStorage(moveTemporaryFileToPermanentStorageRequestV2: MoveTemporaryFileToPermanentStorageRequest): Try[SuccessResponseV2] = { if (moveTemporaryFileToPermanentStorageRequestV2.internalFilename == MockSipiConnector.FAILURE_FILENAME) { Failure(SipiException("Sipi failed to move file to permanent storage")) } else { @@ -107,7 +74,7 @@ class MockSipiConnector extends Actor with ActorLogging { } } - private def deleteTemporaryFileV2(deleteTemporaryFileRequestV2: DeleteTemporaryFileRequestV2): Try[SuccessResponseV2] = { + private def deleteTemporaryFile(deleteTemporaryFileRequestV2: DeleteTemporaryFileRequest): Try[SuccessResponseV2] = { if (deleteTemporaryFileRequestV2.internalFilename == MockSipiConnector.FAILURE_FILENAME) { Failure(SipiException("Sipi failed to delete temporary file")) } else { diff --git a/webapi/src/test/scala/org/knora/webapi/util/StringFormatterSpec.scala b/webapi/src/test/scala/org/knora/webapi/util/StringFormatterSpec.scala index 930a96aa4a..ca509afef5 100644 --- a/webapi/src/test/scala/org/knora/webapi/util/StringFormatterSpec.scala +++ b/webapi/src/test/scala/org/knora/webapi/util/StringFormatterSpec.scala @@ -917,7 +917,7 @@ class StringFormatterSpec extends CoreSpec() { val shortcode = SharedTestDataV1.imagesProjectInfo.shortcode val shortname = SharedTestDataV1.imagesProjectInfo.shortname val expected = s"http://www.knora.org/data/$shortcode/$shortname" - val result = stringFormatter.projectDataNamedGraph(SharedTestDataV1.imagesProjectInfo) + val result = stringFormatter.projectDataNamedGraphV1(SharedTestDataV1.imagesProjectInfo) result should be(expected) // check consistency of our test data From a072e38792062c2a5bc906f76917567484b60e0d Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 5 Mar 2019 16:16:56 +0100 Subject: [PATCH 06/24] feature (sipi): Support cookie authentication in upload.lua. --- sipi/scripts/get_knora_session.lua | 3 +-- sipi/scripts/upload.lua | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/sipi/scripts/get_knora_session.lua b/sipi/scripts/get_knora_session.lua index e93d0c69e1..7982297ab6 100644 --- a/sipi/scripts/get_knora_session.lua +++ b/sipi/scripts/get_knora_session.lua @@ -37,8 +37,7 @@ function get_session_id(cookie) -- space is also treated as a separator -- returns nil if it cannot find the session id (pattern does not match) server.log("extracted cookie: " .. cookie, server.loglevel.LOG_DEBUG) - print("extracted cookie: " .. cookie) - session_id = string.match(cookie, "KnoraAuthentication=([^%s;]+)") + local session_id = string.match(cookie, "KnoraAuthentication=([^%s;]+)") return session_id diff --git a/sipi/scripts/upload.lua b/sipi/scripts/upload.lua index 8d21e00b4e..8bc6f54b07 100644 --- a/sipi/scripts/upload.lua +++ b/sipi/scripts/upload.lua @@ -33,12 +33,24 @@ if not success then return end --- Check for a valid JSON Web Token from Knora. +-- Check for a valid JSON Web Token or authentication cookie from Knora. local token = get_knora_token() if token == nil then - return + local cookie_OK = false + + for cookie_index, cookie in pairs(server.cookies) do + if get_session_id(cookie) ~= nil then + cookie_OK = true + server.log("Knora cookie OK", server.loglevel.LOG_DEBUG) + break + end + end + + if not cookie_OK then + return + end end -- Create the temporary directory if necessary. From 663c0dd58b0c9976018170f15e612a51c582a1ba Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 5 Mar 2019 18:01:51 +0100 Subject: [PATCH 07/24] feature (sipi): Remove obsolete routes from Sipi configs. --- sipi/config/sipi.knora-docker-config.lua | 15 ----------- sipi/config/sipi.knora-docker-it-config.lua | 15 ----------- .../sipi.knora-docker-no-auth-config.lua | 25 ------------------- sipi/config/sipi.knora-docker-test-config.lua | 15 ----------- sipi/config/sipi.knora-local-config.lua | 15 ----------- 5 files changed, 85 deletions(-) diff --git a/sipi/config/sipi.knora-docker-config.lua b/sipi/config/sipi.knora-docker-config.lua index 73fdd8f10f..4db7586f12 100644 --- a/sipi/config/sipi.knora-docker-config.lua +++ b/sipi/config/sipi.knora-docker-config.lua @@ -163,21 +163,6 @@ fileserver = { -- executes the given script defined below -- routes = { - { - method = 'POST', - route = '/make_thumbnail', - script = 'make_thumbnail.lua' - }, - { - method = 'POST', - route = '/convert_from_binaries', - script = 'convert_from_binaries.lua' - }, - { - method = 'POST', - route = '/convert_from_file', - script = 'convert_from_file.lua' - }, { method = 'POST', route = '/upload', diff --git a/sipi/config/sipi.knora-docker-it-config.lua b/sipi/config/sipi.knora-docker-it-config.lua index e6d97abed5..34819ca254 100644 --- a/sipi/config/sipi.knora-docker-it-config.lua +++ b/sipi/config/sipi.knora-docker-it-config.lua @@ -161,21 +161,6 @@ fileserver = { -- Custom routes. Each route is URL path associated with a Lua script. -- routes = { - { - method = 'POST', - route = '/make_thumbnail', - script = 'make_thumbnail.lua' - }, - { - method = 'POST', - route = '/convert_from_binaries', - script = 'convert_from_binaries.lua' - }, - { - method = 'POST', - route = '/convert_from_file', - script = 'convert_from_file.lua' - }, { method = 'POST', route = '/upload', diff --git a/sipi/config/sipi.knora-docker-no-auth-config.lua b/sipi/config/sipi.knora-docker-no-auth-config.lua index 455081b8b2..2dd3c5fe34 100644 --- a/sipi/config/sipi.knora-docker-no-auth-config.lua +++ b/sipi/config/sipi.knora-docker-no-auth-config.lua @@ -179,31 +179,6 @@ fileserver = { -- Custom routes. Each route is URL path associated with a Lua script. -- routes = { - { - method = 'POST', - route = '/make_thumbnail', - script = 'make_thumbnail.lua' - }, - { - method = 'POST', - route = '/convert_from_binaries', - script = 'convert_from_binaries.lua' - }, - { - method = 'POST', - route = '/convert_from_file', - script = 'convert_from_file.lua' - }, - --{ - -- method = 'POST', - -- route = '/Knora_login', - -- script = 'Knora_login.lua' - --}, - --{ - -- method = 'POST', - -- route = '/Knora_logout', - -- script = 'Knora_logout.lua' - --}, { method = 'GET', route = '/test_functions', diff --git a/sipi/config/sipi.knora-docker-test-config.lua b/sipi/config/sipi.knora-docker-test-config.lua index a7b64bf8c2..9bd791b2ad 100644 --- a/sipi/config/sipi.knora-docker-test-config.lua +++ b/sipi/config/sipi.knora-docker-test-config.lua @@ -178,21 +178,6 @@ fileserver = { -- Custom routes. Each route is URL path associated with a Lua script. -- routes = { - { - method = 'POST', - route = '/make_thumbnail', - script = 'make_thumbnail.lua' - }, - { - method = 'POST', - route = '/convert_from_binaries', - script = 'convert_from_binaries.lua' - }, - { - method = 'POST', - route = '/convert_from_file', - script = 'convert_from_file.lua' - }, { method = 'POST', route = '/admin_upload', diff --git a/sipi/config/sipi.knora-local-config.lua b/sipi/config/sipi.knora-local-config.lua index aa1694edf3..d9fb591b7e 100644 --- a/sipi/config/sipi.knora-local-config.lua +++ b/sipi/config/sipi.knora-local-config.lua @@ -162,21 +162,6 @@ fileserver = { -- executes the given script defined below -- routes = { - { - method = 'POST', - route = '/make_thumbnail', - script = 'make_thumbnail.lua' - }, - { - method = 'POST', - route = '/convert_from_binaries', - script = 'convert_from_binaries.lua' - }, - { - method = 'POST', - route = '/convert_from_file', - script = 'convert_from_file.lua' - }, { method = 'POST', route = '/upload', From bd83b659cacd7bf84257fe1578b9f6f036e6a3a3 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Wed, 6 Mar 2019 13:52:28 +0100 Subject: [PATCH 08/24] refactor (salsah1): Use Sipi /upload route. --- docs/src/paradox/00-release-notes/next.md | 9 + .../03-apis/api-v1/adding-resources.md | 62 +++++-- .../paradox/03-apis/api-v1/changing-values.md | 51 ++---- .../paradox/03-apis/api-v2/editing-values.md | 4 +- .../05-internals/design/api-v2/sipi.md | 2 +- .../paradox/07-sipi/setup-sipi-for-knora.md | 28 +-- docs/src/paradox/07-sipi/sipi-and-knora.md | 162 +++--------------- salsah1/src/public/index.html | 4 + salsah1/src/public/js/jquery.location.js | 9 +- salsah1/src/public/js/jquery.propedit.js | 6 +- salsah1/src/public/js/jquery.resadd.js | 6 +- .../basicMessageComponents.ts | 23 +-- .../sampleRequests/sampleChangeValues.ts | 6 +- .../sampleRequests/sampleCreateResources.ts | 6 +- sipi/scripts/jwt.lua | 11 +- sipi/scripts/upload.lua | 18 +- .../org/knora/webapi/ITKnoraLiveSpec.scala | 4 +- .../e2e/v2/KnoraSipiIntegrationV2ITSpec.scala | 2 +- .../webapi/responders/v1/ValueUtilV1.scala | 2 +- 19 files changed, 130 insertions(+), 285 deletions(-) diff --git a/docs/src/paradox/00-release-notes/next.md b/docs/src/paradox/00-release-notes/next.md index 72eb16f822..973e986574 100644 --- a/docs/src/paradox/00-release-notes/next.md +++ b/docs/src/paradox/00-release-notes/next.md @@ -15,3 +15,12 @@ Also, please change the **HINT** to the appropriate level: ## HINT => MAJOR CHANGE - FIX: Unescape standoff string attributes when verifying text value update (@github[#1242](#1242)) + +- MAJOR: Change API v1 file uploads to work like API v2 (@github[#1233](#1233)). To enable + Knora and Sipi tow work without sharing a filesystem, the procedure + for uploading files in API v1 has changed; see + @ref:[Adding Resources with Image Files](../03-apis/api-v1/adding-resources.md#adding-resources-with-image-files) + and @ref:[Bulk Import with Image Files](../03-apis/api-v1/adding-resources.md#bulk-import-with-image-files). + Also, for consistency with API v2 responses, the `temporaryBaseIIIFUrl` returned by the Sipi + `/upload` route no longer includes the image filename; see + @ref:[Upload Files to Sipi](../03-apis/api-v2/editing-values.md#upload-files-to-sipi). diff --git a/docs/src/paradox/03-apis/api-v1/adding-resources.md b/docs/src/paradox/03-apis/api-v1/adding-resources.md index 341c25c509..68fd095946 100644 --- a/docs/src/paradox/03-apis/api-v1/adding-resources.md +++ b/docs/src/paradox/03-apis/api-v1/adding-resources.md @@ -49,20 +49,54 @@ The request header's content type has to be set to `application/json`. ## Adding Resources with Image Files -Certain resource classes can have attached image files. To attach -attach a file to a resource, you must first submit to the file to Sipi, then -submit the file's metadata to Knora (see @ref:[Sipi and Knora](../../07-sipi/sipi-and-knora.md)). +The first step is to upload an image file to Sipi, using a +`multipart/form-data` request, where `sipihost` represents the host and +port on which Sipi is running: + +``` +HTTP POST to http://sipihost/upload?token=TOKEN +``` + +The `TOKEN` is the `sid` returned by Knora in response to the +client's login request (see @ref:[Authentication](authentication.md)). +The request must contain a body part providing the file as well as a parameter +`filename`, providing the file's original filename, which both Knora and Sipi will +store; these filenames can be descriptive and need not be unique. + +Sipi will then convert the uploaded image file to JPEG 2000 format and store +it in a temporary location. If this is successful, it will return a JSON +response that looks something like this: + +```json +{ + "uploadedFiles": [{ + "originalFilename": "manuscript-1234-page-1.tiff", + "internalFilename": "3UIsXH9bP0j-BV0D4sN51Xz.jp2", + "temporaryBaseIIIFUrl": "http://sipihost/tmp" + }] +} +``` + +This provides: + +- the `originalFilename`, which we submitted when uploading the file +- the unique `internalFilename` that Sipi has randomly generated for the file +- the `temporaryBaseIIIFUrl`, which we can use to construct a IIIF URL for + previewing the file + +The client may now wish to get a thumbnail of the uploaded image, to allow +the user to confirm that the correct files have been uploaded. This can be done +by adding the filename and IIIF parameters to `temporaryBaseIIIFUrl`. For example, to get +a JPG thumbnail image whose width and height are at most 128 pixels wide, you would request +`http://sipihost/tmp/3UIsXH9bP0j-BV0D4sN51Xz.jp2/full/!128,128/0/default.jpg`. The request to Knora works similarly to -[Adding Resources Without Image Files](#adding-resources-without-a-digital-representation). The JSON format is described -in the TypeScript interface `createResourceWithRepresentationRequest` in -module `createResourceFormats`. The request header's content type has to +[Adding Resources Without Image Files](#adding-resources-without-image-files), +with the addition of `file`, whose value is the `internalFilename` that Sipi returned. +See the TypeScript interface `createResourceWithRepresentationRequest` in +module `createResourceFormats` for details. The request header's content type must be set to `application/json`. -In addition to [Adding Resources Without Image Files](#adding-resources-without-a-digital-representation), the -(temporary) name of the file, its original name, and mime type have to -be provided (see @ref:[GUI Case](../../07-sipi/sipi-and-knora.md#gui-case)). - ## Response to a Resource Creation When a resource has been successfully created, Knora sends back a JSON @@ -76,7 +110,7 @@ The JSON format of the response is described in the TypeScript interface ## Changing a Resource's Label A resource's label can be changed by making a PUT request to the path -segments `resources/label`. The resource's Iri has to be provided in the +segments `resources/label`. The resource's IRI has to be provided in the URL (as its last segment). The new label has to submitted as JSON in the HTTP request's body. @@ -374,8 +408,8 @@ contains the IRI of the target resource. To attach an image file to a resource, we must provide the element `knoraXmlImport:file` before the property elements. In this -element, we must give the filename that Sipi returned for the file in -@ref:[2. Upload Files to Sipi](#2-upload-files-to-sipi). +element, we must provide a `filename` attribute, containing the `internalFilename` +that Sipi returned for the file in @ref:[2. Upload Files to Sipi](#2-upload-files-to-sipi). ```xml @@ -390,7 +424,7 @@ element, we must give the filename that Sipi returned for the file in a page with an image - + Chlaus 1a diff --git a/docs/src/paradox/03-apis/api-v1/changing-values.md b/docs/src/paradox/03-apis/api-v1/changing-values.md index efc3ac70d6..beaca1e156 100644 --- a/docs/src/paradox/03-apis/api-v1/changing-values.md +++ b/docs/src/paradox/03-apis/api-v1/changing-values.md @@ -56,52 +56,21 @@ has to be used in order to create a new value (all these TypeScript interfaces a ## Modifying a File Value -In order to exchange a file value (digital representation of a -resource), the path segment `filevalue` has to be used. The IRI of the -resource whose file value is to be exchanged has to be appended: +To change a file value, the client first uploads the new file to +Sipi, following the procedure described in +@ref:[Adding Resources with Image Files](adding-resources.md#adding-resources-with-image-files). + +Then the client sends a request to Knora, using this following route: ``` HTTP PUT to http://host/filevalue/resourceIRI ``` -Please note that the resource IRI has to be URL encoded. - -There are two ways to change a file of a resource: Either by submitting -directly the binaries of the file in a HTTP Multipart request or by -indicating the location of the file. The two cases are referred to as -non-GUI case and GUI case (TODO: add a link to "Sipi and Knora"). - -### Including the binaries (non-GUI case) - -Here, a HTTP MULTIPART request has to be made simply providing the -binaries (without JSON): - -```python -#!/usr/bin/env python3 - -import requests, json, urllib - -# the name of the file to be submitted -filename = 'myimage.tif' - -# a tuple containing the file's name, its binaries and its mimetype -files = {'file': (filename, open(filename, 'rb'), "image/tiff")} - -resIri = urllib.parse.quote_plus('http://rdfh.ch/xy') - -r = requests.put("http://host/filevalue/" + resIri, - files=files) -``` - -Please note that the file has to be read in binary mode (by default it -would be read in text mode). - -### Indicating the location of a file (GUI case) - -Here, simply the location of the new file has to be submitted as JSON. -The JSON format is described in the TypeScript interface -`changeFileValueRequest` in module `changeValueFormats`. The request -header's content type has to set to `application/json`. +Here, `resourceIRI` is the URL-encoded IRI of the resource whose file value is +to be changed. The body of the request is a JSON object described in the TypeScript +interface `changeFileValueRequest` in module `changeValueFormats`, and contains +`file`, whose value is the `internalFilename` that Sipi returned. The request header's +content type must be set to `application/json`. ## Response on Value Change diff --git a/docs/src/paradox/03-apis/api-v2/editing-values.md b/docs/src/paradox/03-apis/api-v2/editing-values.md index dbbf9ffbf3..ec587b9fea 100644 --- a/docs/src/paradox/03-apis/api-v2/editing-values.md +++ b/docs/src/paradox/03-apis/api-v2/editing-values.md @@ -218,11 +218,11 @@ response that looks something like this: "uploadedFiles": [{ "originalFilename": "manuscript-1234-page-1.tiff", "internalFilename": "3UIsXH9bP0j-BV0D4sN51Xz.jp2", - "temporaryBaseIIIFUrl": "http://sipihost/tmp/3UIsXH9bP0j-BV0D4sN51Xz.jp2" + "temporaryBaseIIIFUrl": "http://sipihost/tmp" }, { "originalFilename": "manuscript-1234-page-2.tiff", "internalFilename": "2RvJgguglpe-B45EOk0Gx8H.jp2", - "temporaryBaseIIIFUrl": "http://sipihost/tmp/2RvJgguglpe-B45EOk0Gx8H.jp2" + "temporaryBaseIIIFUrl": "http://sipihost/tmp" }] } ``` diff --git a/docs/src/paradox/05-internals/design/api-v2/sipi.md b/docs/src/paradox/05-internals/design/api-v2/sipi.md index c8bc5ca8c4..d0ec9ac1db 100644 --- a/docs/src/paradox/05-internals/design/api-v2/sipi.md +++ b/docs/src/paradox/05-internals/design/api-v2/sipi.md @@ -43,7 +43,7 @@ are described below. The `upload.lua` script is available at Sipi's `upload` route. It processes one or more file uploads submitted to Sipi. It converts uploaded images to JPEG 2000 format, and stores them in Sipi's `tmp` directory. The usage of this script is described in -@ref:[Creating File Values](../../../03-apis/api-v2/editing-values.md#creating-file-values). +@ref:[Upload Files to Sipi](../../../03-apis/api-v2/editing-values.md#upload-files-to-sipi). Each time `upload.lua` processes a request, it also deletes old temporary files from `tmp` and (recursively) from any subdirectories. The maximum allowed age of diff --git a/docs/src/paradox/07-sipi/setup-sipi-for-knora.md b/docs/src/paradox/07-sipi/setup-sipi-for-knora.md index 05171dbe37..546b14258b 100644 --- a/docs/src/paradox/07-sipi/setup-sipi-for-knora.md +++ b/docs/src/paradox/07-sipi/setup-sipi-for-knora.md @@ -45,11 +45,10 @@ Whenever a file is requested from Sipi (e.g. a browser trying to dereference an image link served by Knora), a preflight function is called. This function is defined in `sipi.init-knora.lua` present in the Sipi root directory. It takes three parameters: `prefix`, `identifier` -(the name of the requested file), and `cookie`. File links created by -Knora use the prefix `knora`, e.g. -`http://localhost:1024/knora/incunabula_0000000002.jp2/full/2613,3505/0/default.jpg`. +(the name of the requested file), and `cookie`. The prefix is the shortcode +of the project that the resource containing the file value belongs to. -Given these information, Sipi asks Knora about the current's users +Given this information, Sipi asks Knora about the current's users permissions on the given file. The cookie contains the current user's Knora session id, so Knora can match Sipi's request with a given user profile and determine the permissions this user has on the file. If the @@ -60,8 +59,8 @@ refuses to serve the file. However, all of this behaviour is defined in the preflight function in Sipi and not controlled by Knora. Knora only provides the permission code. -See @ref:[Sharing the Session ID with Sipi](sipi-and-knora.md#sharing-the-session-id-with-sipi) for more -information about sharing the session id. +See @ref:[Authentication of Users with Sipi](sipi-and-knora.md#authentication-of-users-with-sipi) for more +information about sharing the session ID. ## Using Sipi in Test Mode @@ -76,24 +75,11 @@ $ docker run --rm -it --add-host webapihost:$DOCKERHOST -v $PWD/config:/sipi/con ``` Then always the same test file will be served which is included in Sipi. In test mode, Sipi will -not aks Knora about the user's permission on the requested file. - -## Using Sipi in production behind a proxy - -For SIPI to work with Salsah1 (non-angular) GUI, we need to define an additional set of -environment variables if we want to run SIPI behind a proxy: - -- `SIPI_EXTERNAL_PROTOCOL=https` -- `SIPI_EXTERNAL_HOSTNAME=iiif.example.org` -- `SIPI_EXTERNAL_PORT=443` - -These variables are only used by `make_thumbnail.lua`: - -@@snip[make_thumbnail.lua](../../../../sipi/scripts/make_thumbnail.lua) { #snip_marker } +not ask Knora about the user's permission on the requested file. ## Additional Sipi Environment Variables -Additionaly, these environment variables can be used to further configure sipi: +Additionally, these environment variables can be used to further configure sipi: - `SIPI_WEBAPI_HOSTNAME=localhost`: overrides `knora_path` in Sipi's config - `SIPI_WEBAPI_PORT=3333`: overrides `knora_port` in Sipi's config diff --git a/docs/src/paradox/07-sipi/sipi-and-knora.md b/docs/src/paradox/07-sipi/sipi-and-knora.md index 376b23776d..c1eb6848e6 100644 --- a/docs/src/paradox/07-sipi/sipi-and-knora.md +++ b/docs/src/paradox/07-sipi/sipi-and-knora.md @@ -19,8 +19,6 @@ License along with Knora. If not, see . # Interaction Between Sipi and Knora -TODO: reorganise this to make clear that it describes Knora API v1. - ## General Remarks Knora and Sipi (Simple Image Presentation Interface) are two @@ -39,126 +37,29 @@ structure, format conversions, and serving) is taken care of by Sipi. ## Adding Files to Knora -The file is directly sent to Sipi (route: -`create_thumbnail`) to calculate a thumbnail hosted by Sipi which then -gets displayed to the user in the browser. Sipi copies the original file -into a temporary directory and keeps it there (for later processing in -another request). In its answer (JSON), Sipi returns: - -- `preview_path`: the path to the thumbnail (accessible to a - web-browser) -- `filename`: the name of the temporarily stored original file - (managed by Sipi) -- `original_mimetype`: mime type of the original file -- `original_filename`: the original name of the file submitted by - the client - -Once the user finally wants to attach the file to a resource, the -request is sent to Knora's API providing all the required parameters to -create the resource along with additional information about the file to -be attached. **However, the file itself is not submitted to the Knora -Api, but its filename returned by Sipi (from the `create_thumbnail` -response).** - -#### Create a new Resource with a Digital Representation - -The POST request is handled in `ResourcesRouteV1.scala` and parsed to a -`CreateResourceApiRequestV1`. Information about the file is sent -separately from the other resource parameters (properties) under the -name `file`: - -- `originalFilename`: original name of the file (returned by Sipi - when creating the thumbnail) -- `originalMimeType`: original mime type of the file (returned by - Sipi when creating the thumbnail) -- `filename`: name of the temporarily stored original file (returned - by Sipi when creating the thumbnail) - -In the route, a `SipiResponderConversionFileRequestV1` is created -representing the information about the file to be attached to the new -resource. Along with the other parameters, it is sent to the resources -responder. - -Once a `SipiResponderConversionFileRequestV1` has been created -and passed to the resources responder, the GUI and the non-GUI case can -be handled in a very similar way. This is why they are both -implementations of the trait `SipiResponderConversionRequestV1`. - -The resource responder calls the ontology responder to check if all -required properties were submitted for the given resource type. Also it -is checked if the given resource type may have a digital representation. -The resources responder then sends a message to Sipi connector, which makes -a request to the Sipi server. - -Sipi's response contains the following information: - - - `file_type`: the type of the file that has been handled by Sipi - (image | video | audio | text | binary) - - `mimetype_full`: mime type of the image - - `original_mimetype`: the mime type of the original file - - `original_filename`: the name of the original file - - `nx_full`, `ny_full`: the x and y dimensions of the image - - `filename_full`: the internal filename of the image (needed to request the image from Sipi) - -The `file_type` is important because representations for resources are -restricted to media types: image, audio, video or a generic binary file. -If a resource type requires an image representations (subclass of -`StillImageRepresentation`), the `file_type` has to be an image. -Otherwise, the ontology's restrictions would be violated. Because of -this requirement, there is a construct `fileType2FileValueProperty` -mapping file types to file value properties. Also all the possible file -types are defined in enumeration. - -Depending on the given file type, Sipi responder can create the apt -message (here: `StillImageFileValueV1`) to save the data to the -triplestore. - -#### Change the Digital Representation of a Resource - -The request is taken care of in `ValuesRouteV1.scala`. The PUT request -is handled in path `v1/filevalue/{resIri}` which receives the resource -Iri as a part of the URL: *The submitted file will update the existing -file value of the given resource.* - -The file parameters are submitted as json and are parsed into a -`ChangeFileValueApiRequestV1`. To represent the conversion request for -the Sipi responder, a `SipiResponderConversionFileRequestV1` is created. -A `ChangeFileValueRequestV1` containing the resource Iri and the message -for Sipi is then created and sent to the values responder. - -In the values responder, `ChangeFileValueRequestV1` is passed to the -method `changeFileValueV1`. Unlike ordinary value change requests, the -Iris of the value objects to be updated are not known yet. Because of -this, all the existing file values of the given resource Iri have to be -queried first. - -With the file values being returned, we actually know about the current -Iris of the value objects. Now the Sipi responder is called to handle -the file conversion request (see @ref:[Further Processing in the Resources Responder](#further-processing-in-the-resources-responder)). -After that, it is checked that the `file_type` returned by Sipi responder -corresponds to the property type of the existing file values. For -example, if the `file_type` is an image, the property pointing to the -current file values must be a `hasStillImageFileValue`. Otherwise, the -user submitted a non image file that has to be rejected. - -Depending on the `file_type`, messages of type `ChangeValueRequestV1` -can be created. For each existing file value, such a message is -instantiated containing the current value Iri and the new value to be -created (returned by the sipi responder). These messages are passed to -`changeValueV1` because with the described handling done in -`changeFileValueV1`, the file values can be changed like any other value -type. - -In case of success, a `ChangeFileValueResponseV1` is sent back to the -client, containing a list of the single `ChangeValueResponseV1`. +A file is first uploaded to Sipi, then its metadata is submitted to +Knora. The implementation of this procedure is described in +@ref:[Knora and Sipi](../05-internals/design/api-v2/sipi.md). Instructions +for the client are given in +@ref:[Creating File Values](../03-apis/api-v2/editing-values.md#creating-file-values) +(for Knora API v2) and in +@ref:[Adding Resources with Image Files](../03-apis/api-v1/adding-resources.md#adding-resources-with-image-files) +(for API v1). ## Retrieving Files from Sipi -### URL creation +### File URLs in API v2 + +In Knora API v2, image file URLs are provided in [IIIF](https://iiif.io/) format. In the simple +@ref:[ontology schema](../03-apis/api-v2/introduction.md#api-schema), a file value is simply +a IIIF URL that can be used to retrieve the file from Sipi. In the complex schema, +it is a `StillImageFileValue` with additional properties that the client can use to construct +different IIIF URLs, e.g. at different resolutions. See the `knora-api` ontology for details. -Binary representions of Knora locations are served by Sipi. For each -file value, Knora creates several locations representing different -quality levels: +### File URLs in API v1 + +In API v1, for each file value, Knora creates several Sipi URLs for accessing the file at different +resolutions: ``` "resinfo": { @@ -200,30 +101,15 @@ quality levels: ``` Each of these paths has to be handled by the browser by making a call to -Sipi, obtaining the binary representation in the desired quality. To -deal with different image quality levels, Sipi implements the [IIIF -standard](http://iiif.io/api/image/2.0/). The different quality level -paths are created by Knora in `ValueUtilV1`. - -Whenever Sipi serves a binary representation of a Knora file value -(indicated by using the prefix `knora` in the path), it has to make a -request to Knora's Sipi responder to get the user's permissions on the -requested file. Sipi's request to Knora contains a cookie with the Knora -session id the user has obtained when logging in to Knora: As a response -to a successful login, Knora returns the user's session id and this id -is automatically sent to Sipi by the browser, setting a second cookie -for the communication with Sipi. The reason the Knora session id is set -in two cookies, is the fact that cookies can not be shared among -different domains. Since Knora and Sipi are likely to be running under -different domains, this solution offers the necessary flexibility. - -## Authentication of users with Sipi +Sipi, obtaining the binary representation in the desired quality. + +## Authentication of Users with Sipi Whenever a file is requested, Sipi asks Knora about the current user's permissions on the given file. This is achieved by sharing the Knora session cookie with Sipi. When the user logs in to Knora using his browser (using either `V1` or `V2` authentication route), a session cookie containing a JWT token representing -the user is stored in the user's client. This session cookie is then read by Sipi and used to query for +the user is stored in the user's client. This session cookie is then read by Sipi and used to ask Knora for the user's image permissions. For the session cookie to be sent to Sipi, both the Knora API and Sipi endpoints need to -be under the same domain, e.g., `api.example.com` and `iiif.example.com`. \ No newline at end of file +be under the same domain, e.g., `api.example.com` and `iiif.example.com`. diff --git a/salsah1/src/public/index.html b/salsah1/src/public/index.html index 9d729a23c8..3332494525 100644 --- a/salsah1/src/public/index.html +++ b/salsah1/src/public/index.html @@ -212,7 +212,9 @@ // When retrieving a file from Sipi (e.g. an IIIF URL), Sipi can send the session id to Knora with the request, // identifying the user that is making the request to Knora. SALSAH.userprofile = data.userProfile; + window.sessionStorage.setItem('userprofile', JSON.stringify(SALSAH.userprofile)); + window.sessionStorage.setItem('token', data.sid); $('#dologin').simpledialog('loginbox', 'close'); @@ -254,6 +256,8 @@ SALSAH.ApiDelete('session', function(data) { if (data.status == ApiErrors.OK) { window.sessionStorage.removeItem('userprofile'); + window.sessionStorage.removeItem('token'); + SALSAH.userprofile = null; $('#dologut').simpledialog('logoutbox', 'close'); diff --git a/salsah1/src/public/js/jquery.location.js b/salsah1/src/public/js/jquery.location.js index 370a798a67..d9a1633bb5 100644 --- a/salsah1/src/public/js/jquery.location.js +++ b/salsah1/src/public/js/jquery.location.js @@ -91,7 +91,7 @@ $.ajax({ type:'POST', - url: SIPI_URL + "/make_thumbnail", + url: SIPI_URL + "/upload?token=" + window.sessionStorage.getItem('token'), data: fd, cache: false, contentType: false, @@ -104,8 +104,11 @@ $this.find('.progressNumber').remove(); $this.find('.uploadButton').remove(); + var uploadedFile = data["uploadedFiles"][0]; + var preview_path = uploadedFile["temporaryBaseIIIFUrl"] + "/" + uploadedFile["internalFilename"] + "/full/!128,128/0/default.jpg"; + $this.append($('
').addClass('thumbNail') - .append($('', {src: data.preview_path, style: "image-orientation: from-image"})) + .append($('', {src: preview_path, style: "image-orientation: from-image"})) .append($('
')) .append(data.original_filename) ); @@ -113,7 +116,7 @@ //$this.append($('').attr({'type': 'hidden'}).addClass('fileData').val(event.target.responseText)); - localdata.sipi_response = data; + localdata.sipi_response = uploadedFile; }, error: function(jqXHR, textStatus, errorThrown) { if (errorThrown !== undefined && jqXHR !== undefined && jqXHR.responseJSON !== undefined) { diff --git a/salsah1/src/public/js/jquery.propedit.js b/salsah1/src/public/js/jquery.propedit.js index 1d708ed4f9..f7668282ef 100644 --- a/salsah1/src/public/js/jquery.propedit.js +++ b/salsah1/src/public/js/jquery.propedit.js @@ -1358,11 +1358,7 @@ } else { data = { - file: { - originalFilename: sipi_response["original_filename"], - originalMimeType: sipi_response["original_mimetype"], - filename: sipi_response["filename"] - } + file: sipi_response["internalFilename"] }; SALSAH.ApiPut('filevalue/' + encodeURIComponent(res_id), data, function(data) { if (data.status == ApiErrors.OK) { diff --git a/salsah1/src/public/js/jquery.resadd.js b/salsah1/src/public/js/jquery.resadd.js index 562dc2115e..4a41168fd9 100644 --- a/salsah1/src/public/js/jquery.resadd.js +++ b/salsah1/src/public/js/jquery.resadd.js @@ -1031,11 +1031,7 @@ ele = form.find('[name="' + propname + '"]'); var sipi_response = ele.location('value'); - file = { - originalFilename: sipi_response["original_filename"], - originalMimeType: sipi_response["original_mimetype"], - filename: sipi_response["filename"] - }; + file = sipi_response["internalFilename"]; break; diff --git a/salsah1/src/typescript_interfaces/basicMessageComponents.ts b/salsah1/src/typescript_interfaces/basicMessageComponents.ts index 4a48642d4f..61b77d4072 100644 --- a/salsah1/src/typescript_interfaces/basicMessageComponents.ts +++ b/salsah1/src/typescript_interfaces/basicMessageComponents.ts @@ -384,31 +384,14 @@ export module basicMessageComponents { } /** - * Describes a file value (for GUI-case) + * Describes a file value. */ export interface createOrChangeFileValueRequest { /** - * Describes a file value (for GUI-case) + * The internal filename returned by Sipi. */ - file: { - - /** - * The file's original name - */ - originalFilename: string; - - /** - * The original mime type of the file - */ - originalMimeType: string; - - /** - * The file's temporary name - */ - filename: string; - - } + file: string; } /** diff --git a/salsah1/src/typescript_interfaces/sampleRequests/sampleChangeValues.ts b/salsah1/src/typescript_interfaces/sampleRequests/sampleChangeValues.ts index 554af6e450..f77ebf465c 100644 --- a/salsah1/src/typescript_interfaces/sampleRequests/sampleChangeValues.ts +++ b/salsah1/src/typescript_interfaces/sampleRequests/sampleChangeValues.ts @@ -24,11 +24,7 @@ let changeIntervalValue: changeValueFormats.changeIntervalValueRequest = {"inter let changeIntervalValueResponse: changeValueFormats.changeValueResponse = {"id":"http://rdfh.ch/a-thing/values/G58MBZ5ES7yxmKX2l5QTPg","status":0,"comment":null,"rights":8,"value":{"timeval1":0,"timeval2":36000}}; let changeFileValueRequest: changeValueFormats.changeFileValueRequest = { - 'file': { - 'originalFilename' : "myfile.jpg", - 'originalMimeType' : "image/jpeg", - 'filename' : "tmpname.jpg" - } + "file": "3UIsXH9bP0j-BV0D4sN51Xz.jp2" }; let changeFileValueResponse: changeValueFormats.changeFileValueResponse = {"locations":[{"duration":0,"nx":128,"path":"http://localhost:1024/knora/5XTEI1z10A2-D8ojQHrMiUz.jpg/full/full/0/default.jpg","ny":72,"fps":0,"format_name":"JPEG","origname":"2016-06-26+12.26.45.jpg","protocol":"file"},{"duration":0,"nx":3264,"path":"http://localhost:1024/knora/5XTEI1z10A2-D8ojQHrMiUz.jpx/full/3264,1836/0/default.jpg","ny":1836,"fps":0,"format_name":"JPEG2000","origname":"2016-06-26+12.26.45.jpg","protocol":"file"}],"status":0}; diff --git a/salsah1/src/typescript_interfaces/sampleRequests/sampleCreateResources.ts b/salsah1/src/typescript_interfaces/sampleRequests/sampleCreateResources.ts index 48a079bc82..3bb9f8c809 100644 --- a/salsah1/src/typescript_interfaces/sampleRequests/sampleCreateResources.ts +++ b/salsah1/src/typescript_interfaces/sampleRequests/sampleCreateResources.ts @@ -50,11 +50,7 @@ let thingWithFile: createResourceFormats.createResourceWithRepresentationRequest "http://www.knora.org/ontology/0001/anything#hasListItem": [{"hlist_value":"http://rdfh.ch/anything/treeList10"}], "http://www.knora.org/ontology/0001/anything#hasInterval": [{"interval_value": [1000000000000000.0000000000000001, 1000000000000000.0000000000000002]}] }, - "file": { - 'originalFilename' : "myfile.jpg", - 'originalMimeType' : "image/jpeg", - 'filename' : "tmp.jpg" - } + "file": "3UIsXH9bP0j-BV0D4sN51Xz.jp2" }; let createResourceResponse: createResourceFormats.createResourceResponse = {"res_id":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw","results":{"http://www.knora.org/ontology/0001/anything#hasDecimal":[{"value":{"dateval1":null,"ival":null,"dateprecision1":null,"textval":{"string":"3.3"},"person_id":{"string":"http://rdfh.ch/users/91e19f1e01"},"property_id":{"string":"http://www.knora.org/ontology/0001/anything#hasDecimal"},"calendar":null,"timeval2":null,"dval":{"decimal":3.3},"dateval2":null,"order":{"integer":1},"resource_id":{"string":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw"},"timeval1":null,"dateprecision2":null},"id":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw/values/1WQOgjJWS86laX0BYKoyGw"}],"http://www.knora.org/ontology/0001/anything#hasColor":[{"value":{"dateval1":null,"ival":null,"dateprecision1":null,"textval":{"string":"#ff3333"},"person_id":{"string":"http://rdfh.ch/users/91e19f1e01"},"property_id":{"string":"http://www.knora.org/ontology/0001/anything#hasColor"},"calendar":null,"timeval2":null,"dval":null,"dateval2":null,"order":{"integer":1},"resource_id":{"string":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw"},"timeval1":null,"dateprecision2":null},"id":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw/values/x5EBVPVHTReZsz-UCnZR0g"}],"http://www.knora.org/ontology/0001/anything#hasInteger":[{"value":{"dateval1":null,"ival":{"integer":1},"dateprecision1":null,"textval":{"string":"1"},"person_id":{"string":"http://rdfh.ch/users/91e19f1e01"},"property_id":{"string":"http://www.knora.org/ontology/0001/anything#hasInteger"},"calendar":null,"timeval2":null,"dval":null,"dateval2":null,"order":{"integer":1},"resource_id":{"string":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw"},"timeval1":null,"dateprecision2":null},"id":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw/values/6mbnuiGeSIWaB_sXw3-cJA"}],"http://www.knora.org/ontology/0001/anything#hasInterval":[{"value":{"dateval1":null,"ival":null,"dateprecision1":null,"textval":{"string":"IntervalValueV1(0,0)"},"person_id":{"string":"http://rdfh.ch/users/91e19f1e01"},"property_id":{"string":"http://www.knora.org/ontology/0001/anything#hasInterval"},"calendar":null,"timeval2":null,"dval":null,"dateval2":null,"order":{"integer":1},"resource_id":{"string":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw"},"timeval1":null,"dateprecision2":null},"id":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw/values/VRlXZ-taQoKonY4y6_Y9wQ"}],"http://www.knora.org/ontology/0001/anything#hasDate":[{"value":{"dateval1":{"string":"2016-07-14"},"ival":null,"dateprecision1":{"string":"DAY"},"textval":{"string":"2016-07-14"},"person_id":{"string":"http://rdfh.ch/users/91e19f1e01"},"property_id":{"string":"http://www.knora.org/ontology/0001/anything#hasDate"},"calendar":{"string":"GREGORIAN"},"timeval2":null,"dval":null,"dateval2":{"string":"2016-07-14 CE"},"order":{"integer":1},"resource_id":{"string":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw"},"timeval1":null,"dateprecision2":{"string":"DAY"}},"id":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw/values/M15JNh0rRjGvYrL7G257EQ"}],"http://www.knora.org/ontology/0001/anything#hasListItem":[{"value":{"dateval1":null,"ival":null,"dateprecision1":null,"textval":{"string":"http://rdfh.ch/anything/treeList01"},"person_id":{"string":"http://rdfh.ch/users/91e19f1e01"},"property_id":{"string":"http://www.knora.org/ontology/0001/anything#hasListItem"},"calendar":null,"timeval2":null,"dval":null,"dateval2":null,"order":{"integer":1},"resource_id":{"string":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw"},"timeval1":null,"dateprecision2":null},"id":"http://rdfh.ch/Bc7yXd3ETJ6SjNuq86eBdw/values/FuZ9sVvzQgywFhkf5SuE8Q"}]},"status":0} diff --git a/sipi/scripts/jwt.lua b/sipi/scripts/jwt.lua index bf172d437d..cc80a3c1a5 100644 --- a/sipi/scripts/jwt.lua +++ b/sipi/scripts/jwt.lua @@ -25,7 +25,6 @@ function get_knora_token() local token = get_token() if token == nil then - send_error(401, "Not a Knora token") return nil end @@ -57,15 +56,15 @@ function get_token() local expiration_date = token["exp"] if expiration_date == nil then - send_error(401, "Token has no expiry date") - return nil + send_error(401, "Token has no expiry date") + return nil end local systime = server.systime() if (expiration_date <= systime) then - send_error(401, "Expired token") - return nil + send_error(401, "Expired token") + return nil end local audience = token["aud"] @@ -76,4 +75,4 @@ function get_token() end return token -end +end \ No newline at end of file diff --git a/sipi/scripts/upload.lua b/sipi/scripts/upload.lua index 8bc6f54b07..dcd44ddf83 100644 --- a/sipi/scripts/upload.lua +++ b/sipi/scripts/upload.lua @@ -33,24 +33,12 @@ if not success then return end --- Check for a valid JSON Web Token or authentication cookie from Knora. +-- Check for a valid JSON Web Token from Knora. local token = get_knora_token() if token == nil then - local cookie_OK = false - - for cookie_index, cookie in pairs(server.cookies) do - if get_session_id(cookie) ~= nil then - cookie_OK = true - server.log("Knora cookie OK", server.loglevel.LOG_DEBUG) - break - end - end - - if not cookie_OK then - return - end + return end -- Create the temporary directory if necessary. @@ -130,7 +118,7 @@ for image_index, image_params in pairs(server.uploads) do end -- Create a IIIF base URL for the converted file. - local iiif_base_url = protocol .. server.host .. '/tmp/' .. jp2_filename + local iiif_base_url = protocol .. server.host .. '/tmp' -- Construct response data about the file that was uploaded. local this_file_upload_data = {} diff --git a/webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala b/webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala index e1bb0a9a4d..4c51dcde1c 100644 --- a/webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/ITKnoraLiveSpec.scala @@ -176,7 +176,7 @@ class ITKnoraLiveSpec(_system: ActorSystem) extends Core with KnoraService with * * @param originalFilename the original filename that was submitted to Sipi. * @param internalFilename Sipi's internal filename for the stored temporary file. - * @param temporaryBaseIIIFUrl the base URL at which the temporary file can be accessed. + * @param temporaryBaseIIIFUrl the base URL for constructing a IIIF URL for accessing the temporary file. */ protected case class SipiUploadResponseEntry(originalFilename: String, internalFilename: String, temporaryBaseIIIFUrl: String) @@ -236,7 +236,7 @@ class ITKnoraLiveSpec(_system: ActorSystem) extends Core with KnoraService with // Request the temporary image from Sipi. for (responseEntry <- sipiUploadResponse.uploadedFiles) { - val sipiGetTmpFileRequest = Get(responseEntry.temporaryBaseIIIFUrl + "/full/full/0/default.jpg") + val sipiGetTmpFileRequest = Get(responseEntry.temporaryBaseIIIFUrl + "/" + responseEntry.internalFilename + "/full/full/0/default.jpg") checkResponseOK(sipiGetTmpFileRequest) } diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala index 91f9b85116..bebc8cd6ae 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v2/KnoraSipiIntegrationV2ITSpec.scala @@ -272,7 +272,7 @@ class KnoraSipiIntegrationV2ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV assert(knoraPostResponse.status == StatusCodes.Forbidden) // Request the temporary image from Sipi. - val sipiGetTmpFileRequest = Get(temporaryBaseIIIFUrl) + val sipiGetTmpFileRequest = Get(temporaryBaseIIIFUrl + "/" + internalFilename + "/full/full/0/default.jpg") val sipiResponse = singleAwaitingRequest(sipiGetTmpFileRequest) assert(sipiResponse.status == StatusCodes.NotFound) } diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala b/webapi/src/main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala index 4abb7e1bb4..0233665eb9 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala @@ -73,7 +73,7 @@ class ValueUtilV1(private val settings: SettingsImpl) { } def makeSipiImagePreviewGetUrlFromFilename(projectShortcode: String, filename: String): String = { - s"${settings.externalSipiIIIFGetUrl}/$projectShortcode/$filename/full/full/0/default.jpg" + s"${settings.externalSipiIIIFGetUrl}/$projectShortcode/$filename/full/!128,128/0/default.jpg" } /** From c4beaadd444cde13a50d104cf31d55e11545c896 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 28 Mar 2019 15:20:58 +0100 Subject: [PATCH 09/24] fix(sipi): Fix compile error. --- .../knora/webapi/messages/store/sipimessages/SipiMessages.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala b/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala index 175ee65770..82287d9a92 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala @@ -20,6 +20,7 @@ package org.knora.webapi.messages.store.sipimessages import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport +import org.knora.webapi.SipiException import org.knora.webapi.messages.admin.responder.usersmessages.UserADM import spray.json._ From 894e7e1315e44e5b5b0f46c393e8476768331387 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Mon, 24 Jun 2019 17:07:24 +0200 Subject: [PATCH 10/24] test(sipi): Fix tests. --- .../org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala | 2 -- .../scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala | 2 -- 2 files changed, 4 deletions(-) diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala index fe7804978a..6a4de1eeaa 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala @@ -173,8 +173,6 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV loginToken = lr.token loginToken.nonEmpty should be(true) - - log.debug("token: {}", loginToken) } "create an 'incunabula:page' with parameters" in { diff --git a/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala index bd3210ab04..3c0bed8388 100644 --- a/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/other/v1/DrawingsGodsV1ITSpec.scala @@ -80,8 +80,6 @@ class DrawingsGodsV1ITSpec extends ITKnoraLiveSpec(DrawingsGodsV1ITSpec.config) loginToken = lr.token loginToken.nonEmpty should be(true) - - log.debug("token: {}", loginToken) } "be able to create a resource, only find one DOAP (with combined resource class / property), and have permission to access the image" in { From 6f33295423f1da8007949dda6562cb4e4926e7f0 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 28 Nov 2019 13:38:21 +0100 Subject: [PATCH 11/24] fix(api-v1): Fix compile errors. --- .../05-internals/design/api-v2/sipi.md | 2 +- .../e2e/v1/KnoraSipiIntegrationV1ITSpec.scala | 26 +------------------ .../store/sipimessages/SipiMessages.scala | 2 +- .../valuemessages/ValueMessagesV1.scala | 4 +-- .../webapi/routing/v1/ResourcesRouteV1.scala | 26 +++++++++---------- .../webapi/routing/v1/ValuesRouteV1.scala | 14 +++++----- 6 files changed, 25 insertions(+), 49 deletions(-) diff --git a/docs/src/paradox/05-internals/design/api-v2/sipi.md b/docs/src/paradox/05-internals/design/api-v2/sipi.md index d0ec9ac1db..896e83c741 100644 --- a/docs/src/paradox/05-internals/design/api-v2/sipi.md +++ b/docs/src/paradox/05-internals/design/api-v2/sipi.md @@ -96,7 +96,7 @@ If it encounters an error, it returns `SipiException`. to create or change a file value. The request includes Sipi's internal filename. 3. During parsing of this JSON-LD request, a `StillImageFileValueContentV2` is constructed to represent the file value. During the construction of this - object, a `GetImageMetadataRequestV2` is sent to `SipiConnector`, which + object, a `GetFileMetadataRequestV2` is sent to `SipiConnector`, which uses Sipi's built-in `knora.json` route to get the rest of the file's metadata. 4. A responder (`ResourcesResponderV2` or `ValuesResponderV2`) validates diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala index 114bfd3afd..6ca330a359 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala @@ -132,7 +132,7 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV case _ => false } - case _ => throw InvalidApiJsonException("bulk import response should have memeber 'createdResources'") + case _ => throw InvalidApiJsonException("bulk import response should have member 'createdResources'") } if (resIriOption.nonEmpty) { @@ -176,30 +176,6 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV loginToken.nonEmpty should be(true) } - "reject an 'incunabula:page' with binary data if the file extension is incorrect" ignore { // Ignored because of issue #1531. - // The image to be uploaded. - val fileToSend = new File(pathToMarblesWithWrongExtension) - assert(fileToSend.exists(), s"File $pathToMarblesWithWrongExtension does not exist") - - // A multipart/form-data request containing the image. - val formData = Multipart.FormData( - Multipart.FormData.BodyPart( - "file", - HttpEntity.fromPath(MediaTypes.`image/tiff`, fileToSend.toPath), - Map("filename" -> fileToSend.getName) - ) - ) - - // Send the image in a PUT request to the Knora API server. - val knoraPutRequest = Put(baseApiUrl + "/v1/filevalue/" + URLEncoder.encode(firstPageIri.get, "UTF-8"), formData) ~> addCredentials(BasicHttpCredentials(username, password)) - - val exception = intercept[AssertionException] { - checkResponseOK(knoraPutRequest) - } - - assert(exception.getMessage.contains("MIME type and/or file extension are inconsistent")) - } - "create an 'incunabula:page' with parameters" in { // Upload the image to Sipi. val sipiUploadResponse: SipiUploadResponse = uploadToSipi( diff --git a/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala b/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala index d492abed5a..0846c9ecf2 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/store/sipimessages/SipiMessages.scala @@ -71,7 +71,7 @@ case class GetFileMetadataResponseV2(originalFilename: Option[String], } object GetFileMetadataResponseV2JsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { - implicit val getImageMetadataResponseV2Format: RootJsonFormat[GetFileMetadataResponseV2] = jsonFormat6(GetFileMetadataResponseV2) + implicit val GetFileMetadataResponseV2Format: RootJsonFormat[GetFileMetadataResponseV2] = jsonFormat6(GetFileMetadataResponseV2) } /** diff --git a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala index 9d9abc5a93..a0e4cb6d8f 100644 --- a/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/messages/v1/responder/valuemessages/ValueMessagesV1.scala @@ -1449,8 +1449,8 @@ case class StillImageFileValueV1(internalMimeType: String, fileValue = FileValueV2( internalFilename = internalFilename, internalMimeType = internalMimeType, - originalFilename = originalFilename, - originalMimeType = internalMimeType + originalFilename = Some(originalFilename), + originalMimeType = Some(internalMimeType) ), dimX = dimX, dimY = dimY diff --git a/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala b/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala index fa7bd44954..0298e10b2c 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/v1/ResourcesRouteV1.scala @@ -38,7 +38,7 @@ import javax.xml.validation.{Schema, SchemaFactory, Validator} import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.projectsmessages.{ProjectGetRequestADM, ProjectGetResponseADM, ProjectIdentifierADM} import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataRequest, GetImageMetadataResponseV2} +import org.knora.webapi.messages.store.sipimessages.{GetFileMetadataRequest, GetFileMetadataResponseV2} import org.knora.webapi.messages.v1.responder.ontologymessages._ import org.knora.webapi.messages.v1.responder.resourcemessages.ResourceV1JsonProtocol._ import org.knora.webapi.messages.v1.responder.resourcemessages._ @@ -227,17 +227,17 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) val tempFileUrl = stringFormatter.makeSipiTempFileUrl(settings, filename) for { - imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequest(fileUrl = tempFileUrl, requestingUser = userADM)).mapTo[GetImageMetadataResponseV2] + fileMetadataResponse: GetFileMetadataResponseV2 <- (storeManager ? GetFileMetadataRequest(fileUrl = tempFileUrl, requestingUser = userADM)).mapTo[GetFileMetadataResponseV2] // TODO: check that the file stored is an image. } yield Some(StillImageFileValueV1( internalFilename = filename, - internalMimeType = "image/jp2", - originalFilename = imageMetadataResponse.originalFilename, - originalMimeType = Some(imageMetadataResponse.originalMimeType), + internalMimeType = fileMetadataResponse.internalMimeType, + originalFilename = fileMetadataResponse.originalFilename.getOrElse(throw SipiException(s"Sipi did not return the original filename of the image")), + originalMimeType = fileMetadataResponse.originalMimeType, projectShortcode = projectShortcode, - dimX = imageMetadataResponse.width, - dimY = imageMetadataResponse.height + dimX = fileMetadataResponse.width.getOrElse(throw SipiException(s"Sipi did not return the width of the image")), + dimY = fileMetadataResponse.height.getOrElse(throw SipiException(s"Sipi did not return the height of the image")) )) case None => FastFuture.successful(None) @@ -290,17 +290,17 @@ class ResourcesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) val tempFileUrl = stringFormatter.makeSipiTempFileUrl(settings, filename) for { - imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequest(fileUrl = tempFileUrl, requestingUser = userProfile)).mapTo[GetImageMetadataResponseV2] + fileMetadataResponse: GetFileMetadataResponseV2 <- (storeManager ? GetFileMetadataRequest(fileUrl = tempFileUrl, requestingUser = userProfile)).mapTo[GetFileMetadataResponseV2] // TODO: check that the file stored is an image. } yield Some(StillImageFileValueV1( internalFilename = filename, - internalMimeType = "image/jp2", - originalFilename = imageMetadataResponse.originalFilename, - originalMimeType = Some(imageMetadataResponse.originalMimeType), + internalMimeType = fileMetadataResponse.internalMimeType, + originalFilename = fileMetadataResponse.originalFilename.getOrElse(throw SipiException(s"Sipi did not return the original filename of the image")), + originalMimeType = fileMetadataResponse.originalMimeType, projectShortcode = projectShortcode, - dimX = imageMetadataResponse.width, - dimY = imageMetadataResponse.height + dimX = fileMetadataResponse.width.getOrElse(throw SipiException(s"Sipi did not return the width of the image")), + dimY = fileMetadataResponse.height.getOrElse(throw SipiException(s"Sipi did not return the height of the image")) )) case None => FastFuture.successful(None) diff --git a/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala b/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala index 81ed919323..5f5ed0427c 100644 --- a/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/routing/v1/ValuesRouteV1.scala @@ -28,7 +28,7 @@ import akka.pattern._ import akka.stream.ActorMaterializer import org.knora.webapi._ import org.knora.webapi.messages.admin.responder.usersmessages.UserADM -import org.knora.webapi.messages.store.sipimessages.{GetImageMetadataRequest, GetImageMetadataResponseV2} +import org.knora.webapi.messages.store.sipimessages.{GetFileMetadataRequest, GetFileMetadataResponseV2} import org.knora.webapi.messages.v1.responder.resourcemessages.{ResourceInfoGetRequestV1, ResourceInfoResponseV1} import org.knora.webapi.messages.v1.responder.valuemessages.ApiValueV1JsonProtocol._ import org.knora.webapi.messages.v1.responder.valuemessages._ @@ -312,19 +312,19 @@ class ValuesRouteV1(routeData: KnoraRouteData) extends KnoraRoute(routeData) wit val tempFileUrl = stringFormatter.makeSipiTempFileUrl(settings, apiRequest.file) for { - imageMetadataResponse: GetImageMetadataResponseV2 <- (storeManager ? GetImageMetadataRequest(fileUrl = tempFileUrl, requestingUser = userADM)).mapTo[GetImageMetadataResponseV2] + fileMetadataResponse: GetFileMetadataResponseV2 <- (storeManager ? GetFileMetadataRequest(fileUrl = tempFileUrl, requestingUser = userADM)).mapTo[GetFileMetadataResponseV2] // TODO: check that the file stored is an image. } yield ChangeFileValueRequestV1( resourceIri = resourceIri, file = StillImageFileValueV1( internalFilename = apiRequest.file, - internalMimeType = "image/jp2", - originalFilename = imageMetadataResponse.originalFilename, - originalMimeType = Some(imageMetadataResponse.originalMimeType), + internalMimeType = fileMetadataResponse.internalMimeType, + originalFilename = fileMetadataResponse.originalFilename.getOrElse(throw SipiException(s"Sipi did not return the original filename of the image")), + originalMimeType = fileMetadataResponse.originalMimeType, projectShortcode = projectShortcode, - dimX = imageMetadataResponse.width, - dimY = imageMetadataResponse.height + dimX = fileMetadataResponse.width.getOrElse(throw SipiException(s"Sipi did not return the width of the image")), + dimY = fileMetadataResponse.height.getOrElse(throw SipiException(s"Sipi did not return the height of the image")) ), apiRequestID = UUID.randomUUID, userProfile = userADM From 1c99630d85188e3e2f8c4b3c2ba3eecdc19055aa Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 28 Nov 2019 14:00:59 +0100 Subject: [PATCH 12/24] fix(settings): Remove obsolete settings. --- webapi/src/main/scala/org/knora/webapi/Settings.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/webapi/src/main/scala/org/knora/webapi/Settings.scala b/webapi/src/main/scala/org/knora/webapi/Settings.scala index cdf94c4174..35972484f5 100644 --- a/webapi/src/main/scala/org/knora/webapi/Settings.scala +++ b/webapi/src/main/scala/org/knora/webapi/Settings.scala @@ -101,15 +101,8 @@ class SettingsImpl(config: Config) extends Extension { val externalSipiHost: String = config.getString("app.sipi.external-host") val externalSipiPort: Int = config.getInt("app.sipi.external-port") val externalSipiBaseUrl: String = externalSipiProtocol + "://" + externalSipiHost + (if (externalSipiPort != 80) ":" + externalSipiPort else "") - - val sipiFileServerPrefix: String = config.getString("app.sipi.file-server-path") - val externalSipiIIIFGetUrl: String = externalSipiBaseUrl - - val internalSipiImageConversionUrlV1: String = s"$internalSipiBaseUrl" - val sipiFileConversionRouteV1: String = config.getString("app.sipi.v1.file-conversion-route") - val sipiFileMetadataRouteV2: String = config.getString("app.sipi.v2.file-metadata-route") val sipiMoveFileRouteV2: String = config.getString("app.sipi.v2.move-file-route") val sipiDeleteTempFileRouteV2: String = config.getString("app.sipi.v2.delete-temp-file-route") From 58134fb683358dbe0c1e8b70cdfbec967000eab8 Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 28 Nov 2019 14:53:21 +0100 Subject: [PATCH 13/24] fix(api-v1): Accept image/jpx MIME type. --- .../main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/webapi/src/main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala b/webapi/src/main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala index 00b350369f..144ca21ba7 100644 --- a/webapi/src/main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala +++ b/webapi/src/main/scala/org/knora/webapi/responders/v1/ValueUtilV1.scala @@ -106,6 +106,7 @@ class ValueUtilV1(private val settings: SettingsImpl) { "application/octet-stream" -> "BINARY-UNKNOWN", "image/jpeg" -> "JPEG", "image/jp2" -> "JPEG2000", + "image/jpx" -> "JPEG2000", "application/pdf" -> "PDF", "application/postscript" -> "POSTSCRIPT", "application/vnd.ms-powerpoint" -> "PPT", From 7260876fe6cc40595e0843a31166cd5641b9003c Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Thu, 28 Nov 2019 15:55:26 +0100 Subject: [PATCH 14/24] fix(settings): Fix typo. --- webapi/src/main/resources/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapi/src/main/resources/application.conf b/webapi/src/main/resources/application.conf index 849180df47..2d5fc5f327 100644 --- a/webapi/src/main/resources/application.conf +++ b/webapi/src/main/resources/application.conf @@ -285,7 +285,7 @@ app { external-port = 1024 external-port = ${?KNORA_WEBAPI_SIPI_EXTERNAL_PORT} - file-server-path = "server"å + file-server-path = "server" v2 { file-metadata-route = "knora.json" From 28ab3008856362b93ab89db5567d0b47e731169b Mon Sep 17 00:00:00 2001 From: Benjamin Geer Date: Tue, 18 Feb 2020 13:25:58 +0100 Subject: [PATCH 15/24] test(api-v1): Disable test. --- .../knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala index 94f6086b2a..f497e6da62 100644 --- a/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala +++ b/webapi/src/it/scala/org/knora/webapi/e2e/v1/KnoraSipiIntegrationV1ITSpec.scala @@ -690,7 +690,9 @@ class KnoraSipiIntegrationV1ITSpec extends ITKnoraLiveSpec(KnoraSipiIntegrationV } - "provide a helpful error message if an XSLT file is not found" in { + "provide a helpful error message if an XSLT file is not found" ignore { + // TODO: re-enables this with the preceding test, when we can upload non-image files to Sipi (PR #1206). + val missingHeaderXSLTIri = "http://rdfh.ch/0801/608NfPLCRpeYnkXKABC5mg" val letterTEIRequest: HttpRequest = Get(baseApiUrl + "/v2/tei/" + URLEncoder.encode(letterIri.get, "UTF-8") + From 946281af9a12e8ce242039b3ce677dde490cef8a Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Fri, 17 Apr 2020 21:54:56 +0200 Subject: [PATCH 16/24] triggering github-ci --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9cb7af25e9..51e666047d 100644 --- a/README.md +++ b/README.md @@ -199,3 +199,4 @@ YourKit supports open source projects with its full-featured Java Profiler. YourKit, LLC is the creator of YourKit Java Profiler and YourKit .NET Profiler, innovative and intelligent tools for profiling Java and .NET applications. + From 8b31606b483e494954db7ddcc943dcb9b6df3cf9 Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Sat, 18 Apr 2020 06:43:34 +0200 Subject: [PATCH 17/24] ci: add debugging output --- .github/workflows/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4e129f903c..9cc88c58ec 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,6 +77,10 @@ jobs: run: | make env-file make print-env-file + - name: Disk Free + run: | + df -i + df -h - name: Run API unit tests run: make test-unit-ci @@ -108,6 +112,10 @@ jobs: run: | make env-file make print-env-file + - name: Disk Free + run: | + df -i + df -h - name: run API E2E tests run: make test-e2e-ci @@ -139,6 +147,10 @@ jobs: run: | make env-file make print-env-file + - name: Disk Free + run: | + df -i + df -h - name: run API integration tests run: make test-it-ci @@ -175,6 +187,10 @@ jobs: run: | make env-file make print-env-file + - name: Disk Free + run: | + df -i + df -h - name: setup required node version uses: actions/setup-node@v1 with: @@ -199,6 +215,10 @@ jobs: ${{ runner.OS }}-build-${{ env.cache-name }}- ${{ runner.OS }}-build- ${{ runner.OS }}- + - name: Disk Free + run: | + df -i + df -h - name: run upgrade tests run: sbt "upgrade/test" @@ -231,6 +251,10 @@ jobs: ${{ runner.OS }}- - name: install requirements run: sudo apt-get install expect + - name: Disk Free + run: | + df -i + df -h - name: build all docker images run: | make build-all-scala From eb9d5b29b3e8eeba89d74ad534b0bd3359f705c2 Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Mon, 20 Apr 2020 08:10:54 +0200 Subject: [PATCH 18/24] build: bump sbt version --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index c0bab04941..797e7ccfdb 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.2.8 +sbt.version=1.3.10 From 7fdc5ffbe50f22e1a0b333af2b20cf0ed19fc0c6 Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Mon, 20 Apr 2020 09:24:24 +0200 Subject: [PATCH 19/24] ci: add debugging output --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9cc88c58ec..7c28a71cd8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,6 +81,7 @@ jobs: run: | df -i df -h + du -sh /var/lib/docker - name: Run API unit tests run: make test-unit-ci @@ -116,6 +117,7 @@ jobs: run: | df -i df -h + du -sh /var/lib/docker - name: run API E2E tests run: make test-e2e-ci @@ -151,6 +153,7 @@ jobs: run: | df -i df -h + du -sh /var/lib/docker - name: run API integration tests run: make test-it-ci @@ -191,6 +194,7 @@ jobs: run: | df -i df -h + du -sh /var/lib/docker - name: setup required node version uses: actions/setup-node@v1 with: @@ -219,6 +223,7 @@ jobs: run: | df -i df -h + du -sh /var/lib/docker - name: run upgrade tests run: sbt "upgrade/test" @@ -255,6 +260,7 @@ jobs: run: | df -i df -h + du -sh /var/lib/docker - name: build all docker images run: | make build-all-scala From 319ee8fb1080be3256ef0be5afbe6c3c5ac61271 Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Mon, 20 Apr 2020 09:31:43 +0200 Subject: [PATCH 20/24] ci: add debugging output --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c28a71cd8..5cd7209f36 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,6 +82,8 @@ jobs: df -i df -h du -sh /var/lib/docker + docker system prune --all --force --volumes + df -h - name: Run API unit tests run: make test-unit-ci From b88a982c0532c16af9b1e7d327f69c11ffc86db6 Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Mon, 20 Apr 2020 09:33:16 +0200 Subject: [PATCH 21/24] ci: add debugging output --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5cd7209f36..b457a9cf1e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,11 +79,11 @@ jobs: make print-env-file - name: Disk Free run: | - df -i - df -h - du -sh /var/lib/docker - docker system prune --all --force --volumes - df -h + sudo df -i + sudo df -h + sudo du -sh /var/lib/docker + sudo docker system prune --all --force --volumes + sudo df -h - name: Run API unit tests run: make test-unit-ci From 0541ee1b515199e5cc23ed384f494778f2ae9606 Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Mon, 20 Apr 2020 09:56:10 +0200 Subject: [PATCH 22/24] ci: add debugging output --- .github/workflows/main.yml | 40 ++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b457a9cf1e..cf0e922936 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,9 +117,11 @@ jobs: make print-env-file - name: Disk Free run: | - df -i - df -h - du -sh /var/lib/docker + sudo df -i + sudo df -h + sudo du -sh /var/lib/docker + sudo docker system prune --all --force --volumes + sudo df -h - name: run API E2E tests run: make test-e2e-ci @@ -153,9 +155,11 @@ jobs: make print-env-file - name: Disk Free run: | - df -i - df -h - du -sh /var/lib/docker + sudo df -i + sudo df -h + sudo du -sh /var/lib/docker + sudo docker system prune --all --force --volumes + sudo df -h - name: run API integration tests run: make test-it-ci @@ -194,9 +198,11 @@ jobs: make print-env-file - name: Disk Free run: | - df -i - df -h - du -sh /var/lib/docker + sudo df -i + sudo df -h + sudo du -sh /var/lib/docker + sudo docker system prune --all --force --volumes + sudo df -h - name: setup required node version uses: actions/setup-node@v1 with: @@ -223,9 +229,11 @@ jobs: ${{ runner.OS }}- - name: Disk Free run: | - df -i - df -h - du -sh /var/lib/docker + sudo df -i + sudo df -h + sudo du -sh /var/lib/docker + sudo docker system prune --all --force --volumes + sudo df -h - name: run upgrade tests run: sbt "upgrade/test" @@ -260,9 +268,11 @@ jobs: run: sudo apt-get install expect - name: Disk Free run: | - df -i - df -h - du -sh /var/lib/docker + sudo df -i + sudo df -h + sudo du -sh /var/lib/docker + sudo docker system prune --all --force --volumes + sudo df -h - name: build all docker images run: | make build-all-scala From 5fcf0c7d8c52d308304d7335c85d9591713dceda Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Thu, 25 Jun 2020 06:04:38 +0200 Subject: [PATCH 23/24] build: bump sipi version --- KnoraBuild.sbt | 2 +- project/Dependencies.scala | 2 +- vars.mk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/KnoraBuild.sbt b/KnoraBuild.sbt index 80355d17a0..ca1ad3cbd7 100644 --- a/KnoraBuild.sbt +++ b/KnoraBuild.sbt @@ -300,7 +300,7 @@ lazy val knoraSipi: Project = knoraModule("knora-sipi") Docker / dockerExposedPorts ++= Seq(1024), Docker / dockerCommands := Seq( // FIXME: Someday find out how to reference here Dependencies.Versions.sipiImage - Cmd("FROM", "dhlabbasel/sipi:v2.0.1"), + Cmd("FROM", "daschswiss/sipi:v3.0.0-rc.3"), Cmd("LABEL", s"""MAINTAINER="${maintainer.value}""""), Cmd("COPY", "opt/docker/scripts", "/sipi/scripts"), ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index a6261a9356..27898e65de 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -49,7 +49,7 @@ object Dependencies { akkaHttpVersion := "10.1.12", jenaVersion := "3.4.0", metricsVersion := "4.0.1", - sipiImage := "dhlabbasel/sipi:v2.0.1", + sipiImage := "daschswiss/sipi:v3.0.0-rc.3", gdbSEImage := "daschswiss/graphdb:9.0.0-se", gdbFreeImage := "daschswiss/graphdb:9.0.0-free" ) diff --git a/vars.mk b/vars.mk index dedffb0000..8f0c9748b0 100644 --- a/vars.mk +++ b/vars.mk @@ -1,4 +1,4 @@ -SIPI_VERSION := 2.0.1 +SIPI_VERSION := 3.0.0-rc.3 GRAPHDB_SE_VERSION := 9.0.0 GRAPHDB_FREE_VERSION := 9.0.0 _GRAPHDB_HEAP_SIZE := 5G From d20885727575bd9dfdf0fec068653786544dd246 Mon Sep 17 00:00:00 2001 From: Ivan Subotic <400790+subotic@users.noreply.github.com> Date: Thu, 25 Jun 2020 10:39:28 +0200 Subject: [PATCH 24/24] build: bump sipi version --- docker/knora-sipi.template.dockerfile | 2 +- sipi/docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/knora-sipi.template.dockerfile b/docker/knora-sipi.template.dockerfile index c3084d97f1..025e5c1780 100644 --- a/docker/knora-sipi.template.dockerfile +++ b/docker/knora-sipi.template.dockerfile @@ -1,4 +1,4 @@ -FROM dhlabbasel/sipi:v@SIPI_VERSION@ +FROM daschswiss/sipi:v@SIPI_VERSION@ COPY stage/scripts /sipi/scripts diff --git a/sipi/docker-compose.yml b/sipi/docker-compose.yml index 305f91830c..af935cf862 100644 --- a/sipi/docker-compose.yml +++ b/sipi/docker-compose.yml @@ -4,7 +4,7 @@ version: '3' services: # sipi using default (production-like) configuration with additional routes for testing sipi: - image: dhlabbasel/sipi:${SIPI_VERSION} + image: daschswiss/sipi:${SIPI_VERSION} container_name: sipi ports: - "1024:1024" @@ -26,7 +26,7 @@ services: # sipi using configuration which disables authentication sipi-no-auth: - image: dhlabbasel/sipi:${SIPI_VERSION} + image: daschswiss/sipi:${SIPI_VERSION} container_name: sipi-no-auth ports: - "1024:1024"