feat(api-v2): Add support for PDF files (#1206)

docs/src/paradox/03-apis/api-v2/editing-values.md

@@ -188,8 +188,12 @@ provided.
 Knora supports the storage of certain types of data as files, using
 [Sipi](https://github.com/dhlab-basel/Sipi)
 (see @ref:[FileValue](../../02-knora-ontologies/knora-base.md#filevalue)).
-Knora API v2 currently supports using Sipi to store image files. Support for
-other types of files will be added in the near future.
+Knora API v2 currently supports using Sipi to store the following types of files:
+
+* Images (JPEG, JPEG2000, TIFF, PNG), which are stored internally as JPEG2000
+* PDF
+
+Support for other types of files will be added in the future.
 
 The following sections describe the steps for creating a file value.
 
@@ -209,9 +213,10 @@ must contain a parameter `filename`, providing the file's original filename,
 which both Knora and Sipi will store; these filenames can be descriptive
 and need not be unique.
 
-Sipi will then convert the uploaded image files to JPEG 2000 format and store
-them in a temporary location. If this is successful, it will return a JSON
-response that looks something like this:
+Sipi stores the file in a temporary location. If the file is an image, it is
+converted first to JPEG2000 format, and the converted file is stored.
+
+Sipi then returns a JSON response that looks something like this:
 
 ```json
 {
@@ -235,9 +240,9 @@ array with two elements. For each file, we have:
 - the `temporaryBaseIIIFUrl`, which we can use to construct a IIIF URL for
   previewing the file
 
-The client may now wish to get a thumbnail of each uploaded image, to allow
-the user to confirm that the correct files have been uploaded. This can be done
-by adding IIIF parameters to `temporaryBaseIIIFUrl`. For example, to get
+In the case of an image file, the client may now wish to get a thumbnail of each
+uploaded image, to allow the user to confirm that the correct files have been uploaded.
+This can be done by adding IIIF parameters to `temporaryBaseIIIFUrl`. For example, to get
 a JPG thumbnail image that is 150 pixels wide, you would add
 `/full/150,/0/default.jpg`.
 
@@ -284,6 +289,11 @@ request to Knora is valid, Knora saves the file value in the triplestore and
 instructs Sipi to move the file to permanent storage. Otherwise, the
 temporary file that was stored by Sipi is deleted.
 
+If you're submitting a PDF document, use the resource class
+`knora-api:DocumentRepresentation`, which has the property
+`knora-api:hasDocumentFileValue`, pointing to a
+`knora-api:DocumentFileValue`.
+
 ## Updating a Value
 
 To update a value, use this route:

knora-ontologies/knora-base.ttl

@@ -33,7 +33,7 @@
 
     :attachedToProject knora-admin:SystemProject ;
 
-    :ontologyVersion "PR 1440" .
+    :ontologyVersion "knora-base v6" .
 
 
 
@@ -1115,6 +1115,18 @@
 
 
 
+###  http://www.knora.org/ontology/knora-base#pageCount
+
+:pageCount rdf:type owl:DatatypeProperty ;
+
+      :subjectClassConstraint :FileValue ;
+
+      rdfs:subPropertyOf :valueHas ;
+
+      :objectDatatypeConstraint xsd:integer .
+
+
+
 ###  http://www.knora.org/ontology/knora-base#duration
 
 :duration rdf:type owl:DatatypeProperty ;
@@ -1768,8 +1780,19 @@
 
 :DocumentFileValue rdf:type owl:Class ;
 
-                   rdfs:subClassOf :FileValue .
-
+                   rdfs:subClassOf :FileValue ,
+                   [ rdf:type owl:Restriction ;
+                     owl:onProperty :pageCount ;
+                     owl:cardinality "1"^^xsd:nonNegativeInteger
+                   ] ,
+                   [ rdf:type owl:Restriction ;
+                     owl:onProperty :dimX ;
+                     owl:maxCardinality "1"^^xsd:nonNegativeInteger
+                   ] ,
+                   [ rdf:type owl:Restriction ;
+                     owl:onProperty :dimY ;
+                     owl:maxCardinality "1"^^xsd:nonNegativeInteger
+                   ] .
 
 
 ###  http://www.knora.org/ontology/knora-base#DocumentRepresentation
@@ -1833,10 +1856,6 @@
 :FileValue rdf:type owl:Class ;
 
            rdfs:subClassOf :Value ,
-                           [ rdf:type owl:Restriction ;
-                             owl:onProperty :originalMimeType ;
-                             owl:cardinality "1"^^xsd:nonNegativeInteger
-                           ] ,
                            [ rdf:type owl:Restriction ;
                              owl:onProperty :internalFilename ;
                              owl:cardinality "1"^^xsd:nonNegativeInteger
@@ -1847,7 +1866,11 @@
                            ] ,
                            [ rdf:type owl:Restriction ;
                              owl:onProperty :originalFilename ;
-                             owl:cardinality "1"^^xsd:nonNegativeInteger
+                             owl:maxCardinality "1"^^xsd:nonNegativeInteger
+                           ] ,
+                           [ rdf:type owl:Restriction ;
+                             owl:onProperty :originalMimeType ;
+                             owl:maxCardinality "1"^^xsd:nonNegativeInteger
                            ] .
 
 

sipi/config/sipi.knora-docker-config.lua

@@ -207,8 +207,8 @@ routes = {
     },
     {
         method = 'GET',
-        route = '/test_mediatype',
-        script = 'test_mediatype.lua'
+        route = '/test_file_info',
+        script = 'test_file_info.lua'
     },
     {
         method = 'GET',

sipi/config/sipi.knora-docker-no-auth-config.lua

@@ -211,8 +211,8 @@ routes = {
     },
     {
         method = 'GET',
-        route = '/test_mediatype',
-        script = 'test_mediatype.lua'
+        route = '/test_file_info',
+        script = 'test_file_info.lua'
     },
     {
         method = 'GET',

sipi/config/sipi.knora-docker-test-config.lua

@@ -220,8 +220,8 @@ routes = {
     },
     {
         method = 'GET',
-        route = '/test_mediatype',
-        script = 'test_mediatype.lua'
+        route = '/test_file_info',
+        script = 'test_file_info.lua'
     },
     {
         method = 'GET',

sipi/config/sipi.knora-local-config.lua

@@ -199,8 +199,8 @@ routes = {
     },
     {
         method = 'GET',
-        route = '/test_mediatype',
-        script = 'test_mediatype.lua'
+        route = '/test_file_info',
+        script = 'test_file_info.lua'
     },
     {
         method = 'GET',

sipi/docker-compose.yml

@@ -4,7 +4,7 @@ version: '3'
 services:
   # sipi using default (production-like) configuration with additional routes for testing
   sipi:
-    image: dhlabbasel/sipi:v2.0.1
+    image: dhlabbasel/sipi:develop
     container_name: sipi
     ports:
       - "1024:1024"

sipi/images/thumbs/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

sipi/scripts/admin_upload.lua

@@ -35,15 +35,13 @@ for findex,fparam in pairs(server.uploads) do
     admindir = config.docroot .. '/admin/'
     local success, exists = server.fs.exists(admindir)
     if not success then
-        server.log("server.fs.exists() failed: " .. exists, server.loglevel.LOG_ERR)
-        send_error(500, "Internal server error")
+        send_error(500, "server.fs.exists() failed: " .. exists)
         return false
     end
     if not exists then
         local success, errmsg = server.fs.mkdir(admindir, 511)
         if not success then
-            server.log("server.fs.mkdir() failed: " .. errmsg, server.loglevel.LOG_ERR)
-            send_error(500, "Admin directory could not be created on server")
+            send_error(500, "server.fs.mkdir() failed: " .. errmsg)
             return false
         end
     end
@@ -55,10 +53,10 @@ for findex,fparam in pairs(server.uploads) do
     origname = fparam["origname"]:gsub("%s+", "-")
 
     adminpath =  admindir .. uuid62 .. '-' .. origname
-    local success, errmsg = server.copyTmpfile(findex, adminpath)
+    local errmsg
+    success, errmsg = server.copyTmpfile(findex, adminpath)
     if not success then
-        server.log(errmsg, server.loglevel.error)
-        send_error(500, "Couldn't upload file: " .. result)
+        send_error(500, "Couldn't upload file: " .. errmsg)
         return false
     else
         files[findex] = uuid62 .. '-' .. origname

sipi/scripts/cache.lua

@@ -20,6 +20,8 @@ if not authorize_api('admin.sipi.org', 'administrator', config.adminuser) then
 end
 
 if server.method == 'GET' then
+    local flist
+
     if server.get and (server.get.sort == 'atasc') then
         flist = cache.filelist('AT_ASC')
     elseif server.get and (server.get.sort == 'atdesc') then
@@ -29,14 +31,12 @@ if server.method == 'GET' then
     elseif server.get and (server.get.sort == 'fsdesc') then
         flist = cache.filelist('FS_DESC')
     else
-       flist = cache.filelist('AT_ASC')
+        flist = cache.filelist('AT_ASC')
     end
 
-
     local success, jsonstr = server.table_to_json(flist)
     if not success then
-        server.sendStatus(500)
-        server.log(jsonstr, server.loglevel.err)
+        send_error(500, jsonstr)
         return false
     end
 
@@ -47,34 +47,39 @@ elseif server.method == 'DELETE' then
     if server.content and server.content_type == 'application/json' then
         local success, todel = server.json_to_table(server.content)
         if not success then
-            server.sendStatus(500)
-            server.log(todel, server.loglevel.err)
+            send_error(500, todel)
             return false
         end
-        for index,canonical in pairs(todel) do
+
+        for index, canonical in pairs(todel) do
             cache.delete(canonical)
         end
+
         result = {
             status = 'OK'
         }
-        local success, jsonresult = server.table_to_json(result)
+
+        local jsonresult
+        success, jsonresult = server.table_to_json(result)
         server.sendHeader('Content-type', 'application/json')
-        server.sendStatus(200);
+        server.sendStatus(200)
         server.print(jsonresult)
     else
         local n = cache.purge()
+
         result = {
             status = 'OK',
             n = n
         }
+
         local success, jsonresult = server.table_to_json(result)
         if not success then
-            server.sendStatus(500)
-            server.log(jsonstr, server.loglevel.err)
+            send_error(500, jsonresult)
             return false
         end
+
         server.sendHeader('Content-type', 'application/json')
-        server.sendStatus(200);
+        server.sendStatus(200)
         server.print(jsonresult)
     end
 end

sipi/scripts/clean_temp_dir.lua

@@ -46,21 +46,22 @@ function clean_dir_entries(dir_path, current_time)
         end
 
         local entry_path = dir_path .. "/" .. entry
-        local success, entry_type = server.fs.ftype(entry_path)
+        local entry_type
+        success, entry_type = server.fs.ftype(entry_path)
 
         if not success then
             server.log(entry_type, server.loglevel.LOG_ERR)
             return false
         end
 
         if entry_type == "FILE" then
-            local success = maybe_delete_temp_file(entry_path, current_time)
+            success = maybe_delete_temp_file(entry_path, current_time)
 
             if not success then
                 return false
             end
         elseif entry_type == "DIRECTORY" then
-            local success = clean_dir_entries(entry_path, current_time)
+            success = clean_dir_entries(entry_path, current_time)
 
             if not success then
                 return false
@@ -91,7 +92,8 @@ function maybe_delete_temp_file(file_path, current_time)
 
         if file_age > config.max_temp_file_age then
             server.log("clean_temp_dir: removing " .. file_path, server.loglevel.LOG_DEBUG)
-            local success, error_msg = server.fs.unlink(file_path)
+            local error_msg
+            success, error_msg = server.fs.unlink(file_path)
 
             if not success then
                 -- If we couldn't delete the file, maybe it has already been deleted.