File upload backend

docker-compose.yaml

@@ -35,6 +35,17 @@ services:
             POSTGRES_USER: postgres
             POSTGRES_PASSWORD: welkom123
 
+    ## Recompiles on source code changes.
+    ## Do ensure that the two Gradle containers have independent Home directories and Project cache directories.
+    ztor-build-once:
+        image: gradle:8.3.0-jdk20
+        working_dir: /home/gradle/ztor
+        user: gradle
+        volumes:
+            - ./ztor:/home/gradle/ztor
+            - gradle-build-once-cache:/home/gradle/.gradle
+        command: gradle buildFatJar --no-daemon
+
     ## Recompiles on source code changes.
     ## Do ensure that the two Gradle containers have independent Home directories and Project cache directories.
     ztor-build:
@@ -59,9 +70,12 @@ services:
             - gradle-run-cache:/home/gradle/.gradle
         restart: on-failure
         command: gradle --project-cache-dir=/tmp/gradle run
+        env_file:
+            - ./ztor/local.env
         environment:
             POSTGRES_URL: jdbc:postgresql://postgres:5432/postgres
             POSTGRES_USER: postgres
+            # TODO: security risk, make developer set it in local.env
             POSTGRES_PASSWORD: welkom123
 #        depends_on:
 #            - ztor-build
@@ -104,6 +118,7 @@ services:
 
 volumes:
     gradle-build-cache:
+    gradle-build-once-cache:
     gradle-run-cache:
     gradle-cmd-cache:
     postgres:

frontend/src/components/company-survey-v2/electricity.tsx

@@ -11,7 +11,7 @@ import {OldNumberInput} from './generic/old-number-input'
 import {Supply} from './supply'
 
 enum ConsumptionSpec {
-    AUTHORIZATION = "AUTHORIZATION",
+    SPECTRAL_AUTHORIZATION = "SPECTRAL_AUTHORIZATION",
     UPLOAD_QUARTER_HOURLY_VALUES = "UPLOAD_QUARTER_HOURLY_VALUES",
     ANNUAL_VALUES = "ANNUAL_VALUES",
 }
@@ -44,7 +44,7 @@ export const Electricity = ({form, prefix, hasSupplyName}: {
             </LabelRow>
             <LabelRow label="Hoe wilt u het elektriciteitsprofiel van deze netaansluiting doorgeven?">
                 <Radio.Group onChange={e => setConsumptionSpec(e.target.value)} value={consumptionSpec}>
-                    <Radio value={ConsumptionSpec.AUTHORIZATION} css={{display: 'block'}}>Machting ophalen meetdata invullen (voorkeur)</Radio>
+                    <Radio value={ConsumptionSpec.SPECTRAL_AUTHORIZATION} css={{display: 'block'}}>Machting voor het ophalen van de meetdata bij Spectral</Radio>
                     <Radio value={ConsumptionSpec.UPLOAD_QUARTER_HOURLY_VALUES} css={{display: 'block'}}>Kwartierwaarden uploaden</Radio>
                     <Radio value={ConsumptionSpec.ANNUAL_VALUES} css={{display: 'block'}}>Jaarverbruik invullen</Radio>
                 </Radio.Group>

ztor/.gitignore

@@ -5,6 +5,7 @@ build/
 !gradle/wrapper/gradle-wrapper.jar
 !**/src/main/**/build/
 !**/src/test/**/build/
+local.env
 
 ### STS ###
 .apt_generated

ztor/build.gradle.kts

@@ -23,6 +23,8 @@ repositories {
 val exposed_version = "0.43.0"
 
 dependencies {
+    implementation(platform("com.azure:azure-sdk-bom:1.2.18"))
+    implementation("com.azure:azure-storage-blob:12.25.0")
     implementation("io.ktor:ktor-server-cors-jvm")
     implementation("io.ktor:ktor-server-call-logging-jvm")
     implementation("io.ktor:ktor-server-core-jvm")

ztor/example.local.env

@@ -0,0 +1,5 @@
+# To save time-series CSV files supplied by customers.
+# Specifically quarter-hourly electricity usage and hourly gas usage.
+AZURE_STORAGE_ACCOUNT_NAME=
+AZURE_STORAGE_ACCOUNT_KEY=
+AZURE_STORAGE_CONTAINER=

ztor/src/main/kotlin/com/zenmo/Application.kt

@@ -5,7 +5,19 @@ import io.ktor.server.application.*
 import io.ktor.server.engine.*
 import io.ktor.server.netty.*
 
-fun main() {
+fun main(args: Array<String>) {
+    if (args.count() == 1) {
+        if (args[0] == "create-schema") {
+            println("Creating database schema...")
+            val db = connectToPostgres(false)
+            createSchema(db)
+            println("Schema created!")
+            return
+        }
+
+        println("Unknown argument: ${args[0]}")
+    }
+
     embeddedServer(Netty, port = 8082, host = "0.0.0.0", module = Application::module)
             .start(wait = true)
 }
@@ -17,4 +29,5 @@ fun Application.module() {
     configureDatabases()
     configureRouting()
     configureStatusPages()
+    configureUpload()
 }

ztor/src/main/kotlin/com/zenmo/Errors.kt

@@ -0,0 +1,9 @@
+package com.zenmo
+
+fun errorMessageToJson(message: String?): Any {
+    return mapOf(
+        "error" to mapOf(
+            "message" to message
+        )
+    )
+}

ztor/src/main/kotlin/com/zenmo/companysurvey/BlobStorage.kt

@@ -0,0 +1,85 @@
+package com.zenmo.companysurvey
+
+import com.azure.storage.blob.BlobServiceClientBuilder
+import com.azure.storage.blob.sas.BlobSasPermission
+import com.azure.storage.blob.sas.BlobServiceSasSignatureValues
+import com.azure.storage.common.sas.SasProtocol
+import java.time.Instant
+import java.time.OffsetDateTime
+import java.time.ZoneId
+import java.time.ZoneOffset
+import java.time.format.DateTimeFormatter
+import kotlin.streams.asSequence
+
+enum class BlobPurpose {
+    NATURAL_GAS_VALUES,
+    ELECTRICITY_VALUES,
+    ELECTRICITY_AUTHORIZATION;
+
+    fun toNamePart(): String {
+        return this.toString().lowercase().replace("_", "-")
+    }
+}
+
+class BlobStorage(
+    private val azureAccountName: String = System.getenv("AZURE_STORAGE_ACCOUNT_NAME"),
+    private val azureAccountKey: String = System.getenv("AZURE_STORAGE_ACCOUNT_KEY"),
+    private val containerName: String = System.getenv("AZURE_STORAGE_CONTAINER"),
+) {
+    val blobServiceClient = BlobServiceClientBuilder()
+        .connectionString("DefaultEndpointsProtocol=https;AccountName=$azureAccountName;AccountKey=$azureAccountKey")
+        .buildClient()
+
+    /**
+     * You can use the result of this function to directly upload a file to Azure Blob Storage.
+     * You only need to set the `x-ms-blob-type` header to `BlockBlob`.
+     */
+    fun getBlobSasUrl(
+        blobPurpose: BlobPurpose,
+        project: String,
+        company: String,
+        fileName: String,
+    ): String {
+        if (project === "") {
+            throw Exception("Company name cannot be empty")
+        }
+
+        if (company === "") {
+            throw Exception("Company name cannot be empty")
+        }
+
+        if (fileName === "") {
+            throw Exception("File name cannot be empty")
+        }
+
+        val dateStamp = DateTimeFormatter
+            .ofPattern("yyyy-MM-dd")
+            .withZone(ZoneId.of("Europe/Amsterdam"))
+            .format(Instant.now())
+
+        val blobName = "${dateStamp}_${project}_${company}_${blobPurpose.toNamePart()}_${randomString(3u)}_${fileName}"
+
+        val blobClient = blobServiceClient.getBlobContainerClient(containerName).getBlobClient(blobName)
+        // We would like to (pre-)set some metadata but this is not (easily) possible.
+
+        val permissions = BlobSasPermission().setReadPermission(true).setWritePermission(true).setCreatePermission(true)
+
+        val expiryTime = OffsetDateTime.now().plusDays(1)
+
+        val sasSignatureValues = BlobServiceSasSignatureValues(expiryTime, permissions)
+            .setProtocol(SasProtocol.HTTPS_HTTP)
+
+        val sasToken = blobClient.generateSas(sasSignatureValues)
+
+        return "${blobClient.blobUrl}?$sasToken"
+    }
+}
+
+fun randomString(length: UInt): String {
+    val chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+
+    return java.util.Random().ints(length.toLong(), 0, chars.length)
+        .asSequence()
+        .map(chars::get)
+        .joinToString("")
+}

ztor/src/main/kotlin/com/zenmo/companysurvey/SurveyRepository.kt

@@ -18,6 +18,7 @@ class SurveyRepository(
         transaction(db) {
             CompanySurveyTable.insert {
                 it[id] = surveyId
+                it[project] = survey.project
                 it[companyName] = survey.companyName
                 it[personName] = survey.personName
                 it[email] = survey.email

ztor/src/main/kotlin/com/zenmo/companysurvey/dto/Survey.kt

@@ -10,6 +10,7 @@ import kotlinx.serialization.Serializable
 @Serializable
 data class Survey(
     val created: Instant = Clock.System.now(),
+    val project: String,
     val companyName: String,
     val personName: String,
     val email: String = "",

ztor/src/main/kotlin/com/zenmo/companysurvey/table/Survey.kt

@@ -9,6 +9,8 @@ object CompanySurveyTable: Table("company_survey") {
     val id = uuid("id").autoGenerate()
     // Can be fetched at https://energiekeregio.nl/api/v1/zenmo?details=15989
     val energiekeRegioId = uinteger("energieke_regio_id").nullable()
+    // ZEnMo project name
+    val project = varchar("project", 50)
 
     val companyName = varchar("company_name", 50)
     val personName = varchar("person_name", 50)

ztor/src/main/kotlin/com/zenmo/plugins/Databases.kt

@@ -6,6 +6,7 @@ import com.zenmo.energieprestatieonline.RawPandTable
 import com.zenmo.companysurvey.table.CompanySurveyGridConnectionTable
 import com.zenmo.companysurvey.table.CompanySurveyTable
 import com.zenmo.dbutil.createEnumTypeSql
+import com.zenmo.errorMessageToJson
 import io.ktor.http.*
 import io.ktor.serialization.*
 import io.ktor.server.application.*
@@ -15,7 +16,6 @@ import io.ktor.server.response.*
 import io.ktor.server.routing.*
 import java.sql.*
 import kotlinx.coroutines.*
-import kotlinx.serialization.SerializationException
 import org.jetbrains.exposed.sql.*
 import org.jetbrains.exposed.sql.transactions.transaction
 import java.util.*
@@ -51,14 +51,6 @@ fun Application.configureDatabases(): Database {
     return db
 }
 
-fun errorMessageToJson(message: String?): Any {
-    return mapOf(
-        "error" to mapOf(
-            "message" to message
-        )
-    )
-}
-
 fun createSchema(db: Database) {
     transaction(db) {
         exec(createEnumTypeSql(KleinverbruikElectricityConnectionCapacity::class.java))

ztor/src/main/kotlin/com/zenmo/plugins/Upload.kt

@@ -0,0 +1,42 @@
+package com.zenmo.plugins
+
+import io.ktor.server.application.*
+import io.ktor.server.response.*
+import io.ktor.server.routing.*
+import com.azure.storage.blob.BlobServiceClientBuilder
+import com.azure.storage.blob.sas.BlobSasPermission
+import com.azure.storage.blob.sas.BlobServiceSasSignatureValues
+import com.azure.storage.common.sas.SasProtocol
+import com.zenmo.companysurvey.BlobPurpose
+import com.zenmo.companysurvey.BlobStorage
+import com.zenmo.errorMessageToJson
+import io.ktor.http.*
+import io.ktor.server.plugins.*
+import java.time.OffsetDateTime
+
+fun Application.configureUpload() {
+    val blobStorage = BlobStorage()
+
+    routing {
+        // Get a shared access signature (SAS) token that can be used to upload a blob.
+        get("/upload-url") {
+            val queryParams = call.request.queryParameters
+
+            val required = listOf("project", "company", "fileName", "purpose")
+            val missing = required.filter { !queryParams.contains(it) }
+            if (missing.isNotEmpty()) {
+                call.respond(HttpStatusCode.BadRequest, errorMessageToJson("Missing query parameters: ${missing.joinToString(", ")}"))
+                return@get
+            }
+
+            val url = blobStorage.getBlobSasUrl(
+                BlobPurpose.valueOf(queryParams["purpose"]!!),
+                queryParams["project"]!!,
+                queryParams["company"]!!,
+                queryParams["fileName"]!!,
+            )
+
+            call.respond(mapOf("uploadUrl" to url));
+        }
+    }
+}

ztor/src/test/kotlin/com/zenmo/companysurvey/RepositoryTest.kt

@@ -28,6 +28,7 @@ class RepositoryTest {
         val repo = SurveyRepository(db)
         val survey = Survey(
             companyName = "Zenmo",
+            project = "Project",
             personName = "John Doe",
             email = "[email protected]",
             gridConnections = emptyList(),
@@ -69,6 +70,7 @@ class RepositoryTest {
         val repo = SurveyRepository(db)
         val survey = Survey(
             companyName = "Zenmo",
+            project = "Project",
             personName = "John Doe",
             email = "[email protected]",
             transport = Transport(
@@ -162,6 +164,7 @@ class RepositoryTest {
                     ),
                     mainConsumptionProcess = "Main consumption process",
                     electrificationPlans = "Electrification plans",
+                    consumptionFlexibility = "Consumption flexibility",
                 )
             )
         )