Exports and imports

assets/js/liveview/live_socket.js

@@ -19,12 +19,30 @@ if (csrfToken && websocketUrl) {
       })
     }
   }
+  let Uploaders = {}
+  Uploaders.S3 = function (entries, onViewError) {
+    entries.forEach(entry => {
+      let xhr = new XMLHttpRequest()
+      onViewError(() => xhr.abort())
+      xhr.onload = () => xhr.status === 200 ? entry.progress(100) : entry.error()
+      xhr.onerror = () => entry.error()
+      xhr.upload.addEventListener("progress", (event) => {
+        if (event.lengthComputable) {
+          let percent = Math.round((event.loaded / event.total) * 100)
+          if (percent < 100) { entry.progress(percent) }
+        }
+      })
+      let url = entry.meta.url
+      xhr.open("PUT", url, true)
+      xhr.send(entry.file)
+    })
+  }
   let token = csrfToken.getAttribute("content")
   let url = websocketUrl.getAttribute("content")
   let liveUrl = (url === "") ? "/live" : new URL("/live", url).href;
   let liveSocket = new LiveSocket(liveUrl, Socket, {
     heartbeatIntervalMs: 10000,
-    params: { _csrf_token: token }, hooks: Hooks, dom: {
+    params: { _csrf_token: token }, hooks: Hooks, uploaders: Uploaders, dom: {
       // for alpinejs integration
       onBeforeElUpdated(from, to) {
         if (from._x_dataStack) {

config/.env.dev

@@ -20,3 +20,12 @@ GOOGLE_CLIENT_SECRET=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
 
 PROMEX_DISABLED=false
 SITE_DEFAULT_INGEST_THRESHOLD=1000000
+
+S3_DISABLED=false
+S3_ACCESS_KEY_ID=minioadmin
+S3_SECRET_ACCESS_KEY=minioadmin
+S3_REGION=us-east-1
+S3_ENDPOINT=http://localhost:9000
+S3_EXPORTS_BUCKET=exports
+S3_IMPORTS_BUCKET=imports
+S3_HOST_FOR_CLICKHOUSE=172.19.0.4

config/.env.test

@@ -15,3 +15,12 @@ IP_GEOLOCATION_DB=test/priv/GeoLite2-City-Test.mmdb
 SITE_DEFAULT_INGEST_THRESHOLD=1000000
 GOOGLE_CLIENT_ID=fake_client_id
 GOOGLE_CLIENT_SECRET=fake_client_secret
+
+S3_DISABLED=false
+S3_ACCESS_KEY_ID=minioadmin
+S3_SECRET_ACCESS_KEY=minioadmin
+S3_REGION=us-east-1
+S3_ENDPOINT=http://localhost:9000
+S3_EXPORTS_BUCKET=exports
+S3_IMPORTS_BUCKET=imports
+S3_HOST_FOR_CLICKHOUSE=172.19.0.4

config/runtime.exs

@@ -701,3 +701,74 @@ if not is_selfhost do
 
   config :plausible, Plausible.Site, default_ingest_threshold: site_default_ingest_threshold
 end
+
+s3_disabled? =
+  config_dir
+  |> get_var_from_path_or_env("S3_DISABLED", "true")
+  |> String.to_existing_atom()
+
+unless s3_disabled? do
+  s3_env = [
+    %{
+      name: "S3_ACCESS_KEY_ID",
+      example: "AKIAIOSFODNN7EXAMPLE"
+    },
+    %{
+      name: "S3_SECRET_ACCESS_KEY",
+      example: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+    },
+    %{
+      name: "S3_REGION",
+      example: "us-east-1"
+    },
+    %{
+      name: "S3_ENDPOINT",
+      example: "https://<ACCOUNT_ID>.r2.cloudflarestorage.com"
+    },
+    %{
+      name: "S3_EXPORTS_BUCKET",
+      example: "my-exports-bucket"
+    },
+    %{
+      name: "S3_IMPORTS_BUCKET",
+      example: "my-imports-bucket"
+    }
+  ]
+
+  s3_env =
+    Enum.map(s3_env, fn var ->
+      Map.put(var, :value, get_var_from_path_or_env(config_dir, var.name))
+    end)
+
+  s3_missing_env = Enum.filter(s3_env, &is_nil(&1.value))
+
+  unless s3_missing_env == [] do
+    raise """
+    Missing S3 configuration. Please set #{s3_missing_env |> Enum.map(& &1.name) |> Enum.join(", ")} environment variable(s):
+
+    #{s3_missing_env |> Enum.map(fn %{name: name, example: example} -> "\t#{name}=#{example}" end) |> Enum.join("\n")}
+    """
+  end
+
+  s3_env_value = fn name ->
+    s3_env |> Enum.find(&(&1.name == name)) |> Map.fetch!(:value)
+  end
+
+  config :ex_aws,
+    http_client: Plausible.S3.Client,
+    access_key_id: s3_env_value.("S3_ACCESS_KEY_ID"),
+    secret_access_key: s3_env_value.("S3_SECRET_ACCESS_KEY"),
+    region: s3_env_value.("S3_REGION")
+
+  %URI{scheme: s3_scheme, host: s3_host, port: s3_port} = URI.parse(s3_env_value.("S3_ENDPOINT"))
+
+  config :ex_aws, :s3,
+    scheme: s3_scheme <> "://",
+    host: s3_host,
+    port: s3_port
+
+  config :plausible, Plausible.S3,
+    exports_bucket: s3_env_value.("S3_EXPORTS_BUCKET"),
+    imports_bucket: s3_env_value.("S3_IMPORTS_BUCKET"),
+    host_for_clickhouse: get_var_from_path_or_env(config_dir, "S3_HOST_FOR_CLICKHOUSE")
+end