Merge branch 'umami-software:master' into master

.gitignore

@@ -22,6 +22,7 @@ node_modules
 # misc
 .DS_Store
 .idea
+.yarn
 *.iml
 *.log
 .vscode

README.md

@@ -11,7 +11,7 @@ A detailed getting started guide can be found at [https://umami.is/docs/](https:
 ### Requirements
 
 - A server with Node.js version 16.13 or newer
-- A database. Umami supports [MySQL](https://www.mysql.com/) and [Postgresql](https://www.postgresql.org/) databases.
+- A database. Umami supports [MySQL](https://www.mysql.com/) (minimum v8.0) and [Postgresql](https://www.postgresql.org/) (minimum v12.14) databases.
 
 ### Install Yarn
 
@@ -72,13 +72,13 @@ docker compose up -d
 Alternatively, to pull just the Umami Docker image with PostgreSQL support:
 
 ```bash
-docker pull ghcr.io/umami-software/umami:postgresql-latest
+docker pull docker.umami.is/umami-software/umami:postgresql-latest
 ```
 
 Or with MySQL support:
 
 ```bash
-docker pull ghcr.io/umami-software/umami:mysql-latest
+docker pull docker.umami.is/umami-software/umami:mysql-latest
 ```
 
 ## Getting updates
@@ -101,3 +101,4 @@ docker compose up --force-recreate
 ## License
 
 MIT
+

cypress.config.ts

@@ -4,4 +4,9 @@ export default defineConfig({
   e2e: {
     baseUrl: 'http://localhost:3000',
   },
+  // default username / password on init
+  env: {
+    umami_user: 'admin',
+    umami_password: 'umami',
+  },
 });

cypress/docker-compose.yml

@@ -43,9 +43,10 @@ services:
       - CYPRESS_umami_user=admin
       - CYPRESS_umami_password=umami
     volumes:
-      - ../tsconfig.json:/tsconfig.json
+      - ./tsconfig.json:/tsconfig.json
       - ../cypress.config.ts:/cypress.config.ts
       - ./:/cypress
       - ../node_modules/:/node_modules
+      - ../src/lib/crypto.ts:/src/lib/crypto.ts
 volumes:
   umami-db-data:

cypress/e2e/login.cy.ts

@@ -6,8 +6,12 @@ describe('Login tests', () => {
     },
     () => {
       cy.visit('/login');
-      cy.getDataTest('input-username').find('input').type(Cypress.env('umami_user'));
-      cy.getDataTest('input-password').find('input').type(Cypress.env('umami_password'));
+      cy.getDataTest('input-username').find('input').click();
+      cy.getDataTest('input-username').find('input').type(Cypress.env('umami_user'), { delay: 50 });
+      cy.getDataTest('input-password').find('input').click();
+      cy.getDataTest('input-password')
+        .find('input')
+        .type(Cypress.env('umami_password'), { delay: 50 });
       cy.getDataTest('button-submit').click();
       cy.url().should('eq', Cypress.config().baseUrl + '/dashboard');
       cy.getDataTest('button-profile').click();

cypress/e2e/website.cy.ts

@@ -10,8 +10,10 @@ describe('Website tests', () => {
     cy.visit('/settings/websites');
     cy.getDataTest('button-website-add').click();
     cy.contains(/Add website/i).should('be.visible');
-    cy.getDataTest('input-name').find('input').wait(500).type('Add test', { delay: 50 });
-    cy.getDataTest('input-domain').find('input').wait(500).type('addtest.com', { delay: 50 });
+    cy.getDataTest('input-name').find('input').click();
+    cy.getDataTest('input-name').find('input').type('Add test', { delay: 50 });
+    cy.getDataTest('input-domain').find('input').click();
+    cy.getDataTest('input-domain').find('input').type('addtest.com', { delay: 50 });
     cy.getDataTest('button-submit').click();
     cy.get('td[label="Name"]').should('contain.text', 'Add test');
     cy.get('td[label="Domain"]').should('contain.text', 'addtest.com');
@@ -26,27 +28,23 @@ describe('Website tests', () => {
         cy.deleteWebsite(websiteId);
       });
     cy.visit('/settings/websites');
-    cy.contains('Add test').should('not.exist');
+    cy.contains(/Add test/i).should('not.exist');
   });
 
-  it.only('Edit a website', () => {
+  it('Edit a website', () => {
     // prep data
     cy.addWebsite('Update test', 'updatetest.com');
     cy.visit('/settings/websites');
 
     // edit website
     cy.getDataTest('link-button-edit').first().click();
     cy.contains(/Details/i).should('be.visible');
-    cy.getDataTest('input-name')
-      .find('input')
-      .wait(500)
-      .clear()
-      .type('Updated website', { delay: 50 });
-    cy.getDataTest('input-domain')
-      .find('input')
-      .wait(500)
-      .clear()
-      .type('updatedwebsite.com', { delay: 50 });
+    cy.getDataTest('input-name').find('input').click();
+    cy.getDataTest('input-name').find('input').clear();
+    cy.getDataTest('input-name').find('input').type('Updated website', { delay: 50 });
+    cy.getDataTest('input-domain').find('input').click();
+    cy.getDataTest('input-domain').find('input').clear();
+    cy.getDataTest('input-domain').find('input').type('updatedwebsite.com', { delay: 50 });
     cy.getDataTest('button-submit').click({ force: true });
     cy.getDataTest('input-name').find('input').should('have.value', 'Updated website');
     cy.getDataTest('input-domain').find('input').should('have.value', 'updatedwebsite.com');
@@ -69,7 +67,7 @@ describe('Website tests', () => {
         cy.deleteWebsite(websiteId);
       });
     cy.visit('/settings/websites');
-    cy.contains('Add test').should('not.exist');
+    cy.contains(/Add test/i).should('not.exist');
   });
 
   it('Delete a website', () => {
@@ -86,6 +84,6 @@ describe('Website tests', () => {
     cy.contains(/Type DELETE in the box below to confirm./i).should('be.visible');
     cy.get('input[name="confirm"').type('DELETE');
     cy.get('button[type="submit"]').click();
-    cy.contains('Delete test').should('not.exist');
+    cy.contains(/Delete test/i).should('not.exist');
   });
 });

db/clickhouse/migrations/02_add_visit_id.sql

@@ -0,0 +1,90 @@
+CREATE TABLE umami.website_event_join
+(
+    session_id UUID,
+    visit_id UUID,
+    created_at DateTime('UTC')
+)
+    engine = MergeTree
+        ORDER BY (session_id, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.website_event_join
+SELECT DISTINCT
+    s.session_id,
+    generateUUIDv4() visit_id,
+    s.created_at
+FROM (SELECT DISTINCT session_id,
+        date_trunc('hour', created_at) created_at
+    FROM website_event) s;
+
+-- create new table
+CREATE TABLE umami.website_event_new
+(
+    website_id UUID,
+    session_id UUID,
+    visit_id UUID,
+    event_id UUID,
+    hostname LowCardinality(String),
+    browser LowCardinality(String),
+    os LowCardinality(String),
+    device LowCardinality(String),
+    screen LowCardinality(String),
+    language LowCardinality(String),
+    country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    subdivision2 LowCardinality(String),
+    city String,
+    url_path String,
+    url_query String,
+    referrer_path String,
+    referrer_query String,
+    referrer_domain String,
+    page_title String,
+    event_type UInt32,
+    event_name String,
+    created_at DateTime('UTC'),
+    job_id UUID
+)
+    engine = MergeTree
+        ORDER BY (website_id, session_id, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.website_event_new
+SELECT we.website_id,
+    we.session_id,
+    j.visit_id,
+    we.event_id,
+    we.hostname,
+    we.browser,
+    we.os,
+    we.device,
+    we.screen,
+    we.language,
+    we.country,
+    we.subdivision1,
+    we.subdivision2,
+    we.city,
+    we.url_path,
+    we.url_query,
+    we.referrer_path,
+    we.referrer_query,
+    we.referrer_domain,
+    we.page_title,
+    we.event_type,
+    we.event_name,
+    we.created_at,
+    we.job_id
+FROM umami.website_event we
+JOIN umami.website_event_join j
+    ON we.session_id = j.session_id
+        and date_trunc('hour', we.created_at) = j.created_at
+
+RENAME TABLE umami.website_event TO umami.website_event_old;
+RENAME TABLE umami.website_event_new TO umami.website_event;
+
+/*
+
+ DROP TABLE umami.website_event_old
+ DROP TABLE umami.website_event_join
+
+ */

db/clickhouse/migrations/03_session_data.sql

@@ -0,0 +1,57 @@
+CREATE TABLE umami.event_data_new
+(
+    website_id UUID,
+    session_id UUID,
+    event_id UUID,
+    url_path String,
+    event_name String,
+    data_key String,
+    string_value Nullable(String),
+    number_value Nullable(Decimal64(4)),
+    date_value Nullable(DateTime('UTC')),
+    data_type UInt32,
+    created_at DateTime('UTC'),
+    job_id Nullable(UUID)
+)
+    engine = MergeTree
+        ORDER BY (website_id, event_id, data_key, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.event_data_new
+SELECT website_id,
+    session_id,
+    event_id,
+    url_path,
+    event_name,
+    event_key,
+    string_value,
+    number_value,
+    date_value,
+    data_type,
+    created_at,
+    NULL
+FROM umami.event_data;
+
+CREATE TABLE umami.session_data
+(
+    website_id UUID,
+    session_id UUID,
+    data_key String,
+    string_value Nullable(String),
+    number_value Nullable(Decimal64(4)),
+    date_value Nullable(DateTime('UTC')),
+    data_type UInt32,
+    created_at DateTime('UTC'),
+    job_id Nullable(UUID)
+)
+    engine = MergeTree
+        ORDER BY (website_id, session_id, data_key, created_at)
+        SETTINGS index_granularity = 8192;
+
+RENAME TABLE umami.event_data TO umami.event_data_old;
+RENAME TABLE umami.event_data_new TO umami.event_data;
+
+/*
+DROP TABLE umami.event_data_old
+ */
+

db/clickhouse/schema.sql

@@ -3,6 +3,7 @@ CREATE TABLE umami.website_event
 (
     website_id UUID,
     session_id UUID,
+    visit_id UUID,
     event_id UUID,
     --sessions
     hostname LowCardinality(String),
@@ -26,7 +27,7 @@ CREATE TABLE umami.website_event
     event_type UInt32,
     event_name String,
     created_at DateTime('UTC'),
-    job_id UUID
+    job_id Nullable(UUID)
 )
     engine = MergeTree
         ORDER BY (website_id, session_id, created_at)
@@ -39,14 +40,30 @@ CREATE TABLE umami.event_data
     event_id UUID,
     url_path String,
     event_name String,
-    event_key String,
+    data_key String,
     string_value Nullable(String),
-    number_value Nullable(Decimal64(4)), --922337203685477.5625
+    number_value Nullable(Decimal64(4)),
     date_value Nullable(DateTime('UTC')),
     data_type UInt32,
     created_at DateTime('UTC'),
-    job_id UUID
+    job_id Nullable(UUID)
 )
     engine = MergeTree
-        ORDER BY (website_id, event_id, event_key, created_at)
+        ORDER BY (website_id, event_id, data_key, created_at)
+        SETTINGS index_granularity = 8192;
+
+CREATE TABLE umami.session_data
+(
+    website_id UUID,
+    session_id UUID,
+    data_key String,
+    string_value Nullable(String),
+    number_value Nullable(Decimal64(4)),
+    date_value Nullable(DateTime('UTC')),
+    data_type UInt32,
+    created_at DateTime('UTC'),
+    job_id Nullable(UUID)
+)
+    engine = MergeTree
+        ORDER BY (website_id, session_id, data_key, created_at)
         SETTINGS index_granularity = 8192;

db/mysql/migrations/05_add_visit_id/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE `website_event` ADD COLUMN `visit_id` VARCHAR(36) NULL;
+
+UPDATE `website_event` we
+JOIN (SELECT DISTINCT
+        s.session_id,
+        s.visit_time,
+        BIN_TO_UUID(RANDOM_BYTES(16) & 0xffffffffffff0fff3fffffffffffffff | 0x00000000000040008000000000000000) uuid
+    FROM (SELECT DISTINCT session_id,
+            DATE_FORMAT(created_at, '%Y-%m-%d %H:00:00') visit_time
+        FROM `website_event`) s) a
+    ON we.session_id = a.session_id and DATE_FORMAT(we.created_at, '%Y-%m-%d %H:00:00') = a.visit_time
+SET we.visit_id = a.uuid
+WHERE we.visit_id IS NULL;
+
+ALTER TABLE `website_event` MODIFY `visit_id` VARCHAR(36) NOT NULL;
+
+-- CreateIndex
+CREATE INDEX `website_event_visit_id_idx` ON `website_event`(`visit_id`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_visit_id_created_at_idx` ON `website_event`(`website_id`, `visit_id`, `created_at`);

db/mysql/migrations/06_session_data/migration.sql

@@ -0,0 +1,20 @@
+-- DropIndex
+DROP INDEX `event_data_website_id_created_at_event_key_idx` ON `event_data`;
+
+-- DropIndex
+DROP INDEX `event_data_website_id_website_event_id_created_at_idx` ON `event_data`;
+
+-- AlterTable
+ALTER TABLE `event_data` RENAME COLUMN `event_key` TO `data_key`;
+
+-- AlterTable
+ALTER TABLE `session_data` RENAME COLUMN `event_key` TO `data_key`;
+
+-- CreateIndex
+CREATE INDEX `event_data_website_id_created_at_data_key_idx` ON `event_data`(`website_id`, `created_at`, `data_key`);
+
+-- CreateIndex
+CREATE INDEX `session_data_session_id_created_at_idx` ON `session_data`(`session_id`, `created_at`);
+
+-- CreateIndex
+CREATE INDEX `session_data_website_id_created_at_data_key_idx` ON `session_data`(`website_id`, `created_at`, `data_key`);