Merge pull request #556 from metabrainz/master

Merge master into production

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,28 +6,25 @@
     depending on your change. Keep or delete what is relevant for your change.
     Remember that it helps us review if you give more helpful info for us to
     understand your change.
-    
+
     Ensure that you've read through and followed the Contributing Guidelines, in
     ./github/CONTRIBUTING.md.
 -->
 
 # Summary
 
-<!--
-    Update the checkbox with an [x] for the type of contribution you are making.
--->
-
 * This is a…
-    * [x] Bug fix
-    * [ ] Feature addition
-    * [ ] Refactoring
-    * [ ] Minor / simple change (like a typo)
-    * [ ] Other
+    * ( ) Bug fix
+    * (x) Feature addition
+    * ( ) Refactoring
+    * ( ) Minor / simple change (like a typo)
+    * ( ) Other
+
 * **Describe this change in 1-2 sentences**:
 
 # Problem
 
-<!-- 
+<!--
     Anything that helps us understand why you are making this change goes here.
     What problem are you trying to fix? What does this change address?
 -->

.gitignore

@@ -28,3 +28,5 @@ pyvenv.cfg
 .venv
 pip-selfcheck.json
 
+# Javascript
+node_modules

.travis.yml

@@ -1,12 +1,10 @@
-sudo: required
-
 language: python
 
 services:
   - docker
 
 install:
-  - cp listenbrainz/rtd_config.py listenbrainz/config.py
+  - cp listenbrainz/config.py.sample listenbrainz/config.py
   # Build the unit test docker-compose project
   - docker-compose -f docker/docker-compose.test.yml -p listenbrainz_test build
   - docker-compose -f docker/docker-compose.test.yml -p listenbrainz_test run --rm listenbrainz

Dockerfile

@@ -1,6 +1,6 @@
-FROM metabrainz/python:3.6
+FROM metabrainz/python:3.7
 
-ENV DOCKERIZE_VERSION v0.2.0
+ENV DOCKERIZE_VERSION v0.6.1
 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
@@ -15,29 +15,31 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 # PostgreSQL client
-RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
+RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 ENV PG_MAJOR 9.5
 RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list
 RUN apt-get update \
     && apt-get install -y --no-install-recommends postgresql-client-$PG_MAJOR \
     && rm -rf /var/lib/apt/lists/*
 
+# Node
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
+    apt-get install -y nodejs && rm -rf /var/lib/apt/lists/*
+
 RUN mkdir /code
 WORKDIR /code
 
-RUN pip3 install setuptools==36.0.1
-
 # MessyBrainz
 RUN git clone https://github.com/metabrainz/messybrainz-server.git messybrainz
 WORKDIR /code/messybrainz
-RUN pip3 install -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
 RUN python3 setup.py install
 
 RUN mkdir /code/listenbrainz
 WORKDIR /code/listenbrainz
 
 COPY requirements.txt /code/listenbrainz/
-RUN pip3 install -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
 
 # Now install our code, which may change frequently
 COPY . /code/listenbrainz/

Dockerfile.prod

@@ -1,8 +1,8 @@
-FROM metabrainz/python:3.6
+FROM metabrainz/python:3.7
 
 ARG deploy_env
 
-ENV DOCKERIZE_VERSION v0.2.0
+ENV DOCKERIZE_VERSION v0.6.1
 RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
@@ -27,25 +27,36 @@ RUN apt-get update \
     && apt-get install -y --no-install-recommends postgresql-client-$PG_MAJOR \
     && rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install uWSGI==2.0.13.1
+RUN pip3 install uWSGI==2.0.18
 
 RUN mkdir /code
 WORKDIR /code
 
 # MessyBrainz
 RUN git clone https://github.com/metabrainz/messybrainz-server.git messybrainz
 WORKDIR /code/messybrainz
+RUN git checkout production
 RUN pip3 install -r requirements.txt
 RUN python3 setup.py install
 
 # ListenBrainz
 WORKDIR /code/listenbrainz
 COPY ./requirements.txt .
 RUN pip3 install -r requirements.txt
-RUN pip3 install setuptools==36.0.1
+
+# Node
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
+    apt-get install -y nodejs && rm -rf /var/lib/apt/lists/*
+RUN mkdir /static
+WORKDIR /static
+COPY package.json package-lock.json webpack.config.js ./listenbrainz/webserver/static /static/
+RUN npm install && npm run build:prod && ./node_modules/less/bin/lessc --clean-css /static/css/main.less > /static/css/main.css && \
+    rm -rf node_modules js/*.jsx *.json webpack.config.js && npm cache clean --force
 
 COPY . /code/listenbrainz
 WORKDIR /code/listenbrainz
+RUN rm -rf ./listenbrainz/webserver/static/
+
 
 # Sometimes the local copy of config.py[c] gets in the way. Better nuke it to not conflict.
 RUN rm -f /code/listenbrainz/listenbrainz/config.py /code/listenbrainz/listenbrainz/config.pyc
@@ -64,18 +75,6 @@ RUN touch /var/log/stats.log /var/log/dump_create.log && chown listenbrainz:list
 # http://smarden.org/runit/runsv.8.html
 RUN touch /etc/service/cron/down
 
-# Node
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
-RUN apt-get install -y nodejs
-RUN npm install -g [email protected]
-RUN npm install less-plugin-clean-css
-
-# Compile the CSS
-# Note: this will not persist because /code/listenbrainz is a volume
-# After bringing up a container, run this again inside the container
-# to compile the less files
-RUN lessc --clean-css listenbrainz/webserver/static/css/main.less > listenbrainz/webserver/static/css/main.css
-
 # Consul Template service is already set up with the base image.
 # Just need to copy the configuration.
 COPY ./docker/consul-template.conf /etc/consul-template.conf

Dockerfile.webpack

@@ -0,0 +1,7 @@
+FROM node:10.15-alpine
+
+RUN mkdir /code
+WORKDIR /code
+
+COPY package.json package-lock.json webpack.config.js /code/
+RUN npm install

admin/sql/create_extensions.sql

@@ -1 +1 @@
-CREATE EXTENSION "uuid-ossp";
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

admin/sql/create_foreign_keys.sql

@@ -18,4 +18,16 @@ ALTER TABLE statistics.user
     REFERENCES "user" (id)
     ON DELETE CASCADE;
 
+ALTER TABLE spotify_auth
+    ADD CONSTRAINT spotify_auth_user_id_foreign_key
+    FOREIGN KEY (user_id)
+    REFERENCES "user" (id)
+    ON DELETE CASCADE;
+
+ALTER TABLE follow_list
+    ADD CONSTRAINT follow_list_user_id_foreign_key
+    FOREIGN KEY (creator)
+    REFERENCES "user" (id)
+    ON DELETE CASCADE;
+
 COMMIT;

admin/sql/create_indexes.sql

@@ -2,6 +2,7 @@ BEGIN;
 
 CREATE UNIQUE INDEX auth_token_ndx_user ON "user" (auth_token);
 CREATE UNIQUE INDEX lower_musicbrainz_id_ndx_user ON "user" (lower(musicbrainz_id));
+CREATE UNIQUE INDEX login_id_ndx_user ON "user" (login_id);
 
 CREATE UNIQUE INDEX token_ndx_token ON api_compat.token (token);
 CREATE UNIQUE INDEX token_api_key_ndx_token ON api_compat.token (token, api_key);
@@ -14,4 +15,9 @@ CREATE UNIQUE INDEX msid_ndx_artist_stats ON statistics.artist (msid);
 CREATE UNIQUE INDEX msid_ndx_release_stats ON statistics.release (msid);
 CREATE UNIQUE INDEX msid_ndx_recording_stats ON statistics.recording (msid);
 
+CREATE INDEX latest_listened_at_spotify_auth ON spotify_auth (latest_listened_at DESC NULLS LAST);
+
+CREATE INDEX creator_ndx_follow_list ON follow_list (creator);
+CREATE INDEX last_saved_ndx_follow_list ON follow_list (last_saved DESC);
+
 COMMIT;

admin/sql/create_primary_keys.sql

@@ -2,6 +2,8 @@ BEGIN;
 
 ALTER TABLE "user" ADD CONSTRAINT user_pkey PRIMARY KEY (id);
 ALTER TABLE api_compat.session ADD CONSTRAINT session_pkey PRIMARY KEY (sid);
+ALTER TABLE spotify_auth ADD CONSTRAINT spotify_auth_pkey PRIMARY KEY (user_id);
+ALTER TABLE follow_list ADD CONSTRAINT follow_list_pkey PRIMARY KEY (id);
 
 ALTER TABLE statistics.user ADD CONSTRAINT stats_user_pkey PRIMARY KEY (user_id);
 ALTER TABLE statistics.artist ADD CONSTRAINT stats_artist_pkey PRIMARY KEY (id);

admin/sql/create_tables.sql

@@ -1,17 +1,31 @@
 BEGIN;
 
 CREATE TABLE "user" (
-  id             SERIAL,
-  created        TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
-  musicbrainz_id VARCHAR NOT NULL,
-  auth_token     VARCHAR,
-  last_login     TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-  latest_import  TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT TIMESTAMP 'epoch',
-  gdpr_agreed    TIMESTAMP WITH TIME ZONE,
-  musicbrainz_row_id INTEGER NOT NULL
+  id                    SERIAL,
+  created               TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+  musicbrainz_id        VARCHAR NOT NULL,
+  auth_token            VARCHAR,
+  last_login            TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+  latest_import         TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT TIMESTAMP 'epoch',
+  gdpr_agreed           TIMESTAMP WITH TIME ZONE,
+  musicbrainz_row_id    INTEGER NOT NULL,
+  login_id              TEXT NOT NULL DEFAULT uuid_generate_v4()::text
 );
 ALTER TABLE "user" ADD CONSTRAINT user_musicbrainz_id_key UNIQUE (musicbrainz_id);
 ALTER TABLE "user" ADD CONSTRAINT user_musicbrainz_row_id_key UNIQUE (musicbrainz_row_id);
+ALTER TABLE "user" ADD CONSTRAINT user_login_id_key UNIQUE (login_id);
+
+CREATE TABLE spotify_auth (
+  user_id                   INTEGER NOT NULL, -- PK and FK to user.id
+  user_token                VARCHAR NOT NULL,
+  token_expires             TIMESTAMP WITH TIME ZONE,
+  refresh_token             VARCHAR NOT NULL,
+  last_updated              TIMESTAMP WITH TIME ZONE,
+  latest_listened_at        TIMESTAMP WITH TIME ZONE,
+  record_listens            BOOLEAN DEFAULT TRUE,
+  error_message             VARCHAR,
+  permission                VARCHAR NOT NULL
+);
 
 CREATE TABLE api_compat.token (
      id               SERIAL,
@@ -74,10 +88,20 @@ CREATE TABLE statistics.recording (
 );
 ALTER TABLE statistics.recording ADD CONSTRAINT recording_stats_msid_uniq UNIQUE (msid);
 
-
 CREATE TABLE data_dump (
   id          SERIAL,
   created     TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
 );
 
+CREATE TABLE follow_list (
+  id                SERIAL, -- PK
+  name              TEXT NOT NULL,
+  creator           INTEGER NOT NULL, -- FK to "user".id
+  private           BOOLEAN NOT NULL DEFAULT FALSE,
+  members           INTEGER ARRAY NOT NULL DEFAULT ARRAY[]::INTEGER[],
+  created           TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+  last_saved        TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+);
+ALTER TABLE follow_list ADD CONSTRAINT follow_list_name_creator_key UNIQUE (name, creator);
+
 COMMIT;

admin/sql/drop_tables.sql

@@ -2,5 +2,7 @@ BEGIN;
 
 DROP TABLE IF EXISTS "user"               CASCADE;
 DROP TABLE IF EXISTS data_dump            CASCADE;
+DROP TABLE IF EXISTS spotify_auth         CASCADE;
+DROP TABLE IF EXISTS follow_list          CASCADE;
 
 COMMIT;

admin/sql/updates/2018-06-22-spotify-listen-importer.sql

@@ -0,0 +1,18 @@
+BEGIN;
+
+CREATE TABLE spotify_auth (
+  user_id                   INTEGER NOT NULL, -- PK and FK to user.id
+  user_token                VARCHAR NOT NULL,
+  token_expires             TIMESTAMP WITH TIME ZONE,
+  refresh_token             VARCHAR NOT NULL,
+  last_updated              TIMESTAMP WITH TIME ZONE,
+  latest_listened_at        TIMESTAMP WITH TIME ZONE,
+  active                    BOOLEAN DEFAULT TRUE,
+  error_message             VARCHAR
+);
+
+ALTER TABLE spotify_auth ADD CONSTRAINT spotify_auth_user_id_foreign_key FOREIGN KEY (user_id) REFERENCES "user" (id);
+ALTER TABLE spotify_auth ADD CONSTRAINT spotify_auth_pkey PRIMARY KEY (user_id);
+CREATE INDEX latest_listened_at_spotify_auth ON spotify_auth (latest_listened_at DESC NULLS LAST);
+
+COMMIT;

admin/sql/updates/2018-11-17-add-on-delete-cascade-to-spotify-fk.sql

@@ -0,0 +1,12 @@
+BEGIN;
+
+ALTER TABLE spotify_auth
+    DROP CONSTRAINT spotify_auth_user_id_foreign_key;
+
+ALTER TABLE spotify_auth
+    ADD CONSTRAINT spotify_auth_user_id_foreign_key
+    FOREIGN KEY (user_id)
+    REFERENCES "user" (id)
+    ON DELETE CASCADE;
+
+COMMIT;

admin/sql/updates/2019-01-04-add-user-login-id.sql

@@ -0,0 +1,6 @@
+BEGIN;
+
+-- Add column for alternative user id for login purpose to user table
+ALTER TABLE "user" ADD COLUMN user_login_id UUID NOT NULL DEFAULT uuid_generate_v4();
+
+COMMIT;

admin/sql/updates/2019-02-08-save-spotify-permissions.sql

@@ -0,0 +1,14 @@
+BEGIN;
+
+-- rename active to record_listens
+ALTER TABLE spotify_auth RENAME active TO record_listens;
+
+-- add column permission to table
+ALTER TABLE spotify_auth ADD COLUMN permission VARCHAR;
+-- set value to 'user-read-recently-played' for all current users
+UPDATE spotify_auth
+   SET permission = 'user-read-recently-played';
+-- set the column as not null
+ALTER TABLE spotify_auth ALTER COLUMN permission SET NOT NULL;
+
+COMMIT;

admin/sql/updates/2019-02-13-change-login-id-type.sql

@@ -0,0 +1,13 @@
+BEGIN;
+
+ALTER TABLE "user" DROP COLUMN user_login_id;
+ALTER TABLE "user" ADD COLUMN login_id TEXT NOT NULL DEFAULT uuid_generate_v4()::text;
+
+UPDATE "user"
+   SET login_id = id::text;
+
+ALTER TABLE "user" ADD CONSTRAINT user_login_id_key UNIQUE (login_id);
+CREATE UNIQUE INDEX login_id_ndx_user ON "user" (login_id);
+
+COMMIT;
+

admin/sql/updates/2019-02-26-add-follow-list.sql

@@ -0,0 +1,25 @@
+BEGIN;
+
+CREATE TABLE follow_list (
+  id                SERIAL, -- PK
+  name              TEXT NOT NULL,
+  creator           INTEGER NOT NULL, -- FK to "user".id
+  private           BOOLEAN NOT NULL DEFAULT FALSE,
+  members           INTEGER ARRAY NOT NULL DEFAULT ARRAY[]::INTEGER[],
+  created           TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+  last_saved        TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+);
+
+ALTER TABLE follow_list ADD CONSTRAINT follow_list_name_creator_key UNIQUE (name, creator);
+ALTER TABLE follow_list ADD CONSTRAINT follow_list_pkey PRIMARY KEY (id);
+
+ALTER TABLE follow_list
+    ADD CONSTRAINT follow_list_user_id_foreign_key
+    FOREIGN KEY (creator)
+    REFERENCES "user" (id)
+    ON DELETE CASCADE;
+
+CREATE INDEX creator_ndx_follow_list ON follow_list (creator);
+CREATE INDEX last_saved_ndx_follow_list ON follow_list (last_saved DESC);
+
+COMMIT;