Merge pull request #235 from Joystream/dev

Release: `v2.0.0`
Joystream · Aug 8, 2023 · c719595 · c719595
2 parents d4e215a + 3d9f5e1
commit c719595
Show file tree

Hide file tree

Showing 54 changed files with 1,372 additions and 628 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
@@ -29,7 +29,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
@@ -24,7 +24,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,15 @@
+### 2.0.0
+
+- Updates minimum required NodeJS version to `v18`.
+- Adds `referredChannels` field in `GET /channels/{joystreamChannelId}` endpoint's response.
+- Adds configuration option to enable/disable syncing by operators.
+- Introduced feature to sync videos of only `Verified` channels.
+- Introduced feature for per channel based sync limits.
+- **FIX**: Adds validation against multiple signups by same Joystream channel.
+- **FIX**: Hide elasticsearch credentials in the logs.
+- **FIX**: Make `avatar` optional in create membership (`POST /membership`) request.
+- **FIX**: Youtube Quota usage query endpoint.
+
 ### 1.5.0
 
 - Adds new `POST /membership` endpoint for captcha-free membership creation of verified YPP users. Requires Faucet endpoint (`joystream.faucet.endpoint`) and Bearer Authentication token (`joystream.faucet.captchaBypassKey`) in the configuration, as the request handler calls Joystream Faucet as an authenticated actor.

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Use the specified image as base
-FROM node:14
+FROM node:18
 
 # Set the working directory to /youtube-synch
 WORKDIR /youtube-synch

diff --git a/config.yml b/config.yml
@@ -1,8 +1,19 @@
 endpoints:
   queryNode: http://localhost:8080/graphql
   joystreamNodeWs: ws://localhost:9944
-directories:
-  assets: ./local/data
+sync:
+  enable: true
+  downloadsDir: ./local/data
+  intervals: # in minutes
+    youtubePolling: 30
+    contentProcessing: 1
+  limits:
+    dailyApiQuota:
+      sync: 9500
+      signup: 500
+    maxConcurrentDownloads: 50
+    maxConcurrentUploads: 50
+    storage: 100G
 logs:
   file:
     level: debug
@@ -17,9 +28,6 @@ logs:
   #   auth:
   #     username: elastic-username
   #     password: elastic-password
-intervals: # in minutes
-  youtubePolling: 30
-  contentProcessing: 1
 youtube:
   clientId: google-client-id
   clientSecret: google-client-secret
@@ -31,16 +39,6 @@ aws:
   credentials:
     accessKeyId: access-key-id
     secretAccessKey: secret-access-key
-limits:
-  # Total daily Youtube quota is 10,000 which is shared between signup(5%) and sync(98%) services
-  dailyApiQuota:
-    sync: 9500
-    signup: 500
-  # How many videos should be concurrently downloaded from Youtube to be prepared for upload to Joystream
-  maxConcurrentDownloads: 50
-  # Max no. of videos that should be concurrently uploaded to Joystream's storage node
-  maxConcurrentUploads: 50
-  storage: 100G
 httpApi:
   port: 3001
   ownerKey: ypp-owner-key
@@ -50,7 +48,6 @@ joystream:
     captchaBypassKey: some-random-key
   app:
     name: app-name
-    # App auth key used for signing App Actions
     accountSeed: 'example_string_seed'
   channelCollaborator:
     memberId: collaborator-member-id

diff --git a/docs/config/definition-properties-joystream-properties-app.md b/docs/config/definition-properties-joystream-properties-app.md
@@ -11,7 +11,7 @@
 
 ## name
 
-Name of the application
+Name of the app
 
 `name`
 
@@ -29,7 +29,7 @@ Name of the application
 
 ## accountSeed
 
-Specifies the application auth key's string seed for generating ed25519 keypair
+Specifies the app auth key's string seed, for generating ed25519 keypair, to be used for signing App Actions
 
 `accountSeed`
 

diff --git a/...efinition-properties-joystream-properties-faucet-properties-captchabypasskey.md b/...efinition-properties-joystream-properties-faucet-properties-captchabypasskey.md
@@ -0,0 +1,3 @@
+## captchaBypassKey Type
+
+`string`
diff --git a/...config/definition-properties-joystream-properties-faucet-properties-endpoint.md b/...config/definition-properties-joystream-properties-faucet-properties-endpoint.md
@@ -0,0 +1,3 @@
+## endpoint Type
+
+`string`
diff --git a/docs/config/definition-properties-joystream-properties-faucet.md b/docs/config/definition-properties-joystream-properties-faucet.md
@@ -0,0 +1,46 @@
+## faucet Type
+
+`object` ([Details](definition-properties-joystream-properties-faucet.md))
+
+# faucet Properties
+
+| Property                              | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                                   |
+| :------------------------------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [endpoint](#endpoint)                 | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-endpoint.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/endpoint")                 |
+| [captchaBypassKey](#captchabypasskey) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-captchabypasskey.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/captchaBypassKey") |
+
+## endpoint
+
+Joystream's faucet URL
+
+`endpoint`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-endpoint.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/endpoint")
+
+### endpoint Type
+
+`string`
+
+## captchaBypassKey
+
+Bearer Authentication Key needed to bypass captcha verification on Faucet
+
+`captchaBypassKey`
+
+*   is required
+
+*   Type: `string`
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-captchabypasskey.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/captchaBypassKey")
+
+### captchaBypassKey Type
+
+`string`
diff --git a/docs/config/definition-properties-joystream.md b/docs/config/definition-properties-joystream.md
@@ -6,12 +6,31 @@
 
 | Property                                    | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                                             |
 | :------------------------------------------ | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [faucet](#faucet)                           | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet")                                                                   |
 | [app](#app)                                 | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-app.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/app")                                                                         |
 | [channelCollaborator](#channelcollaborator) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-joystream-channel-collaborator-used-for-syncing-the-content.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/channelCollaborator") |
 
+## faucet
+
+Joystream's faucet configuration (needed for captcha-free membership creation)
+
+`faucet`
+
+*   is required
+
+*   Type: `object` ([Details](definition-properties-joystream-properties-faucet.md))
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet")
+
+### faucet Type
+
+`object` ([Details](definition-properties-joystream-properties-faucet.md))
+
 ## app
 
-Joystream metaprotocol application specific configuration
+Joystream Metaprotocol App specific configuration
 
 `app`
 

diff --git a/...ion-properties-yt-synch-syncronization-related-settings-if-properties-enable.md b/...ion-properties-yt-synch-syncronization-related-settings-if-properties-enable.md
@@ -0,0 +1,11 @@
+## enable Type
+
+unknown
+
+## enable Constraints
+
+**constant**: the value of this property must be equal to:
+
+```json
+true
+```
diff --git a/...definition-properties-yt-synch-syncronization-related-settings-if-properties.md b/...definition-properties-yt-synch-syncronization-related-settings-if-properties.md
@@ -0,0 +1,3 @@
+## properties Type
+
+unknown
diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-if.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-if.md
@@ -0,0 +1,35 @@
+## if Type
+
+unknown
+
+# if Properties
+
+| Property          | Type          | Required | Nullable       | Defined by                                                                                                                                                                                                           |
+| :---------------- | :------------ | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [enable](#enable) | Not specified | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-if-properties-enable.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/if/properties/enable") |
+
+## enable
+
+
+
+`enable`
+
+*   is optional
+
+*   Type: unknown
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-if-properties-enable.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/if/properties/enable")
+
+### enable Type
+
+unknown
+
+### enable Constraints
+
+**constant**: the value of this property must be equal to:
+
+```json
+true
+```
diff --git a/...-properties-yt-synch-syncronization-related-settings-properties-downloadsdir.md b/...-properties-yt-synch-syncronization-related-settings-properties-downloadsdir.md
@@ -0,0 +1,3 @@
+## downloadsDir Type
+
+`string`
diff --git a/...nition-properties-yt-synch-syncronization-related-settings-properties-enable.md b/...nition-properties-yt-synch-syncronization-related-settings-properties-enable.md
@@ -0,0 +1,11 @@
+## enable Type
+
+`boolean`
+
+## enable Default Value
+
+The default value is:
+
+```json
+true
+```
diff --git a/...onization-related-settings-properties-intervals-properties-contentprocessing.md b/...onization-related-settings-properties-intervals-properties-contentprocessing.md
@@ -0,0 +1,7 @@
+## contentProcessing Type
+
+`integer`
+
+## contentProcessing Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
diff --git a/...ncronization-related-settings-properties-intervals-properties-youtubepolling.md b/...ncronization-related-settings-properties-intervals-properties-youtubepolling.md
@@ -0,0 +1,7 @@
+## youtubePolling Type
+
+`integer`
+
+## youtubePolling Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
diff --git a/...ion-properties-yt-synch-syncronization-related-settings-properties-intervals.md b/...ion-properties-yt-synch-syncronization-related-settings-properties-intervals.md
@@ -0,0 +1,54 @@
+## intervals Type
+
+`object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md))
+
+# intervals Properties
+
+| Property                                | Type      | Required | Nullable       | Defined by                                                                                                                                                                                                                                                                     |
+| :-------------------------------------- | :-------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [youtubePolling](#youtubepolling)       | `integer` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-youtubepolling.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals/properties/youtubePolling")       |
+| [contentProcessing](#contentprocessing) | `integer` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-contentprocessing.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals/properties/contentProcessing") |
+
+## youtubePolling
+
+After how many minutes, the polling service should poll the Youtube api for channels state update
+
+`youtubePolling`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-youtubepolling.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals/properties/youtubePolling")
+
+### youtubePolling Type
+
+`integer`
+
+### youtubePolling Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
+
+## contentProcessing
+
+After how many minutes, the service should scan the database for new content to start downloading, on-chain creation & uploading to storage node
+
+`contentProcessing`
+
+*   is required
+
+*   Type: `integer`
+
+*   cannot be null
+
+*   defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-contentprocessing.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals/properties/contentProcessing")
+
+### contentProcessing Type
+
+`integer`
+
+### contentProcessing Constraints
+
+**minimum**: the value of this number must greater than or equal to: `1`
diff --git a/...ization-related-settings-properties-limits-properties-maxconcurrentdownloads.md b/...ization-related-settings-properties-limits-properties-maxconcurrentdownloads.md
@@ -0,0 +1,11 @@
+## maxConcurrentDownloads Type
+
+`number`
+
+## maxConcurrentDownloads Default Value
+
+The default value is:
+
+```json
+50
+```
diff --git a/...onization-related-settings-properties-limits-properties-maxconcurrentuploads.md b/...onization-related-settings-properties-limits-properties-maxconcurrentuploads.md
@@ -0,0 +1,11 @@
+## maxConcurrentUploads Type
+
+`number`
+
+## maxConcurrentUploads Default Value
+
+The default value is:
+
+```json
+50
+```
diff --git a/...ube-api-quota-rationing-scheme-for-youtube-partner-program-properties-signup.md b/...ube-api-quota-rationing-scheme-for-youtube-partner-program-properties-signup.md
@@ -0,0 +1,11 @@
+## signup Type
+
+`number`
+
+## signup Default Value
+
+The default value is:
+
+```json
+500
+```