diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f67467d2..65eb8270 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [14.x] + node-version: [18.x] steps: - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} @@ -29,7 +29,7 @@ jobs: runs-on: macos-latest strategy: matrix: - node-version: [14.x] + node-version: [18.x] steps: - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7077eb69..d05ca7d2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [14.x] + node-version: [18.x] steps: - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} @@ -24,7 +24,7 @@ jobs: runs-on: macos-latest strategy: matrix: - node-version: [14.x] + node-version: [18.x] steps: - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index dff558a2..ea7533d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +### 2.0.0 + +- Updates minimum required NodeJS version to `v18`. +- Adds `referredChannels` field in `GET /channels/{joystreamChannelId}` endpoint's response. +- Adds configuration option to enable/disable syncing by operators. +- Introduced feature to sync videos of only `Verified` channels. +- Introduced feature for per channel based sync limits. +- **FIX**: Adds validation against multiple signups by same Joystream channel. +- **FIX**: Hide elasticsearch credentials in the logs. +- **FIX**: Make `avatar` optional in create membership (`POST /membership`) request. +- **FIX**: Youtube Quota usage query endpoint. + ### 1.5.0 - Adds new `POST /membership` endpoint for captcha-free membership creation of verified YPP users. Requires Faucet endpoint (`joystream.faucet.endpoint`) and Bearer Authentication token (`joystream.faucet.captchaBypassKey`) in the configuration, as the request handler calls Joystream Faucet as an authenticated actor. diff --git a/Dockerfile b/Dockerfile index c2f084ba..d533ea0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Use the specified image as base -FROM node:14 +FROM node:18 # Set the working directory to /youtube-synch WORKDIR /youtube-synch diff --git a/config.yml b/config.yml index 0d31b4c6..d8722265 100644 --- a/config.yml +++ b/config.yml @@ -1,8 +1,19 @@ endpoints: queryNode: http://localhost:8080/graphql joystreamNodeWs: ws://localhost:9944 -directories: - assets: ./local/data +sync: + enable: true + downloadsDir: ./local/data + intervals: # in minutes + youtubePolling: 30 + contentProcessing: 1 + limits: + dailyApiQuota: + sync: 9500 + signup: 500 + maxConcurrentDownloads: 50 + maxConcurrentUploads: 50 + storage: 100G logs: file: level: debug @@ -17,9 +28,6 @@ logs: # auth: # username: elastic-username # password: elastic-password -intervals: # in minutes - youtubePolling: 30 - contentProcessing: 1 youtube: clientId: google-client-id clientSecret: google-client-secret @@ -31,16 +39,6 @@ aws: credentials: accessKeyId: access-key-id secretAccessKey: secret-access-key -limits: - # Total daily Youtube quota is 10,000 which is shared between signup(5%) and sync(98%) services - dailyApiQuota: - sync: 9500 - signup: 500 - # How many videos should be concurrently downloaded from Youtube to be prepared for upload to Joystream - maxConcurrentDownloads: 50 - # Max no. of videos that should be concurrently uploaded to Joystream's storage node - maxConcurrentUploads: 50 - storage: 100G httpApi: port: 3001 ownerKey: ypp-owner-key @@ -50,7 +48,6 @@ joystream: captchaBypassKey: some-random-key app: name: app-name - # App auth key used for signing App Actions accountSeed: 'example_string_seed' channelCollaborator: memberId: collaborator-member-id diff --git a/docs/config/definition-properties-joystream-properties-app.md b/docs/config/definition-properties-joystream-properties-app.md index be24108e..026dd6cf 100644 --- a/docs/config/definition-properties-joystream-properties-app.md +++ b/docs/config/definition-properties-joystream-properties-app.md @@ -11,7 +11,7 @@ ## name -Name of the application +Name of the app `name` @@ -29,7 +29,7 @@ Name of the application ## accountSeed -Specifies the application auth key's string seed for generating ed25519 keypair +Specifies the app auth key's string seed, for generating ed25519 keypair, to be used for signing App Actions `accountSeed` diff --git a/docs/config/definition-properties-joystream-properties-faucet-properties-captchabypasskey.md b/docs/config/definition-properties-joystream-properties-faucet-properties-captchabypasskey.md new file mode 100644 index 00000000..c450d7b9 --- /dev/null +++ b/docs/config/definition-properties-joystream-properties-faucet-properties-captchabypasskey.md @@ -0,0 +1,3 @@ +## captchaBypassKey Type + +`string` diff --git a/docs/config/definition-properties-joystream-properties-faucet-properties-endpoint.md b/docs/config/definition-properties-joystream-properties-faucet-properties-endpoint.md new file mode 100644 index 00000000..00e8b7f7 --- /dev/null +++ b/docs/config/definition-properties-joystream-properties-faucet-properties-endpoint.md @@ -0,0 +1,3 @@ +## endpoint Type + +`string` diff --git a/docs/config/definition-properties-joystream-properties-faucet.md b/docs/config/definition-properties-joystream-properties-faucet.md new file mode 100644 index 00000000..3abaa799 --- /dev/null +++ b/docs/config/definition-properties-joystream-properties-faucet.md @@ -0,0 +1,46 @@ +## faucet Type + +`object` ([Details](definition-properties-joystream-properties-faucet.md)) + +# faucet Properties + +| Property | Type | Required | Nullable | Defined by | +| :------------------------------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [endpoint](#endpoint) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-endpoint.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/endpoint") | +| [captchaBypassKey](#captchabypasskey) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-captchabypasskey.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/captchaBypassKey") | + +## endpoint + +Joystream's faucet URL + +`endpoint` + +* is required + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-endpoint.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/endpoint") + +### endpoint Type + +`string` + +## captchaBypassKey + +Bearer Authentication Key needed to bypass captcha verification on Faucet + +`captchaBypassKey` + +* is required + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-captchabypasskey.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/captchaBypassKey") + +### captchaBypassKey Type + +`string` diff --git a/docs/config/definition-properties-joystream.md b/docs/config/definition-properties-joystream.md index b297c7d3..c7b016db 100644 --- a/docs/config/definition-properties-joystream.md +++ b/docs/config/definition-properties-joystream.md @@ -6,12 +6,31 @@ | Property | Type | Required | Nullable | Defined by | | :------------------------------------------ | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [faucet](#faucet) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet") | | [app](#app) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-app.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/app") | | [channelCollaborator](#channelcollaborator) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-joystream-channel-collaborator-used-for-syncing-the-content.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/channelCollaborator") | +## faucet + +Joystream's faucet configuration (needed for captcha-free membership creation) + +`faucet` + +* is required + +* Type: `object` ([Details](definition-properties-joystream-properties-faucet.md)) + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet") + +### faucet Type + +`object` ([Details](definition-properties-joystream-properties-faucet.md)) + ## app -Joystream metaprotocol application specific configuration +Joystream Metaprotocol App specific configuration `app` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-if-properties-enable.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-if-properties-enable.md new file mode 100644 index 00000000..6c619b30 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-if-properties-enable.md @@ -0,0 +1,11 @@ +## enable Type + +unknown + +## enable Constraints + +**constant**: the value of this property must be equal to: + +```json +true +``` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-if-properties.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-if-properties.md new file mode 100644 index 00000000..c89940c6 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-if-properties.md @@ -0,0 +1,3 @@ +## properties Type + +unknown diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-if.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-if.md new file mode 100644 index 00000000..24777048 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-if.md @@ -0,0 +1,35 @@ +## if Type + +unknown + +# if Properties + +| Property | Type | Required | Nullable | Defined by | +| :---------------- | :------------ | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [enable](#enable) | Not specified | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-if-properties-enable.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/if/properties/enable") | + +## enable + + + +`enable` + +* is optional + +* Type: unknown + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-if-properties-enable.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/if/properties/enable") + +### enable Type + +unknown + +### enable Constraints + +**constant**: the value of this property must be equal to: + +```json +true +``` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-downloadsdir.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-downloadsdir.md new file mode 100644 index 00000000..56d87436 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-downloadsdir.md @@ -0,0 +1,3 @@ +## downloadsDir Type + +`string` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-enable.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-enable.md new file mode 100644 index 00000000..c1a32bad --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-enable.md @@ -0,0 +1,11 @@ +## enable Type + +`boolean` + +## enable Default Value + +The default value is: + +```json +true +``` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-contentprocessing.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-contentprocessing.md new file mode 100644 index 00000000..69f9131b --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-contentprocessing.md @@ -0,0 +1,7 @@ +## contentProcessing Type + +`integer` + +## contentProcessing Constraints + +**minimum**: the value of this number must greater than or equal to: `1` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-youtubepolling.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-youtubepolling.md new file mode 100644 index 00000000..ca51cf21 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-youtubepolling.md @@ -0,0 +1,7 @@ +## youtubePolling Type + +`integer` + +## youtubePolling Constraints + +**minimum**: the value of this number must greater than or equal to: `1` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md new file mode 100644 index 00000000..2786cdb8 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md @@ -0,0 +1,54 @@ +## intervals Type + +`object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md)) + +# intervals Properties + +| Property | Type | Required | Nullable | Defined by | +| :-------------------------------------- | :-------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [youtubePolling](#youtubepolling) | `integer` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-youtubepolling.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals/properties/youtubePolling") | +| [contentProcessing](#contentprocessing) | `integer` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-contentprocessing.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals/properties/contentProcessing") | + +## youtubePolling + +After how many minutes, the polling service should poll the Youtube api for channels state update + +`youtubePolling` + +* is required + +* Type: `integer` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-youtubepolling.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals/properties/youtubePolling") + +### youtubePolling Type + +`integer` + +### youtubePolling Constraints + +**minimum**: the value of this number must greater than or equal to: `1` + +## contentProcessing + +After how many minutes, the service should scan the database for new content to start downloading, on-chain creation & uploading to storage node + +`contentProcessing` + +* is required + +* Type: `integer` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals-properties-contentprocessing.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals/properties/contentProcessing") + +### contentProcessing Type + +`integer` + +### contentProcessing Constraints + +**minimum**: the value of this number must greater than or equal to: `1` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md new file mode 100644 index 00000000..49c3b535 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md @@ -0,0 +1,11 @@ +## maxConcurrentDownloads Type + +`number` + +## maxConcurrentDownloads Default Value + +The default value is: + +```json +50 +``` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md new file mode 100644 index 00000000..69300541 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md @@ -0,0 +1,11 @@ +## maxConcurrentUploads Type + +`number` + +## maxConcurrentUploads Default Value + +The default value is: + +```json +50 +``` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-signup.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-signup.md new file mode 100644 index 00000000..2034f3ff --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-signup.md @@ -0,0 +1,11 @@ +## signup Type + +`number` + +## signup Default Value + +The default value is: + +```json +500 +``` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-sync.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-sync.md new file mode 100644 index 00000000..9666a894 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-sync.md @@ -0,0 +1,11 @@ +## sync Type + +`number` + +## sync Default Value + +The default value is: + +```json +9500 +``` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md new file mode 100644 index 00000000..7fe47de6 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md @@ -0,0 +1,62 @@ +## dailyApiQuota Type + +`object` ([Specifies daily Youtube API quota rationing scheme for Youtube Partner Program](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md)) + +# dailyApiQuota Properties + +| Property | Type | Required | Nullable | Defined by | +| :---------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [sync](#sync) | `number` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-sync.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota/properties/sync") | +| [signup](#signup) | `number` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-signup.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota/properties/signup") | + +## sync + + + +`sync` + +* is required + +* Type: `number` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-sync.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota/properties/sync") + +### sync Type + +`number` + +### sync Default Value + +The default value is: + +```json +9500 +``` + +## signup + + + +`signup` + +* is required + +* Type: `number` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program-properties-signup.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota/properties/signup") + +### signup Type + +`number` + +### signup Default Value + +The default value is: + +```json +500 +``` diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md new file mode 100644 index 00000000..03b17786 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md @@ -0,0 +1,13 @@ +## storage Type + +`string` + +## storage Constraints + +**pattern**: the string must match the following regular expression: + +```regexp +^[0-9]+(B|K|M|G|T)$ +``` + +[try pattern](https://regexr.com/?expression=%5E%5B0-9%5D%2B\(B%7CK%7CM%7CG%7CT\)%24 "try regular expression with regexr.com") diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits.md new file mode 100644 index 00000000..64a5f51d --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-properties-limits.md @@ -0,0 +1,110 @@ +## limits Type + +`object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-limits.md)) + +# limits Properties + +| Property | Type | Required | Nullable | Defined by | +| :------------------------------------------------ | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [dailyApiQuota](#dailyapiquota) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota") | +| [maxConcurrentDownloads](#maxconcurrentdownloads) | `number` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentDownloads") | +| [maxConcurrentUploads](#maxconcurrentuploads) | `number` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentUploads") | +| [storage](#storage) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/storage") | + +## dailyApiQuota + +Specifies daily Youtube API quota rationing scheme for Youtube Partner Program + +`dailyApiQuota` + +* is required + +* Type: `object` ([Specifies daily Youtube API quota rationing scheme for Youtube Partner Program](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md)) + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/dailyApiQuota") + +### dailyApiQuota Type + +`object` ([Specifies daily Youtube API quota rationing scheme for Youtube Partner Program](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-specifies-daily-youtube-api-quota-rationing-scheme-for-youtube-partner-program.md)) + +## maxConcurrentDownloads + +Max no. of videos that should be concurrently downloaded from Youtube to be prepared for upload to Joystream + +`maxConcurrentDownloads` + +* is required + +* Type: `number` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentdownloads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentDownloads") + +### maxConcurrentDownloads Type + +`number` + +### maxConcurrentDownloads Default Value + +The default value is: + +```json +50 +``` + +## maxConcurrentUploads + +Max no. of videos that should be concurrently uploaded to Joystream's storage node + +`maxConcurrentUploads` + +* is required + +* Type: `number` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-maxconcurrentuploads.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/maxConcurrentUploads") + +### maxConcurrentUploads Type + +`number` + +### maxConcurrentUploads Default Value + +The default value is: + +```json +50 +``` + +## storage + +Maximum total size of all downloaded assets stored in `downloadsDir` + +`storage` + +* is required + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits-properties-storage.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits/properties/storage") + +### storage Type + +`string` + +### storage Constraints + +**pattern**: the string must match the following regular expression: + +```regexp +^[0-9]+(B|K|M|G|T)$ +``` + +[try pattern](https://regexr.com/?expression=%5E%5B0-9%5D%2B\(B%7CK%7CM%7CG%7CT\)%24 "try regular expression with regexr.com") diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings-then.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings-then.md new file mode 100644 index 00000000..139b21d9 --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings-then.md @@ -0,0 +1,3 @@ +## then Type + +unknown diff --git a/docs/config/definition-properties-yt-synch-syncronization-related-settings.md b/docs/config/definition-properties-yt-synch-syncronization-related-settings.md new file mode 100644 index 00000000..1e0578dd --- /dev/null +++ b/docs/config/definition-properties-yt-synch-syncronization-related-settings.md @@ -0,0 +1,92 @@ +## sync Type + +`object` ([YT-synch syncronization related settings](definition-properties-yt-synch-syncronization-related-settings.md)) + +# sync Properties + +| Property | Type | Required | Nullable | Defined by | +| :---------------------------- | :-------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [enable](#enable) | `boolean` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-enable.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/enable") | +| [downloadsDir](#downloadsdir) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-downloadsdir.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/downloadsDir") | +| [intervals](#intervals) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals") | +| [limits](#limits) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits") | + +## enable + +Option to enable/disable syncing while starting the service + +`enable` + +* is required + +* Type: `boolean` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-enable.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/enable") + +### enable Type + +`boolean` + +### enable Default Value + +The default value is: + +```json +true +``` + +## downloadsDir + +Path to a directory where all the downloaded assets will be stored + +`downloadsDir` + +* is optional + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-downloadsdir.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/downloadsDir") + +### downloadsDir Type + +`string` + +## intervals + +Specifies how often periodic tasks (for example youtube state polling) are executed. + +`intervals` + +* is optional + +* Type: `object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md)) + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/intervals") + +### intervals Type + +`object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-intervals.md)) + +## limits + +Specifies youtube-synch service limits. + +`limits` + +* is optional + +* Type: `object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-limits.md)) + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings-properties-limits.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync/properties/limits") + +### limits Type + +`object` ([Details](definition-properties-yt-synch-syncronization-related-settings-properties-limits.md)) diff --git a/docs/config/definition.md b/docs/config/definition.md index 7f2564b4..62cb9547 100644 --- a/docs/config/definition.md +++ b/docs/config/definition.md @@ -8,14 +8,12 @@ | :-------------------------------------------------------------- | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [joystream](#joystream) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream") | | [endpoints](#endpoints) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-endpoints.md "https://joystream.org/schemas/youtube-synch/config#/properties/endpoints") | -| [directories](#directories) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-directories.md "https://joystream.org/schemas/youtube-synch/config#/properties/directories") | | [logs](#logs) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-logs.md "https://joystream.org/schemas/youtube-synch/config#/properties/logs") | -| [limits](#limits) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-limits.md "https://joystream.org/schemas/youtube-synch/config#/properties/limits") | -| [intervals](#intervals) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-intervals.md "https://joystream.org/schemas/youtube-synch/config#/properties/intervals") | | [youtube](#youtube) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube") | | [aws](#aws) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-aws-configurations-needed-to-connect-with-dynamodb-instance.md "https://joystream.org/schemas/youtube-synch/config#/properties/aws") | | [creatorOnboardingRequirements](#creatoronboardingrequirements) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-creatoronboardingrequirements.md "https://joystream.org/schemas/youtube-synch/config#/properties/creatorOnboardingRequirements") | | [httpApi](#httpapi) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-public-api-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/httpApi") | +| [sync](#sync) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync") | ## joystream @@ -53,24 +51,6 @@ Specifies external endpoints that the distributor node will connect to `object` ([Details](definition-properties-endpoints.md)) -## directories - -Specifies paths where node's data will be stored - -`directories` - -* is required - -* Type: `object` ([Details](definition-properties-directories.md)) - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-directories.md "https://joystream.org/schemas/youtube-synch/config#/properties/directories") - -### directories Type - -`object` ([Details](definition-properties-directories.md)) - ## logs Specifies the logging configuration @@ -89,42 +69,6 @@ Specifies the logging configuration `object` ([Details](definition-properties-logs.md)) -## limits - -Specifies youtube-synch service limits. - -`limits` - -* is required - -* Type: `object` ([Details](definition-properties-limits.md)) - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-limits.md "https://joystream.org/schemas/youtube-synch/config#/properties/limits") - -### limits Type - -`object` ([Details](definition-properties-limits.md)) - -## intervals - -Specifies how often periodic tasks (for example youtube state polling) are executed. - -`intervals` - -* is required - -* Type: `object` ([Details](definition-properties-intervals.md)) - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-intervals.md "https://joystream.org/schemas/youtube-synch/config#/properties/intervals") - -### intervals Type - -`object` ([Details](definition-properties-intervals.md)) - ## youtube Youtube Oauth2 Client configuration @@ -196,3 +140,21 @@ Public api configuration ### httpApi Type `object` ([Public api configuration](definition-properties-public-api-configuration.md)) + +## sync + +YT-synch's syncronization related settings + +`sync` + +* is required + +* Type: `object` ([YT-synch syncronization related settings](definition-properties-yt-synch-syncronization-related-settings.md)) + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-yt-synch-syncronization-related-settings.md "https://joystream.org/schemas/youtube-synch/config#/properties/sync") + +### sync Type + +`object` ([YT-synch syncronization related settings](definition-properties-yt-synch-syncronization-related-settings.md)) diff --git a/package.json b/package.json index af698b68..c3dd8717 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "youtube-sync", - "version": "1.5.0", + "version": "2.0.0", "license": "MIT", "scripts": { "postpack": "rm -f oclif.manifest.json", @@ -116,7 +116,7 @@ "inquirer": "^8.1.2", "inquirer-date-prompt": "^2.0.0", "iso8601-duration": "^2.1.1", - "json-schema-to-typescript": "^10.1.4", + "json-schema-to-typescript": "^13.0.2", "lodash": "^4.17.21", "moment-timezone": "^0.5.43", "multihashes": "^4.0.3", @@ -139,11 +139,11 @@ "ytpl": "^2.3.0" }, "engines": { - "node": ">=14.0.0", - "yarn": "^1.22.15" + "node": ">=18.6.0", + "yarn": "^1.22.19" }, "volta": { - "node": "14.18.0", - "yarn": "1.22.15" + "node": "18.6.0", + "yarn": "1.22.19" } } diff --git a/src/app/index.ts b/src/app/index.ts index c8c49175..bed92206 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -34,35 +34,36 @@ export class Service { this.logger = this.logging.createLogger('Server') this.queryNodeApi = new QueryNodeApi(config.endpoints.queryNode, this.logging) this.dynamodbService = new DynamodbService(this.config.aws) - this.youtubeApi = YoutubeApi.create(this.config) + this.youtubeApi = YoutubeApi.create(this.config, this.dynamodbService.repo.stats) this.joystreamClient = new JoystreamClient(config, this.youtubeApi, this.queryNodeApi, this.logging) - this.youtubePollingService = new YoutubePollingService( - config, - this.logging, - this.youtubeApi, - this.dynamodbService, - this.joystreamClient - ) - this.contentDownloadService = new ContentDownloadService( - config, - this.logging, - this.dynamodbService, - this.youtubeApi - ) - this.contentCreationService = new ContentCreationService( - config, - this.logging, - this.dynamodbService, - this.contentDownloadService, - this.joystreamClient - ) - this.contentUploadService = new ContentUploadService( - config, - this.logging, - this.dynamodbService, - this.contentDownloadService, - this.queryNodeApi - ) + + if (config.sync.enable) { + this.youtubePollingService = new YoutubePollingService( + this.logging, + this.youtubeApi, + this.dynamodbService, + this.joystreamClient + ) + this.contentDownloadService = new ContentDownloadService( + config.sync, + this.logging, + this.dynamodbService, + this.youtubeApi + ) + this.contentCreationService = new ContentCreationService( + this.logging, + this.dynamodbService, + this.contentDownloadService, + this.joystreamClient + ) + this.contentUploadService = new ContentUploadService( + config.sync, + this.logging, + this.dynamodbService, + this.contentDownloadService, + this.queryNodeApi + ) + } } private checkConfigDir(name: string, path: string): void { @@ -87,18 +88,38 @@ export class Service { } private checkConfigDirectories(): void { - Object.entries(this.config.directories).forEach(([name, path]) => this.checkConfigDir(name, path)) + if (this.config.sync.enable) { + this.checkConfigDir('sync.downloadsDir', this.config.sync.downloadsDir) + } if (this.config.logs?.file) { this.checkConfigDir('logs.file.path', this.config.logs.file.path) } } + private async startSync(): Promise { + if (this.config.sync.enable) { + const { + intervals: { youtubePolling, contentProcessing }, + } = this.config.sync + this.logger.verbose('Starting the Youtube-Synch service', { config: this.hideSecrets(this.config) }) + // Null-assertion is safe here since intervals won't be not null due to Ajv schema validation + await this.youtubePollingService.start(youtubePolling) + await this.contentDownloadService.start(contentProcessing) + await this.contentCreationService.start(contentProcessing) + await this.contentUploadService.start(contentProcessing) + } + } + private hideSecrets(config: Config): DisplaySafeConfig { const displaySafeConfig = { ...config, youtube: _.mapValues(config.youtube, () => '###SECRET###' as const), httpApi: _.mapValues(config.httpApi, () => '###SECRET###' as const), joystream: _.mapValues(config.joystream, () => '###SECRET###' as const), + logs: { + ...config.logs, + elastic: _.mapValues(config.logs?.elastic, () => '###SECRET###' as const), + }, aws: { ...config.aws, credentials: _.mapValues(config.aws?.credentials, () => '###SECRET###' as const), @@ -110,13 +131,9 @@ export class Service { public async start(): Promise { try { - this.checkConfigDirectories() await bootstrapHttpApi(this.config, this.logging, this.dynamodbService, this.queryNodeApi, this.youtubeApi) - this.logger.verbose('Starting the Youtube-Synch service', { config: this.hideSecrets(this.config) }) - await this.youtubePollingService.start() - await this.contentDownloadService.start() - await this.contentCreationService.start() - await this.contentUploadService.start() + this.checkConfigDirectories() + await this.startSync() } catch (err) { this.logger.error('Youtube-Synch service initialization failed!', { err }) process.exit(-1) diff --git a/src/cli/commands/sync/addUnauthorizedChannelForSyncing.ts b/src/cli/commands/sync/addUnauthorizedChannelForSyncing.ts index 0257dd8e..716de8d5 100644 --- a/src/cli/commands/sync/addUnauthorizedChannelForSyncing.ts +++ b/src/cli/commands/sync/addUnauthorizedChannelForSyncing.ts @@ -104,7 +104,7 @@ export default class AddUnauthorizedChannelForSyncing extends RuntimeApiCommandB // Ensure Youtube Channel is not already being synced. try { - const ytCh = await dynamo.channels.getByChannelId(ytChannel.author.channelID) + const ytCh = await dynamo.channels.getById(ytChannel.author.channelID) if (ytCh.performUnauthorizedSync) { await this.requireConfirmation( 'This youtube channel is already being synced. Do you want to redo the syncing?', @@ -140,7 +140,7 @@ export default class AddUnauthorizedChannelForSyncing extends RuntimeApiCommandB }) } try { - const { title } = await dynamo.channels.getByJoystreamChannelId(joystreamChannelId) + const { title } = await dynamo.channels.getByJoystreamId(joystreamChannelId) // Allow to re-sync the same JS channel with the same YT channel if (title !== ytChannel.author.name) { throw new CLIError( @@ -195,7 +195,7 @@ export default class AddUnauthorizedChannelForSyncing extends RuntimeApiCommandB privacyStatus: 'public', uploadStatus: 'processed', joystreamChannelId: c.joystreamChannelId, - liveBroadcastContent: video.isLive ? 'live' : 'none', + liveStreamingDetails: video.isLive || undefined, state: 'New', viewCount: 0, } as YtVideo) diff --git a/src/infrastructure/index.ts b/src/infrastructure/index.ts index 02967655..01a77282 100644 --- a/src/infrastructure/index.ts +++ b/src/infrastructure/index.ts @@ -50,7 +50,7 @@ const channelsTable = new aws.dynamodb.Table('channels', { }, { name: 'phantomKey-createdAt-index', - hashKey: nameof('phantomKey'), // we'll have a single value partition + hashKey: nameof('phantomKey'), // we'll have a single value partition to enable sorting on createdAt rangeKey: nameof('createdAt'), projectionType: 'ALL', }, @@ -59,6 +59,11 @@ const channelsTable = new aws.dynamodb.Table('channels', { name: 'id-index', projectionType: 'ALL', }, + { + hashKey: nameof('referrerChannelId'), + name: 'referrerChannelId-index', + projectionType: 'ALL', + }, ], billingMode: 'PAY_PER_REQUEST', }) diff --git a/src/repository/channel.ts b/src/repository/channel.ts index da9e426b..daa60005 100644 --- a/src/repository/channel.ts +++ b/src/repository/channel.ts @@ -45,7 +45,13 @@ function createChannelModel(tablePrefix: ResourcePrefix) { joystreamChannelLanguageIso: String, // Referrer's Joystream Channel ID - referrerChannelId: Number, + referrerChannelId: { + type: Number, + index: { + type: 'global', + name: 'referrerChannelId-index', + }, + }, // Channel's title title: String, @@ -87,6 +93,9 @@ function createChannelModel(tablePrefix: ResourcePrefix) { }, }, + // total size of historical videos (videos that were published on Youtube before YPP signup) synced + historicalVideoSyncedSize: Number, + thumbnails: { type: Object, schema: { @@ -98,6 +107,9 @@ function createChannelModel(tablePrefix: ResourcePrefix) { }, }, + // Banner or Background image URL + bannerImageUrl: String, + // user access token obtained from authorization code after successful authentication userAccessToken: String, @@ -109,7 +121,14 @@ function createChannelModel(tablePrefix: ResourcePrefix) { // Should this channel be ingested for automated Youtube/Joystream syncing? shouldBeIngested: { type: Boolean, - default: true, + default: false, + }, + + // Should this channel be ingested for automated Youtube/Joystream syncing? (operator managed flag) + // Both `shouldBeIngested` and `allowOperatorIngestion` should be set for sync to work. + allowOperatorIngestion: { + type: Boolean, + default: false, }, // Should this channel be ingested for automated Youtube/Joystream syncing without explicit authorization granted to app? @@ -241,7 +260,7 @@ export class ChannelsService { * @param joystreamChannelId * @returns Returns channel by joystreamChannelId */ - async getByJoystreamChannelId(joystreamChannelId: number): Promise { + async getByJoystreamId(joystreamChannelId: number): Promise { const [result] = await this.channelsRepository.query({ joystreamChannelId }, (q) => q.sort('descending').using('joystreamChannelId-createdAt-index') ) @@ -251,11 +270,40 @@ export class ChannelsService { return result } + /** + * @param joystreamChannelId + * @returns Returns partner channel by joystreamChannelId (if any) + */ + async findPartnerChannelByJoystreamId(joystreamChannelId: number): Promise { + const [result] = await this.channelsRepository.query({ joystreamChannelId }, (q) => + q + .sort('descending') + .filter('yppStatus') + .eq('Verified') + .or() + .filter('yppStatus') + .eq('Unverified') + .using('joystreamChannelId-createdAt-index') + ) + return result || undefined + } + + /** + * @param joystreamChannelId + * @returns Returns list of all channels referred by given joystream channel + */ + async getReferredChannels(referrerChannelId: number): Promise { + const results = await this.channelsRepository.query({ referrerChannelId }, (q) => + q.sort('descending').using('referrerChannelId-index') + ) + return results + } + /** * @param channelId * @returns Returns channel by youtube channelId */ - async getByChannelId(channelId: string): Promise { + async getById(channelId: string): Promise { const result = await this.channelsRepository.get(channelId.toString()) if (!result) { throw new Error(`Could not find channel with id ${channelId}`) diff --git a/src/repository/video.ts b/src/repository/video.ts index cac10dfc..3f2506b2 100644 --- a/src/repository/video.ts +++ b/src/repository/video.ts @@ -5,7 +5,7 @@ import { AnyItem } from 'dynamoose/dist/Item' import { Query, QueryResponse, Scan, ScanResponse } from 'dynamoose/dist/ItemRetriever' import { omit } from 'ramda' import { DYNAMO_MODEL_OPTIONS, IRepository, mapTo } from '.' -import { ResourcePrefix, VideoState, YtVideo, videoStates } from '../types/youtube' +import { ResourcePrefix, VideoState, YtChannel, YtVideo, videoStates } from '../types/youtube' function videoRepository(tablePrefix: ResourcePrefix) { const videoSchema = new dynamoose.Schema( @@ -79,9 +79,6 @@ function videoRepository(tablePrefix: ResourcePrefix) { // Video's container container: String, - // Indicates if the video is an upcoming/active live broadcast. else it's "none" - liveBroadcastContent: String, - // joystream video ID in `VideoCreated` event response, returned from joystream runtime after creating a video joystreamVideo: { type: Object, @@ -256,6 +253,16 @@ export class VideosService { return [...(await this.getVideosInState('VideoCreationFailed')), ...(await this.getVideosInState('New'))] } + async getHistoricalUnsyncedVideosOfChannel(channel: YtChannel): Promise { + const videos = await this.videosRepository.query({ channelId: channel.id }, (q) => + q.using('channelId-publishedAt-index') + ) + + return videos.filter( + (v) => new Date(v.publishedAt) < channel.createdAt && (v.state === 'New' || v.state === 'VideoCreationFailed') + ) + } + /** * * @param video diff --git a/src/schemas/config.ts b/src/schemas/config.ts index 2a68ccb8..d4610085 100644 --- a/src/schemas/config.ts +++ b/src/schemas/config.ts @@ -1,30 +1,22 @@ -import { JSONSchema4 } from 'json-schema' +import { JSONSchema7 } from 'json-schema' import * as winston from 'winston' import { objectSchema } from './utils' +import { boolean } from '@oclif/command/lib/flags' export const byteSizeUnits = ['B', 'K', 'M', 'G', 'T'] export const byteSizeRegex = new RegExp(`^[0-9]+(${byteSizeUnits.join('|')})$`) -const logLevelSchema: JSONSchema4 = { +const logLevelSchema: JSONSchema7 = { description: 'Minimum level of logs sent to this output', type: 'string', enum: [...Object.keys(winston.config.npm.levels)], } -export const configSchema: JSONSchema4 = objectSchema({ +export const configSchema: JSONSchema7 = objectSchema({ '$id': 'https://joystream.org/schemas/youtube-synch/config', title: 'Youtube Sync node configuration', description: 'Configuration schema for Youtube synch service node', - required: [ - 'joystream', - 'endpoints', - 'directories', - 'limits', - 'intervals', - 'youtube', - 'creatorOnboardingRequirements', - 'httpApi', - ], + required: ['joystream', 'endpoints', 'youtube', 'creatorOnboardingRequirements', 'httpApi', 'sync'], properties: { joystream: objectSchema({ description: 'Joystream network related configuration', @@ -41,11 +33,11 @@ export const configSchema: JSONSchema4 = objectSchema({ required: ['endpoint', 'captchaBypassKey'], }), app: objectSchema({ - description: 'Joystream metaprotocol application specific configuration', + description: 'Joystream Metaprotocol App specific configuration', properties: { - name: { type: 'string', description: 'Name of the application' }, + name: { type: 'string', description: 'Name of the app' }, accountSeed: { - description: `Specifies the application auth key's string seed for generating ed25519 keypair`, + description: `Specifies the app auth key's string seed, for generating ed25519 keypair, to be used for signing App Actions`, type: 'string', }, }, @@ -103,16 +95,6 @@ export const configSchema: JSONSchema4 = objectSchema({ }, required: ['queryNode', 'joystreamNodeWs'], }), - directories: objectSchema({ - description: "Specifies paths where node's data will be stored", - properties: { - assets: { - description: 'Path to a directory where all the cached assets will be stored', - type: 'string', - }, - }, - required: ['assets'], - }), logs: objectSchema({ description: 'Specifies the logging configuration', properties: { @@ -182,55 +164,6 @@ export const configSchema: JSONSchema4 = objectSchema({ }, required: [], }), - limits: objectSchema({ - description: 'Specifies youtube-synch service limits.', - properties: { - dailyApiQuota: objectSchema({ - title: 'Specifies daily Youtube API quota rationing scheme for Youtube Partner Program', - description: 'Specifies daily Youtube API quota rationing scheme for Youtube Partner Program', - properties: { - sync: { type: 'number', default: 9500 }, - signup: { type: 'number', default: 500 }, - }, - required: ['sync', 'signup'], - }), - maxConcurrentDownloads: { - description: - 'Max no. of videos that should be concurrently downloaded from Youtube to be prepared for upload to Joystream', - type: 'number', - default: 50, - }, - maxConcurrentUploads: { - description: `Max no. of videos that should be concurrently uploaded to Joystream's storage node`, - type: 'number', - default: 50, - }, - storage: { - description: 'Maximum total size of all downloaded assets stored in `directories.assets`', - type: 'string', - pattern: byteSizeRegex.source, - }, - }, - required: ['dailyApiQuota', 'maxConcurrentDownloads', 'maxConcurrentUploads', 'storage'], - }), - intervals: objectSchema({ - description: 'Specifies how often periodic tasks (for example youtube state polling) are executed.', - properties: { - youtubePolling: { - description: - 'After how many minutes, the polling service should poll the Youtube api for channels state update', - type: 'integer', - minimum: 1, - }, - contentProcessing: { - description: - 'After how many minutes, the service should scan the database for new content to start downloading, on-chain creation & uploading to storage node', - type: 'integer', - minimum: 1, - }, - }, - required: ['youtubePolling', 'contentProcessing'], - }), youtube: objectSchema({ title: 'Youtube Oauth2 Client configuration', description: 'Youtube Oauth2 Client configuration', @@ -315,6 +248,77 @@ export const configSchema: JSONSchema4 = objectSchema({ }, required: ['port', 'ownerKey'], }), + sync: objectSchema({ + title: `YT-synch syncronization related settings`, + description: `YT-synch's syncronization related settings`, + properties: { + enable: { + description: 'Option to enable/disable syncing while starting the service', + type: 'boolean', + default: true, + }, + downloadsDir: { + description: 'Path to a directory where all the downloaded assets will be stored', + type: 'string', + }, + intervals: objectSchema({ + description: 'Specifies how often periodic tasks (for example youtube state polling) are executed.', + properties: { + youtubePolling: { + description: + 'After how many minutes, the polling service should poll the Youtube api for channels state update', + type: 'integer', + minimum: 1, + }, + contentProcessing: { + description: + 'After how many minutes, the service should scan the database for new content to start downloading, on-chain creation & uploading to storage node', + type: 'integer', + minimum: 1, + }, + }, + required: ['youtubePolling', 'contentProcessing'], + }), + limits: objectSchema({ + description: 'Specifies youtube-synch service limits.', + properties: { + dailyApiQuota: objectSchema({ + title: 'Specifies daily Youtube API quota rationing scheme for Youtube Partner Program', + description: 'Specifies daily Youtube API quota rationing scheme for Youtube Partner Program', + properties: { + sync: { type: 'number', default: 9500 }, + signup: { type: 'number', default: 500 }, + }, + required: ['sync', 'signup'], + }), + maxConcurrentDownloads: { + description: + 'Max no. of videos that should be concurrently downloaded from Youtube to be prepared for upload to Joystream', + type: 'number', + default: 50, + }, + maxConcurrentUploads: { + description: `Max no. of videos that should be concurrently uploaded to Joystream's storage node`, + type: 'number', + default: 50, + }, + storage: { + description: 'Maximum total size of all downloaded assets stored in `downloadsDir`', + type: 'string', + pattern: byteSizeRegex.source, + }, + }, + required: ['dailyApiQuota', 'maxConcurrentDownloads', 'maxConcurrentUploads', 'storage'], + }), + }, + if: { + properties: { enable: { const: true } }, + }, + then: { + required: ['downloadsDir', 'intervals', 'limits'], + }, + required: ['enable'], + }), }, }) diff --git a/src/schemas/scripts/generateTypes.ts b/src/schemas/scripts/generateTypes.ts index aacc25c2..4219a45a 100644 --- a/src/schemas/scripts/generateTypes.ts +++ b/src/schemas/scripts/generateTypes.ts @@ -1,3 +1,4 @@ +import { JSONSchema4 } from 'json-schema' import fs from 'fs' import { compile } from 'json-schema-to-typescript' import path from 'path' @@ -8,7 +9,7 @@ const prettierConfig = require('@joystream/prettier-config') Object.entries(schemas).forEach(([schemaKey, schema]) => { // eslint-disable-next-line @typescript-eslint/no-floating-promises - compile(schema, `${schemaKey}Json`, { + compile(schema as JSONSchema4, `${schemaKey}Json`, { style: prettierConfig, ignoreMinAndMaxItems: true, }) diff --git a/src/schemas/utils.ts b/src/schemas/utils.ts index 4c212224..1e0c4195 100644 --- a/src/schemas/utils.ts +++ b/src/schemas/utils.ts @@ -1,16 +1,20 @@ -import { JSONSchema4 } from 'json-schema' +import { JSONSchema7 } from 'json-schema' export function objectSchema< - P extends NonNullable, - D extends JSONSchema4['dependencies'] + P extends NonNullable, + D extends JSONSchema7['dependencies'], + If extends JSONSchema7['if'], + Then extends JSONSchema7['then'] >(props: { $id?: string title?: string description?: string properties: P dependencies?: D + if?: If, + then?: Then, required: Array -}): JSONSchema4 { +}): JSONSchema7 { return { type: 'object', additionalProperties: false, diff --git a/src/services/httpApi/api-spec.json b/src/services/httpApi/api-spec.json index 4d9cc795..d9428246 100644 --- a/src/services/httpApi/api-spec.json +++ b/src/services/httpApi/api-spec.json @@ -836,6 +836,12 @@ "referrerChannelId": { "type": "number" }, + "referredChannels": { + "type": "array", + "items": { + "type": "string" + } + }, "videoCategoryId": { "type": "string" }, @@ -862,6 +868,7 @@ "yppStatus", "joystreamChannelId", "referrerChannelId", + "referredChannels", "videoCategoryId", "language", "thumbnails", diff --git a/src/services/httpApi/controllers/channels.ts b/src/services/httpApi/controllers/channels.ts index 0b4fda3f..b0dcb0ff 100644 --- a/src/services/httpApi/controllers/channels.ts +++ b/src/services/httpApi/controllers/channels.ts @@ -60,6 +60,10 @@ export class ChannelsController { referrerChannelId, } = channelInfo + /** + * Input Validation + */ + if (referrerChannelId === joystreamChannelId) { throw new Error('Referrer channel cannot be the same as the channel being verified.') } @@ -72,14 +76,27 @@ export class ChannelsController { throw new Error('Invalid request author. Permission denied.') } + // ensure that Joystream channel exists + const jsChannel = await this.qnApi.getChannelById(joystreamChannelId.toString()) + if (!jsChannel) { + throw new Error(`Joystream Channel ${joystreamChannelId} does not exist.`) + } + + // ensure that Joystream channel isn't associated with any other participant channel + const existingJsChannel = await this.dynamodbService.channels.findPartnerChannelByJoystreamId(joystreamChannelId) + if (existingJsChannel) { + throw new Error( + `Joystream Channel ${joystreamChannelId} already connected with Youtube Channel ${existingJsChannel.id}.` + ) + } + // get channel from user const { channel } = await this.youtubeApi.getVerifiedChannel(user) // reset authorization code to prevent repeated save channel requests by authorization code re-use const updatedUser: YtUser = { ...user, email, authorizationCode: randomBytes(10).toString('hex') } - const joystreamChannelLanguageIso = (await this.qnApi.getChannelById(joystreamChannelId.toString()))?.language - ?.iso + const joystreamChannelLanguageIso = jsChannel.language?.iso const updatedChannel: YtChannel = { ...channel, email, @@ -106,8 +123,9 @@ export class ChannelsController { @ApiOperation({ description: 'Retrieves channel by joystreamChannelId' }) async get(@Param('joystreamChannelId', ParseIntPipe) id: number) { try { - const channel = await this.dynamodbService.channels.getByJoystreamChannelId(id) - return new ChannelDto(channel) + const channel = await this.dynamodbService.channels.getByJoystreamId(id) + const referredChannels = await this.dynamodbService.channels.getReferredChannels(id) + return new ChannelDto(channel, referredChannels) } catch (error) { const message = error instanceof Error ? error.message : error throw new NotFoundException(message) @@ -136,7 +154,7 @@ export class ChannelsController { @Body() { message, signature }: IngestChannelDto ) { try { - const channel = await this.dynamodbService.channels.getByJoystreamChannelId(id) + const channel = await this.dynamodbService.channels.getByJoystreamId(id) // Ensure channel is not suspended or opted out if (channel.yppStatus === 'Suspended' || channel.yppStatus === 'OptedOut') { @@ -180,7 +198,7 @@ export class ChannelsController { @Body() { message, signature }: OptoutChannelDto ) { try { - const channel = await this.dynamodbService.channels.getByJoystreamChannelId(id) + const channel = await this.dynamodbService.channels.getByJoystreamId(id) // Ensure channel is not suspended if (channel.yppStatus === 'Suspended') { @@ -228,14 +246,14 @@ export class ChannelsController { try { for (const { joystreamChannelId, isSuspended } of channels) { - const channel = await this.dynamodbService.channels.getByJoystreamChannelId(joystreamChannelId) + const channel = await this.dynamodbService.channels.getByJoystreamId(joystreamChannelId) // if channel is being suspended then its YT ingestion/syncing should also be stopped if (isSuspended) { await this.dynamodbService.channels.save({ ...channel, yppStatus: 'Suspended', - shouldBeIngested: false, + allowOperatorIngestion: false, }) } else { // if channel suspension is revoked then its YT ingestion/syncing should not be resumed @@ -263,14 +281,18 @@ export class ChannelsController { try { for (const { joystreamChannelId, isVerified } of channels) { - const channel = await this.dynamodbService.channels.getByJoystreamChannelId(joystreamChannelId) + const channel = await this.dynamodbService.channels.getByJoystreamId(joystreamChannelId) // channel is being verified if (isVerified) { - await this.dynamodbService.channels.save({ ...channel, yppStatus: 'Verified' }) + await this.dynamodbService.channels.save({ ...channel, yppStatus: 'Verified', allowOperatorIngestion: true }) } else { // channel is being unverified - await this.dynamodbService.channels.save({ ...channel, yppStatus: 'Unverified' }) + await this.dynamodbService.channels.save({ + ...channel, + yppStatus: 'Unverified', + allowOperatorIngestion: false, + }) } } } catch (error) { @@ -286,7 +308,7 @@ export class ChannelsController { }) async getVideos(@Param('joystreamChannelId', ParseIntPipe) id: number): Promise { try { - const channelId = (await this.dynamodbService.channels.getByJoystreamChannelId(id)).id + const channelId = (await this.dynamodbService.channels.getByJoystreamId(id)).id const result = await this.dynamodbService.repo.videos.query({ channelId }, (q) => q.sort('descending')) return result } catch (error) { diff --git a/src/services/httpApi/controllers/users.ts b/src/services/httpApi/controllers/users.ts index a188640d..ce10215c 100644 --- a/src/services/httpApi/controllers/users.ts +++ b/src/services/httpApi/controllers/users.ts @@ -62,7 +62,7 @@ export class UsersController { // Get existing user record from db (if any) const existingUser = await this.dynamodbService.repo.users.get(user.id) - // save user & set joystreamMemberId if user already existed + // save user & set joystreamMemberId if user already existed await this.dynamodbService.users.save({ ...user, joystreamMemberId: existingUser?.joystreamMemberId }) // return verified user diff --git a/src/services/httpApi/dtos.ts b/src/services/httpApi/dtos.ts index 16159c2c..4cd91fbf 100644 --- a/src/services/httpApi/dtos.ts +++ b/src/services/httpApi/dtos.ts @@ -48,18 +48,18 @@ export class ChannelDto { @ApiProperty() youtubeChannelId: string @ApiProperty() title: string @ApiProperty() description: string - @ApiProperty() aggregatedStats: number @ApiProperty() shouldBeIngested: boolean @ApiProperty() yppStatus: string @ApiProperty() joystreamChannelId: number @ApiProperty() referrerChannelId?: number + @ApiProperty() referredChannels: ReferredChannelDto[] @ApiProperty() videoCategoryId: string @ApiProperty() language: string @ApiProperty() thumbnails: ThumbnailsDto @ApiProperty() subscribersCount: number @ApiProperty() createdAt: Date - constructor(channel: YtChannel) { + constructor(channel: YtChannel, referredChannels?: YtChannel[]) { this.youtubeChannelId = channel.id this.title = channel.title this.description = channel.description @@ -70,9 +70,25 @@ export class ChannelDto { this.language = channel.language this.shouldBeIngested = channel.shouldBeIngested this.yppStatus = channel.yppStatus - this.aggregatedStats = channel.aggregatedStats this.thumbnails = channel.thumbnails this.createdAt = new Date(channel.createdAt) + this.referredChannels = referredChannels?.map((c) => new ReferredChannelDto(c)) || [] + } +} + +class ReferredChannelDto { + @ApiProperty() joystreamChannelId: number + @ApiProperty() title: string + @ApiProperty() subscribersCount: number + @ApiProperty() yppStatus: string + @ApiProperty() createdAt: Date + + constructor(referrerChannel: YtChannel) { + this.joystreamChannelId = referrerChannel.joystreamChannelId + this.title = referrerChannel.title + this.subscribersCount = referrerChannel.statistics.subscriberCount + this.yppStatus = referrerChannel.yppStatus + this.createdAt = new Date(referrerChannel.createdAt) } } @@ -170,7 +186,7 @@ export class CreateMembershipRequest { @IsString() @ApiProperty({ required: true }) handle: string // Membership avatar URL - @IsUrl({ require_tld: false }) @ApiProperty({ required: true }) avatar: string + @IsOptional() @IsUrl({ require_tld: false }) @ApiProperty({ required: true }) avatar: string // `about` information to associate with new Membership @ApiProperty() about: string diff --git a/src/services/runtime/client.ts b/src/services/runtime/client.ts index 8a3c50b3..e0be2fc6 100644 --- a/src/services/runtime/client.ts +++ b/src/services/runtime/client.ts @@ -100,13 +100,13 @@ export class JoystreamClient { return isCollaboratorSet } - async createVideo(video: YtVideo, videoFilePath: string): Promise<[YtVideo, BN]> { + async createVideo(video: YtVideo, videoFilePath: string): Promise<[YtVideo, BN, number]> { const collaborator = await this.qnApi.memberById(this.collaboratorId) if (!collaborator) { throw new Error(`Joystream member with id ${this.collaboratorId} not found`) } // Video metadata & assets - const { meta: rawAction, assets } = await this.prepareVideoInput(this.runtimeApi, video, videoFilePath) + const { meta: rawAction, assets, size } = await this.prepareVideoInput(this.runtimeApi, video, videoFilePath) const creatorId = video.joystreamChannelId.toString() const nonce = (await this.qnApi.getChannelById(creatorId || ''))?.totalVideosCreated || 0 @@ -137,6 +137,7 @@ export class JoystreamClient { }, }, createdVideo.createdInBlock, + size, ] } @@ -163,7 +164,7 @@ export class JoystreamClient { api: RuntimeApi, video: YtVideo, filePath: string - ): Promise<{ meta: Bytes; assets: Option }> { + ): Promise<{ meta: Bytes; assets: Option; size: number }> { const inputAssets: VideoInputAssets = {} const videoHashStream = fs.createReadStream(filePath) const { hash: videoHash, size: videoSize } = await computeFileHashAndSize(videoHashStream) @@ -212,14 +213,21 @@ export class JoystreamClient { const meta = metadataToBytes(ContentMetadata, { videoMetadata }) - return { meta, assets } + return { meta, assets, size: videoSize + thumbnailPhotoSize } } } export async function getThumbnailAsset(thumbnails: Thumbnails) { - // * We are using `medium` thumbnail because it has correct aspect ratio for Atlas (16/9) - const response = await axios.get(thumbnails.medium, { responseType: 'stream' }) - return response.data + try { + // * We are using `medium` thumbnail because it has correct aspect ratio for Atlas (16/9) + const response = await axios.get(thumbnails.medium, { responseType: 'stream' }) + return response.data + } catch (error) { + if (axios.isAxiosError(error)) { + throw error.toJSON() + } + throw error + } } async function prepareAssetsForExtrinsic( diff --git a/src/services/syncProcessing/ContentCreationService.ts b/src/services/syncProcessing/ContentCreationService.ts index 291e4dbf..0f73f370 100644 --- a/src/services/syncProcessing/ContentCreationService.ts +++ b/src/services/syncProcessing/ContentCreationService.ts @@ -4,19 +4,18 @@ import pWaitFor from 'p-wait-for' import sleep from 'sleep-promise' import { Logger } from 'winston' import { IDynamodbService } from '../../repository' -import { ReadonlyConfig } from '../../types' import { VideoCreationTask } from '../../types/youtube' import { LoggingService } from '../logging' import { JoystreamClient } from '../runtime/client' import { ContentDownloadService } from './ContentDownloadService' import { PriorityQueue } from './PriorityQueue' +import { SyncLimits } from './syncLimitations' // TODO: keep hash calculation separate from extrinsic calling // Video content creation/processing service export class ContentCreationService { private readonly DEFAULT_SUDO_PRIORITY = 10 - private config: ReadonlyConfig private logger: Logger private joystreamClient: JoystreamClient private dynamodbService: IDynamodbService @@ -26,13 +25,11 @@ export class ContentCreationService { private activeTaskId: string // video Id of the currently running video creation task constructor( - config: ReadonlyConfig, logging: LoggingService, dynamodbService: IDynamodbService, contentDownloadService: ContentDownloadService, joystreamClient: JoystreamClient ) { - this.config = config this.logger = logging.createLogger('ContentCreationService') this.dynamodbService = dynamodbService this.joystreamClient = joystreamClient @@ -43,13 +40,13 @@ export class ContentCreationService { }) } - async start() { + async start(interval: number) { this.logger.info(`Starting Video creation service.`) await this.ensureContentStateConsistency() // start video creation service - setTimeout(async () => this.createContentWithInterval(this.config.intervals.contentProcessing), 0) + setTimeout(async () => this.createContentWithInterval(interval), 0) } private async pendingOnchainCreationVideos() { @@ -92,8 +89,25 @@ export class ContentCreationService { await Promise.all( pendingOnchainCreationVideosByChannel.map(async ({ channelId, unsyncedVideos }) => { // Get total videos of channel - const { videoCount } = (await this.dynamodbService.channels.getByChannelId(channelId)).statistics - const percentageOfCreatorBacklogNotSynched = (unsyncedVideos.length * 100) / videoCount + const channel = await this.dynamodbService.channels.getById(channelId) + + const isSyncEnable = channel.shouldBeIngested && channel.allowOperatorIngestion + if (!isSyncEnable) { + this.logger.warn( + `Syncing is disabled for channel ${channel.joystreamChannelId}. Removing ` + + `all videos from syncing queue & deleting the records from the database.` + ) + // Remove all videos from queue + unsyncedVideos.forEach((v) => this.queue.cancel(v as VideoCreationTask)) + // Remove all the videos from db too (so that they wont be requeued) + await Promise.all(unsyncedVideos.map((v) => this.dynamodbService.videos.delete(v))) + // Remove the downloaded video file + await Promise.all(unsyncedVideos.map((v) => this.contentDownloadService.removeVideoFile(v.id))) + return + } + + const totalVideos = Math.min(channel.statistics.videoCount, SyncLimits.videoCap(channel)) + const percentageOfCreatorBacklogNotSynched = (unsyncedVideos.length * 100) / totalVideos for (const v of unsyncedVideos) { const rank = this.queue.calculateVideoRank( @@ -126,7 +140,7 @@ export class ContentCreationService { `${video.id} from syncing & deleting it's record from the database.` ) await this.dynamodbService.videos.delete(video) - await this.contentDownloadService.removeVideoFile(video.id) + await this.contentDownloadService.removeVideoFile(video.id) // TODO: remove this after adding `isSyncEnable` check? return } @@ -151,14 +165,26 @@ export class ContentCreationService { `Inconsistent state. Youtube video ${video.id} was already created on Joystream but the service tried to recreate it.`, { videoId: video.id, channelId: video.joystreamChannelId } ) - process.exit(-1) + await this.ensureContentStateConsistency() + return + // process.exit(-1) } await this.dynamodbService.videos.updateState(video, 'CreatingVideo') - const [createdVideo, createdInBlock] = await this.joystreamClient.createVideo(video, video.filePath) + const [createdVideo, createdInBlock, size] = await this.joystreamClient.createVideo(video, video.filePath) this.lastVideoCreationBlockByChannelId.set(video.joystreamChannelId, createdInBlock) await this.dynamodbService.videos.updateState(createdVideo, 'VideoCreated') this.logger.info(`Video created on chain.`, { videoId: video.id, channelId: video.joystreamChannelId }) + + // Update channel size limit + const channel = await this.dynamodbService.channels.getById(video.channelId) + const isHistoricalVideo = new Date(video.publishedAt) < channel.createdAt + if (isHistoricalVideo) { + await this.dynamodbService.channels.save({ + ...channel, + historicalVideoSyncedSize: (channel.historicalVideoSyncedSize || 0) + size, + }) + } } catch (err) { this.logger.error(`Got error processing video`, { videoId: video.id, err }) await this.dynamodbService.videos.updateState(video, 'VideoCreationFailed') diff --git a/src/services/syncProcessing/ContentDownloadService.ts b/src/services/syncProcessing/ContentDownloadService.ts index 0c82d1bc..ce5a4179 100644 --- a/src/services/syncProcessing/ContentDownloadService.ts +++ b/src/services/syncProcessing/ContentDownloadService.ts @@ -10,12 +10,13 @@ import { VideoDownloadTask } from '../../types/youtube' import { LoggingService } from '../logging' import { IYoutubeApi } from '../youtube/api' import { PriorityQueue } from './PriorityQueue' +import { SyncLimits } from './syncLimitations' // Youtube videos download service export class ContentDownloadService { private readonly DEFAULT_SUDO_PRIORITY = 10 - private config: ReadonlyConfig + private syncConfig: Required private logger: Logger private youtubeApi: IYoutubeApi private dynamodbService: IDynamodbService @@ -30,17 +31,17 @@ export class ContentDownloadService { } public get freeSpace(): number { - const freeSpace = this.config.limits.storage - this.contentSizeSum + const freeSpace = this.syncConfig.limits.storage - this.contentSizeSum return freeSpace > 0 ? freeSpace : 0 } public constructor( - config: ReadonlyConfig, + syncConfig: Required, logging: LoggingService, dynamodbService: IDynamodbService, youtubeApi: IYoutubeApi ) { - this.config = config + this.syncConfig = syncConfig this.logger = logging.createLogger('ContentDownloadService') this.dynamodbService = dynamodbService this.youtubeApi = youtubeApi @@ -52,18 +53,18 @@ export class ContentDownloadService { (video: VideoDownloadTask, cb) => { cb(null, video.priorityScore) }, - this.config.limits.maxConcurrentDownloads + this.syncConfig.limits.maxConcurrentDownloads ) } - async start() { + async start(interval: number) { this.logger.info(`Starting Video download service.`) // Resolve already downloaded videos this.resolveDownloadedVideos() // start video creation service - setTimeout(async () => this.downloadContentWithInterval(this.config.intervals.contentProcessing), 0) + setTimeout(async () => this.downloadContentWithInterval(interval), 0) } public getVideoFilePath(resourceId: string): string | undefined { @@ -80,7 +81,7 @@ export class ContentDownloadService { public async removeVideoFile(resourceId: string) { try { - const dir = this.config.directories.assets + const dir = this.syncConfig.downloadsDir const size = this.fileSize(resourceId) const files = await fsPromises.readdir(dir) for (const file of files) { @@ -103,12 +104,12 @@ export class ContentDownloadService { private setVideoFilePath(resourceId: string, fileExt: string) { this.downloadedVideoPathByResourceId.set( resourceId, - path.join(this.config.directories.assets, `${resourceId}.${fileExt}`) + path.join(this.syncConfig.downloadsDir, `${resourceId}.${fileExt}`) ) } private resolveDownloadedVideos() { - const videoDownloadsDir = this.config.directories.assets + const videoDownloadsDir = this.syncConfig.downloadsDir const resolvedDownloads = fs .readdirSync(videoDownloadsDir) .map((filePath) => filePath.split('.')) @@ -172,8 +173,22 @@ export class ContentDownloadService { await Promise.all( pendingDownloadVideosByChannel.map(async ({ channelId, unsyncedVideos }) => { // Get total videos of channel - const { videoCount } = (await this.dynamodbService.channels.getByChannelId(channelId)).statistics - const percentageOfCreatorBacklogNotSynched = (unsyncedVideos.length * 100) / videoCount + const channel = await this.dynamodbService.channels.getById(channelId) + + const isSyncEnable = channel.shouldBeIngested && channel.allowOperatorIngestion + if (!isSyncEnable) { + this.logger.warn( + `Syncing is disabled for channel ${channel.joystreamChannelId}. Removing ` + + `all videos from download queue & deleting the records from the database.` + ) + // Remove all videos from queue + unsyncedVideos.forEach((v) => this.downloadQueue.cancel(v as VideoDownloadTask)) + // Remove all the videos from db too (so that they wont be requeued) + await Promise.all(unsyncedVideos.map((v) => this.dynamodbService.videos.delete(v))) + } + + const totalVideos = Math.min(channel.statistics.videoCount, SyncLimits.videoCap(channel)) + const percentageOfCreatorBacklogNotSynched = (unsyncedVideos.length * 100) / totalVideos for (const v of unsyncedVideos) { const rank = this.downloadQueue.calculateVideoRank( @@ -197,14 +212,41 @@ export class ContentDownloadService { this.activeDownloadsIds = videos.map((v) => v.id) this.activeDownloadsCount = videos.length - await Promise.allSettled( + await Promise.all( videos.map(async (video) => { try { // download the video from youtube this.logger.info(`Downloading video`, { videoId: video.id, channelId: video.joystreamChannelId }) - const { ext: fileExt } = await this.youtubeApi.downloadVideo(video.url, this.config.directories.assets) + const { ext: fileExt } = await this.youtubeApi.downloadVideo(video.url, this.syncConfig.downloadsDir) this.setVideoFilePath(video.id, fileExt) - this.contentSizeSum += this.fileSize(video.id) + const size = this.fileSize(video.id) + + // ensure syncing this video won't violate per channel limits + const channel = await this.dynamodbService.channels.getById(video.channelId) + const isHistoricalVideo = new Date(video.publishedAt) < channel.createdAt + if (isHistoricalVideo) { + const hasLimitReached = channel.historicalVideoSyncedSize + size > SyncLimits.sizeCap(channel) + + if (hasLimitReached) { + // TODO: should this check be in ContentCreationService + // delete this video file + await this.removeVideoFile(video.id) + // delete all tracked historical videos that hasn't been created yet + const videos = await this.dynamodbService.videos.getHistoricalUnsyncedVideosOfChannel(channel) + this.logger.warn( + `Size limit for channel's historical has reached. Removing all historical not-downloaded videos` + ) + await Promise.all( + videos.map(async (v) => !this.getVideoFilePath(v.id) && (await this.dynamodbService.videos.delete(v))) + ) + + return + } + } + + // TODO: remove all historical not-downloaded videos if limit reached + + this.contentSizeSum += size this.logger.info(`Video downloaded.`, { videoId: video.id, channelId: video.joystreamChannelId }) } catch (err) { const errorMsg = (err as Error).message @@ -218,16 +260,19 @@ export class ContentDownloadService { this.logger.warn(`Video visibility was set to private. Skipping from syncing...`, { videoId: video.id, }) - } else if (errorMsg.includes('Postprocessing: Conversion failed!')) { + } else if (errorMsg.includes('Postprocessing:')) { await this.dynamodbService.videos.updateState(video, 'VideoUnavailable') this.logger.error(`Video Postprocessing error. Skipping from syncing...`, { videoId: video.id, }) + } else if (errorMsg.includes('The downloaded file is empty')) { + await this.dynamodbService.videos.updateState(video, 'VideoUnavailable') + this.logger.error(`The downloaded file is empty. Skipping from syncing...`, { + videoId: video.id, + }) } else { this.logger.error(`Got error downloading video. Retrying...`, { videoId: video.id, err }) } - - await this.removeVideoFile(video.id) } finally { this.activeDownloadsCount-- } diff --git a/src/services/syncProcessing/ContentUploadService.ts b/src/services/syncProcessing/ContentUploadService.ts index daad2726..5a228e14 100644 --- a/src/services/syncProcessing/ContentUploadService.ts +++ b/src/services/syncProcessing/ContentUploadService.ts @@ -9,7 +9,7 @@ import { ContentDownloadService } from './ContentDownloadService' // Video content upload service export class ContentUploadService { - private config: ReadonlyConfig + private syncConfig: Required private logger: Logger private logging: LoggingService private dynamodbService: IDynamodbService @@ -18,13 +18,13 @@ export class ContentUploadService { private queryNodeApi: QueryNodeApi public constructor( - config: ReadonlyConfig, + syncConfig: Required, logging: LoggingService, dynamodbService: IDynamodbService, contentDownloadService: ContentDownloadService, queryNodeApi: QueryNodeApi ) { - this.config = config + this.syncConfig = syncConfig this.logger = logging.createLogger('ContentUploadService') this.logging = logging this.dynamodbService = dynamodbService @@ -33,13 +33,13 @@ export class ContentUploadService { this.storageNodeApi = new StorageNodeApi(this.logging, this.queryNodeApi) } - async start() { + async start(interval: number) { this.logger.info(`Starting service to upload video assets to storage-node.`) await this.ensureUploadStateConsistency() // start assets upload service - setTimeout(async () => this.uploadAssetsWithInterval(this.config.intervals.contentProcessing), 0) + setTimeout(async () => this.uploadAssetsWithInterval(interval), 0) } /** @@ -74,7 +74,7 @@ export class ContentUploadService { await sleep(sleepInterval) try { this.logger.info(`Resume service....`) - await this.uploadPendingAssets(this.config.limits.maxConcurrentUploads) + await this.uploadPendingAssets(this.syncConfig.limits.maxConcurrentUploads) } catch (err) { this.logger.error(`Critical Upload error`, { err }) } diff --git a/src/services/syncProcessing/PriorityQueue.ts b/src/services/syncProcessing/PriorityQueue.ts index ed66253d..d1b11f33 100644 --- a/src/services/syncProcessing/PriorityQueue.ts +++ b/src/services/syncProcessing/PriorityQueue.ts @@ -12,11 +12,11 @@ export class PriorityQueue void ) => void, - priorityFunc: (task: Task, cb: (error: any, priority: number) => void) => void, + priority: (task: Task, cb: (error: any, priority: number) => void) => void, batchSize?: ProcessingType extends 'batchProcessor' ? number : never ) { this.queue = new Queue(processingFunc, { - priority: priorityFunc, + priority, batchSize, }) } @@ -25,6 +25,10 @@ export class PriorityQueue this.runPollingWithInterval(this.config.intervals.youtubePolling), 0) + setTimeout(async () => this.runPollingWithInterval(pollingInterval), 0) } - // get IDs of all new videos of the channel - private async getNewVideosIds(channel: YtChannel, allVideosIds: string[]): Promise { + // get IDs of all videos of a channel that are still not tracked in DB + private async getUntrackedVideosIds( + channel: YtChannel, + videosIds: YtDlpFlatPlaylistOutput + ): Promise { // Get all the existing videos const existingVideos = await this.dynamodbService.repo.videos.query({ channelId: channel.id }, (q) => q) - - return _.difference( - allVideosIds, - existingVideos.map((v) => v.id) - ) + return _.differenceBy(videosIds, existingVideos, 'id') } /** @@ -71,7 +67,12 @@ export class YoutubePollingService { `Completed Channels Ingestion. Videos of ${channels.length} channels will be prepared for syncing in this polling cycle....` ) - await Promise.allSettled(channels.map((channel) => this.performVideosIngestion(channel))) + // Ingest videos of channels in a batch of 50 to limit IO/CPU resource consumption + const channelsBatch = _.chunk(channels, 50) + // Process each batch + for (let i = 0; i < channelsBatch.length; i++) { + await Promise.all(channelsBatch[i].map((channel) => this.performVideosIngestion(channel))) + } } catch (err) { this.logger.error(`Critical Polling error`, { err }) } @@ -84,90 +85,101 @@ export class YoutubePollingService { private async performChannelsIngestion(): Promise { // get all channels that need to be ingested const channelsWithSyncEnabled = async () => - await this.dynamodbService.repo.channels.scan('shouldBeIngested', (s) => - // * Unauthorized channels add by infra operator are exempted from periodic - // * ingestion as we don't have access to their access/refresh tokens - s.eq(true).and().filter('performUnauthorizedSync').eq(false) + await this.dynamodbService.repo.channels.scan('yppStatus', (s) => + s + .eq('Verified') + .and() + .filter('shouldBeIngested') + .eq(true) + .and() + .filter('allowOperatorIngestion') + .eq(true) + .and() + // * Unauthorized channels add by infra operator are exempted from periodic + // * ingestion as we don't have access to their access/refresh tokens + .filter('performUnauthorizedSync') + .eq(false) ) // updated channel objects with uptodate info - const updatedChannels: YtChannel[] = [] const channelsToBeIngested = await channelsWithSyncEnabled() - for (const ch of channelsToBeIngested) { - try { - const uptodateChannel = await this.youtubeApi.getChannel({ - id: ch.userId, - accessToken: ch.userAccessToken, - refreshToken: ch.userRefreshToken, + const updatedChannels = ( + await Promise.all( + channelsToBeIngested.map(async (ch) => { + try { + const uptodateChannel = await this.youtubeApi.getChannel({ + id: ch.userId, + accessToken: ch.userAccessToken, + refreshToken: ch.userRefreshToken, + }) + + // ensure that Ypp collaborator member is still set as channel's collaborator + const isCollaboratorSet = await this.joystreamClient.doesChannelHaveCollaborator(ch.joystreamChannelId) + if (!isCollaboratorSet) { + this.logger.warn( + `Joystream Channel ${ch.joystreamChannelId} has either not set or revoked Ypp collaborator member ` + + `as channel's collaborator. Corresponding Youtube Channel '${ch.id}' is being opted out from Ypp program.` + ) + return { + ...ch, + yppStatus: 'OptedOut', + shouldBeIngested: false, + lastActedAt: new Date(), + } + } + + // Update the current channel record if it changed + if (!_.isEqual(ch.statistics, uptodateChannel.statistics)) { + return { ...ch, statistics: uptodateChannel.statistics } + } + } catch (err: unknown) { + // if app permission is revoked by user from Google account then set `shouldBeIngested` to false & OptOut channel from + // Ypp program, because then trying to fetch user channel will throw error with code 400 and 'invalid_grant' message + if (err instanceof GaxiosError && err.code === '400' && err.response?.data?.error === 'invalid_grant') { + this.logger.warn( + `Opting out '${ch.id}' from YPP program as their owner has revoked the permissions from Google settings` + ) + return { + ...ch, + yppStatus: 'OptedOut', + shouldBeIngested: false, + lastActedAt: new Date(), + } + // ! Although type of `err.code` is string, the api api response returns it as number. + } else if ( + err instanceof GaxiosError && + err.code === (403 as any) && + (err as GaxiosError).response?.data?.error?.errors[0]?.reason === 'authenticatedUserAccountSuspended' + ) { + this.logger.warn( + `Opting out '${ch.id}' from YPP program as their Youtube channel has been terminated by the Youtube.` + ) + return { + ...ch, + yppStatus: 'OptedOut', + shouldBeIngested: false, + lastActedAt: new Date(), + } + } else if (err instanceof YoutubeApiError && err.code === ExitCodes.YoutubeApi.CHANNEL_NOT_FOUND) { + this.logger.warn(`Opting out '${ch.id}' from YPP program as Channel is not found on Youtube.`) + return { + ...ch, + yppStatus: ' OptedOut', + shouldBeIngested: false, + lastActedAt: new Date(), + } + } else if ( + err instanceof YoutubeApiError && + err.code === ExitCodes.YoutubeApi.YOUTUBE_QUOTA_LIMIT_EXCEEDED + ) { + this.logger.info('Youtube quota limit exceeded, skipping polling for now.') + return + } + this.logger.error('Failed to fetch updated channel info', { err, channelId: ch.joystreamChannelId }) + } }) - - // ensure that Ypp collaborator member is still set as channel's collaborator - const isCollaboratorSet = await this.joystreamClient.doesChannelHaveCollaborator(ch.joystreamChannelId) - if (!isCollaboratorSet) { - this.logger.warn( - `Joystream Channel ${ch.joystreamChannelId} has either not set or revoked Ypp collaborator member ` + - `as channel's collaborator. Corresponding Youtube Channel '${ch.id}' is being opted out from Ypp program.` - ) - updatedChannels.push({ - ...ch, - yppStatus: 'OptedOut', - shouldBeIngested: false, - lastActedAt: new Date(), - }) - continue - } - - // Update the current channel record if it changed - if (!_.isEqual(ch.statistics, uptodateChannel.statistics)) { - updatedChannels.push({ ...ch, statistics: uptodateChannel.statistics }) - } - } catch (err: unknown) { - // if app permission is revoked by user from Google account then set `shouldBeIngested` to false & OptOut channel from - // Ypp program, because then trying to fetch user channel will throw error with code 400 and 'invalid_grant' message - if (err instanceof GaxiosError && err.code === '400' && err.response?.data?.error === 'invalid_grant') { - this.logger.warn( - `Opting out '${ch.id}' from YPP program as their owner has revoked the permissions from Google settings` - ) - updatedChannels.push({ - ...ch, - yppStatus: 'OptedOut', - shouldBeIngested: false, - lastActedAt: new Date(), - }) - continue - // ! Although type of `err.code` is string, the api api response returns it as number. - } else if ( - err instanceof GaxiosError && - err.code === (403 as any) && - (err as GaxiosError).response?.data?.error?.errors[0]?.reason === 'authenticatedUserAccountSuspended' - ) { - this.logger.warn( - `Opting out '${ch.id}' from YPP program as their Youtube channel has been terminated by the Youtube.` - ) - updatedChannels.push({ - ...ch, - yppStatus: 'OptedOut', - shouldBeIngested: false, - lastActedAt: new Date(), - }) - continue - } else if (err instanceof YoutubeApiError && err.code === ExitCodes.YoutubeApi.CHANNEL_NOT_FOUND) { - this.logger.warn(`Opting out '${ch.id}' from YPP program as Channel is not found on Youtube.`) - updatedChannels.push({ - ...ch, - yppStatus: 'OptedOut', - shouldBeIngested: false, - lastActedAt: new Date(), - }) - continue - } else if (err instanceof YoutubeApiError && err.code === ExitCodes.YoutubeApi.YOUTUBE_QUOTA_LIMIT_EXCEEDED) { - this.logger.info('Youtube quota limit exceeded, skipping polling for now.') - return [] - } - updatedChannels.push(ch) - this.logger.error('Failed to fetch updated channel info', { err, channelId: ch.joystreamChannelId }) - } - } + ) + ).filter((ch): ch is YtChannel => ch !== undefined) // save updated channels await this.dynamodbService.repo.channels.upsertAll(updatedChannels) @@ -177,18 +189,34 @@ export class YoutubePollingService { private async performVideosIngestion(channel: YtChannel) { try { - // get all sync-able videos of the channel - const allVideosIds = await this.ytdlpClient.getAllVideosIds(channel) + // + const historicalVideosCountLimit = SyncLimits.videoCap(channel) + + // get all sync-able videos within the channel limits + const videosIds = await this.ytdlpClient.getVideosIds(channel, historicalVideosCountLimit) + + // get all video Ids that are not yet being tracked + let untrackedVideosIds = await this.getUntrackedVideosIds(channel, videosIds) - // get all new video Ids that are not yet being tracked - const newVideosIds = await this.getNewVideosIds(channel, allVideosIds) + // if size limit has reached, don't track new historical videos + if (SyncLimits.hasSizeLimitReached(channel)) { + untrackedVideosIds = untrackedVideosIds.filter((v) => v.publishedAt >= channel.createdAt) + } - // get all new videos that are not yet being tracked - const newVideos = await this.youtubeApi.getVideos(channel, newVideosIds) + // get all videos that are not yet being tracked + const untrackedVideos = await this.youtubeApi.getVideos( + channel, + untrackedVideosIds.map((v) => v.id) + ) // save all new videos to DB including - await this.dynamodbService.repo.videos.upsertAll(newVideos) + await this.dynamodbService.repo.videos.upsertAll(untrackedVideos) } catch (err) { + if (err instanceof YoutubeApiError && err.code === ExitCodes.YoutubeApi.YOUTUBE_QUOTA_LIMIT_EXCEEDED) { + this.logger.info('Youtube quota limit exceeded, skipping polling for now.') + return + } + this.logger.error('Failed to ingest videos for channel', { err, channelId: channel.joystreamChannelId }) } } diff --git a/src/services/syncProcessing/syncLimitations.ts b/src/services/syncProcessing/syncLimitations.ts new file mode 100644 index 00000000..d88a2552 --- /dev/null +++ b/src/services/syncProcessing/syncLimitations.ts @@ -0,0 +1,27 @@ +import { YtChannel } from '../../types/youtube' + +export class SyncLimits { + static videoCap(ch: YtChannel): number { + if (ch.statistics.subscriberCount < 5000) { + return 100 + } else if (ch.statistics.subscriberCount < 50000) { + return 250 + } else { + return 1000 + } + } + + static sizeCap(ch: YtChannel): number { + if (ch.statistics.subscriberCount < 5000) { + return 10_000_000_000 // 10 GB + } else if (ch.statistics.subscriberCount < 50000) { + return 100_000_000_000 // 100 GB + } else { + return 1_000_000_000_000 // 1 TB + } + } + + static hasSizeLimitReached(ch: YtChannel) { + return ch.historicalVideoSyncedSize >= this.sizeCap(ch) + } +} diff --git a/src/services/youtube/api.ts b/src/services/youtube/api.ts index 15772c8a..e97ebe02 100644 --- a/src/services/youtube/api.ts +++ b/src/services/youtube/api.ts @@ -13,11 +13,11 @@ import path from 'path' import pkgDir from 'pkg-dir' import { promisify } from 'util' import ytdl from 'youtube-dl-exec' +import { StatsRepository } from '../../repository' import { ReadonlyConfig, WithRequired, formattedJSON } from '../../types' import { ExitCodes, YoutubeApiError } from '../../types/errors' -import { YtChannel, YtUser, YtVideo } from '../../types/youtube' +import { YtChannel, YtDlpFlatPlaylistOutput, YtUser, YtVideo } from '../../types/youtube' -import Schema$PlaylistItem = youtube_v3.Schema$PlaylistItem import Schema$Video = youtube_v3.Schema$Video import Schema$Channel = youtube_v3.Schema$Channel @@ -30,19 +30,44 @@ export class YtDlpClient { this.ytdlpPath = `${pkgDir.sync(__dirname)}/node_modules/youtube-dl-exec/bin/yt-dlp` } - async getAllVideosIds(channel: YtChannel): Promise { + async getVideosIds( + channel: YtChannel, + limit?: number, + order: 'first' | 'last' = 'first' + ): Promise { try { + if (limit === undefined && order !== undefined) { + throw new Error('Order should only be provided if limit is provided') + } + + let limitOption = '' + if (limit) { + limitOption = order === 'first' ? `-I :${limit}` : `-I -${limit}:-1` + } + const { stdout } = await this.exec( - `${this.ytdlpPath} --flat-playlist --get-id https://www.youtube.com/playlist?list=${channel.uploadsPlaylistId}` + `${this.ytdlpPath} --extractor-args "youtubetab:approximate_date" -J --flat-playlist ${limitOption} https://www.youtube.com/playlist?list=${channel.uploadsPlaylistId}`, + { maxBuffer: Number.MAX_SAFE_INTEGER } ) - // Each line of stdout is a video ID - const videoIDs = stdout.split('\n').filter((id: unknown) => id) // Remove any empty lines - return videoIDs + const videos: YtDlpFlatPlaylistOutput = [] + JSON.parse(stdout).entries.forEach((category: any) => { + if (category.entries) { + category.entries.forEach((video: any) => { + videos.push({ id: video.id, publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */ }) + }) + } else { + videos.push({ id: category.id, publishedAt: new Date(category.timestamp * 1000) /** Convert UNIX to date */ }) + } + }) + + return videos } catch (err) { throw err } } + + async a() {} } export interface IYoutubeApi { @@ -52,7 +77,6 @@ export interface IYoutubeApi { user: Pick ): Promise<{ channel: YtChannel; errors: YoutubeApiError[] }> getVideos(channel: YtChannel, ids: string[]): Promise - getAllVideos(channel: YtChannel): Promise downloadVideo(videoUrl: string, outPath: string): ReturnType getCreatorOnboardingRequirements(): ReadonlyConfig['creatorOnboardingRequirements'] } @@ -64,9 +88,11 @@ export interface IQuotaMonitoringClient { class YoutubeClient implements IYoutubeApi { private config: ReadonlyConfig + private ytdlpClient: YtDlpClient constructor(config: ReadonlyConfig) { this.config = config + this.ytdlpClient = new YtDlpClient() } getCreatorOnboardingRequirements() { @@ -191,7 +217,9 @@ class YoutubeClient implements IYoutubeApi { videoCreationTimeCutoff.setHours(videoCreationTimeCutoff.getHours() - minimumVideoAgeHours) // filter all videos that are older than MINIMUM_VIDEO_AGE_HOURS - const videos = (await this.getAllVideos(channel)).filter((v) => new Date(v.publishedAt) < videoCreationTimeCutoff) + const videos = (await this.ytdlpClient.getVideosIds(channel, minimumVideoCount, 'last')).filter( + (v) => new Date(v.publishedAt) < videoCreationTimeCutoff + ) if (videos.length < minimumVideoCount) { errors.push( new YoutubeApiError( @@ -235,15 +263,6 @@ class YoutubeClient implements IYoutubeApi { } } - async getAllVideos(channel: YtChannel) { - const yt = this.getYoutube(channel.userAccessToken, channel.userRefreshToken) - try { - return this.iterateAllVideos(yt, channel) - } catch (error) { - throw new Error(`Failed to fetch videos for channel ${channel.title}. Error: ${error}`) - } - } - async downloadVideo(videoUrl: string, outPath: string): ReturnType { const response = await ytdl(videoUrl, { noWarnings: true, @@ -284,51 +303,6 @@ class YoutubeClient implements IYoutubeApi { return videos } - private async iterateAllVideos(youtube: youtube_v3.Youtube, channel: YtChannel, limit?: number) { - let videos: YtVideo[] = [] - let nextPageToken: string | undefined - - do { - const nextPage = await youtube.playlistItems - .list({ - part: ['contentDetails', 'snippet', 'id', 'status'], - playlistId: channel.uploadsPlaylistId, - maxResults: 50, - pageToken: nextPageToken, - }) - .catch((err) => { - if (err instanceof FetchError && err.code === 'ENOTFOUND') { - throw new YoutubeApiError(ExitCodes.YoutubeApi.YOUTUBE_API_NOT_CONNECTED, err.message) - } - throw err - }) - nextPageToken = nextPage.data.nextPageToken ?? '' - - // Filter `public` videos as only those would be synced - const videosPage = nextPage.data.items?.filter((v) => v.status?.privacyStatus === 'public') ?? [] - - const videosDetailsPage = videosPage.length - ? ( - await youtube.videos - .list({ - id: videosPage?.map((v) => v.snippet?.resourceId?.videoId ?? ``), - part: ['contentDetails', 'fileDetails', 'snippet', 'id', 'status', 'statistics'], - }) - .catch((err) => { - if (err instanceof FetchError && err.code === 'ENOTFOUND') { - throw new YoutubeApiError(ExitCodes.YoutubeApi.YOUTUBE_API_NOT_CONNECTED, err.message) - } - throw err - }) - ).data?.items ?? [] - : [] - - const page = this.mapAllVideos(videosPage, videosDetailsPage, channel) - videos = [...videos, ...page] - } while (nextPageToken && (limit === undefined || videos.length < limit)) - return videos - } - private mapChannels(user: Pick, channels: Schema$Channel[]) { return channels.map( (channel) => @@ -352,12 +326,14 @@ class YoutubeClient implements IYoutubeApi { videoCount: parseInt(channel.statistics?.videoCount ?? '0'), commentCount: parseInt(channel.statistics?.commentCount ?? '0'), }, + historicalVideoSyncedSize: 0, bannerImageUrl: channel.brandingSettings?.image?.bannerExternalUrl, uploadsPlaylistId: channel.contentDetails?.relatedPlaylists?.uploads, language: channel.snippet?.defaultLanguage, - performUnauthorizedSync: false, publishedAt: channel.snippet?.publishedAt, - shouldBeIngested: true, + performUnauthorizedSync: false, + shouldBeIngested: false, + allowOperatorIngestion: false, yppStatus: 'Unverified', createdAt: new Date(), lastActedAt: new Date(), @@ -390,7 +366,7 @@ class YoutubeClient implements IYoutubeApi { joystreamChannelId: channel.joystreamChannelId, privacyStatus: video.status?.privacyStatus, ytRating: video.contentDetails?.contentRating?.ytRating, - liveBroadcastContent: video.snippet?.liveBroadcastContent, + liveStreamingDetails: video.liveStreamingDetails, license: video.status?.license, duration: toSeconds(parse(video.contentDetails?.duration ?? 'PT0S')), container: video.fileDetails?.container, @@ -400,45 +376,9 @@ class YoutubeClient implements IYoutubeApi { } ) // filter out videos that are not public, processed, have live-stream or age-restriction, since those can't be synced yet - .filter((v) => v.uploadStatus === 'processed' && v.liveBroadcastContent === 'none' && v.ytRating === undefined) - ) - } - - private mapAllVideos(videos: Schema$PlaylistItem[], videosDetails: Schema$Video[], channel: YtChannel): YtVideo[] { - return ( - videos - .map( - (video, i) => - { - id: video.snippet?.resourceId?.videoId, - description: video.snippet?.description, - title: video.snippet?.title, - channelId: video.snippet?.channelId, - thumbnails: { - high: video.snippet?.thumbnails?.high?.url, - medium: video.snippet?.thumbnails?.medium?.url, - standard: video.snippet?.thumbnails?.standard?.url, - default: video.snippet?.thumbnails?.default?.url, - }, - url: `https://youtube.com/watch?v=${video.snippet?.resourceId?.videoId}`, - publishedAt: video.contentDetails?.videoPublishedAt, - createdAt: new Date(), - category: channel.videoCategoryId, - languageIso: channel.joystreamChannelLanguageIso, - joystreamChannelId: channel.joystreamChannelId, - privacyStatus: video.status?.privacyStatus, - ytRating: videosDetails[i].contentDetails?.contentRating?.ytRating, - liveBroadcastContent: videosDetails[i].snippet?.liveBroadcastContent, - license: videosDetails[i].status?.license, - duration: toSeconds(parse(videosDetails[i].contentDetails?.duration ?? 'PT0S')), - container: videosDetails[i].fileDetails?.container, - uploadStatus: videosDetails[i].status?.uploadStatus, - viewCount: parseInt(videosDetails[i].statistics?.viewCount ?? '0'), - state: 'New', - } + .filter( + (v) => v.uploadStatus === 'processed' && v.liveStreamingDetails === undefined && v.ytRating === undefined ) - // filter out videos that are not public, processed, have live-stream or age-restriction, since those can't be synced yet - .filter((v) => v.uploadStatus === 'processed' && v.liveBroadcastContent === 'none' && v.ytRating === undefined) ) } } @@ -448,7 +388,7 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { private googleCloudProjectId: string private DEFAULT_MAX_ALLOWED_QUOTA_USAGE = 95 // 95% - constructor(private decorated: IYoutubeApi, private config: ReadonlyConfig) { + constructor(private decorated: IYoutubeApi, private config: ReadonlyConfig, private statsRepo: StatsRepository) { // Use the client id to get the google cloud project id this.googleCloudProjectId = this.config.youtube.clientId.split('-')[0] @@ -545,6 +485,10 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { async getVerifiedChannel(user: Pick) { // These is no api quota check for this operation, as we allow untracked access to channel verification/signup endpoint. const verifiedChannel = await this.decorated.getVerifiedChannel(user) + + // increase used quota count by 1 because only one page is returned + await this.increaseUsedQuota({ signupQuotaIncrement: 1 }) + return verifiedChannel } @@ -559,6 +503,10 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { // get channels from api const channels = await this.decorated.getChannel(user) + + // increase used quota count by 1 because only one page is returned + await this.increaseUsedQuota({ syncQuotaIncrement: 1 }) + return channels } @@ -573,20 +521,10 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { // get videos from api const videos = await this.decorated.getVideos(channel, ids) - return videos - } - async getAllVideos(channel: YtChannel) { - // ensure have some left api quota - if (!(await this.canCallYoutube())) { - throw new YoutubeApiError( - ExitCodes.YoutubeApi.YOUTUBE_QUOTA_LIMIT_EXCEEDED, - 'No more quota left. Please try again later.' - ) - } + // increase used quota count, 1 api call is being used per page of 50 videos + await this.increaseUsedQuota({ syncQuotaIncrement: Math.ceil(videos.length / 50) }) - // get videos from api - const videos = await this.decorated.getAllVideos(channel) return videos } @@ -594,6 +532,17 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { return this.decorated.downloadVideo(videoUrl, outPath) } + private async increaseUsedQuota({ syncQuotaIncrement = 0, signupQuotaIncrement = 0 }) { + // Quota resets at Pacific Time, and pst is 8 hours behind UTC + const stats = await this.statsRepo.getOrSetTodaysStats() + const statsModel = await this.statsRepo.getModel() + + await statsModel.update( + { partition: 'stats', date: stats.date }, + { $ADD: { syncQuotaUsed: syncQuotaIncrement, signupQuotaUsed: signupQuotaIncrement } } + ) + } + private async canCallYoutube(): Promise { if (this.quotaMonitoringClient) { const quotaUsage = await this.getQuotaUsage() @@ -608,7 +557,7 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { } export const YoutubeApi = { - create(config: ReadonlyConfig): IYoutubeApi { - return new QuotaMonitoringClient(new YoutubeClient(config), config) + create(config: ReadonlyConfig, statsRepo: StatsRepository): IYoutubeApi { + return new QuotaMonitoringClient(new YoutubeClient(config), config, statsRepo) }, } diff --git a/src/types/generated/ConfigJson.d.ts b/src/types/generated/ConfigJson.d.ts index 6befe2eb..d627ae82 100644 --- a/src/types/generated/ConfigJson.d.ts +++ b/src/types/generated/ConfigJson.d.ts @@ -1,4 +1,4 @@ -/* tslint:disable */ +/* eslint-disable */ /** * This file was automatically generated by json-schema-to-typescript. * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, @@ -27,15 +27,15 @@ export interface YoutubeSyncNodeConfiguration { captchaBypassKey: string } /** - * Joystream metaprotocol application specific configuration + * Joystream Metaprotocol App specific configuration */ app: { /** - * Name of the application + * Name of the app */ name: string /** - * Specifies the application auth key's string seed for generating ed25519 keypair + * Specifies the app auth key's string seed, for generating ed25519 keypair, to be used for signing App Actions */ accountSeed: string } @@ -54,15 +54,6 @@ export interface YoutubeSyncNodeConfiguration { */ joystreamNodeWs: string } - /** - * Specifies paths where node's data will be stored - */ - directories: { - /** - * Path to a directory where all the cached assets will be stored - */ - assets: string - } /** * Specifies the logging configuration */ @@ -71,37 +62,6 @@ export interface YoutubeSyncNodeConfiguration { console?: ConsoleLoggingOptions elastic?: ElasticsearchLoggingOptions } - /** - * Specifies youtube-synch service limits. - */ - limits: { - dailyApiQuota: SpecifiesDailyYoutubeAPIQuotaRationingSchemeForYoutubePartnerProgram - /** - * Max no. of videos that should be concurrently downloaded from Youtube to be prepared for upload to Joystream - */ - maxConcurrentDownloads: number - /** - * Max no. of videos that should be concurrently uploaded to Joystream's storage node - */ - maxConcurrentUploads: number - /** - * Maximum total size of all downloaded assets stored in `directories.assets` - */ - storage: string - } - /** - * Specifies how often periodic tasks (for example youtube state polling) are executed. - */ - intervals: { - /** - * After how many minutes, the polling service should poll the Youtube api for channels state update - */ - youtubePolling: number - /** - * After how many minutes, the service should scan the database for new content to start downloading, on-chain creation & uploading to storage node - */ - contentProcessing: number - } youtube: YoutubeOauth2ClientConfiguration aws?: AWSConfigurationsNeededToConnectWithDynamoDBInstance /** @@ -126,6 +86,7 @@ export interface YoutubeSyncNodeConfiguration { minimumChannelAgeHours: number } httpApi: PublicApiConfiguration + sync: YTSynchSyncronizationRelatedSettings } /** * Joystream channel collaborators used for syncing the content @@ -134,6 +95,8 @@ export interface JoystreamChannelCollaboratorUsedForSyncingTheContent { memberId: string /** * Specifies the available application auth keys. + * + * @minItems 1 */ account: (SubstrateUri | MnemonicPhrase)[] } @@ -204,13 +167,6 @@ export interface ElasticsearchAuthenticationOptions { */ password: string } -/** - * Specifies daily Youtube API quota rationing scheme for Youtube Partner Program - */ -export interface SpecifiesDailyYoutubeAPIQuotaRationingSchemeForYoutubePartnerProgram { - sync: number - signup: number -} /** * Youtube Oauth2 Client configuration */ @@ -260,3 +216,54 @@ export interface PublicApiConfiguration { port: number ownerKey: string } +/** + * YT-synch's syncronization related settings + */ +export interface YTSynchSyncronizationRelatedSettings { + /** + * Option to enable/disable syncing while starting the service + */ + enable: boolean + /** + * Path to a directory where all the downloaded assets will be stored + */ + downloadsDir?: string + /** + * Specifies how often periodic tasks (for example youtube state polling) are executed. + */ + intervals?: { + /** + * After how many minutes, the polling service should poll the Youtube api for channels state update + */ + youtubePolling: number + /** + * After how many minutes, the service should scan the database for new content to start downloading, on-chain creation & uploading to storage node + */ + contentProcessing: number + } + /** + * Specifies youtube-synch service limits. + */ + limits?: { + dailyApiQuota: SpecifiesDailyYoutubeAPIQuotaRationingSchemeForYoutubePartnerProgram + /** + * Max no. of videos that should be concurrently downloaded from Youtube to be prepared for upload to Joystream + */ + maxConcurrentDownloads: number + /** + * Max no. of videos that should be concurrently uploaded to Joystream's storage node + */ + maxConcurrentUploads: number + /** + * Maximum total size of all downloaded assets stored in `downloadsDir` + */ + storage: string + } +} +/** + * Specifies daily Youtube API quota rationing scheme for Youtube Partner Program + */ +export interface SpecifiesDailyYoutubeAPIQuotaRationingSchemeForYoutubePartnerProgram { + sync: number + signup: number +} diff --git a/src/types/index.ts b/src/types/index.ts index b3528053..e3080a63 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,10 +1,22 @@ import { YoutubeSyncNodeConfiguration } from './generated/ConfigJson' -export type Config = Omit & { +type SyncEnabled = Omit, 'limits'> & { + enable: true + downloadsDir: string // make required when enabled + intervals: NonNullable // make required when enabled + limits: Omit, 'storage'> & { storage: number } +} + +type SyncDisabled = Omit, 'limits'> & { + enable: false + downloadsDir?: string + intervals?: YoutubeSyncNodeConfiguration['sync']['intervals'] + limits?: Omit, 'storage'> & { storage: number } +} + +export type Config = Omit & { version: string - limits: Omit & { - storage: number - } + sync: SyncEnabled | SyncDisabled } export type ReadonlyConfig = DeepReadonly @@ -12,10 +24,11 @@ export type DeepReadonly = { readonly [K in keyof T]: DeepReadonly } type Secret = { [K in keyof T]: '###SECRET###' } -export type DisplaySafeConfig = Omit & { +export type DisplaySafeConfig = Omit & { httpApi: Secret youtube: Secret joystream: Secret + logs?: { elastic: Secret['elastic']> } aws?: { credentials: Secret['credentials']> } } diff --git a/src/types/youtube.ts b/src/types/youtube.ts index b1791e95..9bccefb4 100644 --- a/src/types/youtube.ts +++ b/src/types/youtube.ts @@ -65,18 +65,25 @@ export class YtChannel { videoCount: number } - aggregatedStats: number + // total size of historical videos synced (videos that were published on Youtube before YPP signup) + historicalVideoSyncedSize: number // Channel owner's access token userAccessToken: string // Channel owner's refresh token userRefreshToken: string + + // Channel's playlist ID uploadsPlaylistId: string // Should this channel be ingested for automated Youtube/Joystream syncing? shouldBeIngested: boolean + // Should this channel be ingested for automated Youtube/Joystream syncing? (operator managed flag) + // Both `shouldBeIngested` and `allowOperatorIngestion` should be set for sync to work. + allowOperatorIngestion: boolean + // Should this channel be ingested for automated Youtube/Joystream syncing without explicit authorization granted to app? performUnauthorizedSync: boolean @@ -209,8 +216,8 @@ export class YtVideo { // Media container format container: string - // Indicates if the video is an upcoming/active live broadcast. else it's "none" - liveBroadcastContent: 'upcoming' | 'live' | 'none' + // Indicates if the video is/was a livestream + liveStreamingDetails: any | undefined // joystream video ID in `VideoCreated` event response, returned from joystream runtime after creating a video joystreamVideo: JoystreamVideo @@ -257,6 +264,11 @@ export type VideoCreationTask = YtVideo & { filePath: string } +export type YtDlpFlatPlaylistOutput = { + id: string + publishedAt: Date +}[] + export type FaucetRegisterMembershipParams = { account: string handle: string diff --git a/src/utils/configParser.ts b/src/utils/configParser.ts index 0bbbedf4..a1de6530 100644 --- a/src/utils/configParser.ts +++ b/src/utils/configParser.ts @@ -1,5 +1,5 @@ import fs from 'fs' -import { JSONSchema4, JSONSchema4TypeName } from 'json-schema' +import { JSONSchema7, JSONSchema7TypeName } from 'json-schema' import _ from 'lodash' import path from 'path' import YAML from 'yaml' @@ -23,10 +23,6 @@ export class ConfigParserService { return path.resolve(path.dirname(this.configPath), p) } - public resolveConfigDirectoryPaths(paths: Config['directories']): Config['directories'] { - return _.mapValues(paths, (p) => this.resolvePath(p)) - } - private parseByteSize(byteSize: string) { const intValue = parseInt(byteSize) const unit = byteSize[byteSize.length - 1] @@ -34,15 +30,18 @@ export class ConfigParserService { return intValue * Math.pow(1024, byteSizeUnits.indexOf(unit)) } - private schemaTypeOf(schema: JSONSchema4, path: string[]): JSONSchema4['type'] { + private schemaTypeOf(schema: JSONSchema7, path: string[]): JSONSchema7['type'] | undefined { if (schema.properties && schema.properties[path[0]]) { const item = schema.properties[path[0]] + if (typeof item === 'boolean') { + throw new Error('Unsupported schema type') + } if (path.length > 1) { return this.schemaTypeOf(item, path.slice(1)) } if (item.oneOf) { - const validTypesSet = new Set() - item.oneOf.forEach( + const validTypesSet = new Set() + ;(item.oneOf as JSONSchema7[]).forEach( (s) => Array.isArray(s.type) ? s.type.forEach((t) => validTypesSet.add(t)) @@ -149,18 +148,24 @@ export class ConfigParserService { const configJson = this.validator.validate('Config', inputConfig) // Normalize values - const directories = this.resolveConfigDirectoryPaths(configJson.directories) - const storageLimit = this.parseByteSize(configJson.limits.storage) + const storageLimit = this.parseByteSize(configJson.sync.limits?.storage || '0B') const parsedConfig: Config = { ...configJson, version: this.getNodeVersion(), - directories, - limits: { - ...configJson.limits, - storage: storageLimit, - }, - } + ...(configJson.sync.enable + ? { + sync: { + ...configJson.sync, + downloadsDir: this.resolvePath(configJson.sync.downloadsDir!), + limits: { + ...configJson.sync.limits, + storage: storageLimit, + }, + }, + } + : configJson.sync), + } as Config return parsedConfig } diff --git a/yarn.lock b/yarn.lock index 77445d6d..cfb8462f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -38,16 +38,6 @@ unist-util-inspect "6.0.1" yargs "17.2.1" -"@apidevtools/json-schema-ref-parser@9.0.9": - version "9.0.9" - resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz#d720f9256e3609621280584f2b47ae165359268b" - integrity sha512-GBD2Le9w2+lVFoc4vswGI/TjkNIZSVp7+9xPf+X3uidBfWnAeUWmquteSyt0+VCrhNMWj/FTABISQrD3Z/YA+w== - dependencies: - "@jsdevtools/ono" "^7.1.3" - "@types/json-schema" "^7.0.6" - call-me-maybe "^1.0.1" - js-yaml "^4.1.0" - "@apidevtools/json-schema-ref-parser@^9.0.6": version "9.1.2" resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.1.2.tgz#8ff5386b365d4c9faa7c8b566ff16a46a577d9b8" @@ -1435,6 +1425,16 @@ "@babel/helper-validator-identifier" "^7.18.6" to-fast-properties "^2.0.0" +"@bcherny/json-schema-ref-parser@10.0.5-fork": + version "10.0.5-fork" + resolved "https://registry.yarnpkg.com/@bcherny/json-schema-ref-parser/-/json-schema-ref-parser-10.0.5-fork.tgz#9b5e1e7e07964ea61840174098e634edbe8197bc" + integrity sha512-E/jKbPoca1tfUPj3iSbitDZTGnq6FUFjkH6L8U2oDwSuwK1WhnnVtCG7oFOTg/DDnyoXbQYUiUiGOibHqaGVnw== + dependencies: + "@jsdevtools/ono" "^7.1.3" + "@types/json-schema" "^7.0.6" + call-me-maybe "^1.0.1" + js-yaml "^4.1.0" + "@colors/colors@1.5.0": version "1.5.0" resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" @@ -3712,7 +3712,7 @@ "@types/minimatch" "^5.1.2" "@types/node" "*" -"@types/glob@^7.1.1": +"@types/glob@^7.1.1", "@types/glob@^7.1.3": version "7.2.0" resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.2.0.tgz#bc1b5bf3aa92f25bd5dd39f35c57361bdce5b2eb" integrity sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA== @@ -3753,16 +3753,16 @@ resolved "https://registry.yarnpkg.com/@types/js-yaml/-/js-yaml-4.0.5.tgz#738dd390a6ecc5442f35e7f03fa1431353f7e138" integrity sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA== +"@types/json-schema@^7.0.11", "@types/json-schema@^7.0.8": + version "7.0.12" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb" + integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA== + "@types/json-schema@^7.0.3", "@types/json-schema@^7.0.6", "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== -"@types/json-schema@^7.0.8": - version "7.0.12" - resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb" - integrity sha512-Hr5Jfhc9eYOQNPYO5WLDq/n4jqijdHNlDXjuAQkkt+mWdQR+XJToOHrsD4cPaMXpn6KO7y2+wM8AZEs8VpBLVA== - "@types/json-stable-stringify@^1.0.32": version "1.0.33" resolved "https://registry.yarnpkg.com/@types/json-stable-stringify/-/json-stable-stringify-1.0.33.tgz#099b0712d824d15e2660c20e1c16e6a8381f308c" @@ -3790,10 +3790,10 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.178.tgz#341f6d2247db528d4a13ddbb374bcdc80406f4f8" integrity sha512-0d5Wd09ItQWH1qFbEyQ7oTQ3GZrMfth5JkbN3EvTKLXcHLRDSXeLnlvlOn0wvxVIwK5o2M8JzP/OWz7T3NRsbw== -"@types/lodash@^4.14.168": - version "4.14.191" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.191.tgz#09511e7f7cba275acd8b419ddac8da9a6a79e2fa" - integrity sha512-BdZ5BCCvho3EIXw6wUCXHe7rS53AIDPLE+JzwgT+OsJk53oBfbSmZZ7CX4VaRoN78N+TJpFi9QPlfIVNmJYWxQ== +"@types/lodash@^4.14.182": + version "4.14.196" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.196.tgz#a7c3d6fc52d8d71328b764e28e080b4169ec7a95" + integrity sha512-22y3o88f4a94mKljsZcanlNWPzO0uBsBdzLAngf2tp533LzZcQzb6+eZPJ+vCTt+bqF2XnvT9gejTLsAcJAJyQ== "@types/long@^4.0.0": version "4.0.2" @@ -3870,10 +3870,10 @@ dependencies: "@types/node" "*" -"@types/prettier@^2.1.5": - version "2.4.3" - resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.4.3.tgz#a3c65525b91fca7da00ab1a3ac2b5a2a4afbffbf" - integrity sha512-QzSuZMBuG5u8HqYz01qtMdg/Jfctlnvj1z/lYnIDXs/golxw0fxtRAHd9KrzjR7Yxz1qVeI00o0kiO3PmVdJ9w== +"@types/prettier@^2.6.1": + version "2.7.3" + resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.3.tgz#3e51a17e291d01d17d3fc61422015a933af7a08f" + integrity sha512-+68kP9yzs4LMp7VNh8gdzMSPZFL44MLGqiHWvttYJe+6qnuVr4Ek9wSBQoveqY/r+LwjCcU29kNVkidwim+kYA== "@types/proper-lockfile@^4.1.1": version "4.1.2" @@ -5384,7 +5384,7 @@ clean-stack@^3.0.0: dependencies: escape-string-regexp "4.0.0" -cli-color@^2.0.0: +cli-color@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/cli-color/-/cli-color-2.0.3.tgz#73769ba969080629670f3f2ef69a4bf4e7cc1879" integrity sha512-OkoZnxyC4ERN3zLzZaY9Emb7f/MhBOIpePv0Ycok0fJYT+Ouo00UBEIwsVsr0yoow++n5YWlSUgST9GKhNHiRQ== @@ -7731,12 +7731,12 @@ glob-parent@^6.0.1: dependencies: is-glob "^4.0.3" -glob-promise@^3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/glob-promise/-/glob-promise-3.4.0.tgz#b6b8f084504216f702dc2ce8c9bc9ac8866fdb20" - integrity sha512-q08RJ6O+eJn+dVanerAndJwIcumgbDdYiUT7zFQl3Wm1xD6fBKtah7H8ZJChj4wP+8C+QfeVy8xautR7rdmKEw== +glob-promise@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/glob-promise/-/glob-promise-4.2.2.tgz#15f44bcba0e14219cd93af36da6bb905ff007877" + integrity sha512-xcUzJ8NWN5bktoTIX7eOclO1Npxd/dyVqUJxlLIDasT4C7KZyqlPIwkdJ0Ypiy3p2ZKahTjK4M9uC3sNSfNMzw== dependencies: - "@types/glob" "*" + "@types/glob" "^7.1.3" glob-to-regexp@^0.4.1: version "0.4.1" @@ -9425,33 +9425,25 @@ json-parse-even-better-errors@^2.3.0: resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== -json-schema-ref-parser@^9.0.6: - version "9.0.9" - resolved "https://registry.yarnpkg.com/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz#66ea538e7450b12af342fa3d5b8458bc1e1e013f" - integrity sha512-qcP2lmGy+JUoQJ4DOQeLaZDqH9qSkeGCK3suKWxJXS82dg728Mn3j97azDMaOUmJAN4uCq91LdPx4K7E8F1a7Q== +json-schema-to-typescript@^13.0.2: + version "13.0.2" + resolved "https://registry.yarnpkg.com/json-schema-to-typescript/-/json-schema-to-typescript-13.0.2.tgz#35af1ded746e1bcb1b8f0ec169f24da33bed9eaf" + integrity sha512-TCaEVW4aI2FmMQe7f98mvr3/oiVmXEC1xZjkTZ9L/BSoTXFlC7p64mD5AD2d8XWycNBQZUnHwXL5iVXt1HWwNQ== dependencies: - "@apidevtools/json-schema-ref-parser" "9.0.9" - -json-schema-to-typescript@^10.1.4: - version "10.1.5" - resolved "https://registry.yarnpkg.com/json-schema-to-typescript/-/json-schema-to-typescript-10.1.5.tgz#9ac32808eb4d7c158684e270438ad07256c0cb1c" - integrity sha512-X8bNNksfCQo6LhEuqNxmZr4eZpPjXZajmimciuk8eWXzZlif9Brq7WuMGD/SOhBKcRKP2SGVDNZbC28WQqx9Rg== - dependencies: - "@types/json-schema" "^7.0.6" - "@types/lodash" "^4.14.168" - "@types/prettier" "^2.1.5" - cli-color "^2.0.0" + "@bcherny/json-schema-ref-parser" "10.0.5-fork" + "@types/json-schema" "^7.0.11" + "@types/lodash" "^4.14.182" + "@types/prettier" "^2.6.1" + cli-color "^2.0.2" get-stdin "^8.0.0" glob "^7.1.6" - glob-promise "^3.4.0" - is-glob "^4.0.1" - json-schema-ref-parser "^9.0.6" - json-stringify-safe "^5.0.1" - lodash "^4.17.20" - minimist "^1.2.5" + glob-promise "^4.2.2" + is-glob "^4.0.3" + lodash "^4.17.21" + minimist "^1.2.6" mkdirp "^1.0.4" mz "^2.7.0" - prettier "^2.2.0" + prettier "^2.6.2" json-schema-traverse@^0.4.1: version "0.4.1" @@ -11598,16 +11590,16 @@ prettier-linter-helpers@^1.0.0: dependencies: fast-diff "^1.1.2" -prettier@^2.2.0: - version "2.8.4" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.4.tgz#34dd2595629bfbb79d344ac4a91ff948694463c3" - integrity sha512-vIS4Rlc2FNh0BySk3Wkd6xmwxB0FpOndW5fisM5H8hsZSxU2VWVB5CWIkIjWvrHjIhxk2g3bfMKM87zNTrZddw== - prettier@^2.5.1: version "2.7.1" resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.7.1.tgz#e235806850d057f97bb08368a4f7d899f7760c64" integrity sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g== +prettier@^2.6.2: + version "2.8.8" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.8.tgz#e8c5d7e98a4305ffe3de2e1fc4aca1a71c28b1da" + integrity sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q== + process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"