feat(RL-72): downsamples all images

src/server/package.json

@@ -58,6 +58,7 @@
     "sentiment": "^5.0.2",
     "sequelize": "^6.29.0",
     "sequelize-typescript": "^2.1.5",
+    "sharp": "^0.32.6",
     "swagger-ui-express": "^4.6.2",
     "tsoa": "^5.1.1",
     "user-agents": "^1.0.1307",

src/server/src/services/aws/s3/S3Service.ts

@@ -3,10 +3,16 @@ import p from 'path';
 import { Readable } from 'stream';
 
 import {
+  DeleteObjectCommand,
+  DeleteObjectCommandInput,
   GetObjectCommand,
   GetObjectCommandInput,
+  ListObjectsV2Command,
+  ListObjectsV2CommandInput,
   PutObjectCommand,
   PutObjectCommandInput,
+  RestoreObjectCommand,
+  RestoreObjectCommandInput,
   S3Client,
 } from '@aws-sdk/client-s3';
 import axios from 'axios';
@@ -28,6 +34,11 @@ export type GetOptions = Omit<GetObjectCommandInput, 'Bucket'> & {
   Provider?: string;
 };
 
+export type ListOptions = Omit<ListObjectsV2CommandInput, 'Bucket'> & {
+  Bucket?: string;
+  Provider?: string;
+};
+
 export type PutOptions = Omit<PutObjectCommandInput, 'Body' | 'Bucket' | 'ContentType' | 'Key'> & {
   Bucket?: string;
   Body?: string;
@@ -40,6 +51,17 @@ export type PutOptions = Omit<PutObjectCommandInput, 'Body' | 'Bucket' | 'Conten
   MinSize?: number;
 };
 
+export type DeleteOptions = Omit<DeleteObjectCommandInput, 'Bucket'> & {
+  Bucket?: string;
+  Provider?: string;
+  Folder?: string;
+};
+
+export type RestoreOptions = Omit<RestoreObjectCommandInput, 'Bucket'> & {
+  Bucket?: string;
+  Provider?: string;
+};
+
 export class S3Service extends BaseService {
 
   public static s3Client = new S3Client({
@@ -104,6 +126,7 @@ export class S3Service extends BaseService {
     if (!filepath) {
       filepath = `/tmp/${filename}`;
     }
+    console.log('Downloaded', filepath);
     return new Promise((resolve, reject) => {
       response.data.pipe(fs.createWriteStream(filepath))
         .on('error', reject)
@@ -124,13 +147,38 @@ export class S3Service extends BaseService {
       throw new Error('Malformed key');
     }
     const response = await this.s3Client.send(new GetObjectCommand(params));
+    filepath = `${filepath}.${mime.extension(response.ContentType) || mime.extension(mime.lookup(response.ContentType) || '')}`;
     const stream = response.Body as Readable;
     return new Promise((resolve, reject) => {
       stream.pipe(fs.createWriteStream(filepath))
         .on('error', reject)
         .once('close', () => resolve(filepath));
     });
   }
+
+  public static async listObjects(options: ListOptions = {}) {
+    const params = {
+      ...options,
+      Bucket: options.Bucket || process.env.S3_BUCKET,
+      MaxKeys: options.MaxKeys,
+      Provider: options.Provider || process.env.S3_PROVIDER || 'nyc3.digitaloceanspaces.com',
+    };
+    if (!params.Bucket) {
+      throw new Error('Malformed bucket');
+    }
+    let isTruncated = true;
+    const items: string[] = [];
+    const command = new ListObjectsV2Command(params);
+    while (isTruncated) {
+      const {
+        Contents, IsTruncated, NextContinuationToken, 
+      } = await this.s3Client.send(command);
+      items.push(...Contents.map((c) => c.Key));
+      isTruncated = IsTruncated;
+      command.input.ContinuationToken = NextContinuationToken;
+    }
+    return items;
+  }
 
   public static async putObject(options: PutOptions) {
     const params = {
@@ -159,6 +207,30 @@ export class S3Service extends BaseService {
     };
   }
 
+  public static async deleteObject(options: DeleteOptions) {
+    const params = {
+      ...options,
+      Bucket: options.Bucket || process.env.S3_BUCKET,
+      Key: options.Folder ? [options.Folder, options.Key].join('/') : options.Key,
+      Provider: options.Provider || process.env.S3_PROVIDER || 'nyc3.digitaloceanspaces.com',
+    };
+    const data = await this.s3Client.send(new DeleteObjectCommand(params));
+    return data;
+  }
+
+  public static async restoreObject(options: RestoreOptions) {
+    const params = {
+      ...options,
+      Bucket: options.Bucket || process.env.S3_BUCKET,
+      Provider: options.Provider || process.env.S3_PROVIDER || 'nyc3.digitaloceanspaces.com',
+    };
+    if (!params.Bucket) {
+      throw new Error('Malformed bucket');
+    }
+    const data = await this.s3Client.send(new RestoreObjectCommand(params));
+    return data;
+  }
+
   public static async mirror(url: string, options: PutOptions) {
     try {
       const file = await this.download(url, { 

src/server/src/worker/MediaWorker.ts

@@ -0,0 +1,137 @@
+import fs from 'fs';
+
+import sharp from 'sharp';
+
+import { SummaryMedia } from '../api/v1/schema/models';
+import { SummaryMediaAttributes } from '../api/v1/schema/types';
+import { DBService, S3Service } from '../services';
+
+export async function main() {
+  await DBService.prepare();
+  doWork();
+}
+
+class Size {
+
+  name: string;
+  value: number;
+
+  static xs = new Size('xs', 60);
+  static sm = new Size('sm', 120);
+  static md = new Size('md', 240);
+  static lg = new Size('lg', 360);
+  static xl = new Size('xl', 480);
+  static xxl = new Size('xxl', 720);
+  static xxxl = new Size('xxxl', 1920);
+
+  constructor(name: string, value: number) {
+    this.name = name;
+    this.value = value;
+  }
+
+}
+
+type DownsampleOptions = Pick<SummaryMediaAttributes, 'key' | 'parentId' | 'path'> & {
+  sizes?: Size[];
+};
+
+async function downsampleImage({
+  key,
+  parentId,
+  path,
+  sizes = [Size.xs, Size.sm, Size.md, Size.lg],
+}: DownsampleOptions, folder: string) {
+  // eslint-disable-next-line no-async-promise-executor
+  return new Promise<SummaryMedia[]>(async (resolve, reject) => {
+    const file = await S3Service.getObject({ Key: path });
+    if (!file) {
+      reject('Missing file');
+      return;
+    }
+    const results: SummaryMedia[] = [];
+    for (const [index, size] of sizes.entries()) {
+      const subkey = `${key}@${size.name}`;
+      const media = await SummaryMedia.findOne({
+        where: {
+          key: subkey,
+          parentId,
+        },
+      });
+      if (media) {
+        console.log('media already exists');
+        results.push(media);
+        if (index + 1 === sizes.length) {
+          resolve(results);
+          return;
+        }
+        continue;
+      }
+      const target = file.replace(/(\.\w+)$/, (_, $1) => `@${size.name}${$1}`);
+      sharp(file)
+        .resize(size.value)
+        .jpeg()
+        .toFile(target, async (err) => {
+          if (err) {
+            reject(err);
+            return;
+          }
+          const response = await S3Service.putObject({
+            ACL: 'public-read',
+            Accept: 'image',
+            File: target,
+            Folder: folder,
+          });
+          const media = await SummaryMedia.create({
+            key: subkey,
+            parentId,
+            path: response.key,
+            type: 'image',
+            url: response.url,
+          });
+          try {
+            fs.unlinkSync(file);
+          } catch (e) {
+            console.error(e); 
+          }
+          try {
+            fs.unlinkSync(target);
+          } catch (e) {
+            console.error(e); 
+          }
+          results.push(media);
+          if (index + 1 === sizes.length) {
+            resolve(results);
+          }
+        });
+    }
+  });
+}
+
+export async function doWork() {
+  console.log('fetching objects');
+  try {
+    const items = await S3Service.listObjects();
+    console.log(items.length);
+    for (const item of items) {
+      if (/^img\/s/.test(item)) {
+        const media = await SummaryMedia.findOne({ where: { path: item } });
+        if (!media) {
+          console.log('Unlinking dangling media', item);
+          await S3Service.deleteObject({ Key: item });
+        }
+        if (!/@(?:xs|sm|md|lg|x+l)\.\w+$/.test(item)) {
+          console.log('Downsampling', item);
+          const folders = item.split('/');
+          folders.pop();
+          await downsampleImage(media, folders.join('/'));
+        }
+      }
+    }
+  } catch (e) {
+    console.error(e);
+  } finally {
+    setTimeout(doWork, 3_000);
+  }
+}
+
+main();

src/server/src/worker/index.ts

@@ -6,6 +6,9 @@ export async function main() {
   case 'topics':
     await import('./TopicWorker');
     break;
+  case 'media':
+    await import('./MediaWorker');
+    break;
   case 'notifs':
     await import('./NotificationsWorker');
     break;

src/server/tests/s3.test.ts

@@ -45,5 +45,11 @@ describe('uploads to the s3 bucket', () => {
     console.log(response);
     expect(response).toBeDefined();
   });
+
+  test('list files', async () => {
+    const response = await S3Service.listObjects();
+    console.log(response);
+    expect(response).toBeDefined();
+  });
 
 });