Skip to content

Commit

Permalink
Merge pull request #697 from NoodleOfDeath/dev
Browse files Browse the repository at this point in the history
fix(RL-72): sharp build ~media ~sitemaps
  • Loading branch information
NoodleOfDeath authored Oct 15, 2023
2 parents c5e21fb + 34dbd98 commit 1fd0b22
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 18 deletions.
2 changes: 1 addition & 1 deletion src/server/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"dev:worker": "nodemon --config nodemon.worker.json -x 'ts-node ./serve-worker.ts'",
"build:api": "rimraf dist/serve-api.cjs && esbuild ./src/api --bundle --platform=node --outfile=./dist/serve-api.cjs",
"build:scheduler": "rimraf dist/serve-scheduler.cjs && esbuild ./src/scheduler --bundle --platform=node --outfile=./dist/serve-scheduler.cjs",
"build:worker": "rimraf dist/serve-worker.cjs && esbuild ./src/worker --bundle --platform=node --outfile=./dist/serve-worker.cjs",
"build:worker": "rimraf dist/serve-worker.cjs && esbuild ./src/worker --bundle --platform=node --external:sharp --outfile=./dist/serve-worker.cjs",
"gen:site": "yarn test gen",
"cp:core": "if [[ -d src/core ]]; then rm -rf src/core; fi && mkdir -p ./src/core/utils && cp -rf ../core/src/utils/* ./src/core/utils && cp -rf ../core/src/client/locales ./src/core/locales && cp -rf node_modules/sharp/build .",
"sql": "ts-node ./gen-sql-queries",
Expand Down
104 changes: 103 additions & 1 deletion src/server/src/api/v1/schema/resources/summary/Summary.model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,25 @@ import {
DataType,
Table,
} from 'sequelize-typescript';
import sharp from 'sharp';

import {
PublicSummaryGroup,
SummaryAttributes,
SummaryCreationAttributes,
} from './Summary.types';
import { SummaryInteraction } from './SummaryInteraction.model';
import { SummaryMedia } from './SummaryMedia.model';
import { SummaryMediaAttributes } from './SummaryMedia.types';
import { SummaryRelation } from './SummaryRelation.model';
import { SummarySentiment } from './SummarySentiment.model';
import { PublicSummarySentimentAttributes } from './SummarySentiment.types';
import { QUERIES, QueryKey } from './queries';
import { DeepAiService, SentimentService } from '../../../../../services';
import {
DeepAiService,
S3Service,
SentimentService,
} from '../../../../../services';
import { parseDate } from '../../../../../utils';
import { BulkMetadataResponse } from '../../../controllers';
import { Cache } from '../../system/Cache.model';
Expand Down Expand Up @@ -43,6 +50,30 @@ export type SearchSummariesPayload = {
cacheHalflife?: string;
};

class Size {

name: string;
value: number;

static xs = new Size('xs', 60);
static sm = new Size('sm', 120);
static md = new Size('md', 240);
static lg = new Size('lg', 360);
static xl = new Size('xl', 480);
static xxl = new Size('xxl', 720);
static xxxl = new Size('xxxl', 1920);

constructor(name: string, value: number) {
this.name = name;
this.value = value;
}

}

type DownsampleOptions = Pick<SummaryMediaAttributes, 'key' | 'parentId' | 'path'> & {
sizes?: Size[];
};

function parseTimeInterval(str: string) {
const matches = str.match(/(\d+)\s*(months?|m(?:in(?:ute)?s?)?|h(?:(?:ou)?rs?)?|d(?:ays?)?|w(?:(?:ee)?ks?)?|y(?:(?:ea)?rs?)?)/i);
if (matches && matches[1] && matches[2]) {
Expand Down Expand Up @@ -482,6 +513,77 @@ export class Summary extends Post<SummaryAttributes, SummaryCreationAttributes>
}
}

async generateThumbnails(
{
key,
parentId,
path,
sizes = [Size.xs, Size.sm, Size.md, Size.lg],
}: DownsampleOptions,
folder: string
) {
// eslint-disable-next-line no-async-promise-executor
return new Promise<SummaryMedia[]>(async (resolve, reject) => {
const allMedia = await SummaryMedia.findAll({ where: { parentId: this.id } });
const results: SummaryMedia[] = [];
for (const [i, m] of allMedia.entries()) {
if (!/^img\/s/.test(m.path) || /@(?:xs|sm|md|lg|x+l)\.\w+$/.test(m.path)) {
continue;
}
const file = await S3Service.getObject({ Key: m.path });
if (!file) {
reject('Missing file');
return;
}
for (const [j, size] of sizes.entries()) {
const subkey = `${key}@${size.name}`;
const media = await SummaryMedia.findOne({
where: {
key: subkey,
parentId,
},
});
if (media) {
console.log('media already exists');
results.push(media);
if (i + 1 === allMedia.length && j + 1 === sizes.length) {
resolve(results);
return;
}
continue;
}
const target = file.replace(/(\.\w+)$/, (_, $1) => `@${size.name}${$1}`);
sharp(file)
.resize(size.value)
.jpeg()
.toFile(target, async (err) => {
if (err) {
reject(err);
return;
}
const response = await S3Service.putObject({
ACL: 'public-read',
Accept: 'image',
File: target,
Folder: folder,
});
const media = await SummaryMedia.create({
key: subkey,
parentId,
path: response.key,
type: 'image',
url: response.url,
});
results.push(media);
if (i + 1 === allMedia.length && j + 1 === sizes.length) {
resolve(results);
}
});
}
}
});
}

async generateSentiment(...props: (keyof SummaryAttributes)[]) {
if (props.length === 0) {
props = ['title', 'shortSummary'];
Expand Down
33 changes: 17 additions & 16 deletions src/server/src/worker/MediaWorker.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import fs from 'fs';

import sharp from 'sharp';

import { SummaryMedia } from '../api/v1/schema/models';
import { Summary, SummaryMedia } from '../api/v1/schema/models';
import { SummaryMediaAttributes } from '../api/v1/schema/types';
import { DBService, S3Service } from '../services';

Expand Down Expand Up @@ -88,16 +86,6 @@ async function downsampleImage({
type: 'image',
url: response.url,
});
try {
fs.unlinkSync(file);
} catch (e) {
console.error(e);
}
try {
fs.unlinkSync(target);
} catch (e) {
console.error(e);
}
results.push(media);
if (index + 1 === sizes.length) {
resolve(results);
Expand All @@ -117,13 +105,26 @@ export async function doWork() {
const media = await SummaryMedia.findOne({ where: { path: item } });
if (!media) {
console.log('Unlinking dangling media', item);
await S3Service.deleteObject({ Key: item });
}
try {
await S3Service.deleteObject({ Key: item });
} catch (e) {
console.error(e);
}
} else
if (!/@(?:xs|sm|md|lg|x+l)\.\w+$/.test(item)) {
console.log('Downsampling', item);
const folders = item.split('/');
folders.pop();
await downsampleImage(media, folders.join('/'));
try {
await downsampleImage(media, folders.join('/'));
try {
await Summary.refreshViews();
} catch (e) {
console.error(e);
}
} catch (e) {
console.error(e);
}
}
}
}
Expand Down

0 comments on commit 1fd0b22

Please sign in to comment.