diff --git a/_tools/frontmatter-validator/index.js b/_tools/frontmatter-validator/index.js new file mode 100644 index 0000000..d721362 --- /dev/null +++ b/_tools/frontmatter-validator/index.js @@ -0,0 +1,94 @@ +// frontmatter-validator for dogwood +// +// This script checks that the frontmatter in all your +// Jekyll documents matches a known format. This is important +// since the theme relies on many custom variables. +// +// By default, the script will throw an error if you have +// additional properties to those expected. This catches +// typos like `authors` instead of `author`. Use the +// `--strict=false` option to turn off this behavior. + +const matter = require('gray-matter'); +const fs = require('fs'); +const process = require('process') + +const argv = require('minimist')(process.argv.slice(2)); + +let sourcePath = process.env.npm_config_srcdir || argv.srcdir || '.'; +if (sourcePath.endsWith('/')) sourcePath.slice(0, -1); + +const ZSchema = require("z-schema"); + +let zSchemaOptions = {}; + +let isStrict = true; +if (("npm_config_strict" in process.env && !process.env.npm_config_strict) || + process.argv.includes("--strict=false")) { + isStrict = false; +} + +if (isStrict) { + let strictOptions = { + assumeAdditional: true, + forceItems: true, + forceMinItems: true, + forceMinLength: true, + forceProperties: true, + noEmptyArrays: true, + noEmptyStrings: true, + noExtraKeywords: true, + noTypeless: true, + }; + Object.assign(zSchemaOptions, strictOptions); +} + +const jsonValidator = new ZSchema(zSchemaOptions); + +let hasInvalid = false; + +function validateJson(json, schema) { + const valid = jsonValidator.validate(json, schema); + const errors = jsonValidator.getLastErrors(); + if (!valid) { + errors.forEach(function(error) { + console.log(error); + }); + hasInvalid = true; + } +} + +function validateFrontmatter(path, schemaPath) { + const schema = JSON.parse(fs.readFileSync(schemaPath)); + const dir = fs.opendirSync(path); + let dirent; + while ((dirent = dir.readSync()) !== null) { + const subpath = path + '/' + dirent.name; + if (dirent.isFile()) { + const filedata = fs.readFileSync(subpath); + const info = matter(filedata); + + // ignore files not in Jekyll format + if (Object.keys(info.data).length > 0) { + // parse to string and back again to make sure dates are in string format + let data = JSON.parse(JSON.stringify(info.data)); + validateJson(data, schema); + } + } else if (dirent.isDirectory()) { + validateFrontmatter(subpath, schemaPath); + } + } + dir.closeSync(); +} + +var schemasDir = __dirname + '/schemas'; + +validateFrontmatter(sourcePath + '/_posts', schemasDir + '/post.json'); +validateFrontmatter(sourcePath + '/_people', schemasDir + '/person.json'); +validateFrontmatter(sourcePath + '/_pages', schemasDir + '/page.json'); +validateFrontmatter(sourcePath + '/_redirects', schemasDir + '/redirect.json'); + +if (hasInvalid) { + // nonzero exit for GitHub Actions + process.exit(1); +} \ No newline at end of file diff --git a/_tools/frontmatter-validator/schemas/page.json b/_tools/frontmatter-validator/schemas/page.json new file mode 100644 index 0000000..84cef98 --- /dev/null +++ b/_tools/frontmatter-validator/schemas/page.json @@ -0,0 +1,281 @@ +{ + "title": "Page", + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "temp_title": { + "type": "string" + }, + "short_title": { + "type": "string" + }, + "tagline": { + "type": "string" + }, + "temp_tagline": { + "type": "string" + }, + "blurb": { + "type": "string" + }, + "temp_blurb": { + "type": "string" + }, + "layout": { + "type": "string" + }, + "event": { + "type": "string" + }, + "event_series": { + "type": "string" + }, + "caption": { + "type": "string" + }, + "status": { + "type": "string" + }, + "location": { + "type": "string" + }, + "image": { + "type": "string" + }, + "sign": { + "type": "string" + }, + "logo": { + "type": "string" + }, + "wordmark": { + "type": "string" + }, + "icon": { + "type": "string" + }, + "website": { + "type": "string" + }, + "twitter": { + "type": "string" + }, + "youtube_page": { + "type": "string", + "description": "Path to a page on youtube.com. Different from `youtube` since that parameter is a video ID which is used differently." + }, + "facebook": { + "type": "string" + }, + "slack_channel": { + "type": "string" + }, + "instagram": { + "type": "string" + }, + "email_list": { + "type": "string" + }, + "meetup": { + "type": "string" + }, + "reddit": { + "type": "string" + }, + "mastodon": { + "type": "string" + }, + "donate": { + "type": "string" + }, + "links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "link": { + "type": "string" + } + } + } + }, + "swag_items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "image": { + "type": "string" + } + } + } + }, + "swag_sections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "url": { + "type": "string" + }, + "image": { + "type": "string" + } + } + } + } + } + } + }, + "dropdown_links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "link": { + "type": "string" + } + } + } + }, + "app_links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "link": { + "type": "string" + } + } + } + }, + "footer_links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "link": { + "type": "string" + } + } + } + }, + "buttons": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "link": { + "type": "string" + } + } + } + }, + "sessions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "blurb": { + "type": "string" + }, + "date": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string", + "pattern": "\\d{4}-[01]\\d-[0-3]\\d [0-2]\\d:[0-5]\\d(:[0-5]\\d)? ([+-][0-2]\\d[0-5]\\d)" + } + ] + } + } + } + }, + "section_tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "team_name": { + "type": "string" + }, + "max_posts": { + "type": "integer" + }, + "updated": { + "type": "string", + "format": "date-time" + }, + "timezone": { + "type": "string" + }, + "filter_start": { + "type": "string", + "format": "date-time" + }, + "filter_end": { + "type": "string", + "format": "date-time" + }, + "start": { + "type": "string", + "format": "date-time" + }, + "end": { + "type": "string", + "format": "date-time" + }, + "map": { + "type": "object", + "properties": { + "points": { + "type": "string" + } + } + } + }, + "required": [ + "title" + ] + } \ No newline at end of file diff --git a/_tools/frontmatter-validator/schemas/person.json b/_tools/frontmatter-validator/schemas/person.json new file mode 100644 index 0000000..f2d0cda --- /dev/null +++ b/_tools/frontmatter-validator/schemas/person.json @@ -0,0 +1,77 @@ +{ + "title": "Person", + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "updated": { + "type": "string", + "format": "date-time" + }, + "roles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "at": { + "type": "string" + }, + "from": { + "type": "string", + "format": "date-time" + }, + "to": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "at" + ] + } + }, + "image": { + "type": "string" + }, + "image_remote": { + "type": "string" + }, + "cover": { + "type": "string" + }, + "caption": { + "type": "string" + }, + "linkedin": { + "type": "string" + }, + "twitter": { + "type": "string" + }, + "github": { + "type": "string" + }, + "osm": { + "type": "string" + }, + "website": { + "type": "string" + }, + "wikipedia": { + "type": "string" + }, + "mastodon": { + "type": "string" + }, + "medium": { + "type": "string" + } + }, + "required": [ + "title" + ] + } \ No newline at end of file diff --git a/_tools/frontmatter-validator/schemas/post.json b/_tools/frontmatter-validator/schemas/post.json new file mode 100644 index 0000000..b0c1c14 --- /dev/null +++ b/_tools/frontmatter-validator/schemas/post.json @@ -0,0 +1,110 @@ +{ + "title": "session", + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "link": { + "type": "string" + }, + "permalink": { + "type": "string" + }, + "event": { + "type": "string" + }, + "blurb": { + "type": "string" + }, + "date": { + "oneOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string", + "pattern": "\\d{4}-[01]\\d-[0-3]\\d [0-2]\\d:[0-5]\\d(:[0-5]\\d)? ([+-][0-2]\\d[0-5]\\d)" + } + ] + }, + "image": { + "type": "string" + }, + "image_remote": { + "type": "string" + }, + "caption": { + "type": "string" + }, + "youtube": { + "type": ["string", "null"] + }, + "video_src": { + "type": "string" + }, + "podbean": { + "type": "string" + }, + "register": { + "type": "string" + }, + "slides": { + "type": "string" + }, + "author": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "speaker": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "block": { + "type": "string" + }, + "time": { + "type": "string" + }, + "day": { + "type": "string" + }, + "room": { + "type": "string" + }, + "length": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": [ + "title" + ] + } \ No newline at end of file diff --git a/_tools/frontmatter-validator/schemas/redirect.json b/_tools/frontmatter-validator/schemas/redirect.json new file mode 100644 index 0000000..56113a8 --- /dev/null +++ b/_tools/frontmatter-validator/schemas/redirect.json @@ -0,0 +1,18 @@ +{ + "title": "Redirect", + "type": "object", + "properties": { + "permalink": { + "type": "string" + }, + "redirect": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "required": [ + "redirect" + ] + } \ No newline at end of file diff --git a/_tools/image-cacher/index.js b/_tools/image-cacher/index.js new file mode 100644 index 0000000..a438811 --- /dev/null +++ b/_tools/image-cacher/index.js @@ -0,0 +1,110 @@ +// image-cacher for dogwood +// +// This script fetches remote images referenced in the `image` +// property of `_posts` and `_people` and saves them locally. +// Remote images are sometimes a convenience but they are liable +// to turn into 404s or 403s in time. +// +// Cached images are saved to `/img/posts/cached/` and /img/people/cached/`, +// the `image` property is changed to the local file, and the original `image` +// value is moved to `image_remote` for future reference. +// +// Remember to re-run the thumbnail-generator tool after cacheing images. + +const fs = require('fs'); +const https = require('follow-redirects').https; +const graymatter = require('gray-matter'); +const process = require('process') + +const argv = require('minimist')(process.argv.slice(2)); +let sourcePath = process.env.npm_config_srcdir || argv.srcdir || '.'; +if (sourcePath.endsWith('/')) sourcePath.slice(0, -1); + +const imgDir = 'img'; + +const imgKeys = ['image', 'cover']; + +processFiles('_people', 'people'); +processFiles('_posts', 'posts'); + +async function processFiles(jekyllSubdir, imageSubdir) { + let jekyllDirPath = sourcePath + "/" + jekyllSubdir; + const dir = fs.opendirSync(jekyllDirPath); + let dirent; + while ((dirent = dir.readSync()) !== null) { + const subpath = jekyllDirPath + '/' + dirent.name; + if (dirent.isFile()) { + const filedata = fs.readFileSync(subpath); + const info = graymatter(filedata); + + // ignore files not in Jekyll format + if (Object.keys(info.data).length > 0) { + let newFrontmatter = await cacheRemoteImages(info.data, imageSubdir); + if (newFrontmatter) { + let newFileContent = graymatter.stringify(info, newFrontmatter); + fs.writeFileSync(subpath, newFileContent); + console.log(`updated ${subpath}`); + } + } + } else if (dirent.isDirectory()) { + await processFiles(jekyllSubdir + '/' + dirent.name, imageSubdir); + } + } + dir.closeSync(); +} + +async function cacheRemoteImage(url, toPath) { + + return new Promise((resolve) => { + + var content = ""; + + var req = https.request(url, function(res) { + if (res.statusCode !== 200) { + console.error(`status ${res.statusCode} for ${url}`); + resolve(false); + return; + } + res.setEncoding('binary'); + res.on("data", function (chunk) { + content += chunk; + }); + + res.on("end", function () { + fs.writeFileSync(toPath, content, "binary"); + console.log(`cached ${url}`); + resolve(true); + }); + }); + + req.on('error', (e) => { + console.error(`problem with request: ${e.message}`); + resolve(false); + }); + + req.end(); + }); +} + +async function cacheRemoteImages(frontmatter, imageSubdir) { + + + var didCacheAny = false; + for (var i in imgKeys) { + var key = imgKeys[i]; + let imgUrl = frontmatter[key]; + if (imgUrl && imgUrl.startsWith('http')) { + let slug = encodeURIComponent(imgUrl); + frontmatter[key + "_remote"] = frontmatter[key]; + + var cachedDirPath = imgDir + '/' + imageSubdir + '/cached/'; + if (!fs.existsSync(sourcePath + "/" + cachedDirPath)) fs.mkdirSync(sourcePath + "/" + cachedDirPath); + + let relativeOutPath = '/' + cachedDirPath + slug; + frontmatter[key] = relativeOutPath; + var didCache = await cacheRemoteImage(imgUrl, sourcePath + relativeOutPath); + didCacheAny = didCacheAny || didCache; + } + } + if (didCache) return frontmatter; +} diff --git a/_tools/people-validator/index.js b/_tools/people-validator/index.js new file mode 100644 index 0000000..fc7e5f9 --- /dev/null +++ b/_tools/people-validator/index.js @@ -0,0 +1,99 @@ +// people-validator for dogwood +// +// This script checks to make sure there is a document in the `_people` +// directory for every `author` and `speaker` referenced in your `_posts`. +// This isn't strictly necessary but it enables linking people with the +// various content they're associated with. +// +// Use the --fix=true option to automatically create missing files. +// +// Note that the name is the only unique identifier for a person. This +// means you can only reference one person for each unique name, +// and that different version of the name (Jim vs. Jimmy) will be +// treated as different people. + +const matter = require('gray-matter'); +const fs = require('fs'); +const process = require('process'); + +const argv = require('minimist')(process.argv.slice(2)); + +let shouldFix = process.env.npm_config_fix || argv.fix; +let hasUnfixedIssue = false; + +let sourcePath = process.env.npm_config_srcdir || argv.srcdir || '.'; +if (sourcePath.endsWith('/')) sourcePath.slice(0, -1); + +let expectedPeople = {}; + +function recordPostData(data) { + let peopleKeys = ["author", "speaker"]; + peopleKeys.forEach(function(key) { + let value = data[key]; + if (typeof value === 'string') { + expectedPeople[value] = true; + } else if (Array.isArray(value)) { + value.forEach(function(item) { + if (typeof item === 'string') { + expectedPeople[item] = true; + } + }); + } + }); +} + +function processFiles(path) { + const dir = fs.opendirSync(path); + let dirent; + while ((dirent = dir.readSync()) !== null) { + const subpath = path + '/' + dirent.name; + if (dirent.isFile()) { + const filedata = fs.readFileSync(subpath); + const info = matter(filedata); + + // ignore files not in Jekyll format + if (Object.keys(info.data).length > 0) { + recordPostData(info.data); + } + } else if (dirent.isDirectory()) { + processFiles(subpath); + } + } + dir.closeSync(); +} + +function validatePeopleBasedOnPosts() { + let peopleList = Object.keys(expectedPeople).sort(); + peopleList.forEach(function(name) { + let slug = name.replaceAll(" ", "-").replaceAll(".", "").toLowerCase().normalize("NFD").replace(/\p{Diacritic}/gu, ""); + let path = sourcePath + '/_people/' + slug + '.md'; + if (!fs.existsSync(path)) { + // ignore groups + if (!name.match(/(Team|Committee|Organization|Staff|Board|Community|OpenStreetMap)/gi)) { + if (shouldFix) { + let newFileString = `---\ntitle: "${name}"\n---`; + fs.writeFileSync(path, newFileString); + console.log(`Created file for ${name} at ${path}`); + } else { + console.error(`Missing file for ${name} at ${path}`); + hasUnfixedIssue = true; + } + } + } else { + let personData = matter(fs.readFileSync(path)).data; + // the document title needs to match the name listed in the post meta or else Jekyll can't link them + if (personData.title !== name) { + console.error(`Unexpected name "${personData.title}" for ${name} at ${path}`); + hasUnfixedIssue = true; + } + } + }); +} + +processFiles(sourcePath + '/_posts'); +validatePeopleBasedOnPosts(); + +if (hasUnfixedIssue) { + // nonzero exit for GitHub Actions + process.exit(1); +} \ No newline at end of file diff --git a/_tools/thumbnail-generator/index.js b/_tools/thumbnail-generator/index.js new file mode 100644 index 0000000..59c4bc7 --- /dev/null +++ b/_tools/thumbnail-generator/index.js @@ -0,0 +1,67 @@ +// thumbnail-generator for dogwood +// +// This script creates thumbnail versions of images that are displayed +// at small sizes on some pages in order to reduce load times. +// It is hardcoded for special directories (it works recursively): +// /img/people/ – avatar images for people +// /img/posts/ – header images for posts +// And it saves the output to: +// /img-thumbnails/people/ +// /img-thumbnails/posts/ +// In your Jekyll documents you should specify images like +// "image: /img/posts/john.jpg" And the theme will automatically +// output "/img-thumbnails/posts/john.jpg" in places where smaller +// images are appropriate. +// +// Note that if you specify images in the special directories +// but do not run this script before building your site then +// your images will 404. + +const fs = require('fs'); +const sharp = require('sharp'); +const process = require('process'); + +const argv = require('minimist')(process.argv.slice(2)); +let sourcePath = process.env.npm_config_srcdir || argv.srcdir || '.'; +if (sourcePath.endsWith('/')) sourcePath.slice(0, -1); + +var imgDir = sourcePath + '/img/'; +var imgThumbDir = sourcePath + '/img-thumbnails/'; + +if (!fs.existsSync(imgThumbDir)) fs.mkdirSync(imgThumbDir); + +processFiles('posts', 720, null); +processFiles('people', 48, 48); + +function processFiles(subdir, width, height) { + let inDir = imgDir + subdir; + let outDir = imgThumbDir + subdir; + + if (!fs.existsSync(outDir)) fs.mkdirSync(outDir); + + const dir = fs.opendirSync(inDir); + let dirent; + while ((dirent = dir.readSync()) !== null) { + + //ignore hidden files + if (dirent.name[0] === '.') continue; + + if (dirent.isFile()) { + const subpath = inDir + '/' + dirent.name; + let outPath = outDir + '/' + dirent.name + '.webp'; + const buffer = fs.readFileSync(subpath); + + sharp(buffer, { animated: true, limitInputPixels: false }) + .resize(width, height, { withoutEnlargement: true }) + .toFile(outPath) + .catch(err => { + console.log("cannot create thumbnail for: " + subpath); + console.error(err); + }); + + } else if (dirent.isDirectory()) { + processFiles(subdir + '/' + dirent.name, width, height); + } + } + dir.closeSync(); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..f388929 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "@osmus/dogwood", + "version": "1.0.0", + "scripts": { + "cache:images": "node ./_tools/image-cacher/index.js --srcdir=.", + "cache": "npm-run-all -p cache:*", + "thumbnails": "node ./_tools/thumbnail-generator/index.js --srcdir=.", + "validate:frontmatter": "node ./_tools/frontmatter-validator/index.js --srcdir=.", + "validate:people": "node ./_tools/people-validator/index.js --srcdir=.", + "validate": "npm-run-all -p validate:*", + "fix:people": "node ./_tools/people-validator/index.js --fix=true --srcdir=.", + "fix": "npm-run-all -p fix:*" + }, + "devDependencies": { + "follow-redirects": "^1.15.3", + "gray-matter": "^4.0.3", + "minimist": "^1.2.8", + "npm-run-all": "^4.1.5", + "sharp": "^0.33.2", + "z-schema": "^6.0.1" + } +}