diff --git a/script/prepare-release.js b/script/prepare-release.js index 8afc46e67228e..290db0fb99b61 100755 --- a/script/prepare-release.js +++ b/script/prepare-release.js @@ -12,8 +12,8 @@ const { GitProcess } = require('dugite') const GitHub = require('github') const pass = '\u2713'.green const path = require('path') -const pkg = require('../package.json') const readline = require('readline') +const releaseNotesGenerator = require('./release-notes/index.js') const versionType = args._[0] const targetRepo = versionType === 'nightly' ? 'nightlies' : 'electron' @@ -75,65 +75,10 @@ async function getReleaseNotes (currentBranch) { return 'Nightlies do not get release notes, please compare tags for info' } console.log(`Generating release notes for ${currentBranch}.`) - const githubOpts = { - owner: 'electron', - repo: targetRepo, - base: `v${pkg.version}`, - head: currentBranch - } - let releaseNotes - if (args.automaticRelease) { - releaseNotes = '## Bug Fixes/Changes \n\n' - } else { - releaseNotes = '(placeholder)\n' - } - console.log(`Checking for commits from ${pkg.version} to ${currentBranch}`) - const commitComparison = await github.repos.compareCommits(githubOpts) - .catch(err => { - console.log(`${fail} Error checking for commits from ${pkg.version} to ` + - `${currentBranch}`, err) - process.exit(1) - }) - - if (commitComparison.data.commits.length === 0) { - console.log(`${pass} There are no commits from ${pkg.version} to ` + - `${currentBranch}, skipping release.`) - process.exit(0) + const releaseNotes = await releaseNotesGenerator(currentBranch) + if (releaseNotes.warning) { + console.warn(releaseNotes.warning) } - - let prCount = 0 - const mergeRE = /Merge pull request #(\d+) from .*\n/ - const newlineRE = /(.*)\n*.*/ - const prRE = /(.* )\(#(\d+)\)(?:.*)/ - commitComparison.data.commits.forEach(commitEntry => { - let commitMessage = commitEntry.commit.message - if (commitMessage.indexOf('#') > -1) { - let prMatch = commitMessage.match(mergeRE) - let prNumber - if (prMatch) { - commitMessage = commitMessage.replace(mergeRE, '').replace('\n', '') - const newlineMatch = commitMessage.match(newlineRE) - if (newlineMatch) { - commitMessage = newlineMatch[1] - } - prNumber = prMatch[1] - } else { - prMatch = commitMessage.match(prRE) - if (prMatch) { - commitMessage = prMatch[1].trim() - prNumber = prMatch[2] - } - } - if (prMatch) { - if (commitMessage.substr(commitMessage.length - 1, commitMessage.length) !== '.') { - commitMessage += '.' - } - releaseNotes += `* ${commitMessage} #${prNumber} \n\n` - prCount++ - } - } - }) - console.log(`${pass} Done generating release notes for ${currentBranch}. Found ${prCount} PRs.`) return releaseNotes } @@ -165,12 +110,12 @@ async function createRelease (branchToTarget, isBeta) { githubOpts.body = `Note: This is a nightly release. Please file new issues ` + `for any bugs you find in it.\n \n This release is published to npm ` + `under the nightly tag and can be installed via npm install electron@nightly, ` + - `or npm i electron@${newVersion.substr(1)}.\n \n ${releaseNotes}` + `or npm i electron@${newVersion.substr(1)}.\n \n ${releaseNotes.text}` } else { githubOpts.body = `Note: This is a beta release. Please file new issues ` + `for any bugs you find in it.\n \n This release is published to npm ` + `under the beta tag and can be installed via npm install electron@beta, ` + - `or npm i electron@${newVersion.substr(1)}.\n \n ${releaseNotes}` + `or npm i electron@${newVersion.substr(1)}.\n \n ${releaseNotes.text}` } githubOpts.name = `${githubOpts.name}` githubOpts.prerelease = true @@ -262,7 +207,7 @@ async function prepareRelease (isBeta, notesOnly) { const currentBranch = (args.branch) ? args.branch : await getCurrentBranch(gitDir) if (notesOnly) { const releaseNotes = await getReleaseNotes(currentBranch) - console.log(`Draft release notes are: \n${releaseNotes}`) + console.log(`Draft release notes are: \n${releaseNotes.text}`) } else { const changes = await changesToRelease(currentBranch) if (changes) { diff --git a/script/release-notes/index.js b/script/release-notes/index.js old mode 100644 new mode 100755 index 456d5e037aa61..523ff59d3944a --- a/script/release-notes/index.js +++ b/script/release-notes/index.js @@ -1,472 +1,154 @@ +#!/usr/bin/env node + const { GitProcess } = require('dugite') -const Entities = require('html-entities').AllHtmlEntities -const fetch = require('node-fetch') -const fs = require('fs') -const GitHub = require('github') const path = require('path') const semver = require('semver') -const CACHE_DIR = path.resolve(__dirname, '.cache') -// Fill this with tags to ignore if you are generating release notes for older -// versions -// -// E.g. ['v3.0.0-beta.1'] to generate the release notes for 3.0.0-beta.1 :) from -// the current 3-0-x branch -const EXCLUDE_TAGS = [] +const notesGenerator = require('./notes.js') -const entities = new Entities() -const github = new GitHub() const gitDir = path.resolve(__dirname, '..', '..') -github.authenticate({ type: 'token', token: process.env.ELECTRON_GITHUB_TOKEN }) -let currentBranch -const semanticMap = new Map() -for (const line of fs.readFileSync(path.resolve(__dirname, 'legacy-pr-semantic-map.csv'), 'utf8').split('\n')) { - if (!line) continue - const bits = line.split(',') - if (bits.length !== 2) continue - semanticMap.set(bits[0], bits[1]) -} +const semverify = version => version.replace(/^origin\//, '').replace('x', '0').replace(/-/g, '.') -const getCurrentBranch = async () => { - if (currentBranch) return currentBranch - const gitArgs = ['rev-parse', '--abbrev-ref', 'HEAD'] - const branchDetails = await GitProcess.exec(gitArgs, gitDir) - if (branchDetails.exitCode === 0) { - currentBranch = branchDetails.stdout.trim() - return currentBranch +const runGit = async (args) => { + const response = await GitProcess.exec(args, gitDir) + if (response.exitCode !== 0) { + throw new Error(response.stderr.trim()) } - throw GitProcess.parseError(branchDetails.stderr) + return response.stdout.trim() } -const getBranchOffPoint = async (branchName) => { - const gitArgs = ['merge-base', branchName, 'master'] - const commitDetails = await GitProcess.exec(gitArgs, gitDir) - if (commitDetails.exitCode === 0) { - return commitDetails.stdout.trim() - } - throw GitProcess.parseError(commitDetails.stderr) -} +const tagIsSupported = tag => tag && !tag.includes('nightly') && !tag.includes('unsupported') +const tagIsBeta = tag => tag.includes('beta') +const tagIsStable = tag => tagIsSupported(tag) && !tagIsBeta(tag) -const getTagsOnBranch = async (branchName) => { - const gitArgs = ['tag', '--merged', branchName] - const tagDetails = await GitProcess.exec(gitArgs, gitDir) - if (tagDetails.exitCode === 0) { - return tagDetails.stdout.trim().split('\n').filter(tag => !EXCLUDE_TAGS.includes(tag)) - } - throw GitProcess.parseError(tagDetails.stderr) +const getTagsOf = async (point) => { + return (await runGit(['tag', '--merged', point])) + .split('\n') + .map(tag => tag.trim()) + .filter(tag => semver.valid(tag)) + .sort(semver.compare) } -const memLastKnownRelease = new Map() - -const getLastKnownReleaseOnBranch = async (branchName) => { - if (memLastKnownRelease.has(branchName)) { - return memLastKnownRelease.get(branchName) - } - const tags = await getTagsOnBranch(branchName) - if (!tags.length) { - throw new Error(`Branch ${branchName} has no tags, we have no idea what the last release was`) - } - const branchOffPointTags = await getTagsOnBranch(await getBranchOffPoint(branchName)) - if (branchOffPointTags.length >= tags.length) { - // No release on this branch - return null +const getTagsOnBranch = async (point) => { + const masterTags = await getTagsOf('master') + if (point === 'master') { + return masterTags } - memLastKnownRelease.set(branchName, tags[tags.length - 1]) - // Latest tag is the latest release - return tags[tags.length - 1] -} -const getBranches = async () => { - const gitArgs = ['branch', '--remote'] - const branchDetails = await GitProcess.exec(gitArgs, gitDir) - if (branchDetails.exitCode === 0) { - return branchDetails.stdout.trim().split('\n').map(b => b.trim()).filter(branch => branch !== 'origin/HEAD -> origin/master') - } - throw GitProcess.parseError(branchDetails.stderr) + const masterTagsSet = new Set(masterTags) + return (await getTagsOf(point)).filter(tag => !masterTagsSet.has(tag)) } -const semverify = (v) => v.replace(/^origin\//, '').replace('x', '0').replace(/-/g, '.') - -const getLastReleaseBranch = async () => { - const current = await getCurrentBranch() - const allBranches = await getBranches() - const releaseBranches = allBranches - .filter(branch => /^origin\/[0-9]+-[0-9]+-x$/.test(branch)) - .filter(branch => branch !== current && branch !== `origin/${current}`) - let latest = null - for (const b of releaseBranches) { - if (latest === null) latest = b - if (semver.gt(semverify(b), semverify(latest))) { - latest = b - } - } - return latest +const getBranchOf = async (point) => { + const branches = (await runGit(['branch', '-a', '--contains', point])) + .split('\n') + .map(branch => branch.trim()) + .filter(branch => !!branch) + const current = branches.find(branch => branch.startsWith('* ')) + return current ? current.slice(2) : branches.shift() } -const commitBeforeTag = async (commit, tag) => { - const gitArgs = ['tag', '--contains', commit] - const tagDetails = await GitProcess.exec(gitArgs, gitDir) - if (tagDetails.exitCode === 0) { - return tagDetails.stdout.split('\n').includes(tag) - } - throw GitProcess.parseError(tagDetails.stderr) +const getAllBranches = async () => { + return (await runGit(['branch', '--remote'])) + .split('\n') + .map(branch => branch.trim()) + .filter(branch => !!branch) + .filter(branch => branch !== 'origin/HEAD -> origin/master') + .sort() } -const getCommitsMergedIntoCurrentBranchSincePoint = async (point) => { - return getCommitsBetween(point, 'HEAD') +const getStabilizationBranches = async () => { + return (await getAllBranches()) + .filter(branch => /^origin\/\d+-\d+-x$/.test(branch)) } -const getCommitsBetween = async (point1, point2) => { - const gitArgs = ['rev-list', `${point1}..${point2}`] - const commitsDetails = await GitProcess.exec(gitArgs, gitDir) - if (commitsDetails.exitCode !== 0) { - throw GitProcess.parseError(commitsDetails.stderr) - } - return commitsDetails.stdout.trim().split('\n') -} - -const TITLE_PREFIX = 'Merged Pull Request: ' - -const getCommitDetails = async (commitHash) => { - const commitInfo = await (await fetch(`https://github.com/electron/electron/branch_commits/${commitHash}`)).text() - const bits = commitInfo.split(')')[0].split('>') - const prIdent = bits[bits.length - 1].trim() - if (!prIdent || commitInfo.indexOf('href="/electron/electron/pull') === -1) { - console.warn(`WARNING: Could not track commit "${commitHash}" to a pull request, it may have been committed directly to the branch`) - return null - } - const title = commitInfo.split('title="')[1].split('"')[0] - if (!title.startsWith(TITLE_PREFIX)) { - console.warn(`WARNING: Unknown PR title for commit "${commitHash}" in PR "${prIdent}"`) - return null - } - return { - mergedFrom: prIdent, - prTitle: entities.decode(title.substr(TITLE_PREFIX.length)) - } -} - -const doWork = async (items, fn, concurrent = 5) => { - const results = [] - const toUse = [].concat(items) - let i = 1 - const doBit = async () => { - if (toUse.length === 0) return - console.log(`Running ${i}/${items.length}`) - i += 1 +const getPreviousStabilizationBranch = async (current) => { + const stabilizationBranches = (await getStabilizationBranches()) + .filter(branch => branch !== current && branch !== `origin/${current}`) - const item = toUse.pop() - const index = toUse.length - results[index] = await fn(item) - await doBit() - } - const bits = [] - for (let i = 0; i < concurrent; i += 1) { - bits.push(doBit()) + if (!semver.valid(current)) { + // since we don't seem to be on a stabilization branch right now, + // pick a placeholder name that will yield the newest branch + // as a comparison point. + current = 'v999.999.999' } - await Promise.all(bits) - return results -} - -const notes = new Map() -const NoteType = { - FIX: 'fix', - FEATURE: 'feature', - BREAKING_CHANGE: 'breaking-change', - DOCUMENTATION: 'doc', - OTHER: 'other', - UNKNOWN: 'unknown' -} - -class Note { - constructor (trueTitle, prNumber, ignoreIfInVersion) { - // Self bindings - this.guessType = this.guessType.bind(this) - this.fetchPrInfo = this.fetchPrInfo.bind(this) - this._getPr = this._getPr.bind(this) - - if (!trueTitle.trim()) console.error(prNumber) - - this._ignoreIfInVersion = ignoreIfInVersion - this.reverted = false - if (notes.has(trueTitle)) { - console.warn(`Duplicate PR trueTitle: "${trueTitle}", "${prNumber}" this might cause weird reversions (this would be RARE)`) + let newestMatch = null + for (const branch of stabilizationBranches) { + if (semver.gte(semverify(branch), semverify(current))) { + continue } - - // Memoize - notes.set(trueTitle, this) - - this.originalTitle = trueTitle - this.title = trueTitle - this.prNumber = prNumber - this.stripColon = true - if (this.guessType() !== NoteType.UNKNOWN && this.stripColon) { - this.title = trueTitle.split(':').slice(1).join(':').trim() + if (newestMatch && semver.lte(semverify(branch), semverify(newestMatch))) { + continue } + newestMatch = branch } + return newestMatch +} - guessType () { - if (this.originalTitle.startsWith('fix:') || - this.originalTitle.startsWith('Fix:')) return NoteType.FIX - if (this.originalTitle.startsWith('feat:')) return NoteType.FEATURE - if (this.originalTitle.startsWith('spec:') || - this.originalTitle.startsWith('build:') || - this.originalTitle.startsWith('test:') || - this.originalTitle.startsWith('chore:') || - this.originalTitle.startsWith('deps:') || - this.originalTitle.startsWith('refactor:') || - this.originalTitle.startsWith('tools:') || - this.originalTitle.startsWith('vendor:') || - this.originalTitle.startsWith('perf:') || - this.originalTitle.startsWith('style:') || - this.originalTitle.startsWith('ci')) return NoteType.OTHER - if (this.originalTitle.startsWith('doc:') || - this.originalTitle.startsWith('docs:')) return NoteType.DOCUMENTATION - - this.stripColon = false +const getPreviousPoint = async (point) => { + const currentBranch = await getBranchOf(point) + const currentTag = (await getTagsOf(point)).filter(tag => tagIsSupported(tag)).pop() + const currentIsStable = tagIsStable(currentTag) - if (this.pr && this.pr.data.labels.find(label => label.name === 'semver/breaking-change')) { - return NoteType.BREAKING_CHANGE + try { + // First see if there's an earlier tag on the same branch + // that can serve as a reference point. + let tags = (await getTagsOnBranch(`${point}^`)).filter(tag => tagIsSupported(tag)) + if (currentIsStable) { + tags = tags.filter(tag => tagIsStable(tag)) } - // FIXME: Backported features will not be picked up by this - if (this.pr && this.pr.data.labels.find(label => label.name === 'semver/nonbreaking-feature')) { - return NoteType.FEATURE + if (tags.length) { + return tags.pop() } - - const n = this.prNumber.replace('#', '') - if (semanticMap.has(n)) { - switch (semanticMap.get(n)) { - case 'feat': - return NoteType.FEATURE - case 'fix': - return NoteType.FIX - case 'breaking-change': - return NoteType.BREAKING_CHANGE - case 'doc': - return NoteType.DOCUMENTATION - case 'build': - case 'vendor': - case 'refactor': - case 'spec': - return NoteType.OTHER - default: - throw new Error(`Unknown semantic mapping: ${semanticMap.get(n)}`) - } - } - return NoteType.UNKNOWN - } - - async _getPr (n) { - const cachePath = path.resolve(CACHE_DIR, n) - if (fs.existsSync(cachePath)) { - return JSON.parse(fs.readFileSync(cachePath, 'utf8')) - } else { - try { - const pr = await github.pullRequests.get({ - number: n, - owner: 'electron', - repo: 'electron' - }) - fs.writeFileSync(cachePath, JSON.stringify({ data: pr.data })) - return pr - } catch (err) { - console.info('#### FAILED:', `#${n}`) - throw err - } - } - } - - async fetchPrInfo () { - if (this.pr) return - const n = this.prNumber.replace('#', '') - this.pr = await this._getPr(n) - if (this.pr.data.labels.find(label => label.name === `merged/${this._ignoreIfInVersion.replace('origin/', '')}`)) { - // This means we probably backported this PR, let's try figure out what - // the corresponding backport PR would be by searching through comments - // for trop - let comments - const cacheCommentsPath = path.resolve(CACHE_DIR, `${n}-comments`) - if (fs.existsSync(cacheCommentsPath)) { - comments = JSON.parse(fs.readFileSync(cacheCommentsPath, 'utf8')) - } else { - comments = await github.issues.getComments({ - number: n, - owner: 'electron', - repo: 'electron', - per_page: 100 - }) - fs.writeFileSync(cacheCommentsPath, JSON.stringify({ data: comments.data })) - } - - const tropComment = comments.data.find( - c => ( - new RegExp(`We have automatically backported this PR to "${this._ignoreIfInVersion.replace('origin/', '')}", please check out #[0-9]+`) - ).test(c.body) - ) - - if (tropComment) { - const commentBits = tropComment.body.split('#') - const tropPrNumber = commentBits[commentBits.length - 1] - - const tropPr = await this._getPr(tropPrNumber) - if (tropPr.data.merged && tropPr.data.merge_commit_sha) { - if (await commitBeforeTag(tropPr.data.merge_commit_sha, await getLastKnownReleaseOnBranch(this._ignoreIfInVersion))) { - this.reverted = true - console.log('PR', this.prNumber, 'was backported to a previous version, ignoring from notes') - } - } - } + } catch (error) { + console.log('error', error) + } + + // Otherwise, use the newest stable release that preceeds this branch. + // To reach that you may have to walk past >1 branch, e.g. to get past + // 2-1-x which never had a stable release. + let branch = currentBranch + while (branch) { + const prevBranch = await getPreviousStabilizationBranch(branch) + const tags = (await getTagsOnBranch(prevBranch)).filter(tag => tagIsStable(tag)) + if (tags.length) { + return tags.pop() } + branch = prevBranch } } -Note.findByTrueTitle = (trueTitle) => notes.get(trueTitle) +async function getReleaseNotes (range) { + const rangeList = range.split('..') || ['HEAD'] + const to = rangeList.pop() + const from = rangeList.pop() || (await getPreviousPoint(to)) + console.log(`Generating release notes between ${from} and ${to}`) -class ReleaseNotes { - constructor (ignoreIfInVersion) { - this._ignoreIfInVersion = ignoreIfInVersion - this._handledPrs = new Set() - this._revertedPrs = new Set() - this.other = [] - this.docs = [] - this.fixes = [] - this.features = [] - this.breakingChanges = [] - this.unknown = [] + const notes = await notesGenerator.get(from, to) + const ret = { + text: notesGenerator.render(notes) } - async parseCommits (commitHashes) { - await doWork(commitHashes, async (commit) => { - const info = await getCommitDetails(commit) - if (!info) return - // Only handle each PR once - if (this._handledPrs.has(info.mergedFrom)) return - this._handledPrs.add(info.mergedFrom) - - // Strip the trop backport prefix - const trueTitle = info.prTitle.replace(/^Backport \([0-9]+-[0-9]+-x\) - /, '') - if (this._revertedPrs.has(trueTitle)) return - - // Handle PRs that revert other PRs - if (trueTitle.startsWith('Revert "')) { - const revertedTrueTitle = trueTitle.substr(8, trueTitle.length - 9) - this._revertedPrs.add(revertedTrueTitle) - const existingNote = Note.findByTrueTitle(revertedTrueTitle) - if (existingNote) { - existingNote.reverted = true - } - return - } - - // Add a note for this PR - const note = new Note(trueTitle, info.mergedFrom, this._ignoreIfInVersion) - try { - await note.fetchPrInfo() - } catch (err) { - console.error(commit, info) - throw err - } - switch (note.guessType()) { - case NoteType.FIX: - this.fixes.push(note) - break - case NoteType.FEATURE: - this.features.push(note) - break - case NoteType.BREAKING_CHANGE: - this.breakingChanges.push(note) - break - case NoteType.OTHER: - this.other.push(note) - break - case NoteType.DOCUMENTATION: - this.docs.push(note) - break - case NoteType.UNKNOWN: - default: - this.unknown.push(note) - break - } - }, 20) + if (notes.unknown.length) { + ret.warning = `You have ${notes.unknown.length} unknown release notes. Please fix them before releasing.` } - list (notes) { - if (notes.length === 0) { - return '_There are no items in this section this release_' - } - return notes - .filter(note => !note.reverted) - .sort((a, b) => a.title.toLowerCase().localeCompare(b.title.toLowerCase())) - .map((note) => `* ${note.title.trim()} ${note.prNumber}`).join('\n') - } - - render () { - return ` -# Release Notes - -## Breaking Changes - -${this.list(this.breakingChanges)} - -## Features - -${this.list(this.features)} - -## Fixes - -${this.list(this.fixes)} - -## Other Changes (E.g. Internal refactors or build system updates) - -${this.list(this.other)} - -## Documentation Updates - -Some documentation updates, fixes and reworks: ${ - this.docs.length === 0 - ? '_None in this release_' - : this.docs.sort((a, b) => a.prNumber.localeCompare(b.prNumber)).map(note => note.prNumber).join(', ') -} -${this.unknown.filter(n => !n.reverted).length > 0 - ? `## Unknown (fix these before publishing release) - -${this.list(this.unknown)} -` : ''}` - } + return ret } async function main () { - if (!fs.existsSync(CACHE_DIR)) { - fs.mkdirSync(CACHE_DIR) + if (process.argv.length > 3) { + console.log('Use: script/release-notes/index.js [tag | tag1..tag2]') + return 1 } - const lastReleaseBranch = await getLastReleaseBranch() - - const notes = new ReleaseNotes(lastReleaseBranch) - const lastKnownReleaseInCurrentStream = await getLastKnownReleaseOnBranch(await getCurrentBranch()) - const currentBranchOff = await getBranchOffPoint(await getCurrentBranch()) - - const commits = await getCommitsMergedIntoCurrentBranchSincePoint( - lastKnownReleaseInCurrentStream || currentBranchOff - ) - if (!lastKnownReleaseInCurrentStream) { - // This means we are the first release in our stream - // FIXME: This will not work for minor releases!!!! - - const lastReleaseBranch = await getLastReleaseBranch() - const lastBranchOff = await getBranchOffPoint(lastReleaseBranch) - commits.push(...await getCommitsBetween(lastBranchOff, currentBranchOff)) - } - - await notes.parseCommits(commits) - - console.log(notes.render()) - - const badNotes = notes.unknown.filter(n => !n.reverted).length - if (badNotes > 0) { - throw new Error(`You have ${badNotes.length} unknown release notes, please fix them before releasing`) + const range = process.argv[2] || 'HEAD' + const notes = await getReleaseNotes(range) + console.log(notes.text) + if (notes.warning) { + throw new Error(notes.warning) } } @@ -476,3 +158,5 @@ if (process.mainModule === module) { process.exit(1) }) } + +module.exports = getReleaseNotes diff --git a/script/release-notes/notes.js b/script/release-notes/notes.js new file mode 100644 index 0000000000000..54f3fd64dc190 --- /dev/null +++ b/script/release-notes/notes.js @@ -0,0 +1,599 @@ +#!/usr/bin/env node + +const childProcess = require('child_process') +const fs = require('fs') +const os = require('os') +const path = require('path') + +const { GitProcess } = require('dugite') +const GitHub = require('github') +const semver = require('semver') + +const CACHE_DIR = path.resolve(__dirname, '.cache') +const NO_NOTES = 'No notes' +const FOLLOW_REPOS = [ 'electron/electron', 'electron/libchromiumcontent', 'electron/node' ] +const github = new GitHub() +const gitDir = path.resolve(__dirname, '..', '..') +github.authenticate({ type: 'token', token: process.env.ELECTRON_GITHUB_TOKEN }) + +const breakTypes = new Set(['breaking-change']) +const docTypes = new Set(['doc', 'docs']) +const featTypes = new Set(['feat', 'feature']) +const fixTypes = new Set(['fix']) +const otherTypes = new Set(['spec', 'build', 'test', 'chore', 'deps', 'refactor', 'tools', 'vendor', 'perf', 'style', 'ci']) +const knownTypes = new Set([...breakTypes.keys(), ...docTypes.keys(), ...featTypes.keys(), ...fixTypes.keys(), ...otherTypes.keys()]) + +const semanticMap = new Map() +for (const line of fs.readFileSync(path.resolve(__dirname, 'legacy-pr-semantic-map.csv'), 'utf8').split('\n')) { + if (!line) { + continue + } + const bits = line.split(',') + if (bits.length !== 2) { + continue + } + semanticMap.set(bits[0], bits[1]) +} + +const runGit = async (dir, args) => { + const response = await GitProcess.exec(args, dir) + if (response.exitCode !== 0) { + throw new Error(response.stderr.trim()) + } + return response.stdout.trim() +} + +const getCommonAncestor = async (dir, point1, point2) => { + return runGit(dir, ['merge-base', point1, point2]) +} + +const setPullRequest = (commit, owner, repo, number) => { + if (!owner || !repo || !number) { + throw new Error(JSON.stringify({ owner, repo, number }, null, 2)) + } + + if (!commit.originalPr) { + commit.originalPr = commit.pr + } + + commit.pr = { owner, repo, number } + + if (!commit.originalPr) { + commit.originalPr = commit.pr + } +} + +const getNoteFromBody = body => { + if (!body) { + return null + } + + const NOTE_PREFIX = 'Notes: ' + + let note = body + .split(/\r?\n\r?\n/) // split into paragraphs + .map(paragraph => paragraph.trim()) + .find(paragraph => paragraph.startsWith(NOTE_PREFIX)) + + if (note) { + const placeholder = '' + note = note + .slice(NOTE_PREFIX.length) + .replace(placeholder, '') + .replace(/\r?\n/, ' ') // remove newlines + .trim() + } + + if (note) { + if (note.match(/^[Nn]o[ _-][Nn]otes\.?$/)) { + return NO_NOTES + } + if (note.match(/^[Nn]one\.?$/)) { + return NO_NOTES + } + } + + return note +} + +/** + * Looks for our project's conventions in the commit message: + * + * 'semantic: some description' -- sets type, subject + * 'some description (#99999)' -- sets subject, pr + * 'Fixes #3333' -- sets issueNumber + * 'Merge pull request #99999 from ${branchname}' -- sets pr + * 'This reverts commit ${sha}' -- sets revertHash + * line starting with 'BREAKING CHANGE' in body -- sets breakingChange + * 'Backport of #99999' -- sets pr + */ +const parseCommitMessage = (commitMessage, owner, repo, commit = {}) => { + // split commitMessage into subject & body + let subject = commitMessage + let body = '' + const pos = subject.indexOf('\n') + if (pos !== -1) { + body = subject.slice(pos).trim() + subject = subject.slice(0, pos).trim() + } + + if (!commit.originalSubject) { + commit.originalSubject = subject + } + + if (body) { + commit.body = body + + const note = getNoteFromBody(body) + if (note) { commit.note = note } + } + + // if the subject ends in ' (#dddd)', treat it as a pull request id + let match + if ((match = subject.match(/^(.*)\s\(#(\d+)\)$/))) { + setPullRequest(commit, owner, repo, parseInt(match[2])) + subject = match[1] + } + + // if the subject begins with 'word:', treat it as a semantic commit + if ((match = subject.match(/^(\w+):\s(.*)$/))) { + commit.type = match[1].toLocaleLowerCase() + subject = match[2] + } + + // Check for GitHub commit message that indicates a PR + if ((match = subject.match(/^Merge pull request #(\d+) from (.*)$/))) { + setPullRequest(commit, owner, repo, parseInt(match[1])) + commit.pr.branch = match[2].trim() + } + + // Check for a trop comment that indicates a PR + if ((match = commitMessage.match(/\bBackport of #(\d+)\b/))) { + setPullRequest(commit, owner, repo, parseInt(match[1])) + } + + // https://help.github.com/articles/closing-issues-using-keywords/ + if ((match = subject.match(/\b(?:close|closes|closed|fix|fixes|fixed|resolve|resolves|resolved|for)\s#(\d+)\b/))) { + commit.issueNumber = parseInt(match[1]) + if (!commit.type) { + commit.type = 'fix' + } + } + + // look for 'fixes' in markdown; e.g. 'Fixes [#8952](https://github.com/electron/electron/issues/8952)' + if (!commit.issueNumber && ((match = commitMessage.match(/Fixes \[#(\d+)\]\(https:\/\/github.com\/(\w+)\/(\w+)\/issues\/(\d+)\)/)))) { + commit.issueNumber = parseInt(match[1]) + if (commit.pr && commit.pr.number === commit.issueNumber) { + commit.pr = null + } + if (commit.originalPr && commit.originalPr.number === commit.issueNumber) { + commit.originalPr = null + } + if (!commit.type) { + commit.type = 'fix' + } + } + + // https://www.conventionalcommits.org/en + if (commitMessage + .split(/\r?\n\r?\n/) // split into paragraphs + .map(paragraph => paragraph.trim()) + .some(paragraph => paragraph.startsWith('BREAKING CHANGE'))) { + commit.type = 'breaking-change' + } + + // Check for a reversion commit + if ((match = body.match(/This reverts commit ([a-f0-9]{40})\./))) { + commit.revertHash = match[1] + } + + // Edge case: manual backport where commit has `owner/repo#pull` notation + if (commitMessage.toLowerCase().includes('backport') && + ((match = commitMessage.match(/\b(\w+)\/(\w+)#(\d+)\b/)))) { + const [ , owner, repo, number ] = match + if (FOLLOW_REPOS.includes(`${owner}/${repo}`)) { + setPullRequest(commit, owner, repo, number) + } + } + + // Edge case: manual backport where commit has a link to the backport PR + if (commitMessage.includes('ackport') && + ((match = commitMessage.match(/https:\/\/github\.com\/(\w+)\/(\w+)\/pull\/(\d+)/)))) { + const [ , owner, repo, number ] = match + if (FOLLOW_REPOS.includes(`${owner}/${repo}`)) { + setPullRequest(commit, owner, repo, number) + } + } + + // Legacy commits: pre-semantic commits + if (!commit.type || commit.type === 'chore') { + const commitMessageLC = commitMessage.toLocaleLowerCase() + if ((match = commitMessageLC.match(/\bchore\((\w+)\):/))) { + // example: 'Chore(docs): description' + commit.type = knownTypes.has(match[1]) ? match[1] : 'chore' + } else if (commitMessageLC.match(/\b(?:fix|fixes|fixed)/)) { + // example: 'fix a bug' + commit.type = 'fix' + } else if (commitMessageLC.match(/\[(?:docs|doc)\]/)) { + // example: '[docs] + commit.type = 'doc' + } + } + + commit.subject = subject.trim() + + return commit +} + +const getLocalCommitHashes = async (dir, ref) => { + const args = ['log', '-z', `--format=%H`, ref] + return (await runGit(dir, args)).split(`\0`).map(hash => hash.trim()) +} + +/* + * possible properties: + * breakingChange, email, hash, issueNumber, originalSubject, parentHashes, + * pr { owner, repo, number, branch }, revertHash, subject, type + */ +const getLocalCommitDetails = async (module, point1, point2) => { + const { owner, repo, dir } = module + + const fieldSep = '||' + const format = ['%H', '%P', '%aE', '%B'].join(fieldSep) + const args = ['log', '-z', '--cherry-pick', '--right-only', '--first-parent', `--format=${format}`, `${point1}..${point2}`] + const commits = (await runGit(dir, args)).split(`\0`).map(field => field.trim()) + const details = [] + for (const commit of commits) { + if (!commit) { + continue + } + const [ hash, parentHashes, email, commitMessage ] = commit.split(fieldSep, 4).map(field => field.trim()) + details.push(parseCommitMessage(commitMessage, owner, repo, { + email, + hash, + owner, + repo, + parentHashes: parentHashes.split() + })) + } + return details +} + +const checkCache = async (name, operation) => { + const filename = path.resolve(CACHE_DIR, name) + if (fs.existsSync(filename)) { + return JSON.parse(fs.readFileSync(filename, 'utf8')) + } + const response = await operation() + if (response) { + fs.writeFileSync(filename, JSON.stringify(response)) + } + return response +} + +const getPullRequest = async (number, owner, repo) => { + const name = `${owner}-${repo}-pull-${number}` + return checkCache(name, async () => { + try { + return await github.pullRequests.get({ number, owner, repo }) + } catch (error) { + // Silently eat 404s. + // We can get a bad pull number if someone manually lists + // an issue number in PR number notation, e.g. 'fix: foo (#123)' + if (error.code !== 404) { + throw error + } + } + }) +} + +const addRepoToPool = async (pool, repo, from, to) => { + const commonAncestor = await getCommonAncestor(repo.dir, from, to) + const oldHashes = await getLocalCommitHashes(repo.dir, from) + oldHashes.forEach(hash => { pool.processedHashes.add(hash) }) + const commits = await getLocalCommitDetails(repo, commonAncestor, to) + pool.commits.push(...commits) +} + +/*** +**** Other Repos +***/ + +// other repos - gyp + +const getGypSubmoduleRef = async (dir, point) => { + // example: '160000 commit 028b0af83076cec898f4ebce208b7fadb715656e libchromiumcontent' + const response = await runGit( + path.dirname(dir), + ['ls-tree', '-t', point, path.basename(dir)] + ) + + const line = response.split('\n').filter(line => line.startsWith('160000')).shift() + const tokens = line ? line.split(/\s/).map(token => token.trim()) : null + const ref = tokens && tokens.length >= 3 ? tokens[2] : null + + return ref +} + +const getDependencyCommitsGyp = async (pool, fromRef, toRef) => { + const commits = [] + + const repos = [{ + owner: 'electron', + repo: 'libchromiumcontent', + dir: path.resolve(gitDir, 'vendor', 'libchromiumcontent') + }, { + owner: 'electron', + repo: 'node', + dir: path.resolve(gitDir, 'vendor', 'node') + }] + + for (const repo of repos) { + const from = await getGypSubmoduleRef(repo.dir, fromRef) + const to = await getGypSubmoduleRef(repo.dir, toRef) + await addRepoToPool(pool, repo, from, to) + } + + return commits +} + +// other repos - gn + +const getDepsVariable = async (ref, key) => { + // get a copy of that reference point's DEPS file + const deps = await runGit(gitDir, ['show', `${ref}:DEPS`]) + const filename = path.resolve(os.tmpdir(), 'DEPS') + fs.writeFileSync(filename, deps) + + // query the DEPS file + const response = childProcess.spawnSync( + 'gclient', + ['getdep', '--deps-file', filename, '--var', key], + { encoding: 'utf8' } + ) + + // cleanup + fs.unlinkSync(filename) + return response.stdout.trim() +} + +const getDependencyCommitsGN = async (pool, fromRef, toRef) => { + const repos = [{ // just node + owner: 'electron', + repo: 'node', + dir: path.resolve(gitDir, '..', 'third_party', 'electron_node'), + deps_variable_name: 'node_version' + }] + + for (const repo of repos) { + // the 'DEPS' file holds the dependency reference point + const key = repo.deps_variable_name + const from = await getDepsVariable(fromRef, key) + const to = await getDepsVariable(toRef, key) + await addRepoToPool(pool, repo, from, to) + } +} + +// other repos - controller + +const getDependencyCommits = async (pool, from, to) => { + const filename = path.resolve(gitDir, 'vendor', 'libchromiumcontent') + const useGyp = fs.existsSync(filename) + + return useGyp + ? getDependencyCommitsGyp(pool, from, to) + : getDependencyCommitsGN(pool, from, to) +} + +/*** +**** Main +***/ + +const getNotes = async (fromRef, toRef) => { + if (!fs.existsSync(CACHE_DIR)) { + fs.mkdirSync(CACHE_DIR) + } + + const pool = { + processedHashes: new Set(), + commits: [] + } + + // get the electron/electron commits + const electron = { owner: 'electron', repo: 'electron', dir: gitDir } + await addRepoToPool(pool, electron, fromRef, toRef) + + // Don't include submodules if comparing across major versions; + // there's just too much churn otherwise. + const includeDeps = semver.valid(fromRef) && + semver.valid(toRef) && + semver.major(fromRef) === semver.major(toRef) + + if (includeDeps) { + await getDependencyCommits(pool, fromRef, toRef) + } + + // remove any old commits + pool.commits = pool.commits.filter(commit => !pool.processedHashes.has(commit.hash)) + + // if a commmit _and_ revert occurred in the unprocessed set, skip them both + for (const commit of pool.commits) { + const revertHash = commit.revertHash + if (!revertHash) { + continue + } + + const revert = pool.commits.find(commit => commit.hash === revertHash) + if (!revert) { + continue + } + + commit.note = NO_NOTES + revert.note = NO_NOTES + pool.processedHashes.add(commit.hash) + pool.processedHashes.add(revertHash) + } + + // scrape PRs for release note 'Notes:' comments + for (const commit of pool.commits) { + let pr = commit.pr + while (pr && !commit.note) { + const prData = await getPullRequest(pr.number, pr.owner, pr.repo) + if (!prData || !prData.data) { + break + } + + // try to pull a release note from the pull comment + commit.note = getNoteFromBody(prData.data.body) + if (commit.note) { + break + } + + // if the PR references another PR, maybe follow it + parseCommitMessage(`${prData.data.title}\n\n${prData.data.body}`, pr.owner, pr.repo, commit) + pr = pr.number !== commit.pr.number ? commit.pr : null + } + } + + // remove uninteresting commits + pool.commits = pool.commits + .filter(commit => commit.note !== NO_NOTES) + .filter(commit => !((commit.note || commit.subject).match(/^[Bb]ump v\d+\.\d+\.\d+/))) + + const notes = { + breaks: [], + docs: [], + feat: [], + fix: [], + other: [], + unknown: [], + ref: toRef + } + + pool.commits.forEach(commit => { + const str = commit.type + if (!str) { + notes.unknown.push(commit) + } else if (breakTypes.has(str)) { + notes.breaks.push(commit) + } else if (docTypes.has(str)) { + notes.docs.push(commit) + } else if (featTypes.has(str)) { + notes.feat.push(commit) + } else if (fixTypes.has(str)) { + notes.fix.push(commit) + } else if (otherTypes.has(str)) { + notes.other.push(commit) + } else { + notes.unknown.push(commit) + } + }) + + return notes +} + +/*** +**** Render +***/ + +const renderCommit = commit => { + // clean up the note + let note = commit.note || commit.subject + note = note.trim() + if (note.length !== 0) { + note = note[0].toUpperCase() + note.substr(1) + + if (!note.endsWith('.')) { + note = note + '.' + } + + const commonVerbs = { + 'Added': [ 'Add' ], + 'Backported': [ 'Backport' ], + 'Cleaned': [ 'Clean' ], + 'Disabled': [ 'Disable' ], + 'Ensured': [ 'Ensure' ], + 'Exported': [ 'Export' ], + 'Fixed': [ 'Fix', 'Fixes' ], + 'Handled': [ 'Handle' ], + 'Improved': [ 'Improve' ], + 'Made': [ 'Make' ], + 'Removed': [ 'Remove' ], + 'Repaired': [ 'Repair' ], + 'Reverted': [ 'Revert' ], + 'Stopped': [ 'Stop' ], + 'Updated': [ 'Update' ], + 'Upgraded': [ 'Upgrade' ] + } + for (const [key, values] of Object.entries(commonVerbs)) { + for (const value of values) { + const start = `${value} ` + if (note.startsWith(start)) { + note = `${key} ${note.slice(start.length)}` + } + } + } + } + + // make a GH-markdown-friendly link + let link + const pr = commit.originalPr + if (!pr) { + link = `https://github.com/${commit.owner}/${commit.repo}/commit/${commit.hash}` + } else if (pr.owner === 'electron' && pr.repo === 'electron') { + link = `#${pr.number}` + } else { + link = `[${pr.owner}/${pr.repo}:${pr.number}](https://github.com/${pr.owner}/${pr.repo}/pull/${pr.number})` + } + + return { note, link } +} + +const renderNotes = notes => { + const rendered = [ `# Release Notes for ${notes.ref}\n\n` ] + + const renderSection = (title, commits) => { + if (commits.length === 0) { + return + } + const notes = new Map() + for (const note of commits.map(commit => renderCommit(commit))) { + if (!notes.has(note.note)) { + notes.set(note.note, [note.link]) + } else { + notes.get(note.note).push(note.link) + } + } + rendered.push(`## ${title}\n\n`) + const lines = [] + notes.forEach((links, key) => lines.push(` * ${key} ${links.map(link => link.toString()).sort().join(', ')}\n`)) + rendered.push(...lines.sort(), '\n') + } + + renderSection('Breaking Changes', notes.breaks) + renderSection('Features', notes.feat) + renderSection('Fixes', notes.fix) + renderSection('Other Changes', notes.other) + + if (notes.docs.length) { + const docs = notes.docs.map(commit => { + return commit.pr && commit.pr.number + ? `#${commit.pr.number}` + : `https://github.com/electron/electron/commit/${commit.hash}` + }).sort() + rendered.push('## Documentation\n\n', ` * Documentation changes: ${docs.join(', ')}\n`, '\n') + } + + renderSection('Unknown', notes.unknown) + + return rendered.join('') +} + +/*** +**** Module +***/ + +module.exports = { + get: getNotes, + render: renderNotes +}