From 72c45deb6b0854aa6ad30d1b2b56e8aa11b5ff9d Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Mon, 9 Apr 2018 08:03:59 +0200 Subject: [PATCH] [FEATURE] Add "extension" projects See [RFC0001](https://github.com/SAP/ui5-tooling/pull/4). Identify and apply extensions in dependency tree so that they can influence the actual project processing. --- lib/projectPreprocessor.js | 303 ++++++++++++++++++++++---------- test/lib/projectPreprocessor.js | 84 ++++++++- 2 files changed, 288 insertions(+), 99 deletions(-) diff --git a/lib/projectPreprocessor.js b/lib/projectPreprocessor.js index cb11f17a7..f4f0520c1 100644 --- a/lib/projectPreprocessor.js +++ b/lib/projectPreprocessor.js @@ -3,19 +3,22 @@ const fs = require("graceful-fs"); const path = require("path"); const {promisify} = require("util"); const readFile = promisify(fs.readFile); -const parseYaml = require("js-yaml").safeLoad; +const parseYaml = require("js-yaml").safeLoadAll; const typeRepository = require("@ui5/builder").types.typeRepository; class ProjectPreprocessor { + constructor() { + this.processedProjects = {}; + } + /* Adapt and enhance the project tree: - - Replace duplicate projects further away from the root with those closed to the root + - Replace duplicate projects further away from the root with those closer to the root - Add configuration to projects */ async processTree(tree) { - const processedProjects = {}; const queue = [{ - project: tree, + projects: [tree], parent: null, level: 0 }]; @@ -27,95 +30,224 @@ class ProjectPreprocessor { // Breadth-first search to prefer projects closer to root while (queue.length) { - const {project, parent, level} = queue.shift(); // Get and remove first entry from queue - if (!project.id) { - throw new Error("Encountered project with missing id"); - } - project._level = level; - - // Check whether project ID is already known - const processedProject = processedProjects[project.id]; - if (processedProject) { - if (processedProject.ignored) { - log.verbose(`Dependency of project ${parent.id}, "${project.id}" is flagged as ignored.`); - parent.dependencies.splice(parent.dependencies.indexOf(project), 1); - continue; + const {projects, parent, level} = queue.shift(); // Get and remove first entry from queue + + // Before processing all projects on a level concurrently, we need to set all of them as being processed. + // This prevents transitive dependencies pointing to the same projects from being processed first + // by the dependency lookahead + const projectsToProcess = projects.filter((project) => { + if (!project.id) { + throw new Error("Encountered project with missing id"); + } + if (this.isBeingProcessed(parent, project)) { + return false; } - log.verbose(`Dependency of project ${parent.id}, "${project.id}": Distance to root of ${level}. Will be `+ - `replaced by project with same ID and distance to root of ${processedProject.project._level}.`); + // Flag this project as being processed + this.processedProjects[project.id] = { + project, + // If a project is referenced multiple times in the dependency tree it is replaced + // with the instance that is closest to the root. + // Here we track the parents referencing that project + parents: [parent] + }; + return true; + }); - // Replace with the already processed project (closer to root -> preferred) - parent.dependencies[parent.dependencies.indexOf(project)] = processedProject.project; - processedProject.parents.push(parent); + await Promise.all(projectsToProcess.map(async (project) => { + log.verbose(`Processing project ${project.id} on level ${project._level}...`); - // No further processing needed - continue; - } + project._level = level; - processedProjects[project.id] = { - project, - // If a project is referenced multiple times in the dependency tree, - // it is replaced with the occurrence closest to the root. - // Here we collect the different parents, this single project configuration then has - parents: [parent] - }; + if (project.dependencies && project.dependencies.length) { + // Do a dependency lookahead to apply any extensions that might affect this project + await this.dependencyLookahead(project, project.dependencies); + } - configPromises.push(this.configureProject(project).then((config) => { - if (!config) { + await this.loadProjectConfiguration(project); + // this.applyShims(project); // shims not yet implemented + if (this.isConfigValid(project)) { + await this.applyType(project); + queue.push({ + projects: project.dependencies, + parent: project, + level: level + 1 + }); + } else { if (project === tree) { throw new Error(`Failed to configure root project "${project.id}". Please check verbose log for details.`); } - // No config available // => reject this project by removing it from its parents list of dependencies - log.verbose(`Ignoring project ${project.id} with missing configuration `+ + log.verbose(`Ignoring project ${project.id} with missing configuration ` + "(might be a non-UI5 dependency)"); - const parents = processedProjects[project.id].parents; + + const parents = this.processedProjects[project.id].parents; for (let i = parents.length - 1; i >= 0; i--) { parents[i].dependencies.splice(parents[i].dependencies.indexOf(project), 1); } - processedProjects[project.id] = {ignored: true}; + this.processedProjects[project.id] = {ignored: true}; } })); - - if (project.dependencies) { - queue.push(...project.dependencies.map((depProject) => { - return { - project: depProject, - parent: project, - level: level + 1 - }; - })); - } } return Promise.all(configPromises).then(() => { if (log.isLevelEnabled("verbose")) { const prettyHrtime = require("pretty-hrtime"); const timeDiff = process.hrtime(startTime); - log.verbose(`Processed ${Object.keys(processedProjects).length} projects in ${prettyHrtime(timeDiff)}`); + log.verbose(`Processed ${Object.keys(this.processedProjects).length} projects in ${prettyHrtime(timeDiff)}`); } return tree; }); } - async configureProject(project) { - if (!project.specVersion) { // Project might already be configured (e.g. via inline configuration) + async dependencyLookahead(parent, dependencies) { + return Promise.all(dependencies.map(async (project) => { + if (this.isBeingProcessed(parent, project)) { + return; + } + log.verbose(`Processing dependency lookahead for ${parent.id}: ${project.id}`); + // Temporarily flag project as being processed + this.processedProjects[project.id] = { + project, + parents: [parent] + }; + const {extensions} = await this.loadProjectConfiguration(project); + if (extensions && extensions.length) { + // Project contains additional extensions + // => apply them + await Promise.all(extensions.map((extProject) => { + return this.applyExtension(extProject); + })); + } + + if (project.kind === "extension") { + // Not a project but an extension + // => remove it as from any known projects that depend on it + const parents = this.processedProjects[project.id].parents; + for (let i = parents.length - 1; i >= 0; i--) { + parents[i].dependencies.splice(parents[i].dependencies.indexOf(project), 1); + } + // Also ignore it from further processing by other projects depending on it + this.processedProjects[project.id] = {ignored: true}; + + if (this.isConfigValid(project)) { + // Finally apply the extension + await this.applyExtension(project); + } else { + log.verbose(`Ignoring extension ${project.id} with missing configuration`); + } + } else { + // Project is not an extension: Reset processing status of lookahead to allow the real processing + this.processedProjects[project.id] = null; + } + })); + } + + isBeingProcessed(parent, project) { // Check whether a project is currently being or has already been processed + const processedProject = this.processedProjects[project.id]; + if (processedProject) { + if (processedProject.ignored) { + log.verbose(`Dependency of project ${parent.id}, "${project.id}" is flagged as ignored.`); + parent.dependencies.splice(parent.dependencies.indexOf(project), 1); + return true; + } + log.verbose(`Dependency of project ${parent.id}, "${project.id}": Distance to root of ${parent._level + 1}. Will be `+ + `replaced by project with same ID and distance to root of ${processedProject.project._level}.`); + + // Replace with the already processed project (closer to root -> preferred) + parent.dependencies[parent.dependencies.indexOf(project)] = processedProject.project; + processedProject.parents.push(parent); + + // No further processing needed + return true; + } + return false; + } + + async loadProjectConfiguration(project) { + if (project.specVersion) { // Project might already be configured // Currently, specVersion is the indicator for configured projects - const projectConf = await this.getProjectConfiguration(project); + this.normalizeConfig(project); + return {}; + } + + let configs; + + // A projects configPath property takes precedence over the default "/ui5.yaml" path + const configPath = project.configPath || path.join(project.path, "/ui5.yaml"); + try { + configs = await this.readConfigFile(configPath); + } catch (err) { + const errorText = "Failed to read configuration for project " + + `${project.id} at "${configPath}". Error: ${err.message}`; - if (!projectConf) { - return null; + if (err.code !== "ENOENT") { // Something else than "File or directory does not exist" + throw new Error(errorText); } + log.verbose(errorText); + } + + if (!configs || !configs.length) { + return {}; + } + + for (let i = configs.length - 1; i >= 0; i--) { + this.normalizeConfig(configs[i]); + } + + const projectConfigs = configs.filter((config) => { + return config.kind === "project"; + }); + + const extensionConfigs = configs.filter((config) => { + return config.kind === "extension"; + }); + + const projectClone = JSON.parse(JSON.stringify(project)); + + // While a project can contain multiple configurations, + // from a dependency tree perspective it is always a single project + // This means it can represent one "project", plus multiple extensions or + // one extension, plus multiple extensions + + if (projectConfigs.length === 1) { + // All well, this is the one. Merge config into project + Object.assign(project, projectConfigs[0]); + } else if (projectConfigs.length > 1) { + throw new Error(`Found ${projectConfigs.length} configurations of kind 'project' for ` + + `project ${project.id}. There is only one project per configuration allowed.`); + } else if (projectConfigs.length === 0 && extensionConfigs.length) { + // No project, but extensions + // => choose one to represent the project -> the first one + Object.assign(project, extensionConfigs.shift()); + } else { + throw new Error(`Found ${configs.length} configurations for ` + + `project ${project.id}. None are of valid kind.`); + } + + const extensionProjects = extensionConfigs.map((config) => { + // Clone original project + const configuredProject = JSON.parse(JSON.stringify(projectClone)); + // Enhance project with its configuration - Object.assign(project, projectConf); + Object.assign(configuredProject, config); + }); + + return {extensions: extensionProjects}; + } + + normalizeConfig(config) { + if (!config.kind) { + config.kind = "project"; // default } + } + isConfigValid(project) { if (!project.specVersion) { if (project._level === 0) { throw new Error(`No specification version defined for root project ${project.id}`); } log.verbose(`No specification version defined for project ${project.id}`); - return; // return with empty config + return false; // ignore this project } if (project.specVersion !== "0.1") { @@ -128,52 +260,40 @@ class ProjectPreprocessor { if (project._level === 0) { throw new Error(`No type configured for root project ${project.id}`); } - log.verbose(`No type configured for project ${project.id} (neither in project configuration, nor in any shim)`); - return; // return with empty config + log.verbose(`No type configured for project ${project.id}`); + return false; // ignore this project } - if (project.type === "application" && project._level !== 0) { - // There is only one project of type application allowed + if (project.kind !== "project" && project._level === 0) { + // This is arguable. It is not the concern of ui5-project to define the entry point of a project tree + // On the other hand, there is no known use case for anything else right now and failing early here + // makes sense in that regard + throw new Error(`Root project needs to be of kind "project". ${project.id} is of kind ${project.kind}`); + } + + if (project.kind === "project" && project.type === "application" && project._level !== 0) { + // There is only one project project of type application allowed // That project needs to be the root project log.verbose(`[Warn] Ignoring project ${project.id} with type application`+ ` (distance to root: ${project._level}). Type application is only allowed for the root project`); - return; // return with empty config + return false; // ignore this project } - // Apply type - await this.applyType(project); - return project; + return true; } - async getProjectConfiguration(project) { - // A projects configPath property takes precedence over the default "/ui5.yaml" path - const configPath = project.configPath || path.join(project.path, "/ui5.yaml"); - - let config; + async applyType(project) { + let type; try { - config = await this.readConfigFile(configPath); + type = typeRepository.getType(project.type); } catch (err) { - const errorText = "Failed to read configuration for project " + - `${project.id} at "${configPath}". Error: ${err.message}`; - - if (err.code !== "ENOENT") { // Something else than "File or directory does not exist" - throw new Error(errorText); - } - log.verbose(errorText); - - /* Disabled shimming until shim-plugin is available - // If there is a config shim, use it as fallback - if (configShims[project.id]) { - // It's ok if there is no project configuration in the project if there is a shim for it - log.verbose(`Applying shim for project ${project.id}...`); - config = JSON.parse(JSON.stringify(configShims[project.id])); - } else { - // No configuration available -> return empty config - return null; - }*/ + throw new Error(`Failed to retrieve type for project ${project.id}: ${err.message}`); } + await type.format(project); + } - return config; + async applyExtension(project) { + // TOOD } async readConfigFile(configPath) { @@ -182,11 +302,6 @@ class ProjectPreprocessor { filename: path }); } - - async applyType(project) { - let type = typeRepository.getType(project.type); - return type.format(project); - } } /** diff --git a/test/lib/projectPreprocessor.js b/test/lib/projectPreprocessor.js index 11d36d903..972fa820d 100644 --- a/test/lib/projectPreprocessor.js +++ b/test/lib/projectPreprocessor.js @@ -40,6 +40,7 @@ test("Project with inline configuration", (t) => { }, dependencies: [], id: "application.a", + kind: "project", version: "1.0.0", specVersion: "0.1", path: applicationAPath @@ -47,10 +48,62 @@ test("Project with inline configuration", (t) => { }); }); +test("Projects with extension dependency inline configuration", (t) => { + const tree = { + id: "application.a", + path: applicationAPath, + dependencies: [{ + id: "extension.a", + path: applicationAPath, + dependencies: [], + version: "1.0.0", + specVersion: "0.1", + kind: "extension", + type: "project-type", + metadata: { + name: "z" + } + }], + version: "1.0.0", + specVersion: "0.1", + type: "application", + metadata: { + name: "xy" + } + }; + return projectPreprocessor.processTree(tree).then((parsedTree) => { + t.deepEqual(parsedTree, { + _level: 0, + type: "application", + metadata: { + name: "xy", + }, + resources: { + configuration: { + paths: { + webapp: "webapp" + } + }, + pathMappings: { + "/": "webapp", + } + }, + dependencies: [], + id: "application.a", + kind: "project", + version: "1.0.0", + specVersion: "0.1", + path: applicationAPath + }, "Parsed correctly"); + }); +}); + + test("Project with configPath", (t) => { const tree = { id: "application.a", - path: applicationBPath, // B, not A - just to have something different + path: applicationAPath, + configPath: path.join(applicationBPath, "ui5.yaml"), // B, not A - just to have something different dependencies: [], version: "1.0.0" }; @@ -59,8 +112,7 @@ test("Project with configPath", (t) => { _level: 0, type: "application", metadata: { - name: "application.b", - namespace: "id1" + name: "application.b" }, resources: { configuration: { @@ -74,9 +126,11 @@ test("Project with configPath", (t) => { }, dependencies: [], id: "application.a", + kind: "project", version: "1.0.0", specVersion: "0.1", - path: applicationBPath + path: applicationAPath, + configPath: path.join(applicationBPath, "ui5.yaml") }, "Parsed correctly"); }); }); @@ -107,6 +161,7 @@ test("Project with ui5.yaml at default location", (t) => { }, dependencies: [], id: "application.a", + kind: "project", version: "1.0.0", specVersion: "0.1", path: applicationAPath @@ -141,6 +196,7 @@ test("Project with ui5.yaml at default location and some configuration", (t) => }, dependencies: [], id: "application.c", + kind: "project", version: "1.0.0", specVersion: "0.1", path: applicationCPath @@ -155,7 +211,7 @@ test("Missing configuration for root project", (t) => { dependencies: [] }; return t.throws(projectPreprocessor.processTree(tree), - "Failed to configure root project \"application.a\". Please check verbose log for details.", + "No specification version defined for root project application.a", "Rejected with error"); }); @@ -254,6 +310,7 @@ test("Inconsistent dependencies with same ID", (t) => { return projectPreprocessor.processTree(tree).then((parsedTree) => { t.deepEqual(parsedTree, { id: "application.a", + kind: "project", version: "1.0.0", specVersion: "0.1", path: applicationAPath, @@ -275,6 +332,7 @@ test("Inconsistent dependencies with same ID", (t) => { dependencies: [ { id: "library.d", + kind: "project", version: "1.0.0", specVersion: "0.1", path: libraryDPath, @@ -299,6 +357,7 @@ test("Inconsistent dependencies with same ID", (t) => { dependencies: [ { id: "library.a", + kind: "project", version: "1.0.0", specVersion: "0.1", path: libraryAPath, @@ -326,6 +385,7 @@ test("Inconsistent dependencies with same ID", (t) => { }, { id: "library.a", + kind: "project", version: "1.0.0", specVersion: "0.1", path: libraryAPath, @@ -498,6 +558,7 @@ const treeAWithDefaultYamls = { const expectedTreeAWithInlineConfigs = { "id": "application.a", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": applicationAPath, @@ -519,6 +580,7 @@ const expectedTreeAWithInlineConfigs = { "dependencies": [ { "id": "library.d", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryDPath, @@ -543,6 +605,7 @@ const expectedTreeAWithInlineConfigs = { "dependencies": [ { "id": "library.a", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryAPath, @@ -570,6 +633,7 @@ const expectedTreeAWithInlineConfigs = { }, { "id": "library.a", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryAPath, @@ -600,6 +664,7 @@ const expectedTreeAWithDefaultYamls = expectedTreeAWithInlineConfigs; // This is expectedTreeAWithInlineConfigs with added configPath attributes const expectedTreeAWithConfigPaths = { "id": "application.a", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": applicationAPath, @@ -622,6 +687,7 @@ const expectedTreeAWithConfigPaths = { "dependencies": [ { "id": "library.d", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryDPath, @@ -647,6 +713,7 @@ const expectedTreeAWithConfigPaths = { "dependencies": [ { "id": "library.a", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryAPath, @@ -675,6 +742,7 @@ const expectedTreeAWithConfigPaths = { }, { "id": "library.a", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryAPath, @@ -792,6 +860,7 @@ const treeBWithInlineConfigs = { const expectedTreeBWithInlineConfigs = { "id": "application.b", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": applicationBPath, @@ -814,6 +883,7 @@ const expectedTreeBWithInlineConfigs = { "dependencies": [ { "id": "library.b", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryBPath, @@ -838,6 +908,7 @@ const expectedTreeBWithInlineConfigs = { "dependencies": [ { "id": "library.d", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryDPath, @@ -862,6 +933,7 @@ const expectedTreeBWithInlineConfigs = { "dependencies": [ { "id": "library.a", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryAPath, @@ -891,6 +963,7 @@ const expectedTreeBWithInlineConfigs = { }, { "id": "library.d", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryDPath, @@ -915,6 +988,7 @@ const expectedTreeBWithInlineConfigs = { "dependencies": [ { "id": "library.a", + "kind": "project", "version": "1.0.0", "specVersion": "0.1", "path": libraryAPath,