From 804e6f4b32d004d427eecd0999f11051b7f52392 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Tue, 18 Nov 2025 15:41:39 +0100 Subject: [PATCH 1/7] feat(fs): Enhance API for incremental build, add tracking readers/writers Cherry-picked from https://github.com/SAP/ui5-fs/commit/5651627b91109b8059d4c9401c192e6c9d757e60 JIRA: CPOUI5FOUNDATION-1174 --- packages/fs/lib/DuplexTracker.js | 84 ++++++++++++ .../fs/lib/ReaderCollectionPrioritized.js | 2 +- packages/fs/lib/Resource.js | 127 ++++++++++++++---- packages/fs/lib/ResourceFacade.js | 14 ++ packages/fs/lib/Tracker.js | 69 ++++++++++ packages/fs/lib/adapters/AbstractAdapter.js | 54 ++++++-- packages/fs/lib/adapters/FileSystem.js | 28 ++-- packages/fs/lib/adapters/Memory.js | 34 +---- packages/fs/lib/readers/Filter.js | 4 +- packages/fs/lib/readers/Link.js | 44 +++--- packages/fs/lib/resourceFactory.js | 24 +++- 11 files changed, 372 insertions(+), 112 deletions(-) create mode 100644 packages/fs/lib/DuplexTracker.js create mode 100644 packages/fs/lib/Tracker.js diff --git a/packages/fs/lib/DuplexTracker.js b/packages/fs/lib/DuplexTracker.js new file mode 100644 index 00000000000..2ccdb56b1a4 --- /dev/null +++ b/packages/fs/lib/DuplexTracker.js @@ -0,0 +1,84 @@ +import AbstractReaderWriter from "./AbstractReaderWriter.js"; + +// TODO: Alternative name: Inspector/Interceptor/... + +export default class Trace extends AbstractReaderWriter { + #readerWriter; + #sealed = false; + #pathsRead = []; + #patterns = []; + #resourcesRead = Object.create(null); + #resourcesWritten = Object.create(null); + + constructor(readerWriter) { + super(readerWriter.getName()); + this.#readerWriter = readerWriter; + } + + getResults() { + this.#sealed = true; + return { + requests: { + pathsRead: this.#pathsRead, + patterns: this.#patterns, + }, + resourcesRead: this.#resourcesRead, + resourcesWritten: this.#resourcesWritten, + }; + } + + async _byGlob(virPattern, options, trace) { + if (this.#sealed) { + throw new Error(`Unexpected read operation after reader has been sealed`); + } + if (this.#readerWriter.resolvePattern) { + const resolvedPattern = this.#readerWriter.resolvePattern(virPattern); + this.#patterns.push(resolvedPattern); + } else if (virPattern instanceof Array) { + for (const pattern of virPattern) { + this.#patterns.push(pattern); + } + } else { + this.#patterns.push(virPattern); + } + const resources = await this.#readerWriter._byGlob(virPattern, options, trace); + for (const resource of resources) { + if (!resource.getStatInfo()?.isDirectory()) { + this.#resourcesRead[resource.getOriginalPath()] = resource; + } + } + return resources; + } + + async _byPath(virPath, options, trace) { + if (this.#sealed) { + throw new Error(`Unexpected read operation after reader has been sealed`); + } + if (this.#readerWriter.resolvePath) { + const resolvedPath = this.#readerWriter.resolvePath(virPath); + if (resolvedPath) { + this.#pathsRead.push(resolvedPath); + } + } else { + this.#pathsRead.push(virPath); + } + const resource = await this.#readerWriter._byPath(virPath, options, trace); + if (resource) { + if (!resource.getStatInfo()?.isDirectory()) { + this.#resourcesRead[resource.getOriginalPath()] = resource; + } + } + return resource; + } + + async _write(resource, options) { + if (this.#sealed) { + throw new Error(`Unexpected write operation after writer has been sealed`); + } + if (!resource) { + throw new Error(`Cannot write undefined resource`); + } + this.#resourcesWritten[resource.getOriginalPath()] = resource; + return this.#readerWriter.write(resource, options); + } +} diff --git a/packages/fs/lib/ReaderCollectionPrioritized.js b/packages/fs/lib/ReaderCollectionPrioritized.js index 680b71357ca..8f235f22148 100644 --- a/packages/fs/lib/ReaderCollectionPrioritized.js +++ b/packages/fs/lib/ReaderCollectionPrioritized.js @@ -68,7 +68,7 @@ class ReaderCollectionPrioritized extends AbstractReader { * @returns {Promise<@ui5/fs/Resource|null>} * Promise resolving to a single resource or null if no resource is found */ - _byPath(virPath, options, trace) { + async _byPath(virPath, options, trace) { const that = this; const byPath = (i) => { if (i > this._readers.length - 1) { diff --git a/packages/fs/lib/Resource.js b/packages/fs/lib/Resource.js index c43edc2716f..1cefc2ce490 100644 --- a/packages/fs/lib/Resource.js +++ b/packages/fs/lib/Resource.js @@ -1,9 +1,8 @@ import stream from "node:stream"; +import crypto from "node:crypto"; import clone from "clone"; import posixPath from "node:path/posix"; -const fnTrue = () => true; -const fnFalse = () => false; const ALLOWED_SOURCE_METADATA_KEYS = ["adapter", "fsPath", "contentModified"]; /** @@ -100,24 +99,6 @@ class Resource { this.#project = project; - this.#statInfo = statInfo || { // TODO - isFile: fnTrue, - isDirectory: fnFalse, - isBlockDevice: fnFalse, - isCharacterDevice: fnFalse, - isSymbolicLink: fnFalse, - isFIFO: fnFalse, - isSocket: fnFalse, - atimeMs: new Date().getTime(), - mtimeMs: new Date().getTime(), - ctimeMs: new Date().getTime(), - birthtimeMs: new Date().getTime(), - atime: new Date(), - mtime: new Date(), - ctime: new Date(), - birthtime: new Date() - }; - if (createStream) { this.#createStream = createStream; } else if (stream) { @@ -130,6 +111,15 @@ class Resource { this.#setBuffer(Buffer.from(string, "utf8")); } + if (statInfo) { + this.#statInfo = parseStat(statInfo); + } else { + if (createStream || stream) { + throw new Error("Unable to create Resource: Please provide statInfo for stream content"); + } + this.#statInfo = createStat(this.#buffer.byteLength); + } + // Tracing: this.#collections = []; } @@ -164,6 +154,7 @@ class Resource { setBuffer(buffer) { this.#sourceMetadata.contentModified = true; this.#isModified = true; + this.#updateStatInfo(buffer); this.#setBuffer(buffer); } @@ -269,6 +260,21 @@ class Resource { this.#streamDrained = false; } + async getHash() { + if (this.#statInfo.isDirectory()) { + return; + } + const buffer = await this.getBuffer(); + return crypto.createHash("md5").update(buffer).digest("hex"); + } + + #updateStatInfo(buffer) { + const now = new Date(); + this.#statInfo.mtimeMs = now.getTime(); + this.#statInfo.mtime = now; + this.#statInfo.size = buffer.byteLength; + } + /** * Gets the virtual resources path * @@ -279,6 +285,10 @@ class Resource { return this.#path; } + getOriginalPath() { + return this.#path; + } + /** * Sets the virtual resources path * @@ -318,6 +328,10 @@ class Resource { return this.#statInfo; } + getLastModified() { + + } + /** * Size in bytes allocated by the underlying buffer. * @@ -325,12 +339,13 @@ class Resource { * @returns {Promise} size in bytes, 0 if there is no content yet */ async getSize() { - // if resource does not have any content it should have 0 bytes - if (!this.#buffer && !this.#createStream && !this.#stream) { - return 0; - } - const buffer = await this.getBuffer(); - return buffer.byteLength; + return this.#statInfo.size; + // // if resource does not have any content it should have 0 bytes + // if (!this.#buffer && !this.#createStream && !this.#stream) { + // return 0; + // } + // const buffer = await this.getBuffer(); + // return buffer.byteLength; } /** @@ -356,7 +371,7 @@ class Resource { async #getCloneOptions() { const options = { path: this.#path, - statInfo: clone(this.#statInfo), + statInfo: this.#statInfo, // Will be cloned in constructor sourceMetadata: clone(this.#sourceMetadata) }; @@ -495,4 +510,62 @@ class Resource { } } +const fnTrue = function() { + return true; +}; +const fnFalse = function() { + return false; +}; + +/** + * Parses a Node.js stat object to a UI5 Tooling stat object + * + * @param {fs.Stats} statInfo Node.js stat + * @returns {object} UI5 Tooling stat +*/ +function parseStat(statInfo) { + return { + isFile: statInfo.isFile.bind(statInfo), + isDirectory: statInfo.isDirectory.bind(statInfo), + isBlockDevice: statInfo.isBlockDevice.bind(statInfo), + isCharacterDevice: statInfo.isCharacterDevice.bind(statInfo), + isSymbolicLink: statInfo.isSymbolicLink.bind(statInfo), + isFIFO: statInfo.isFIFO.bind(statInfo), + isSocket: statInfo.isSocket.bind(statInfo), + ino: statInfo.ino, + size: statInfo.size, + atimeMs: statInfo.atimeMs, + mtimeMs: statInfo.mtimeMs, + ctimeMs: statInfo.ctimeMs, + birthtimeMs: statInfo.birthtimeMs, + atime: statInfo.atime, + mtime: statInfo.mtime, + ctime: statInfo.ctime, + birthtime: statInfo.birthtime, + }; +} + +function createStat(size) { + const now = new Date(); + return { + isFile: fnTrue, + isDirectory: fnFalse, + isBlockDevice: fnFalse, + isCharacterDevice: fnFalse, + isSymbolicLink: fnFalse, + isFIFO: fnFalse, + isSocket: fnFalse, + ino: 0, + size, + atimeMs: now.getTime(), + mtimeMs: now.getTime(), + ctimeMs: now.getTime(), + birthtimeMs: now.getTime(), + atime: now, + mtime: now, + ctime: now, + birthtime: now, + }; +} + export default Resource; diff --git a/packages/fs/lib/ResourceFacade.js b/packages/fs/lib/ResourceFacade.js index 58ba37b2a4d..9604b56acd1 100644 --- a/packages/fs/lib/ResourceFacade.js +++ b/packages/fs/lib/ResourceFacade.js @@ -45,6 +45,16 @@ class ResourceFacade { return this.#path; } + /** + * Gets the resources path + * + * @public + * @returns {string} (Virtual) path of the resource + */ + getOriginalPath() { + return this.#resource.getPath(); + } + /** * Gets the resource name * @@ -150,6 +160,10 @@ class ResourceFacade { return this.#resource.setStream(stream); } + getHash() { + return this.#resource.getHash(); + } + /** * Gets the resources stat info. * Note that a resources stat information is not updated when the resource is being modified. diff --git a/packages/fs/lib/Tracker.js b/packages/fs/lib/Tracker.js new file mode 100644 index 00000000000..ed19019e364 --- /dev/null +++ b/packages/fs/lib/Tracker.js @@ -0,0 +1,69 @@ +import AbstractReader from "./AbstractReader.js"; + +export default class Trace extends AbstractReader { + #reader; + #sealed = false; + #pathsRead = []; + #patterns = []; + #resourcesRead = Object.create(null); + + constructor(reader) { + super(reader.getName()); + this.#reader = reader; + } + + getResults() { + this.#sealed = true; + return { + requests: { + pathsRead: this.#pathsRead, + patterns: this.#patterns, + }, + resourcesRead: this.#resourcesRead, + }; + } + + async _byGlob(virPattern, options, trace) { + if (this.#sealed) { + throw new Error(`Unexpected read operation after reader has been sealed`); + } + if (this.#reader.resolvePattern) { + const resolvedPattern = this.#reader.resolvePattern(virPattern); + this.#patterns.push(resolvedPattern); + } else if (virPattern instanceof Array) { + for (const pattern of virPattern) { + this.#patterns.push(pattern); + } + } else { + this.#patterns.push(virPattern); + } + const resources = await this.#reader._byGlob(virPattern, options, trace); + for (const resource of resources) { + if (!resource.getStatInfo()?.isDirectory()) { + this.#resourcesRead[resource.getOriginalPath()] = resource; + } + } + return resources; + } + + async _byPath(virPath, options, trace) { + if (this.#sealed) { + throw new Error(`Unexpected read operation after reader has been sealed`); + } + if (this.#reader.resolvePath) { + const resolvedPath = this.#reader.resolvePath(virPath); + if (resolvedPath) { + this.#pathsRead.push(resolvedPath); + } + } else { + this.#pathsRead.push(virPath); + } + const resource = await this.#reader._byPath(virPath, options, trace); + if (resource) { + if (!resource.getStatInfo()?.isDirectory()) { + this.#resourcesRead[resource.getOriginalPath()] = resource; + } + } + return resource; + } +} diff --git a/packages/fs/lib/adapters/AbstractAdapter.js b/packages/fs/lib/adapters/AbstractAdapter.js index 96cf4154250..4d2387de80b 100644 --- a/packages/fs/lib/adapters/AbstractAdapter.js +++ b/packages/fs/lib/adapters/AbstractAdapter.js @@ -17,20 +17,20 @@ import Resource from "../Resource.js"; */ class AbstractAdapter extends AbstractReaderWriter { /** - * The constructor * * @public * @param {object} parameters Parameters + * @param {string} parameters.name * @param {string} parameters.virBasePath * Virtual base path. Must be absolute, POSIX-style, and must end with a slash * @param {string[]} [parameters.excludes] List of glob patterns to exclude * @param {object} [parameters.project] Experimental, internal parameter. Do not use */ - constructor({virBasePath, excludes = [], project}) { + constructor({name, virBasePath, excludes = [], project}) { if (new.target === AbstractAdapter) { throw new TypeError("Class 'AbstractAdapter' is abstract"); } - super(); + super(name); if (!virBasePath) { throw new Error(`Unable to create adapter: Missing parameter 'virBasePath'`); @@ -81,17 +81,7 @@ class AbstractAdapter extends AbstractReaderWriter { if (patterns[i] && idx !== -1 && idx < this._virBaseDir.length) { const subPath = patterns[i]; return [ - this._createResource({ - statInfo: { // TODO: make closer to fs stat info - isDirectory: function() { - return true; - } - }, - source: { - adapter: "Abstract" - }, - path: subPath - }) + this._createDirectoryResource(subPath) ]; } } @@ -201,6 +191,10 @@ class AbstractAdapter extends AbstractReaderWriter { if (this._project) { parameters.project = this._project; } + if (!parameters.source) { + parameters.source = Object.create(null); + } + parameters.source.adapter = this.constructor.name; return new Resource(parameters); } @@ -289,6 +283,38 @@ class AbstractAdapter extends AbstractReaderWriter { const relPath = virPath.substr(this._virBasePath.length); return relPath; } + + _createDirectoryResource(dirPath) { + const now = new Date(); + const fnFalse = function() { + return false; + }; + const fnTrue = function() { + return true; + }; + const statInfo = { + isFile: fnFalse, + isDirectory: fnTrue, + isBlockDevice: fnFalse, + isCharacterDevice: fnFalse, + isSymbolicLink: fnFalse, + isFIFO: fnFalse, + isSocket: fnFalse, + size: 0, + atimeMs: now.getTime(), + mtimeMs: now.getTime(), + ctimeMs: now.getTime(), + birthtimeMs: now.getTime(), + atime: now, + mtime: now, + ctime: now, + birthtime: now, + }; + return this._createResource({ + statInfo: statInfo, + path: dirPath, + }); + } } export default AbstractAdapter; diff --git a/packages/fs/lib/adapters/FileSystem.js b/packages/fs/lib/adapters/FileSystem.js index d086fac40dd..284d95d84a4 100644 --- a/packages/fs/lib/adapters/FileSystem.js +++ b/packages/fs/lib/adapters/FileSystem.js @@ -12,7 +12,7 @@ import {PassThrough} from "node:stream"; import AbstractAdapter from "./AbstractAdapter.js"; const READ_ONLY_MODE = 0o444; -const ADAPTER_NAME = "FileSystem"; + /** * File system resource adapter * @@ -23,9 +23,9 @@ const ADAPTER_NAME = "FileSystem"; */ class FileSystem extends AbstractAdapter { /** - * The Constructor. * * @param {object} parameters Parameters + * @param {string} parameters.name * @param {string} parameters.virBasePath * Virtual base path. Must be absolute, POSIX-style, and must end with a slash * @param {string} parameters.fsBasePath @@ -35,8 +35,8 @@ class FileSystem extends AbstractAdapter { * Whether to apply any excludes defined in an optional .gitignore in the given fsBasePath directory * @param {@ui5/project/specifications/Project} [parameters.project] Project this adapter belongs to (if any) */ - constructor({virBasePath, project, fsBasePath, excludes, useGitignore=false}) { - super({virBasePath, project, excludes}); + constructor({name, virBasePath, project, fsBasePath, excludes, useGitignore=false}) { + super({name, virBasePath, project, excludes}); if (!fsBasePath) { throw new Error(`Unable to create adapter: Missing parameter 'fsBasePath'`); @@ -80,7 +80,7 @@ class FileSystem extends AbstractAdapter { statInfo: stat, path: this._virBaseDir, sourceMetadata: { - adapter: ADAPTER_NAME, + adapter: this.constructor.name, fsPath: this._fsBasePath }, createStream: () => { @@ -124,7 +124,7 @@ class FileSystem extends AbstractAdapter { statInfo: stat, path: virPath, sourceMetadata: { - adapter: ADAPTER_NAME, + adapter: this.constructor.name, fsPath: fsPath }, createStream: () => { @@ -158,16 +158,8 @@ class FileSystem extends AbstractAdapter { // Neither starts with basePath, nor equals baseDirectory if (!options.nodir && this._virBasePath.startsWith(virPath)) { // Create virtual directories for the virtual base path (which has to exist) - // TODO: Maybe improve this by actually matching the base paths segments to the virPath - return this._createResource({ - project: this._project, - statInfo: { // TODO: make closer to fs stat info - isDirectory: function() { - return true; - } - }, - path: virPath - }); + // FUTURE: Maybe improve this by actually matching the base paths segments to the virPath + return this._createDirectoryResource(virPath); } else { return null; } @@ -200,7 +192,7 @@ class FileSystem extends AbstractAdapter { statInfo, path: virPath, sourceMetadata: { - adapter: ADAPTER_NAME, + adapter: this.constructor.name, fsPath } }; @@ -260,7 +252,7 @@ class FileSystem extends AbstractAdapter { await mkdir(dirPath, {recursive: true}); const sourceMetadata = resource.getSourceMetadata(); - if (sourceMetadata && sourceMetadata.adapter === ADAPTER_NAME && sourceMetadata.fsPath) { + if (sourceMetadata && sourceMetadata.adapter === this.constructor.name && sourceMetadata.fsPath) { // Resource has been created by FileSystem adapter. This means it might require special handling /* The following code covers these four conditions: diff --git a/packages/fs/lib/adapters/Memory.js b/packages/fs/lib/adapters/Memory.js index 35be99cf953..7215e2ab189 100644 --- a/packages/fs/lib/adapters/Memory.js +++ b/packages/fs/lib/adapters/Memory.js @@ -3,8 +3,6 @@ const log = getLogger("resources:adapters:Memory"); import micromatch from "micromatch"; import AbstractAdapter from "./AbstractAdapter.js"; -const ADAPTER_NAME = "Memory"; - /** * Virtual resource Adapter * @@ -15,17 +13,17 @@ const ADAPTER_NAME = "Memory"; */ class Memory extends AbstractAdapter { /** - * The constructor. * * @public * @param {object} parameters Parameters + * @param {string} parameters.name * @param {string} parameters.virBasePath * Virtual base path. Must be absolute, POSIX-style, and must end with a slash * @param {string[]} [parameters.excludes] List of glob patterns to exclude * @param {@ui5/project/specifications/Project} [parameters.project] Project this adapter belongs to (if any) */ - constructor({virBasePath, project, excludes}) { - super({virBasePath, project, excludes}); + constructor({name, virBasePath, project, excludes}) { + super({name, virBasePath, project, excludes}); this._virFiles = Object.create(null); // map full of files this._virDirs = Object.create(null); // map full of directories } @@ -72,18 +70,7 @@ class Memory extends AbstractAdapter { async _runGlob(patterns, options = {nodir: true}, trace) { if (patterns[0] === "" && !options.nodir) { // Match virtual root directory return [ - this._createResource({ - project: this._project, - statInfo: { // TODO: make closer to fs stat info - isDirectory: function() { - return true; - } - }, - sourceMetadata: { - adapter: ADAPTER_NAME - }, - path: this._virBasePath.slice(0, -1) - }) + this._createDirectoryResource(this._virBasePath.slice(0, -1)) ]; } @@ -157,18 +144,7 @@ class Memory extends AbstractAdapter { for (let i = pathSegments.length - 1; i >= 0; i--) { const segment = pathSegments[i]; if (!this._virDirs[segment]) { - this._virDirs[segment] = this._createResource({ - project: this._project, - sourceMetadata: { - adapter: ADAPTER_NAME - }, - statInfo: { // TODO: make closer to fs stat info - isDirectory: function() { - return true; - } - }, - path: this._virBasePath + segment - }); + this._virDirs[segment] = this._createDirectoryResource(this._virBasePath + segment); } } } diff --git a/packages/fs/lib/readers/Filter.js b/packages/fs/lib/readers/Filter.js index b95654daa29..1e4cf31e727 100644 --- a/packages/fs/lib/readers/Filter.js +++ b/packages/fs/lib/readers/Filter.js @@ -27,8 +27,8 @@ class Filter extends AbstractReader { * @param {@ui5/fs/readers/Filter~callback} parameters.callback * Filter function. Will be called for every resource read through this reader. */ - constructor({reader, callback}) { - super(); + constructor({name, reader, callback}) { + super(name); if (!reader) { throw new Error(`Missing parameter "reader"`); } diff --git a/packages/fs/lib/readers/Link.js b/packages/fs/lib/readers/Link.js index dd2d40b3ce3..dd6c56dd16d 100644 --- a/packages/fs/lib/readers/Link.js +++ b/packages/fs/lib/readers/Link.js @@ -44,8 +44,8 @@ class Link extends AbstractReader { * @param {@ui5/fs/AbstractReader} parameters.reader The resource reader or collection to wrap * @param {@ui5/fs/readers/Link/PathMapping} parameters.pathMapping */ - constructor({reader, pathMapping}) { - super(); + constructor({name, reader, pathMapping}) { + super(name); if (!reader) { throw new Error(`Missing parameter "reader"`); } @@ -57,17 +57,7 @@ class Link extends AbstractReader { Link._validatePathMapping(pathMapping); } - /** - * Locates resources by glob. - * - * @private - * @param {string|string[]} patterns glob pattern as string or an array of - * glob patterns for virtual directory structure - * @param {object} options glob options - * @param {@ui5/fs/tracing/Trace} trace Trace instance - * @returns {Promise<@ui5/fs/Resource[]>} Promise resolving to list of resources - */ - async _byGlob(patterns, options, trace) { + resolvePattern(patterns) { if (!(patterns instanceof Array)) { patterns = [patterns]; } @@ -79,7 +69,29 @@ class Link extends AbstractReader { }); // Flatten prefixed patterns - patterns = Array.prototype.concat.apply([], patterns); + return Array.prototype.concat.apply([], patterns); + } + + resolvePath(virPath) { + if (!virPath.startsWith(this._pathMapping.linkPath)) { + return null; + } + const targetPath = this._pathMapping.targetPath + virPath.substr(this._pathMapping.linkPath.length); + return targetPath; + } + + /** + * Locates resources by glob. + * + * @private + * @param {string|string[]} patterns glob pattern as string or an array of + * glob patterns for virtual directory structure + * @param {object} options glob options + * @param {@ui5/fs/tracing/Trace} trace Trace instance + * @returns {Promise<@ui5/fs/Resource[]>} Promise resolving to list of resources + */ + async _byGlob(patterns, options, trace) { + patterns = this.resolvePattern(patterns); // Keep resource's internal path unchanged for now const resources = await this._reader._byGlob(patterns, options, trace); @@ -104,10 +116,10 @@ class Link extends AbstractReader { * @returns {Promise<@ui5/fs/Resource>} Promise resolving to a single resource */ async _byPath(virPath, options, trace) { - if (!virPath.startsWith(this._pathMapping.linkPath)) { + const targetPath = this.resolvePath(virPath); + if (!targetPath) { return null; } - const targetPath = this._pathMapping.targetPath + virPath.substr(this._pathMapping.linkPath.length); log.silly(`byPath: Rewriting virtual path ${virPath} to ${targetPath}`); const resource = await this._reader._byPath(targetPath, options, trace); diff --git a/packages/fs/lib/resourceFactory.js b/packages/fs/lib/resourceFactory.js index ba48d76de1d..c338d0e2bd6 100644 --- a/packages/fs/lib/resourceFactory.js +++ b/packages/fs/lib/resourceFactory.js @@ -9,6 +9,8 @@ import Resource from "./Resource.js"; import WriterCollection from "./WriterCollection.js"; import Filter from "./readers/Filter.js"; import Link from "./readers/Link.js"; +import Tracker from "./Tracker.js"; +import DuplexTracker from "./DuplexTracker.js"; import {getLogger} from "@ui5/logger"; const log = getLogger("resources:resourceFactory"); @@ -26,6 +28,7 @@ const log = getLogger("resources:resourceFactory"); * * @public * @param {object} parameters Parameters + * @param {string} parameters.name * @param {string} parameters.virBasePath Virtual base path. Must be absolute, POSIX-style, and must end with a slash * @param {string} [parameters.fsBasePath] * File System base path. @@ -38,11 +41,11 @@ const log = getLogger("resources:resourceFactory"); * @param {@ui5/project/specifications/Project} [parameters.project] Project this adapter belongs to (if any) * @returns {@ui5/fs/adapters/FileSystem|@ui5/fs/adapters/Memory} File System- or Virtual Adapter */ -export function createAdapter({fsBasePath, virBasePath, project, excludes, useGitignore}) { +export function createAdapter({name, fsBasePath, virBasePath, project, excludes, useGitignore}) { if (fsBasePath) { - return new FsAdapter({fsBasePath, virBasePath, project, excludes, useGitignore}); + return new FsAdapter({name, fsBasePath, virBasePath, project, excludes, useGitignore}); } else { - return new MemAdapter({virBasePath, project, excludes}); + return new MemAdapter({name, virBasePath, project, excludes}); } } @@ -178,15 +181,17 @@ export function createResource(parameters) { export function createWorkspace({reader, writer, virBasePath = "/", name = "workspace"}) { if (!writer) { writer = new MemAdapter({ + name: `Workspace writer for ${name}`, virBasePath }); } - return new DuplexCollection({ + const d = new DuplexCollection({ reader, writer, name }); + return d; } /** @@ -242,12 +247,14 @@ export function createLinkReader(parameters) { * * @public * @param {object} parameters + * @param {string} parameters.name * @param {@ui5/fs/AbstractReader} parameters.reader Single reader or collection of readers * @param {string} parameters.namespace Project namespace * @returns {@ui5/fs/readers/Link} Reader instance */ -export function createFlatReader({reader, namespace}) { +export function createFlatReader({name, reader, namespace}) { return new Link({ + name, reader: reader, pathMapping: { linkPath: `/`, @@ -256,6 +263,13 @@ export function createFlatReader({reader, namespace}) { }); } +export function createTracker(readerWriter) { + if (readerWriter instanceof DuplexCollection) { + return new DuplexTracker(readerWriter); + } + return new Tracker(readerWriter); +} + /** * Normalizes virtual glob patterns by prefixing them with * a given virtual base directory path From 5384508c75f866b0de899a0a306ea0aa38c6bd7c Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Tue, 18 Nov 2025 15:43:27 +0100 Subject: [PATCH 2/7] feat(server): Use incremental build in server Cherry-picked from: https://github.com/SAP/ui5-fs/commit/5651627b91109b8059d4c9401c192e6c9d757e60 JIRA: CPOUI5FOUNDATION-1174 --- .../lib/middleware/MiddlewareManager.js | 2 +- packages/server/lib/server.js | 71 +++++++++++++------ 2 files changed, 49 insertions(+), 24 deletions(-) diff --git a/packages/server/lib/middleware/MiddlewareManager.js b/packages/server/lib/middleware/MiddlewareManager.js index 36894892e4d..a06a2475300 100644 --- a/packages/server/lib/middleware/MiddlewareManager.js +++ b/packages/server/lib/middleware/MiddlewareManager.js @@ -218,7 +218,7 @@ class MiddlewareManager { }); await this.addMiddleware("serveResources"); await this.addMiddleware("testRunner"); - await this.addMiddleware("serveThemes"); + // await this.addMiddleware("serveThemes"); await this.addMiddleware("versionInfo", { mountPath: "/resources/sap-ui-version.json" }); diff --git a/packages/server/lib/server.js b/packages/server/lib/server.js index 3b108a551d7..ea9c544019d 100644 --- a/packages/server/lib/server.js +++ b/packages/server/lib/server.js @@ -1,7 +1,8 @@ import express from "express"; import portscanner from "portscanner"; +import path from "node:path/posix"; import MiddlewareManager from "./middleware/MiddlewareManager.js"; -import {createReaderCollection} from "@ui5/fs/resourceFactory"; +import {createAdapter, createReaderCollection} from "@ui5/fs/resourceFactory"; import ReaderCollectionPrioritized from "@ui5/fs/ReaderCollectionPrioritized"; import {getLogger} from "@ui5/logger"; @@ -136,34 +137,58 @@ export async function serve(graph, { port: requestedPort, changePortIfInUse = false, h2 = false, key, cert, acceptRemoteConnections = false, sendSAPTargetCSP = false, simpleIndex = false, serveCSPReports = false }) { - const rootProject = graph.getRoot(); + // const rootReader = createAdapter({ + // virBasePath: "/", + // }); + // const dependencies = createAdapter({ + // virBasePath: "/", + // }); - const readers = []; - await graph.traverseBreadthFirst(async function({project: dep}) { - if (dep.getName() === rootProject.getName()) { - // Ignore root project - return; - } - readers.push(dep.getReader({style: "runtime"})); + const rootProject = graph.getRoot(); + const watchHandler = await graph.build({ + cacheDir: path.join(rootProject.getRootPath(), ".ui5-cache"), + includedDependencies: ["*"], + watch: true, }); - const dependencies = createReaderCollection({ - name: `Dependency reader collection for project ${rootProject.getName()}`, - readers - }); + async function createReaders() { + const readers = []; + await graph.traverseBreadthFirst(async function({project: dep}) { + if (dep.getName() === rootProject.getName()) { + // Ignore root project + return; + } + readers.push(dep.getReader({style: "runtime"})); + }); - const rootReader = rootProject.getReader({style: "runtime"}); + const dependencies = createReaderCollection({ + name: `Dependency reader collection for project ${rootProject.getName()}`, + readers + }); + + const rootReader = rootProject.getReader({style: "runtime"}); + // TODO change to ReaderCollection once duplicates are sorted out + const combo = new ReaderCollectionPrioritized({ + name: "server - prioritize workspace over dependencies", + readers: [rootReader, dependencies] + }); + const resources = { + rootProject: rootReader, + dependencies: dependencies, + all: combo + }; + return resources; + } + + const resources = await createReaders(); - // TODO change to ReaderCollection once duplicates are sorted out - const combo = new ReaderCollectionPrioritized({ - name: "server - prioritize workspace over dependencies", - readers: [rootReader, dependencies] + watchHandler.on("buildUpdated", async () => { + const newResources = await createReaders(); + // Patch resources + resources.rootProject = newResources.rootProject; + resources.dependencies = newResources.dependencies; + resources.all = newResources.all; }); - const resources = { - rootProject: rootReader, - dependencies: dependencies, - all: combo - }; const middlewareManager = new MiddlewareManager({ graph, From 913da17191907eb7a3e84b1720a63d5231622867 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Tue, 18 Nov 2025 15:46:24 +0100 Subject: [PATCH 3/7] feat(builder): Adapt tasks for incremental build Cherry-picked from: https://github.com/SAP/ui5-builder/commit/ef5a3b2f6ca0339a8ff8c30997e884e462fa6ab9 JIRA: CPOUI5FOUNDATION-1174 --- .../builder/lib/processors/nonAsciiEscaper.js | 2 +- .../builder/lib/processors/stringReplacer.js | 3 +- .../lib/tasks/escapeNonAsciiCharacters.js | 11 ++++-- packages/builder/lib/tasks/minify.js | 14 +++++-- .../builder/lib/tasks/replaceBuildtime.js | 36 +++++++++--------- .../builder/lib/tasks/replaceCopyright.js | 37 ++++++++++--------- packages/builder/lib/tasks/replaceVersion.js | 35 ++++++++++-------- 7 files changed, 80 insertions(+), 58 deletions(-) diff --git a/packages/builder/lib/processors/nonAsciiEscaper.js b/packages/builder/lib/processors/nonAsciiEscaper.js index 858eac7745d..019386dfd35 100644 --- a/packages/builder/lib/processors/nonAsciiEscaper.js +++ b/packages/builder/lib/processors/nonAsciiEscaper.js @@ -83,8 +83,8 @@ async function nonAsciiEscaper({resources, options: {encoding}}) { // only modify the resource's string if it was changed if (escaped.modified) { resource.setString(escaped.string); + return resource; } - return resource; } return Promise.all(resources.map(processResource)); diff --git a/packages/builder/lib/processors/stringReplacer.js b/packages/builder/lib/processors/stringReplacer.js index 2485032cc76..5002d426239 100644 --- a/packages/builder/lib/processors/stringReplacer.js +++ b/packages/builder/lib/processors/stringReplacer.js @@ -23,7 +23,8 @@ export default function({resources, options: {pattern, replacement}}) { const newContent = content.replaceAll(pattern, replacement); if (content !== newContent) { resource.setString(newContent); + return resource; } - return resource; + // return resource; })); } diff --git a/packages/builder/lib/tasks/escapeNonAsciiCharacters.js b/packages/builder/lib/tasks/escapeNonAsciiCharacters.js index 53cb3e8d9f3..73943c04a34 100644 --- a/packages/builder/lib/tasks/escapeNonAsciiCharacters.js +++ b/packages/builder/lib/tasks/escapeNonAsciiCharacters.js @@ -19,12 +19,17 @@ import nonAsciiEscaper from "../processors/nonAsciiEscaper.js"; * @param {string} parameters.options.encoding source file encoding either "UTF-8" or "ISO-8859-1" * @returns {Promise} Promise resolving with undefined once data has been written */ -export default async function({workspace, options: {pattern, encoding}}) { +export default async function({workspace, invalidatedResources, options: {pattern, encoding}}) { if (!encoding) { throw new Error("[escapeNonAsciiCharacters] Mandatory option 'encoding' not provided"); } - const allResources = await workspace.byGlob(pattern); + let allResources; + if (invalidatedResources) { + allResources = await Promise.all(invalidatedResources.map((resource) => workspace.byPath(resource))); + } else { + allResources = await workspace.byGlob(pattern); + } const processedResources = await nonAsciiEscaper({ resources: allResources, @@ -33,5 +38,5 @@ export default async function({workspace, options: {pattern, encoding}}) { } }); - await Promise.all(processedResources.map((resource) => workspace.write(resource))); + await Promise.all(processedResources.map((resource) => resource && workspace.write(resource))); } diff --git a/packages/builder/lib/tasks/minify.js b/packages/builder/lib/tasks/minify.js index 2969ca688dc..f79c3391cd6 100644 --- a/packages/builder/lib/tasks/minify.js +++ b/packages/builder/lib/tasks/minify.js @@ -26,9 +26,17 @@ import fsInterface from "@ui5/fs/fsInterface"; * @returns {Promise} Promise resolving with undefined once data has been written */ export default async function({ - workspace, taskUtil, options: {pattern, omitSourceMapResources = false, useInputSourceMaps = true - }}) { - const resources = await workspace.byGlob(pattern); + workspace, taskUtil, buildCache, + options: {pattern, omitSourceMapResources = false, useInputSourceMaps = true} +}) { + let resources = await workspace.byGlob(pattern); + if (buildCache.hasCache()) { + const changedPaths = buildCache.getChangedProjectResourcePaths(); + resources = resources.filter((resource) => changedPaths.has(resource.getPath())); + } + if (resources.length === 0) { + return; + } const processedResources = await minifier({ resources, fs: fsInterface(workspace), diff --git a/packages/builder/lib/tasks/replaceBuildtime.js b/packages/builder/lib/tasks/replaceBuildtime.js index f4093c0b732..2a3ff1caf22 100644 --- a/packages/builder/lib/tasks/replaceBuildtime.js +++ b/packages/builder/lib/tasks/replaceBuildtime.js @@ -32,22 +32,24 @@ function getTimestamp() { * @param {string} parameters.options.pattern Pattern to locate the files to be processed * @returns {Promise} Promise resolving with undefined once data has been written */ -export default function({workspace, options: {pattern}}) { - const timestamp = getTimestamp(); +export default async function({workspace, buildCache, options: {pattern}}) { + let resources = await workspace.byGlob(pattern); - return workspace.byGlob(pattern) - .then((processedResources) => { - return stringReplacer({ - resources: processedResources, - options: { - pattern: "${buildtime}", - replacement: timestamp - } - }); - }) - .then((processedResources) => { - return Promise.all(processedResources.map((resource) => { - return workspace.write(resource); - })); - }); + if (buildCache.hasCache()) { + const changedPaths = buildCache.getChangedProjectResourcePaths(); + resources = resources.filter((resource) => changedPaths.has(resource.getPath())); + } + const timestamp = getTimestamp(); + const processedResources = await stringReplacer({ + resources, + options: { + pattern: "${buildtime}", + replacement: timestamp + } + }); + return Promise.all(processedResources.map((resource) => { + if (resource) { + return workspace.write(resource); + } + })); } diff --git a/packages/builder/lib/tasks/replaceCopyright.js b/packages/builder/lib/tasks/replaceCopyright.js index 2ccb6a596df..09cd302d9f0 100644 --- a/packages/builder/lib/tasks/replaceCopyright.js +++ b/packages/builder/lib/tasks/replaceCopyright.js @@ -29,27 +29,30 @@ import stringReplacer from "../processors/stringReplacer.js"; * @param {string} parameters.options.pattern Pattern to locate the files to be processed * @returns {Promise} Promise resolving with undefined once data has been written */ -export default function({workspace, options: {copyright, pattern}}) { +export default async function({workspace, buildCache, options: {copyright, pattern}}) { if (!copyright) { - return Promise.resolve(); + return; } // Replace optional placeholder ${currentYear} with the current year copyright = copyright.replace(/(?:\$\{currentYear\})/, new Date().getFullYear()); - return workspace.byGlob(pattern) - .then((processedResources) => { - return stringReplacer({ - resources: processedResources, - options: { - pattern: /(?:\$\{copyright\}|@copyright@)/g, - replacement: copyright - } - }); - }) - .then((processedResources) => { - return Promise.all(processedResources.map((resource) => { - return workspace.write(resource); - })); - }); + let resources = await workspace.byGlob(pattern); + if (buildCache.hasCache()) { + const changedPaths = buildCache.getChangedProjectResourcePaths(); + resources = resources.filter((resource) => changedPaths.has(resource.getPath())); + } + + const processedResources = await stringReplacer({ + resources, + options: { + pattern: /(?:\$\{copyright\}|@copyright@)/g, + replacement: copyright + } + }); + return Promise.all(processedResources.map((resource) => { + if (resource) { + return workspace.write(resource); + } + })); } diff --git a/packages/builder/lib/tasks/replaceVersion.js b/packages/builder/lib/tasks/replaceVersion.js index 699a6221a95..7d1a56ffed1 100644 --- a/packages/builder/lib/tasks/replaceVersion.js +++ b/packages/builder/lib/tasks/replaceVersion.js @@ -19,20 +19,23 @@ import stringReplacer from "../processors/stringReplacer.js"; * @param {string} parameters.options.version Replacement version * @returns {Promise} Promise resolving with undefined once data has been written */ -export default function({workspace, options: {pattern, version}}) { - return workspace.byGlob(pattern) - .then((allResources) => { - return stringReplacer({ - resources: allResources, - options: { - pattern: /\$\{(?:project\.)?version\}/g, - replacement: version - } - }); - }) - .then((processedResources) => { - return Promise.all(processedResources.map((resource) => { - return workspace.write(resource); - })); - }); +export default async function({workspace, buildCache, options: {pattern, version}}) { + let resources = await workspace.byGlob(pattern); + + if (buildCache.hasCache()) { + const changedPaths = buildCache.getChangedProjectResourcePaths(); + resources = resources.filter((resource) => changedPaths.has(resource.getPath())); + } + const processedResources = await stringReplacer({ + resources, + options: { + pattern: /\$\{(?:project\.)?version\}/g, + replacement: version + } + }); + await Promise.all(processedResources.map((resource) => { + if (resource) { + return workspace.write(resource); + } + })); } From 31738e6a2fe2c67729e55f2779025f76c1e59d1b Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Tue, 18 Nov 2025 15:55:28 +0100 Subject: [PATCH 4/7] refactor(project): Align getReader API internals with ComponentProjects Cherry-picked from: https://github.com/SAP/ui5-project/commit/82b20eea1fc5cec4026f8d077cc194408e34d9e7 JIRA: CPOUI5FOUNDATION-1174 --- .../lib/specifications/types/Module.js | 41 +++++++++------- .../lib/specifications/types/ThemeLibrary.js | 48 +++++++++++-------- 2 files changed, 50 insertions(+), 39 deletions(-) diff --git a/packages/project/lib/specifications/types/Module.js b/packages/project/lib/specifications/types/Module.js index 14e6a116442..b10861bb11f 100644 --- a/packages/project/lib/specifications/types/Module.js +++ b/packages/project/lib/specifications/types/Module.js @@ -69,25 +69,10 @@ class Module extends Project { // Apply builder excludes to all styles but "runtime" const excludes = style === "runtime" ? [] : this.getBuilderResourcesExcludes(); - const readers = this._paths.map(({name, virBasePath, fsBasePath}) => { - return resourceFactory.createReader({ - name, - virBasePath, - fsBasePath, - project: this, - excludes - }); - }); - if (readers.length === 1) { - return readers[0]; - } - const readerCollection = resourceFactory.createReaderCollection({ - name: `Reader collection for module project ${this.getName()}`, - readers - }); + const reader = this._getReader(excludes); return resourceFactory.createReaderCollectionPrioritized({ name: `Reader/Writer collection for project ${this.getName()}`, - readers: [this._getWriter(), readerCollection] + readers: [this._getWriter(), reader] }); } @@ -98,7 +83,8 @@ class Module extends Project { * @returns {@ui5/fs/ReaderCollection} A reader collection instance */ getWorkspace() { - const reader = this.getReader(); + const excludes = this.getBuilderResourcesExcludes(); + const reader = this._getReader(excludes); const writer = this._getWriter(); return resourceFactory.createWorkspace({ @@ -107,6 +93,25 @@ class Module extends Project { }); } + _getReader(excludes) { + const readers = this._paths.map(({name, virBasePath, fsBasePath}) => { + return resourceFactory.createReader({ + name, + virBasePath, + fsBasePath, + project: this, + excludes + }); + }); + if (readers.length === 1) { + return readers[0]; + } + return resourceFactory.createReaderCollection({ + name: `Reader collection for module project ${this.getName()}`, + readers + }); + } + _getWriter() { if (!this._writer) { this._writer = resourceFactory.createAdapter({ diff --git a/packages/project/lib/specifications/types/ThemeLibrary.js b/packages/project/lib/specifications/types/ThemeLibrary.js index e0bcd90785d..a7903f9be97 100644 --- a/packages/project/lib/specifications/types/ThemeLibrary.js +++ b/packages/project/lib/specifications/types/ThemeLibrary.js @@ -76,26 +76,7 @@ class ThemeLibrary extends Project { // Apply builder excludes to all styles but "runtime" const excludes = style === "runtime" ? [] : this.getBuilderResourcesExcludes(); - let reader = resourceFactory.createReader({ - fsBasePath: this.getSourcePath(), - virBasePath: "/resources/", - name: `Runtime resources reader for theme-library project ${this.getName()}`, - project: this, - excludes - }); - if (this._testPathExists) { - const testReader = resourceFactory.createReader({ - fsBasePath: fsPath.join(this.getRootPath(), this._testPath), - virBasePath: "/test-resources/", - name: `Runtime test-resources reader for theme-library project ${this.getName()}`, - project: this, - excludes - }); - reader = resourceFactory.createReaderCollection({ - name: `Reader collection for theme-library project ${this.getName()}`, - readers: [reader, testReader] - }); - } + const reader = this._getReader(excludes); const writer = this._getWriter(); return resourceFactory.createReaderCollectionPrioritized({ @@ -115,7 +96,8 @@ class ThemeLibrary extends Project { * @returns {@ui5/fs/DuplexCollection} DuplexCollection */ getWorkspace() { - const reader = this.getReader(); + const excludes = this.getBuilderResourcesExcludes(); + const reader = this._getReader(excludes); const writer = this._getWriter(); return resourceFactory.createWorkspace({ @@ -124,6 +106,30 @@ class ThemeLibrary extends Project { }); } + _getReader(excludes) { + let reader = resourceFactory.createReader({ + fsBasePath: this.getSourcePath(), + virBasePath: "/resources/", + name: `Runtime resources reader for theme-library project ${this.getName()}`, + project: this, + excludes + }); + if (this._testPathExists) { + const testReader = resourceFactory.createReader({ + fsBasePath: fsPath.join(this.getRootPath(), this._testPath), + virBasePath: "/test-resources/", + name: `Runtime test-resources reader for theme-library project ${this.getName()}`, + project: this, + excludes + }); + reader = resourceFactory.createReaderCollection({ + name: `Reader collection for theme-library project ${this.getName()}`, + readers: [reader, testReader] + }); + } + return reader; + } + _getWriter() { if (!this._writer) { this._writer = resourceFactory.createAdapter({ From b535e50c12aeaf266e2370de536cf0944998e87d Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Tue, 18 Nov 2025 15:56:27 +0100 Subject: [PATCH 5/7] refactor(project): Refactor specification-internal workspace handling Prerequisite for versioning support Cherry-picked from: https://github.com/SAP/ui5-project/commit/83b5c4f12dc545357e36366846ad1f6fe94a70e3 JIRA: CPOUI5FOUNDATION-1174 --- .../lib/specifications/ComponentProject.js | 132 +++++++----------- .../project/lib/specifications/Project.js | 89 ++++++++++-- .../lib/specifications/types/Module.js | 72 +++------- .../lib/specifications/types/ThemeLibrary.js | 83 +++-------- 4 files changed, 166 insertions(+), 210 deletions(-) diff --git a/packages/project/lib/specifications/ComponentProject.js b/packages/project/lib/specifications/ComponentProject.js index 5c42ba2c4dc..e40f8a9228b 100644 --- a/packages/project/lib/specifications/ComponentProject.js +++ b/packages/project/lib/specifications/ComponentProject.js @@ -91,39 +91,7 @@ class ComponentProject extends Project { /* === Resource Access === */ - /** - * Get a [ReaderCollection]{@link @ui5/fs/ReaderCollection} for accessing all resources of the - * project in the specified "style": - * - *
    - *
  • buildtime: Resource paths are always prefixed with /resources/ - * or /test-resources/ followed by the project's namespace. - * Any configured build-excludes are applied
    • - *
  • dist: Resource paths always match with what the UI5 runtime expects. - * This means that paths generally depend on the project type. Applications for example use a "flat"-like - * structure, while libraries use a "buildtime"-like structure. - * Any configured build-excludes are applied
    • - *
  • runtime: Resource paths always match with what the UI5 runtime expects. - * This means that paths generally depend on the project type. Applications for example use a "flat"-like - * structure, while libraries use a "buildtime"-like structure. - * This style is typically used for serving resources directly. Therefore, build-excludes are not applied - *
  • flat: Resource paths are never prefixed and namespaces are omitted if possible. Note that - * project types like "theme-library", which can have multiple namespaces, can't omit them. - * Any configured build-excludes are applied
    • - *
- * - * If project resources have been changed through the means of a workspace, those changes - * are reflected in the provided reader too. - * - * Resource readers always use POSIX-style paths. - * - * @public - * @param {object} [options] - * @param {string} [options.style=buildtime] Path style to access resources. - * Can be "buildtime", "dist", "runtime" or "flat" - * @returns {@ui5/fs/ReaderCollection} A reader collection instance - */ - getReader({style = "buildtime"} = {}) { + _getStyledReader(style) { // TODO: Additional style 'ABAP' using "sap.platform.abap".uri from manifest.json? // Apply builder excludes to all styles but "runtime" @@ -161,7 +129,7 @@ class ComponentProject extends Project { throw new Error(`Unknown path mapping style ${style}`); } - reader = this._addWriter(reader, style); + // reader = this._addWriter(reader, style, writer); return reader; } @@ -183,52 +151,49 @@ class ComponentProject extends Project { throw new Error(`_getTestReader must be implemented by subclass ${this.constructor.name}`); } - /** - * Get a resource reader/writer for accessing and modifying a project's resources - * - * @public - * @returns {@ui5/fs/ReaderCollection} A reader collection instance - */ - getWorkspace() { - // Workspace is always of style "buildtime" - // Therefore builder resource-excludes are always to be applied - const excludes = this.getBuilderResourcesExcludes(); - return resourceFactory.createWorkspace({ - name: `Workspace for project ${this.getName()}`, - reader: this._getReader(excludes), - writer: this._getWriter().collection - }); - } + // /** + // * Get a resource reader/writer for accessing and modifying a project's resources + // * + // * @public + // * @returns {@ui5/fs/ReaderCollection} A reader collection instance + // */ + // getWorkspace() { + // // Workspace is always of style "buildtime" + // // Therefore builder resource-excludes are always to be applied + // const excludes = this.getBuilderResourcesExcludes(); + // return resourceFactory.createWorkspace({ + // name: `Workspace for project ${this.getName()}`, + // reader: this._getPlainReader(excludes), + // writer: this._getWriter().collection + // }); + // } _getWriter() { - if (!this._writers) { - // writer is always of style "buildtime" - const namespaceWriter = resourceFactory.createAdapter({ - virBasePath: "/", - project: this - }); + // writer is always of style "buildtime" + const namespaceWriter = resourceFactory.createAdapter({ + virBasePath: "/", + project: this + }); - const generalWriter = resourceFactory.createAdapter({ - virBasePath: "/", - project: this - }); + const generalWriter = resourceFactory.createAdapter({ + virBasePath: "/", + project: this + }); - const collection = resourceFactory.createWriterCollection({ - name: `Writers for project ${this.getName()}`, - writerMapping: { - [`/resources/${this._namespace}/`]: namespaceWriter, - [`/test-resources/${this._namespace}/`]: namespaceWriter, - [`/`]: generalWriter - } - }); + const collection = resourceFactory.createWriterCollection({ + name: `Writers for project ${this.getName()}`, + writerMapping: { + [`/resources/${this._namespace}/`]: namespaceWriter, + [`/test-resources/${this._namespace}/`]: namespaceWriter, + [`/`]: generalWriter + } + }); - this._writers = { - namespaceWriter, - generalWriter, - collection - }; - } - return this._writers; + return { + namespaceWriter, + generalWriter, + collection + }; } _getReader(excludes) { @@ -243,15 +208,15 @@ class ComponentProject extends Project { return reader; } - _addWriter(reader, style) { - const {namespaceWriter, generalWriter} = this._getWriter(); + _addReadersFromWriter(style, readers, writer) { + const {namespaceWriter, generalWriter} = writer; if ((style === "runtime" || style === "dist") && this._isRuntimeNamespaced) { // If the project's type requires a namespace at runtime, the // dist- and runtime-style paths are identical to buildtime-style paths style = "buildtime"; } - const readers = []; + switch (style) { case "buildtime": // Writer already uses buildtime style @@ -279,12 +244,13 @@ class ComponentProject extends Project { default: throw new Error(`Unknown path mapping style ${style}`); } - readers.push(reader); + // return readers; + // readers.push(reader); - return resourceFactory.createReaderCollectionPrioritized({ - name: `Reader/Writer collection for project ${this.getName()}`, - readers - }); + // return resourceFactory.createReaderCollectionPrioritized({ + // name: `Reader/Writer collection for project ${this.getName()}`, + // readers + // }); } /* === Internals === */ diff --git a/packages/project/lib/specifications/Project.js b/packages/project/lib/specifications/Project.js index 17177b08536..40ba76fe635 100644 --- a/packages/project/lib/specifications/Project.js +++ b/packages/project/lib/specifications/Project.js @@ -1,5 +1,6 @@ import Specification from "./Specification.js"; import ResourceTagCollection from "@ui5/fs/internal/ResourceTagCollection"; +import {createWorkspace, createReaderCollectionPrioritized} from "@ui5/fs/resourceFactory"; /** * Project @@ -12,6 +13,12 @@ import ResourceTagCollection from "@ui5/fs/internal/ResourceTagCollection"; * @hideconstructor */ class Project extends Specification { + #latestWriter; + #latestWorkspace; + #latestReader = new Map(); + #writerVersions = []; + #workspaceSealed = false; + constructor(parameters) { super(parameters); if (new.target === Project) { @@ -220,6 +227,7 @@ class Project extends Specification { } /* === Resource Access === */ + /** * Get a [ReaderCollection]{@link @ui5/fs/ReaderCollection} for accessing all resources of the * project in the specified "style": @@ -241,38 +249,93 @@ class Project extends Specification { * Any configured build-excludes are applied * * + * If project resources have been changed through the means of a workspace, those changes + * are reflected in the provided reader too. + * * Resource readers always use POSIX-style paths. * * @public * @param {object} [options] * @param {string} [options.style=buildtime] Path style to access resources. * Can be "buildtime", "dist", "runtime" or "flat" - * @returns {@ui5/fs/ReaderCollection} Reader collection allowing access to all resources of the project + * @returns {@ui5/fs/ReaderCollection} A reader collection instance */ - getReader(options) { - throw new Error(`getReader must be implemented by subclass ${this.constructor.name}`); + getReader({style = "buildtime"} = {}) { + let reader = this.#latestReader.get(style); + if (reader) { + return reader; + } + const readers = []; + this._addReadersFromWriter(style, readers, this.getWriter()); + readers.push(this._getStyledReader(style)); + reader = createReaderCollectionPrioritized({ + name: `Reader collection for project ${this.getName()}`, + readers + }); + this.#latestReader.set(style, reader); + return reader; } - getResourceTagCollection() { - if (!this._resourceTagCollection) { - this._resourceTagCollection = new ResourceTagCollection({ - allowedTags: ["ui5:IsDebugVariant", "ui5:HasDebugVariant"], - allowedNamespaces: ["project"], - tags: this.getBuildManifest()?.tags - }); - } - return this._resourceTagCollection; + getWriter() { + return this.#latestWriter || this.createNewWriterVersion(); + } + + createNewWriterVersion() { + const writer = this._getWriter(); + this.#writerVersions.push(writer); + this.#latestWriter = writer; + + // Invalidate dependents + this.#latestWorkspace = null; + this.#latestReader = new Map(); + + return writer; } /** * Get a [DuplexCollection]{@link @ui5/fs/DuplexCollection} for accessing and modifying a * project's resources. This is always of style buildtime. * + * Once a project has finished building, this method will throw to prevent further modifications + * since those would have no effect. Use the getReader method to access the project's (modified) resources + * * @public * @returns {@ui5/fs/DuplexCollection} DuplexCollection */ getWorkspace() { - throw new Error(`getWorkspace must be implemented by subclass ${this.constructor.name}`); + if (this.#workspaceSealed) { + throw new Error( + `Workspace of project ${this.getName()} has been sealed. Use method #getReader for read-only access`); + } + if (this.#latestWorkspace) { + return this.#latestWorkspace; + } + const excludes = this.getBuilderResourcesExcludes(); // TODO: Do not apply in server context + const writer = this.getWriter(); + this.#latestWorkspace = createWorkspace({ + reader: this._getReader(excludes), + writer: writer.collection || writer + }); + return this.#latestWorkspace; + } + + sealWorkspace() { + this.#workspaceSealed = true; + } + + _addReadersFromWriter(style, readers, writer) { + readers.push(writer); + } + + getResourceTagCollection() { + if (!this._resourceTagCollection) { + this._resourceTagCollection = new ResourceTagCollection({ + allowedTags: ["ui5:IsDebugVariant", "ui5:HasDebugVariant"], + allowedNamespaces: ["project"], + tags: this.getBuildManifest()?.tags + }); + } + return this._resourceTagCollection; } /* === Internals === */ diff --git a/packages/project/lib/specifications/types/Module.js b/packages/project/lib/specifications/types/Module.js index b10861bb11f..a59c464f94a 100644 --- a/packages/project/lib/specifications/types/Module.js +++ b/packages/project/lib/specifications/types/Module.js @@ -33,65 +33,29 @@ class Module extends Project { /* === Resource Access === */ - /** - * Get a [ReaderCollection]{@link @ui5/fs/ReaderCollection} for accessing all resources of the - * project in the specified "style": - * - *
    - *
  • buildtime: Resource paths are always prefixed with /resources/ - * or /test-resources/ followed by the project's namespace. - * Any configured build-excludes are applied
    • - *
  • dist: Resource paths always match with what the UI5 runtime expects. - * This means that paths generally depend on the project type. Applications for example use a "flat"-like - * structure, while libraries use a "buildtime"-like structure. - * Any configured build-excludes are applied
    • - *
  • runtime: Resource paths always match with what the UI5 runtime expects. - * This means that paths generally depend on the project type. Applications for example use a "flat"-like - * structure, while libraries use a "buildtime"-like structure. - * This style is typically used for serving resources directly. Therefore, build-excludes are not applied - *
  • flat: Resource paths are never prefixed and namespaces are omitted if possible. Note that - * project types like "theme-library", which can have multiple namespaces, can't omit them. - * Any configured build-excludes are applied
    • - *
- * - * If project resources have been changed through the means of a workspace, those changes - * are reflected in the provided reader too. - * - * Resource readers always use POSIX-style paths. - * - * @public - * @param {object} [options] - * @param {string} [options.style=buildtime] Path style to access resources. - * Can be "buildtime", "dist", "runtime" or "flat" - * @returns {@ui5/fs/ReaderCollection} A reader collection instance - */ - getReader({style = "buildtime"} = {}) { + _getStyledReader(style) { // Apply builder excludes to all styles but "runtime" const excludes = style === "runtime" ? [] : this.getBuilderResourcesExcludes(); - const reader = this._getReader(excludes); - return resourceFactory.createReaderCollectionPrioritized({ - name: `Reader/Writer collection for project ${this.getName()}`, - readers: [this._getWriter(), reader] - }); + return this._getReader(excludes); } - /** - * Get a resource reader/writer for accessing and modifying a project's resources - * - * @public - * @returns {@ui5/fs/ReaderCollection} A reader collection instance - */ - getWorkspace() { - const excludes = this.getBuilderResourcesExcludes(); - const reader = this._getReader(excludes); - - const writer = this._getWriter(); - return resourceFactory.createWorkspace({ - reader, - writer - }); - } + // /** + // * Get a resource reader/writer for accessing and modifying a project's resources + // * + // * @public + // * @returns {@ui5/fs/ReaderCollection} A reader collection instance + // */ + // getWorkspace() { + // const excludes = this.getBuilderResourcesExcludes(); + // const reader = this._getReader(excludes); + + // const writer = this._getWriter(); + // return resourceFactory.createWorkspace({ + // reader, + // writer + // }); + // } _getReader(excludes) { const readers = this._paths.map(({name, virBasePath, fsBasePath}) => { diff --git a/packages/project/lib/specifications/types/ThemeLibrary.js b/packages/project/lib/specifications/types/ThemeLibrary.js index a7903f9be97..d4644c78885 100644 --- a/packages/project/lib/specifications/types/ThemeLibrary.js +++ b/packages/project/lib/specifications/types/ThemeLibrary.js @@ -40,71 +40,34 @@ class ThemeLibrary extends Project { } /* === Resource Access === */ - /** - * Get a [ReaderCollection]{@link @ui5/fs/ReaderCollection} for accessing all resources of the - * project in the specified "style": - * - *
    - *
  • buildtime: Resource paths are always prefixed with /resources/ - * or /test-resources/ followed by the project's namespace. - * Any configured build-excludes are applied
    • - *
  • dist: Resource paths always match with what the UI5 runtime expects. - * This means that paths generally depend on the project type. Applications for example use a "flat"-like - * structure, while libraries use a "buildtime"-like structure. - * Any configured build-excludes are applied
    • - *
  • runtime: Resource paths always match with what the UI5 runtime expects. - * This means that paths generally depend on the project type. Applications for example use a "flat"-like - * structure, while libraries use a "buildtime"-like structure. - * This style is typically used for serving resources directly. Therefore, build-excludes are not applied - *
  • flat: Resource paths are never prefixed and namespaces are omitted if possible. Note that - * project types like "theme-library", which can have multiple namespaces, can't omit them. - * Any configured build-excludes are applied
    • - *
- * - * If project resources have been changed through the means of a workspace, those changes - * are reflected in the provided reader too. - * - * Resource readers always use POSIX-style paths. - * - * @public - * @param {object} [options] - * @param {string} [options.style=buildtime] Path style to access resources. - * Can be "buildtime", "dist", "runtime" or "flat" - * @returns {@ui5/fs/ReaderCollection} A reader collection instance - */ - getReader({style = "buildtime"} = {}) { + + _getStyledReader(style) { // Apply builder excludes to all styles but "runtime" const excludes = style === "runtime" ? [] : this.getBuilderResourcesExcludes(); - const reader = this._getReader(excludes); - const writer = this._getWriter(); - - return resourceFactory.createReaderCollectionPrioritized({ - name: `Reader/Writer collection for project ${this.getName()}`, - readers: [writer, reader] - }); + return this._getReader(excludes); } - /** - * Get a [DuplexCollection]{@link @ui5/fs/DuplexCollection} for accessing and modifying a - * project's resources. - * - * This is always of style buildtime, wich for theme libraries is identical to style - * runtime. - * - * @public - * @returns {@ui5/fs/DuplexCollection} DuplexCollection - */ - getWorkspace() { - const excludes = this.getBuilderResourcesExcludes(); - const reader = this._getReader(excludes); - - const writer = this._getWriter(); - return resourceFactory.createWorkspace({ - reader, - writer - }); - } + // /** + // * Get a [DuplexCollection]{@link @ui5/fs/DuplexCollection} for accessing and modifying a + // * project's resources. + // * + // * This is always of style buildtime, which for theme libraries is identical to style + // * runtime. + // * + // * @public + // * @returns {@ui5/fs/DuplexCollection} DuplexCollection + // */ + // getWorkspace() { + // const excludes = this.getBuilderResourcesExcludes(); + // const reader = this._getReader(excludes); + + // const writer = this._getWriter(); + // return resourceFactory.createWorkspace({ + // reader, + // writer + // }); + // } _getReader(excludes) { let reader = resourceFactory.createReader({ From 12ce6bae0398a43636f1cffd412a5744a6f5d365 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Tue, 18 Nov 2025 15:57:13 +0100 Subject: [PATCH 6/7] refactor(project): Implement basic incremental build functionality Cherry-picked from: https://github.com/SAP/ui5-project/commit/cb4e858a630fe673cdaf3f991fa8fb6272e45ea2 JIRA: CPOUI5FOUNDATION-1174 --- packages/project/lib/build/ProjectBuilder.js | 144 +++++- packages/project/lib/build/TaskRunner.js | 63 ++- .../project/lib/build/cache/BuildTaskCache.js | 193 ++++++++ .../lib/build/cache/ProjectBuildCache.js | 433 ++++++++++++++++++ .../project/lib/build/helpers/BuildContext.js | 39 +- .../lib/build/helpers/ProjectBuildContext.js | 110 ++++- .../project/lib/build/helpers/WatchHandler.js | 135 ++++++ .../lib/build/helpers/createBuildManifest.js | 89 +++- packages/project/lib/graph/ProjectGraph.js | 88 +++- .../lib/specifications/ComponentProject.js | 17 +- .../project/lib/specifications/Project.js | 291 ++++++++++-- .../lib/specifications/types/Application.js | 25 +- .../lib/specifications/types/Library.js | 43 +- .../lib/specifications/types/Module.js | 10 +- .../lib/specifications/types/ThemeLibrary.js | 23 +- .../project/test/lib/build/ProjectBuilder.js | 2 + packages/project/test/lib/build/TaskRunner.js | 174 +++---- .../lib/build/helpers/ProjectBuildContext.js | 6 +- 18 files changed, 1695 insertions(+), 190 deletions(-) create mode 100644 packages/project/lib/build/cache/BuildTaskCache.js create mode 100644 packages/project/lib/build/cache/ProjectBuildCache.js create mode 100644 packages/project/lib/build/helpers/WatchHandler.js diff --git a/packages/project/lib/build/ProjectBuilder.js b/packages/project/lib/build/ProjectBuilder.js index af1804b6af1..ba4c6ad9aea 100644 --- a/packages/project/lib/build/ProjectBuilder.js +++ b/packages/project/lib/build/ProjectBuilder.js @@ -139,9 +139,11 @@ class ProjectBuilder { async build({ destPath, cleanDest = false, includedDependencies = [], excludedDependencies = [], - dependencyIncludes + dependencyIncludes, + cacheDir, + watch, }) { - if (!destPath) { + if (!destPath && !watch) { throw new Error(`Missing parameter 'destPath'`); } if (dependencyIncludes) { @@ -177,12 +179,15 @@ class ProjectBuilder { } } - const projectBuildContexts = await this._createRequiredBuildContexts(requestedProjects); + const projectBuildContexts = await this._createRequiredBuildContexts(requestedProjects, cacheDir); const cleanupSigHooks = this._registerCleanupSigHooks(); - const fsTarget = resourceFactory.createAdapter({ - fsBasePath: destPath, - virBasePath: "/" - }); + let fsTarget; + if (destPath) { + fsTarget = resourceFactory.createAdapter({ + fsBasePath: destPath, + virBasePath: "/" + }); + } const queue = []; const alreadyBuilt = []; @@ -196,7 +201,7 @@ class ProjectBuilder { // => This project needs to be built or, in case it has already // been built, it's build result needs to be written out (if requested) queue.push(projectBuildContext); - if (!projectBuildContext.requiresBuild()) { + if (!await projectBuildContext.requiresBuild()) { alreadyBuilt.push(projectName); } } @@ -220,8 +225,12 @@ class ProjectBuilder { let msg; if (alreadyBuilt.includes(projectName)) { const buildMetadata = projectBuildContext.getBuildMetadata(); - const ts = new Date(buildMetadata.timestamp).toUTCString(); - msg = `*> ${projectName} /// already built at ${ts}`; + let buildAt = ""; + if (buildMetadata) { + const ts = new Date(buildMetadata.timestamp).toUTCString(); + buildAt = ` at ${ts}`; + } + msg = `*> ${projectName} /// already built${buildAt}`; } else { msg = `=> ${projectName}`; } @@ -231,7 +240,7 @@ class ProjectBuilder { } } - if (cleanDest) { + if (destPath && cleanDest) { this.#log.info(`Cleaning target directory...`); await rmrf(destPath); } @@ -239,8 +248,9 @@ class ProjectBuilder { try { const pWrites = []; for (const projectBuildContext of queue) { - const projectName = projectBuildContext.getProject().getName(); - const projectType = projectBuildContext.getProject().getType(); + const project = projectBuildContext.getProject(); + const projectName = project.getName(); + const projectType = project.getType(); this.#log.verbose(`Processing project ${projectName}...`); // Only build projects that are not already build (i.e. provide a matching build manifest) @@ -248,7 +258,9 @@ class ProjectBuilder { this.#log.skipProjectBuild(projectName, projectType); } else { this.#log.startProjectBuild(projectName, projectType); + project.newVersion(); await projectBuildContext.getTaskRunner().runTasks(); + project.sealWorkspace(); this.#log.endProjectBuild(projectName, projectType); } if (!requestedProjects.includes(projectName)) { @@ -257,8 +269,15 @@ class ProjectBuilder { continue; } - this.#log.verbose(`Writing out files...`); - pWrites.push(this._writeResults(projectBuildContext, fsTarget)); + if (fsTarget) { + this.#log.verbose(`Writing out files...`); + pWrites.push(this._writeResults(projectBuildContext, fsTarget)); + } + + if (cacheDir && !alreadyBuilt.includes(projectName)) { + this.#log.verbose(`Serializing cache...`); + pWrites.push(projectBuildContext.getBuildCache().serializeToDisk()); + } } await Promise.all(pWrites); this.#log.info(`Build succeeded in ${this._getElapsedTime(startTime)}`); @@ -269,9 +288,91 @@ class ProjectBuilder { this._deregisterCleanupSigHooks(cleanupSigHooks); await this._executeCleanupTasks(); } + + if (watch) { + const relevantProjects = queue.map((projectBuildContext) => { + return projectBuildContext.getProject(); + }); + const watchHandler = this._buildContext.initWatchHandler(relevantProjects, async () => { + await this.#update(projectBuildContexts, requestedProjects, fsTarget, cacheDir); + }); + return watchHandler; + + // Register change handler + // this._buildContext.onSourceFileChange(async (event) => { + // await this.#update(projectBuildContexts, requestedProjects, + // fsTarget, + // targetWriterProject, targetWriterDependencies); + // updateOnChange(event); + // }, (err) => { + // updateOnChange(err); + // }); + + // // Start watching + // for (const projectBuildContext of queue) { + // await projectBuildContext.watchFileChanges(); + // } + } + } + + async #update(projectBuildContexts, requestedProjects, fsTarget, cacheDir) { + const queue = []; + await this._graph.traverseDepthFirst(async ({project}) => { + const projectName = project.getName(); + const projectBuildContext = projectBuildContexts.get(projectName); + if (projectBuildContext) { + // Build context exists + // => This project needs to be built or, in case it has already + // been built, it's build result needs to be written out (if requested) + // if (await projectBuildContext.requiresBuild()) { + queue.push(projectBuildContext); + // } + } + }); + + this.#log.setProjects(queue.map((projectBuildContext) => { + return projectBuildContext.getProject().getName(); + })); + + const pWrites = []; + for (const projectBuildContext of queue) { + const project = projectBuildContext.getProject(); + const projectName = project.getName(); + const projectType = project.getType(); + this.#log.verbose(`Updating project ${projectName}...`); + + if (!await projectBuildContext.requiresBuild()) { + this.#log.skipProjectBuild(projectName, projectType); + continue; + } + + this.#log.startProjectBuild(projectName, projectType); + project.newVersion(); + await projectBuildContext.runTasks(); + project.sealWorkspace(); + this.#log.endProjectBuild(projectName, projectType); + if (!requestedProjects.includes(projectName)) { + // Project has not been requested + // => Its resources shall not be part of the build result + continue; + } + + if (fsTarget) { + this.#log.verbose(`Writing out files...`); + pWrites.push(this._writeResults(projectBuildContext, fsTarget)); + } + + if (cacheDir) { + this.#log.verbose(`Updating cache...`); + // TODO: Only serialize if cache has changed + // TODO: Serialize lazily, or based on memory pressure + pWrites.push(projectBuildContext.getBuildCache().serializeToDisk()); + } + } + await Promise.all(pWrites); } - async _createRequiredBuildContexts(requestedProjects) { + async _createRequiredBuildContexts(requestedProjects, cacheDir) { const requiredProjects = new Set(this._graph.getProjectNames().filter((projectName) => { return requestedProjects.includes(projectName); })); @@ -280,13 +381,14 @@ class ProjectBuilder { for (const projectName of requiredProjects) { this.#log.verbose(`Creating build context for project ${projectName}...`); - const projectBuildContext = this._buildContext.createProjectContext({ - project: this._graph.getProject(projectName) + const projectBuildContext = await this._buildContext.createProjectContext({ + project: this._graph.getProject(projectName), + cacheDir, }); projectBuildContexts.set(projectName, projectBuildContext); - if (projectBuildContext.requiresBuild()) { + if (await projectBuildContext.requiresBuild()) { const taskRunner = projectBuildContext.getTaskRunner(); const requiredDependencies = await taskRunner.getRequiredDependencies(); @@ -389,7 +491,9 @@ class ProjectBuilder { const { default: createBuildManifest } = await import("./helpers/createBuildManifest.js"); - const metadata = await createBuildManifest(project, buildConfig, this._buildContext.getTaskRepository()); + const metadata = await createBuildManifest( + project, this._graph, buildConfig, this._buildContext.getTaskRepository(), + projectBuildContext.getBuildCache()); await target.write(resourceFactory.createResource({ path: `/.ui5/build-manifest.json`, string: JSON.stringify(metadata, null, "\t") diff --git a/packages/project/lib/build/TaskRunner.js b/packages/project/lib/build/TaskRunner.js index 0f5677170a3..08473e39b2b 100644 --- a/packages/project/lib/build/TaskRunner.js +++ b/packages/project/lib/build/TaskRunner.js @@ -1,6 +1,6 @@ import {getLogger} from "@ui5/logger"; import composeTaskList from "./helpers/composeTaskList.js"; -import {createReaderCollection} from "@ui5/fs/resourceFactory"; +import {createReaderCollection, createTracker} from "@ui5/fs/resourceFactory"; /** * TaskRunner @@ -21,8 +21,8 @@ class TaskRunner { * @param {@ui5/project/build/ProjectBuilder~BuildConfiguration} parameters.buildConfig * Build configuration */ - constructor({graph, project, log, taskUtil, taskRepository, buildConfig}) { - if (!graph || !project || !log || !taskUtil || !taskRepository || !buildConfig) { + constructor({graph, project, log, cache, taskUtil, taskRepository, buildConfig}) { + if (!graph || !project || !log || !cache || !taskUtil || !taskRepository || !buildConfig) { throw new Error("TaskRunner: One or more mandatory parameters not provided"); } this._project = project; @@ -31,6 +31,7 @@ class TaskRunner { this._taskRepository = taskRepository; this._buildConfig = buildConfig; this._log = log; + this._cache = cache; this._directDependencies = new Set(this._taskUtil.getDependencies()); } @@ -190,20 +191,62 @@ class TaskRunner { options.projectName = this._project.getName(); options.projectNamespace = this._project.getNamespace(); + // TODO: Apply cache and stage handling for custom tasks as well + this._project.useStage(taskName); + + // Check whether any of the relevant resources have changed + if (this._cache.hasCacheForTask(taskName)) { + await this._cache.validateChangedProjectResources( + taskName, this._project.getReader(), this._allDependenciesReader); + if (this._cache.hasValidCacheForTask(taskName)) { + this._log.skipTask(taskName); + return; + } + } + this._log.info( + `Executing task ${taskName} for project ${this._project.getName()}`); + const workspace = createTracker(this._project.getWorkspace()); const params = { - workspace: this._project.getWorkspace(), + workspace, taskUtil: this._taskUtil, - options + options, + buildCache: { + // TODO: Create a proper interface for this + hasCache: () => { + return this._cache.hasCacheForTask(taskName); + }, + getChangedProjectResourcePaths: () => { + return this._cache.getChangedProjectResourcePaths(taskName); + }, + getChangedDependencyResourcePaths: () => { + return this._cache.getChangedDependencyResourcePaths(taskName); + }, + } }; + // const invalidatedResources = this._cache.getDepsOfInvalidatedResourcesForTask(taskName); + // if (invalidatedResources) { + // params.invalidatedResources = invalidatedResources; + // } + let dependencies; if (requiresDependencies) { - params.dependencies = this._allDependenciesReader; + dependencies = createTracker(this._allDependenciesReader); + params.dependencies = dependencies; } if (!taskFunction) { taskFunction = (await this._taskRepository.getTask(taskName)).task; } - return taskFunction(params); + + this._log.startTask(taskName); + this._taskStart = performance.now(); + await taskFunction(params); + if (this._log.isLevelEnabled("perf")) { + this._log.perf( + `Task ${taskName} finished in ${Math.round((performance.now() - this._taskStart))} ms`); + } + this._log.endTask(taskName); + await this._cache.updateTaskResult(taskName, workspace, dependencies); }; } this._tasks[taskName] = { @@ -445,13 +488,15 @@ class TaskRunner { * @returns {Promise} Resolves when task has finished */ async _executeTask(taskName, taskFunction, taskParams) { - this._log.startTask(taskName); + if (this._cache.hasValidCacheForTask(taskName)) { + this._log.skipTask(taskName); + return; + } this._taskStart = performance.now(); await taskFunction(taskParams, this._log); if (this._log.isLevelEnabled("perf")) { this._log.perf(`Task ${taskName} finished in ${Math.round((performance.now() - this._taskStart))} ms`); } - this._log.endTask(taskName); } async _createDependenciesReader(requiredDirectDependencies) { diff --git a/packages/project/lib/build/cache/BuildTaskCache.js b/packages/project/lib/build/cache/BuildTaskCache.js new file mode 100644 index 00000000000..1927b33e58c --- /dev/null +++ b/packages/project/lib/build/cache/BuildTaskCache.js @@ -0,0 +1,193 @@ +import micromatch from "micromatch"; +import {getLogger} from "@ui5/logger"; +const log = getLogger("build:cache:BuildTaskCache"); + +function unionArray(arr, items) { + for (const item of items) { + if (!arr.includes(item)) { + arr.push(item); + } + } +} +function unionObject(target, obj) { + for (const key in obj) { + if (Object.prototype.hasOwnProperty.call(obj, key)) { + target[key] = obj[key]; + } + } +} + +async function createMetadataForResources(resourceMap) { + const metadata = Object.create(null); + await Promise.all(Object.keys(resourceMap).map(async (resourcePath) => { + const resource = resourceMap[resourcePath]; + if (resource.hash) { + // Metadata object + metadata[resourcePath] = resource; + return; + } + // Resource instance + metadata[resourcePath] = { + hash: await resource.getHash(), + lastModified: resource.getStatInfo()?.mtimeMs, + }; + })); + return metadata; +} + +export default class BuildTaskCache { + #projectName; + #taskName; + + // Track which resource paths (and patterns) the task reads + // This is used to check whether a resource change *might* invalidates the task + #projectRequests; + #dependencyRequests; + + // Track metadata for the actual resources the task has read and written + // This is used to check whether a resource has actually changed from the last time the task has been executed (and + // its result has been cached) + // Per resource path, this reflects the last known state of the resource (a task might be executed multiple times, + // i.e. with a small delta of changed resources) + // This map can contain either a resource instance (if the cache has been filled during this session) or an object + // containing the last modified timestamp and an md5 hash of the resource (if the cache has been loaded from disk) + #resourcesRead; + #resourcesWritten; + + constructor(projectName, taskName, {projectRequests, dependencyRequests, resourcesRead, resourcesWritten}) { + this.#projectName = projectName; + this.#taskName = taskName; + + this.#projectRequests = projectRequests ?? { + pathsRead: [], + patterns: [], + }; + + this.#dependencyRequests = dependencyRequests ?? { + pathsRead: [], + patterns: [], + }; + this.#resourcesRead = resourcesRead ?? Object.create(null); + this.#resourcesWritten = resourcesWritten ?? Object.create(null); + } + + getTaskName() { + return this.#taskName; + } + + updateResources(projectRequests, dependencyRequests, resourcesRead, resourcesWritten) { + unionArray(this.#projectRequests.pathsRead, projectRequests.pathsRead); + unionArray(this.#projectRequests.patterns, projectRequests.patterns); + + if (dependencyRequests) { + unionArray(this.#dependencyRequests.pathsRead, dependencyRequests.pathsRead); + unionArray(this.#dependencyRequests.patterns, dependencyRequests.patterns); + } + + unionObject(this.#resourcesRead, resourcesRead); + unionObject(this.#resourcesWritten, resourcesWritten); + } + + async toObject() { + return { + taskName: this.#taskName, + resourceMetadata: { + projectRequests: this.#projectRequests, + dependencyRequests: this.#dependencyRequests, + resourcesRead: await createMetadataForResources(this.#resourcesRead), + resourcesWritten: await createMetadataForResources(this.#resourcesWritten) + } + }; + } + + checkPossiblyInvalidatesTask(projectResourcePaths, dependencyResourcePaths) { + if (this.#isRelevantResourceChange(this.#projectRequests, projectResourcePaths)) { + log.verbose( + `Build cache for task ${this.#taskName} of project ${this.#projectName} possibly invalidated ` + + `by changes made to the following resources ${Array.from(projectResourcePaths).join(", ")}`); + return true; + } + + if (this.#isRelevantResourceChange(this.#dependencyRequests, dependencyResourcePaths)) { + log.verbose( + `Build cache for task ${this.#taskName} of project ${this.#projectName} possibly invalidated ` + + `by changes made to the following resources: ${Array.from(dependencyResourcePaths).join(", ")}`); + return true; + } + + return false; + } + + getReadResourceCacheEntry(searchResourcePath) { + return this.#resourcesRead[searchResourcePath]; + } + + getWrittenResourceCache(searchResourcePath) { + return this.#resourcesWritten[searchResourcePath]; + } + + async isResourceInReadCache(resource) { + const cachedResource = this.#resourcesRead[resource.getPath()]; + if (!cachedResource) { + return false; + } + if (cachedResource.hash) { + return this.#isResourceFingerprintEqual(resource, cachedResource); + } else { + return this.#isResourceEqual(resource, cachedResource); + } + } + + async isResourceInWriteCache(resource) { + const cachedResource = this.#resourcesWritten[resource.getPath()]; + if (!cachedResource) { + return false; + } + if (cachedResource.hash) { + return this.#isResourceFingerprintEqual(resource, cachedResource); + } else { + return this.#isResourceEqual(resource, cachedResource); + } + } + + async #isResourceEqual(resourceA, resourceB) { + if (!resourceA || !resourceB) { + throw new Error("Cannot compare undefined resources"); + } + if (resourceA === resourceB) { + return true; + } + if (resourceA.getStatInfo()?.mtimeMs !== resourceA.getStatInfo()?.mtimeMs) { + return false; + } + if (await resourceA.getString() === await resourceB.getString()) { + return true; + } + return false; + } + + async #isResourceFingerprintEqual(resourceA, resourceBMetadata) { + if (!resourceA || !resourceBMetadata) { + throw new Error("Cannot compare undefined resources"); + } + if (resourceA.getStatInfo()?.mtimeMs !== resourceBMetadata.lastModified) { + return false; + } + if (await resourceA.getHash() === resourceBMetadata.hash) { + return true; + } + return false; + } + + #isRelevantResourceChange({pathsRead, patterns}, changedResourcePaths) { + for (const resourcePath of changedResourcePaths) { + if (pathsRead.includes(resourcePath)) { + return true; + } + if (patterns.length && micromatch.isMatch(resourcePath, patterns)) { + return true; + } + } + return false; + } +} diff --git a/packages/project/lib/build/cache/ProjectBuildCache.js b/packages/project/lib/build/cache/ProjectBuildCache.js new file mode 100644 index 00000000000..3fc87b06afe --- /dev/null +++ b/packages/project/lib/build/cache/ProjectBuildCache.js @@ -0,0 +1,433 @@ +import path from "node:path"; +import {stat} from "node:fs/promises"; +import {createResource, createAdapter} from "@ui5/fs/resourceFactory"; +import {getLogger} from "@ui5/logger"; +import BuildTaskCache from "./BuildTaskCache.js"; +const log = getLogger("build:cache:ProjectBuildCache"); + +/** + * A project's build cache can have multiple states + * - Initial build without existing build manifest or cache: + * * No build manifest + * * Tasks are unknown + * * Resources are unknown + * * No persistence of workspaces + * - Build of project with build manifest + * * (a valid build manifest implies that the project will not be built initially) + * * Tasks are known + * * Resources required and produced by tasks are known + * * No persistence of workspaces + * * => In case of a rebuild, all tasks need to be executed once to restore the workspaces + * - Build of project with build manifest and cache + * * Tasks are known + * * Resources required and produced by tasks are known + * * Workspaces can be restored from cache + */ + +export default class ProjectBuildCache { + #taskCache = new Map(); + #project; + #cacheKey; + #cacheDir; + #cacheRoot; + + #invalidatedTasks = new Map(); + #updatedResources = new Set(); + #restoreFailed = false; + + /** + * + * @param {Project} project Project instance + * @param {string} cacheKey Cache key + * @param {string} [cacheDir] Cache directory + */ + constructor(project, cacheKey, cacheDir) { + this.#project = project; + this.#cacheKey = cacheKey; + this.#cacheDir = cacheDir; + this.#cacheRoot = cacheDir && createAdapter({ + fsBasePath: cacheDir, + virBasePath: "/" + }); + } + + async updateTaskResult(taskName, workspaceTracker, dependencyTracker) { + const projectTrackingResults = workspaceTracker.getResults(); + const dependencyTrackingResults = dependencyTracker?.getResults(); + + const resourcesRead = projectTrackingResults.resourcesRead; + if (dependencyTrackingResults) { + for (const [resourcePath, resource] of Object.entries(dependencyTrackingResults.resourcesRead)) { + resourcesRead[resourcePath] = resource; + } + } + const resourcesWritten = projectTrackingResults.resourcesWritten; + + if (this.#taskCache.has(taskName)) { + log.verbose(`Updating build cache with results of task ${taskName} in project ${this.#project.getName()}`); + const taskCache = this.#taskCache.get(taskName); + + const writtenResourcePaths = Object.keys(resourcesWritten); + if (writtenResourcePaths.length) { + log.verbose(`Task ${taskName} produced ${writtenResourcePaths.length} resources`); + + const changedPaths = new Set((await Promise.all(writtenResourcePaths + .map(async (resourcePath) => { + // Check whether resource content actually changed + if (await taskCache.isResourceInWriteCache(resourcesWritten[resourcePath])) { + return undefined; + } + return resourcePath; + }))).filter((resourcePath) => resourcePath !== undefined)); + + if (!changedPaths.size) { + log.verbose( + `Resources produced by task ${taskName} match with cache from previous executions. ` + + `This task will not invalidate any other tasks`); + return; + } + log.verbose( + `Task ${taskName} produced ${changedPaths.size} resources that might invalidate other tasks`); + for (const resourcePath of changedPaths) { + this.#updatedResources.add(resourcePath); + } + // Check whether other tasks need to be invalidated + const allTasks = Array.from(this.#taskCache.keys()); + const taskIndex = allTasks.indexOf(taskName); + const emptySet = new Set(); + for (let i = taskIndex + 1; i < allTasks.length; i++) { + const nextTaskName = allTasks[i]; + if (!this.#taskCache.get(nextTaskName).checkPossiblyInvalidatesTask(changedPaths, emptySet)) { + continue; + } + if (this.#invalidatedTasks.has(taskName)) { + const {changedDependencyResourcePaths} = + this.#invalidatedTasks.get(taskName); + for (const resourcePath of changedPaths) { + changedDependencyResourcePaths.add(resourcePath); + } + } else { + this.#invalidatedTasks.set(taskName, { + changedProjectResourcePaths: changedPaths, + changedDependencyResourcePaths: emptySet + }); + } + } + } + taskCache.updateResources( + projectTrackingResults.requests, + dependencyTrackingResults?.requests, + resourcesRead, + resourcesWritten + ); + } else { + log.verbose(`Initializing build cache for task ${taskName} in project ${this.#project.getName()}`); + this.#taskCache.set(taskName, + new BuildTaskCache(this.#project.getName(), taskName, { + projectRequests: projectTrackingResults.requests, + dependencyRequests: dependencyTrackingResults?.requests, + resourcesRead, + resourcesWritten + }) + ); + } + + if (this.#invalidatedTasks.has(taskName)) { + this.#invalidatedTasks.delete(taskName); + } + } + + harvestUpdatedResources() { + const updatedResources = new Set(this.#updatedResources); + this.#updatedResources.clear(); + return updatedResources; + } + + resourceChanged(projectResourcePaths, dependencyResourcePaths) { + let taskInvalidated = false; + for (const [taskName, taskCache] of this.#taskCache) { + if (!taskCache.checkPossiblyInvalidatesTask(projectResourcePaths, dependencyResourcePaths)) { + continue; + } + taskInvalidated = true; + if (this.#invalidatedTasks.has(taskName)) { + const {changedProjectResourcePaths, changedDependencyResourcePaths} = + this.#invalidatedTasks.get(taskName); + for (const resourcePath of projectResourcePaths) { + changedProjectResourcePaths.add(resourcePath); + } + for (const resourcePath of dependencyResourcePaths) { + changedDependencyResourcePaths.add(resourcePath); + } + } else { + this.#invalidatedTasks.set(taskName, { + changedProjectResourcePaths: new Set(projectResourcePaths), + changedDependencyResourcePaths: new Set(dependencyResourcePaths) + }); + } + } + return taskInvalidated; + } + + async validateChangedProjectResources(taskName, workspace, dependencies) { + // Check whether the supposedly changed resources for the task have actually changed + if (!this.#invalidatedTasks.has(taskName)) { + return; + } + const {changedProjectResourcePaths, changedDependencyResourcePaths} = this.#invalidatedTasks.get(taskName); + await this._validateChangedResources(taskName, workspace, changedProjectResourcePaths); + await this._validateChangedResources(taskName, dependencies, changedDependencyResourcePaths); + + if (!changedProjectResourcePaths.size && !changedDependencyResourcePaths.size) { + // Task is no longer invalidated + this.#invalidatedTasks.delete(taskName); + } + } + + async _validateChangedResources(taskName, reader, changedResourcePaths) { + for (const resourcePath of changedResourcePaths) { + const resource = await reader.byPath(resourcePath); + if (!resource) { + // Resource was deleted, no need to check further + continue; + } + + const taskCache = this.#taskCache.get(taskName); + if (!taskCache) { + throw new Error(`Failed to validate changed resources for task ${taskName}: Task cache not found`); + } + if (await taskCache.isResourceInReadCache(resource)) { + log.verbose(`Resource content has not changed for task ${taskName}, ` + + `removing ${resourcePath} from set of changed resource paths`); + changedResourcePaths.delete(resourcePath); + } + } + } + + getChangedProjectResourcePaths(taskName) { + return this.#invalidatedTasks.get(taskName)?.changedProjectResourcePaths ?? new Set(); + } + + getChangedDependencyResourcePaths(taskName) { + return this.#invalidatedTasks.get(taskName)?.changedDependencyResourcePaths ?? new Set(); + } + + hasCache() { + return this.#taskCache.size > 0; + } + + /* + Check whether the project's build cache has an entry for the given stage. + This means that the cache has been filled with the output of the given stage. + */ + hasCacheForTask(taskName) { + return this.#taskCache.has(taskName); + } + + hasValidCacheForTask(taskName) { + return this.#taskCache.has(taskName) && !this.#invalidatedTasks.has(taskName); + } + + getCacheForTask(taskName) { + return this.#taskCache.get(taskName); + } + + requiresBuild() { + return !this.hasCache() || this.#invalidatedTasks.size > 0; + } + + async toObject() { + // const globalResourceIndex = Object.create(null); + // function addResourcesToIndex(taskName, resourceMap) { + // for (const resourcePath of Object.keys(resourceMap)) { + // const resource = resourceMap[resourcePath]; + // const resourceKey = `${resourcePath}:${resource.hash}`; + // if (!globalResourceIndex[resourceKey]) { + // globalResourceIndex[resourceKey] = { + // hash: resource.hash, + // lastModified: resource.lastModified, + // tasks: [taskName] + // }; + // } else if (!globalResourceIndex[resourceKey].tasks.includes(taskName)) { + // globalResourceIndex[resourceKey].tasks.push(taskName); + // } + // } + // } + const taskCache = []; + for (const cache of this.#taskCache.values()) { + const cacheObject = await cache.toObject(); + taskCache.push(cacheObject); + // addResourcesToIndex(taskName, cacheObject.resources.project.resourcesRead); + // addResourcesToIndex(taskName, cacheObject.resources.project.resourcesWritten); + // addResourcesToIndex(taskName, cacheObject.resources.dependencies.resourcesRead); + } + // Collect metadata for all relevant source files + const sourceReader = this.#project.getSourceReader(); + // const resourceMetadata = await Promise.all(Array.from(relevantSourceFiles).map(async (resourcePath) => { + const resources = await sourceReader.byGlob("/**/*"); + const sourceMetadata = Object.create(null); + await Promise.all(resources.map(async (resource) => { + sourceMetadata[resource.getOriginalPath()] = { + lastModified: resource.getStatInfo()?.mtimeMs, + hash: await resource.getHash(), + }; + })); + + return { + timestamp: Date.now(), + cacheKey: this.#cacheKey, + taskCache, + sourceMetadata, + // globalResourceIndex, + }; + } + + async #serializeMetadata() { + const serializedCache = await this.toObject(); + const cacheContent = JSON.stringify(serializedCache, null, 2); + const res = createResource({ + path: `/cache-info.json`, + string: cacheContent, + }); + await this.#cacheRoot.write(res); + } + + async #serializeTaskOutputs() { + log.info(`Serializing task outputs for project ${this.#project.getName()}`); + const stageCache = await Promise.all(Array.from(this.#taskCache.keys()).map(async (taskName, idx) => { + const reader = this.#project.getDeltaReader(taskName); + if (!reader) { + log.verbose( + `Skipping serialization of empty writer for task ${taskName} in project ${this.#project.getName()}` + ); + return; + } + const resources = await reader.byGlob("/**/*"); + + const target = createAdapter({ + fsBasePath: path.join(this.#cacheDir, "taskCache", `${idx}-${taskName}`), + virBasePath: "/" + }); + + for (const res of resources) { + await target.write(res); + } + return { + reader: target, + stage: taskName + }; + })); + // Re-import cache as base layer to reduce memory pressure + this.#project.importCachedStages(stageCache.filter((entry) => entry)); + } + + async #checkSourceChanges(sourceMetadata) { + log.verbose(`Checking for source changes for project ${this.#project.getName()}`); + const sourceReader = this.#project.getSourceReader(); + const resources = await sourceReader.byGlob("/**/*"); + const changedResources = new Set(); + for (const resource of resources) { + const resourcePath = resource.getOriginalPath(); + const resourceMetadata = sourceMetadata[resourcePath]; + if (!resourceMetadata) { + // New resource + log.verbose(`New resource: ${resourcePath}`); + changedResources.add(resourcePath); + continue; + } + if (resourceMetadata.lastModified !== resource.getStatInfo()?.mtimeMs) { + log.verbose(`Resource changed: ${resourcePath}`); + changedResources.add(resourcePath); + } + // TODO: Hash-based check can be requested by user and per project + // The performance impact can be quite high for large projects + /* + if (someFlag) { + const currentHash = await resource.getHash(); + if (currentHash !== resourceMetadata.hash) { + log.verbose(`Resource changed: ${resourcePath}`); + changedResources.add(resourcePath); + } + }*/ + } + if (changedResources.size) { + const tasksInvalidated = this.resourceChanged(changedResources, new Set()); + if (tasksInvalidated) { + log.info(`Invalidating tasks due to changed resources for project ${this.#project.getName()}`); + } + } + } + + async #deserializeWriter() { + const cachedStages = await Promise.all(Array.from(this.#taskCache.keys()).map(async (taskName, idx) => { + const fsBasePath = path.join(this.#cacheDir, "taskCache", `${idx}-${taskName}`); + let cacheReader; + if (await exists(fsBasePath)) { + cacheReader = createAdapter({ + name: `Cache reader for task ${taskName} in project ${this.#project.getName()}`, + fsBasePath, + virBasePath: "/", + project: this.#project, + }); + } + + return { + stage: taskName, + reader: cacheReader + }; + })); + this.#project.importCachedStages(cachedStages); + } + + async serializeToDisk() { + if (!this.#cacheRoot) { + log.error("Cannot save cache to disk: No cache persistence available"); + return; + } + await Promise.all([ + await this.#serializeTaskOutputs(), + await this.#serializeMetadata() + ]); + } + + async attemptDeserializationFromDisk() { + if (this.#restoreFailed || !this.#cacheRoot) { + return; + } + const res = await this.#cacheRoot.byPath(`/cache-info.json`); + if (!res) { + this.#restoreFailed = true; + return; + } + const cacheContent = JSON.parse(await res.getString()); + try { + const projectName = this.#project.getName(); + for (const {taskName, resourceMetadata} of cacheContent.taskCache) { + this.#taskCache.set(taskName, new BuildTaskCache(projectName, taskName, resourceMetadata)); + } + await Promise.all([ + this.#checkSourceChanges(cacheContent.sourceMetadata), + this.#deserializeWriter() + ]); + } catch (err) { + throw new Error( + `Failed to restore cache from disk for project ${this.#project.getName()}: ${err.message}`, { + cause: err + }); + } + } +} + +async function exists(filePath) { + try { + await stat(filePath); + return true; + } catch (err) { + // "File or directory does not exist" + if (err.code === "ENOENT") { + return false; + } else { + throw err; + } + } +} diff --git a/packages/project/lib/build/helpers/BuildContext.js b/packages/project/lib/build/helpers/BuildContext.js index 8d8d1e1a329..063aaf30e21 100644 --- a/packages/project/lib/build/helpers/BuildContext.js +++ b/packages/project/lib/build/helpers/BuildContext.js @@ -1,5 +1,8 @@ +import path from "node:path"; import ProjectBuildContext from "./ProjectBuildContext.js"; import OutputStyleEnum from "./ProjectBuilderOutputStyle.js"; +import {createCacheKey} from "./createBuildManifest.js"; +import WatchHandler from "./WatchHandler.js"; /** * Context of a build process @@ -8,11 +11,14 @@ import OutputStyleEnum from "./ProjectBuilderOutputStyle.js"; * @memberof @ui5/project/build/helpers */ class BuildContext { + #watchHandler; + constructor(graph, taskRepository, { // buildConfig selfContained = false, cssVariables = false, jsdoc = false, createBuildManifest = false, + useCache = false, outputStyle = OutputStyleEnum.Default, includedTasks = [], excludedTasks = [], } = {}) { @@ -67,6 +73,7 @@ class BuildContext { outputStyle, includedTasks, excludedTasks, + useCache, }; this._taskRepository = taskRepository; @@ -97,15 +104,43 @@ class BuildContext { return this._graph; } - createProjectContext({project}) { + async createProjectContext({project, cacheDir}) { + const cacheKey = await this.#createCacheKeyForProject(project); + if (cacheDir) { + cacheDir = path.join(cacheDir, cacheKey); + } const projectBuildContext = new ProjectBuildContext({ buildContext: this, - project + project, + cacheKey, + cacheDir, }); this._projectBuildContexts.push(projectBuildContext); return projectBuildContext; } + initWatchHandler(projects, updateBuildResult) { + const watchHandler = new WatchHandler(this, updateBuildResult); + watchHandler.watch(projects); + this.#watchHandler = watchHandler; + return watchHandler; + } + + getWatchHandler() { + return this.#watchHandler; + } + + async #createCacheKeyForProject(project) { + return createCacheKey(project, this._graph, + this.getBuildConfig(), this.getTaskRepository()); + } + + getBuildContext(projectName) { + if (projectName) { + return this._projectBuildContexts.find((ctx) => ctx.getProject().getName() === projectName); + } + } + async executeCleanupTasks(force = false) { await Promise.all(this._projectBuildContexts.map((ctx) => { return ctx.executeCleanupTasks(force); diff --git a/packages/project/lib/build/helpers/ProjectBuildContext.js b/packages/project/lib/build/helpers/ProjectBuildContext.js index 10eb2a67a83..20a9e668150 100644 --- a/packages/project/lib/build/helpers/ProjectBuildContext.js +++ b/packages/project/lib/build/helpers/ProjectBuildContext.js @@ -2,6 +2,7 @@ import ResourceTagCollection from "@ui5/fs/internal/ResourceTagCollection"; import ProjectBuildLogger from "@ui5/logger/internal/loggers/ProjectBuild"; import TaskUtil from "./TaskUtil.js"; import TaskRunner from "../TaskRunner.js"; +import ProjectBuildCache from "../cache/ProjectBuildCache.js"; /** * Build context of a single project. Always part of an overall @@ -11,7 +12,16 @@ import TaskRunner from "../TaskRunner.js"; * @memberof @ui5/project/build/helpers */ class ProjectBuildContext { - constructor({buildContext, project}) { + /** + * + * @param {object} parameters Parameters + * @param {object} parameters.buildContext The build context. + * @param {object} parameters.project The project instance. + * @param {string} parameters.cacheKey The cache key. + * @param {string} parameters.cacheDir The cache directory. + * @throws {Error} Throws an error if 'buildContext' or 'project' is missing. + */ + constructor({buildContext, project, cacheKey, cacheDir}) { if (!buildContext) { throw new Error(`Missing parameter 'buildContext'`); } @@ -25,6 +35,8 @@ class ProjectBuildContext { projectName: project.getName(), projectType: project.getType() }); + this._cacheKey = cacheKey; + this._cache = new ProjectBuildCache(this._project, cacheKey, cacheDir); this._queues = { cleanup: [] }; @@ -33,6 +45,10 @@ class ProjectBuildContext { allowedTags: ["ui5:OmitFromBuildResult", "ui5:IsBundle"], allowedNamespaces: ["build"] }); + const buildManifest = this.#getBuildManifest(); + if (buildManifest) { + this._cache.deserialize(buildManifest.buildManifest.cache); + } } isRootProject() { @@ -111,6 +127,7 @@ class ProjectBuildContext { this._taskRunner = new TaskRunner({ project: this._project, log: this._log, + cache: this._cache, taskUtil: this.getTaskUtil(), graph: this._buildContext.getGraph(), taskRepository: this._buildContext.getTaskRepository(), @@ -126,23 +143,106 @@ class ProjectBuildContext { * * @returns {boolean} True if the project needs to be built */ - requiresBuild() { - return !this._project.getBuildManifest(); + async requiresBuild() { + if (this.#getBuildManifest()) { + return false; + } + + if (!this._cache.hasCache()) { + await this._cache.attemptDeserializationFromDisk(); + } + + return this._cache.requiresBuild(); + } + + async runTasks() { + await this.getTaskRunner().runTasks(); + const updatedResourcePaths = this._cache.harvestUpdatedResources(); + + if (updatedResourcePaths.size === 0) { + return; + } + this._log.verbose( + `Project ${this._project.getName()} updated resources: ${Array.from(updatedResourcePaths).join(", ")}`); + const graph = this._buildContext.getGraph(); + const emptySet = new Set(); + + // Propagate changes to all dependents of the project + for (const {project: dep} of graph.traverseDependents(this._project.getName())) { + const projectBuildContext = this._buildContext.getBuildContext(dep.getName()); + projectBuildContext.getBuildCache().resourceChanged(emptySet, updatedResourcePaths); + } + } + + #getBuildManifest() { + const manifest = this._project.getBuildManifest(); + if (!manifest) { + return; + } + // Check whether the manifest can be used for this build + if (manifest.buildManifest.manifestVersion === "0.1" || manifest.buildManifest.manifestVersion === "0.2") { + // Manifest version 0.1 and 0.2 are always used without further checks for legacy reasons + return manifest; + } + if (manifest.buildManifest.manifestVersion === "0.3" && + manifest.buildManifest.cacheKey === this.getCacheKey()) { + // Manifest version 0.3 is used with a matching cache key + return manifest; + } + // Unknown manifest version can't be used + return; } getBuildMetadata() { - const buildManifest = this._project.getBuildManifest(); + const buildManifest = this.#getBuildManifest(); if (!buildManifest) { return null; } const timeDiff = (new Date().getTime() - new Date(buildManifest.timestamp).getTime()); - // TODO: Format age properly via a new @ui5/logger util module + // TODO: Format age properly return { timestamp: buildManifest.timestamp, age: timeDiff / 1000 + " seconds" }; } + + getBuildCache() { + return this._cache; + } + + getCacheKey() { + return this._cacheKey; + } + + // async watchFileChanges() { + // // const paths = this._project.getSourcePaths(); + // // this._log.verbose(`Watching source paths: ${paths.join(", ")}`); + // // const {default: chokidar} = await import("chokidar"); + // // const watcher = chokidar.watch(paths, { + // // ignoreInitial: true, + // // persistent: false, + // // }); + // // watcher.on("add", async (filePath) => { + // // }); + // // watcher.on("change", async (filePath) => { + // // const resourcePath = this._project.getVirtualPath(filePath); + // // this._log.info(`File changed: ${resourcePath} (${filePath})`); + // // // Inform cache + // // this._cache.fileChanged(resourcePath); + // // // Inform dependents + // // for (const dependent of this._buildContext.getGraph().getTransitiveDependents(this._project.getName())) { + // // await this._buildContext.getProjectBuildContext(dependent).dependencyFileChanged(resourcePath); + // // } + // // // Inform build context + // // await this._buildContext.fileChanged(this._project.getName(), resourcePath); + // // }); + // } + + // dependencyFileChanged(resourcePath) { + // this._log.info(`Dependency file changed: ${resourcePath}`); + // this._cache.fileChanged(resourcePath); + // } } export default ProjectBuildContext; diff --git a/packages/project/lib/build/helpers/WatchHandler.js b/packages/project/lib/build/helpers/WatchHandler.js new file mode 100644 index 00000000000..0a5510a7eba --- /dev/null +++ b/packages/project/lib/build/helpers/WatchHandler.js @@ -0,0 +1,135 @@ +import EventEmitter from "node:events"; +import path from "node:path"; +import {watch} from "node:fs/promises"; +import {getLogger} from "@ui5/logger"; +const log = getLogger("build:helpers:WatchHandler"); + +/** + * Context of a build process + * + * @private + * @memberof @ui5/project/build/helpers + */ +class WatchHandler extends EventEmitter { + #buildContext; + #updateBuildResult; + #abortControllers = []; + #sourceChanges = new Map(); + #fileChangeHandlerTimeout; + + constructor(buildContext, updateBuildResult) { + super(); + this.#buildContext = buildContext; + this.#updateBuildResult = updateBuildResult; + } + + watch(projects) { + for (const project of projects) { + const paths = project.getSourcePaths(); + log.verbose(`Watching source paths: ${paths.join(", ")}`); + + for (const sourceDir of paths) { + const ac = new AbortController(); + const watcher = watch(sourceDir, { + persistent: true, + recursive: true, + signal: ac.signal, + }); + + this.#abortControllers.push(ac); + this.#handleWatchEvents(watcher, sourceDir, project); // Do not await as this would block the loop + } + } + } + + stop() { + for (const ac of this.#abortControllers) { + ac.abort(); + } + } + + async #handleWatchEvents(watcher, basePath, project) { + try { + for await (const {eventType, filename} of watcher) { + log.verbose(`File changed: ${eventType} ${filename}`); + if (filename) { + await this.#fileChanged(project, path.join(basePath, filename.toString())); + } + } + } catch (err) { + if (err.name === "AbortError") { + return; + } + throw err; + } + } + + async #fileChanged(project, filePath) { + // Collect changes (grouped by project), then trigger callbacks (debounced) + const resourcePath = project.getVirtualPath(filePath); + if (!this.#sourceChanges.has(project)) { + this.#sourceChanges.set(project, new Set()); + } + this.#sourceChanges.get(project).add(resourcePath); + + // Trigger callbacks debounced + if (!this.#fileChangeHandlerTimeout) { + this.#fileChangeHandlerTimeout = setTimeout(async () => { + await this.#handleResourceChanges(); + this.#fileChangeHandlerTimeout = null; + }, 100); + } else { + clearTimeout(this.#fileChangeHandlerTimeout); + this.#fileChangeHandlerTimeout = setTimeout(async () => { + await this.#handleResourceChanges(); + this.#fileChangeHandlerTimeout = null; + }, 100); + } + } + + async #handleResourceChanges() { + // Reset file changes before processing + const sourceChanges = this.#sourceChanges; + this.#sourceChanges = new Map(); + const dependencyChanges = new Map(); + let someProjectTasksInvalidated = false; + + const graph = this.#buildContext.getGraph(); + for (const [project, changedResourcePaths] of sourceChanges) { + // Propagate changes to dependents of the project + for (const {project: dep} of graph.traverseDependents(project.getName())) { + const depChanges = dependencyChanges.get(dep); + if (!depChanges) { + dependencyChanges.set(dep, new Set(changedResourcePaths)); + continue; + } + for (const res of changedResourcePaths) { + depChanges.add(res); + } + } + } + + await graph.traverseDepthFirst(({project}) => { + if (!sourceChanges.has(project) && !dependencyChanges.has(project)) { + return; + } + const projectSourceChanges = sourceChanges.get(project) ?? new Set(); + const projectDependencyChanges = dependencyChanges.get(project) ?? new Set(); + const projectBuildContext = this.#buildContext.getBuildContext(project.getName()); + const tasksInvalidated = + projectBuildContext.getBuildCache().resourceChanged(projectSourceChanges, projectDependencyChanges); + + if (tasksInvalidated) { + someProjectTasksInvalidated = true; + } + }); + + if (someProjectTasksInvalidated) { + this.emit("projectInvalidated"); + await this.#updateBuildResult(); + this.emit("buildUpdated"); + } + } +} + +export default WatchHandler; diff --git a/packages/project/lib/build/helpers/createBuildManifest.js b/packages/project/lib/build/helpers/createBuildManifest.js index 998935b3c05..ba19023d54f 100644 --- a/packages/project/lib/build/helpers/createBuildManifest.js +++ b/packages/project/lib/build/helpers/createBuildManifest.js @@ -1,4 +1,5 @@ import {createRequire} from "node:module"; +import crypto from "node:crypto"; // Using CommonsJS require since JSON module imports are still experimental const require = createRequire(import.meta.url); @@ -16,16 +17,33 @@ function getSortedTags(project) { return Object.fromEntries(entities); } -export default async function(project, buildConfig, taskRepository) { +async function collectDepInfo(graph, project) { + const transitiveDependencyInfo = Object.create(null); + for (const depName of graph.getTransitiveDependencies(project.getName())) { + const dep = graph.getProject(depName); + transitiveDependencyInfo[depName] = { + version: dep.getVersion() + }; + } + return transitiveDependencyInfo; +} + +export default async function(project, graph, buildConfig, taskRepository, transitiveDependencyInfo, buildCache) { if (!project) { throw new Error(`Missing parameter 'project'`); } + if (!graph) { + throw new Error(`Missing parameter 'graph'`); + } if (!buildConfig) { throw new Error(`Missing parameter 'buildConfig'`); } if (!taskRepository) { throw new Error(`Missing parameter 'taskRepository'`); } + if (!buildCache) { + throw new Error(`Missing parameter 'buildCache'`); + } const projectName = project.getName(); const type = project.getType(); @@ -44,8 +62,21 @@ export default async function(project, buildConfig, taskRepository) { `Unable to create archive metadata for project ${project.getName()}: ` + `Project type ${type} is currently not supported`); } + let buildManifest; + if (project.isFrameworkProject()) { + buildManifest = await createFrameworkManifest(project, buildConfig, taskRepository); + } else { + buildManifest = { + manifestVersion: "0.3", + timestamp: new Date().toISOString(), + dependencies: collectDepInfo(graph, project), + version: project.getVersion(), + namespace: project.getNamespace(), + tags: getSortedTags(project), + cacheKey: createCacheKey(project, graph, buildConfig, taskRepository), + }; + } - const {builderVersion, fsVersion: builderFsVersion} = await taskRepository.getVersions(); const metadata = { project: { specVersion: project.getSpecVersion().toString(), @@ -59,27 +90,49 @@ export default async function(project, buildConfig, taskRepository) { } } }, - buildManifest: { - manifestVersion: "0.2", - timestamp: new Date().toISOString(), - versions: { - builderVersion: builderVersion, - projectVersion: await getVersion("@ui5/project"), - fsVersion: await getVersion("@ui5/fs"), - }, - buildConfig, - version: project.getVersion(), - namespace: project.getNamespace(), - tags: getSortedTags(project) - } + buildManifest, + buildCache: await buildCache.serialize(), }; - if (metadata.buildManifest.versions.fsVersion !== builderFsVersion) { + return metadata; +} + +async function createFrameworkManifest(project, buildConfig, taskRepository) { + // Use legacy manifest version for framework libraries to ensure compatibility + const {builderVersion, fsVersion: builderFsVersion} = await taskRepository.getVersions(); + const buildManifest = { + manifestVersion: "0.2", + timestamp: new Date().toISOString(), + versions: { + builderVersion: builderVersion, + projectVersion: await getVersion("@ui5/project"), + fsVersion: await getVersion("@ui5/fs"), + }, + buildConfig, + version: project.getVersion(), + namespace: project.getNamespace(), + tags: getSortedTags(project) + }; + + if (buildManifest.versions.fsVersion !== builderFsVersion) { // Added in manifestVersion 0.2: // @ui5/project and @ui5/builder use different versions of @ui5/fs. // This should be mentioned in the build manifest: - metadata.buildManifest.versions.builderFsVersion = builderFsVersion; + buildManifest.versions.builderFsVersion = builderFsVersion; } + return buildManifest; +} - return metadata; +export async function createCacheKey(project, graph, buildConfig, taskRepository) { + const depInfo = collectDepInfo(graph, project); + const {builderVersion, fsVersion: builderFsVersion} = await taskRepository.getVersions(); + const projectVersion = await getVersion("@ui5/project"); + const fsVersion = await getVersion("@ui5/fs"); + + const key = `${builderVersion}-${projectVersion}-${fsVersion}-${builderFsVersion}-` + + `${JSON.stringify(buildConfig)}-${JSON.stringify(depInfo)}`; + const hash = crypto.createHash("sha256").update(key).digest("hex"); + + // Create a hash from the cache key + return `${project.getName()}-${project.getVersion()}-${hash}`; } diff --git a/packages/project/lib/graph/ProjectGraph.js b/packages/project/lib/graph/ProjectGraph.js index ba6967154e6..0d15174e3b3 100644 --- a/packages/project/lib/graph/ProjectGraph.js +++ b/packages/project/lib/graph/ProjectGraph.js @@ -284,6 +284,40 @@ class ProjectGraph { processDependency(projectName); return Array.from(dependencies); } + + getDependents(projectName) { + if (!this._projects.has(projectName)) { + throw new Error( + `Failed to get dependents for project ${projectName}: ` + + `Unable to find project in project graph`); + } + const dependents = []; + for (const [fromProjectName, adjacencies] of this._adjList) { + if (adjacencies.has(projectName)) { + dependents.push(fromProjectName); + } + } + return dependents; + } + + getTransitiveDependents(projectName) { + const dependents = new Set(); + if (!this._projects.has(projectName)) { + throw new Error( + `Failed to get transitive dependents for project ${projectName}: ` + + `Unable to find project in project graph`); + } + const addDependents = (projectName) => { + const projectDependents = this.getDependents(projectName); + projectDependents.forEach((dependent) => { + dependents.add(dependent); + addDependents(dependent); + }); + }; + addDependents(projectName); + return Array.from(dependents); + } + /** * Checks whether a dependency is optional or not. * Currently only used in tests. @@ -475,6 +509,54 @@ class ProjectGraph { })(); } + * traverseDependents(startName, includeStartModule = false) { + if (typeof startName === "boolean") { + includeStartModule = startName; + startName = undefined; + } + if (!startName) { + startName = this._rootProjectName; + } else if (!this.getProject(startName)) { + throw new Error(`Failed to start graph traversal: Could not find project ${startName} in project graph`); + } + + const queue = [{ + projectNames: [startName], + ancestors: [] + }]; + + const visited = Object.create(null); + + while (queue.length) { + const {projectNames, ancestors} = queue.shift(); // Get and remove first entry from queue + + for (const projectName of projectNames) { + this._checkCycle(ancestors, projectName); + if (visited[projectName]) { + continue; + } + + visited[projectName] = true; + + const newAncestors = [...ancestors, projectName]; + const dependents = this.getDependents(projectName); + + queue.push({ + projectNames: dependents, + ancestors: newAncestors + }); + + if (includeStartModule || projectName !== startName) { + // Do not yield the start module itself + yield { + project: this.getProject(projectName), + dependents + }; + } + } + } + } + /** * Join another project graph into this one. * Projects and extensions which already exist in this graph will cause an error to be thrown @@ -558,7 +640,8 @@ class ProjectGraph { dependencyIncludes, selfContained = false, cssVariables = false, jsdoc = false, createBuildManifest = false, includedTasks = [], excludedTasks = [], - outputStyle = OutputStyleEnum.Default + outputStyle = OutputStyleEnum.Default, + cacheDir, watch, }) { this.seal(); // Do not allow further changes to the graph if (this._built) { @@ -579,10 +662,11 @@ class ProjectGraph { includedTasks, excludedTasks, outputStyle, } }); - await builder.build({ + return await builder.build({ destPath, cleanDest, includedDependencies, excludedDependencies, dependencyIncludes, + cacheDir, watch, }); } diff --git a/packages/project/lib/specifications/ComponentProject.js b/packages/project/lib/specifications/ComponentProject.js index e40f8a9228b..34e1fd852ba 100644 --- a/packages/project/lib/specifications/ComponentProject.js +++ b/packages/project/lib/specifications/ComponentProject.js @@ -164,24 +164,26 @@ class ComponentProject extends Project { // return resourceFactory.createWorkspace({ // name: `Workspace for project ${this.getName()}`, // reader: this._getPlainReader(excludes), - // writer: this._getWriter().collection + // writer: this._createWriter().collection // }); // } - _getWriter() { + _createWriter() { // writer is always of style "buildtime" const namespaceWriter = resourceFactory.createAdapter({ + name: `Namespace writer for project ${this.getName()} (${this.getCurrentStage()} stage)`, virBasePath: "/", project: this }); const generalWriter = resourceFactory.createAdapter({ + name: `General writer for project ${this.getName()} (${this.getCurrentStage()} stage)`, virBasePath: "/", project: this }); const collection = resourceFactory.createWriterCollection({ - name: `Writers for project ${this.getName()}`, + name: `Writers for project ${this.getName()} (${this.getCurrentStage()} stage)`, writerMapping: { [`/resources/${this._namespace}/`]: namespaceWriter, [`/test-resources/${this._namespace}/`]: namespaceWriter, @@ -208,8 +210,13 @@ class ComponentProject extends Project { return reader; } - _addReadersFromWriter(style, readers, writer) { - const {namespaceWriter, generalWriter} = writer; + _addWriterToReaders(style, readers, writer) { + let {namespaceWriter, generalWriter} = writer; + if (!namespaceWriter || !generalWriter) { + // TODO: Too hacky + namespaceWriter = writer; + generalWriter = writer; + } if ((style === "runtime" || style === "dist") && this._isRuntimeNamespaced) { // If the project's type requires a namespace at runtime, the diff --git a/packages/project/lib/specifications/Project.js b/packages/project/lib/specifications/Project.js index 40ba76fe635..bc60cf5ca29 100644 --- a/packages/project/lib/specifications/Project.js +++ b/packages/project/lib/specifications/Project.js @@ -13,11 +13,15 @@ import {createWorkspace, createReaderCollectionPrioritized} from "@ui5/fs/resour * @hideconstructor */ class Project extends Specification { - #latestWriter; - #latestWorkspace; - #latestReader = new Map(); - #writerVersions = []; - #workspaceSealed = false; + #currentWriter; + #currentWorkspace; + #currentReader = new Map(); + #currentStage; + #currentVersion = 0; // Writer version (0 is reserved for a possible imported writer cache) + + #stages = [""]; // Stages in order of creation + #writers = new Map(); // Maps stage to a set of writer versions (possibly sparse array) + #workspaceSealed = true; // Project starts as being sealed. Needs to be unsealed using newVersion() constructor(parameters) { super(parameters); @@ -94,6 +98,14 @@ class Project extends Specification { throw new Error(`getSourcePath must be implemented by subclass ${this.constructor.name}`); } + getSourcePaths() { + throw new Error(`getSourcePaths must be implemented by subclass ${this.constructor.name}`); + } + + getVirtualPath() { + throw new Error(`getVirtualPath must be implemented by subclass ${this.constructor.name}`); + } + /** * Get the project's framework name configuration * @@ -261,37 +273,68 @@ class Project extends Specification { * @returns {@ui5/fs/ReaderCollection} A reader collection instance */ getReader({style = "buildtime"} = {}) { - let reader = this.#latestReader.get(style); + let reader = this.#currentReader.get(style); if (reader) { + // Use cached reader return reader; } - const readers = []; - this._addReadersFromWriter(style, readers, this.getWriter()); - readers.push(this._getStyledReader(style)); - reader = createReaderCollectionPrioritized({ - name: `Reader collection for project ${this.getName()}`, - readers - }); - this.#latestReader.set(style, reader); + // const readers = []; + // this._addWriterToReaders(style, readers, this.getWriter()); + // readers.push(this._getStyledReader(style)); + // reader = createReaderCollectionPrioritized({ + // name: `Reader collection for project ${this.getName()}`, + // readers + // }); + reader = this.#getReader(this.#currentStage, this.#currentVersion, style); + this.#currentReader.set(style, reader); return reader; } - getWriter() { - return this.#latestWriter || this.createNewWriterVersion(); + // getCacheReader({style = "buildtime"} = {}) { + // return this.#getReader(this.#currentStage, style, true); + // } + + getSourceReader(style = "buildtime") { + return this._getStyledReader(style); } - createNewWriterVersion() { - const writer = this._getWriter(); - this.#writerVersions.push(writer); - this.#latestWriter = writer; + #getWriter() { + if (this.#currentWriter) { + return this.#currentWriter; + } + + const stage = this.#currentStage; + const currentVersion = this.#currentVersion; - // Invalidate dependents - this.#latestWorkspace = null; - this.#latestReader = new Map(); + if (!this.#writers.has(stage)) { + this.#writers.set(stage, []); + } + const versions = this.#writers.get(stage); + let writer; + if (versions[currentVersion]) { + writer = versions[currentVersion]; + } else { + // Create new writer + writer = this._createWriter(); + versions[currentVersion] = writer; + } + this.#currentWriter = writer; return writer; } + // #createNewWriterStage(stageId) { + // const writer = this._createWriter(); + // this.#writers.set(stageId, writer); + // this.#currentWriter = writer; + + // // Invalidate dependents + // this.#currentWorkspace = null; + // this.#currentReader = new Map(); + + // return writer; + // } + /** * Get a [DuplexCollection]{@link @ui5/fs/DuplexCollection} for accessing and modifying a * project's resources. This is always of style buildtime. @@ -305,25 +348,209 @@ class Project extends Specification { getWorkspace() { if (this.#workspaceSealed) { throw new Error( - `Workspace of project ${this.getName()} has been sealed. Use method #getReader for read-only access`); + `Workspace of project ${this.getName()} has been sealed. This indicates that the project already ` + + `finished building and its content must not be modified further. ` + + `Use method 'getReader' for read-only access`); } - if (this.#latestWorkspace) { - return this.#latestWorkspace; + if (this.#currentWorkspace) { + return this.#currentWorkspace; } - const excludes = this.getBuilderResourcesExcludes(); // TODO: Do not apply in server context - const writer = this.getWriter(); - this.#latestWorkspace = createWorkspace({ - reader: this._getReader(excludes), + const writer = this.#getWriter(); + + // if (this.#stageCacheReaders.has(this.getCurrentStage())) { + // reader = createReaderCollectionPrioritized({ + // name: `Reader collection for project ${this.getName()} stage ${this.getCurrentStage()}`, + // readers: [ + // this.#stageCacheReaders.get(this.getCurrentStage()), + // reader, + // ] + // }); + // } + this.#currentWorkspace = createWorkspace({ + reader: this.getReader(), writer: writer.collection || writer }); - return this.#latestWorkspace; + return this.#currentWorkspace; } + // getWorkspaceForVersion(version) { + // return createWorkspace({ + // reader: this.#getReader(version), + // writer: this.#writerVersions[version].collection || this.#writerVersions[version] + // }); + // } + sealWorkspace() { this.#workspaceSealed = true; + this.useFinalStage(); + } + + newVersion() { + this.#workspaceSealed = false; + this.#currentVersion++; + this.useInitialStage(); + } + + revertToLastVersion() { + if (this.#currentVersion === 0) { + throw new Error(`Unable to revert to previous version: No previous version available`); + } + this.#currentVersion--; + this.useInitialStage(); + + // Remove writer version from all stages + for (const writerVersions of this.#writers.values()) { + if (writerVersions[this.#currentVersion]) { + delete writerVersions[this.#currentVersion]; + } + } + } + + #getReader(stage, version, style = "buildtime") { + const readers = []; + + // Add writers for previous stages as readers + const stageIdx = this.#stages.indexOf(stage); + if (stageIdx > 0) { // Stage 0 has no previous stage + // Collect writers from all preceding stages + for (let i = stageIdx - 1; i >= 0; i--) { + const stageWriters = this.#getWriters(this.#stages[i], version, style); + if (stageWriters) { + readers.push(stageWriters); + } + } + } + + // Always add source reader + readers.push(this._getStyledReader(style)); + + return createReaderCollectionPrioritized({ + name: `Reader collection for stage '${stage}' of project ${this.getName()}`, + readers: readers + }); + } + + useStage(stageId, newWriter = false) { + // if (newWriter && this.#writers.has(stageId)) { + // this.#writers.delete(stageId); + // } + if (stageId === this.#currentStage) { + return; + } + if (!this.#stages.includes(stageId)) { + // Add new stage + this.#stages.push(stageId); + } + + this.#currentStage = stageId; + + // Unset "current" reader/writer + this.#currentReader = new Map(); + this.#currentWriter = null; + this.#currentWorkspace = null; + } + + useInitialStage() { + this.useStage(""); + } + + useFinalStage() { + this.useStage(""); + } + + #getWriters(stage, version, style = "buildtime") { + const readers = []; + const stageWriters = this.#writers.get(stage); + if (!stageWriters?.length) { + return null; + } + for (let i = version; i >= 0; i--) { + if (!stageWriters[i]) { + // Writers is a sparse array, some stages might skip a version + continue; + } + this._addWriterToReaders(style, readers, stageWriters[i]); + } + + return createReaderCollectionPrioritized({ + name: `Collection of all writers for stage '${stage}', version ${version} of project ${this.getName()}`, + readers + }); + } + + getDeltaReader(stage) { + const readers = []; + const stageWriters = this.#writers.get(stage); + if (!stageWriters?.length) { + return null; + } + const version = this.#currentVersion; + for (let i = version; i >= 1; i--) { // Skip version 0 (possibly containing cached writers) + if (!stageWriters[i]) { + // Writers is a sparse array, some stages might skip a version + continue; + } + this._addWriterToReaders("buildtime", readers, stageWriters[i]); + } + + const reader = createReaderCollectionPrioritized({ + name: `Collection of new writers for stage '${stage}', version ${version} of project ${this.getName()}`, + readers + }); + + + // Condense writer versions (TODO: this step is optional but might improve memory consumption) + // this.#condenseVersions(reader); + return reader; + } + + // #condenseVersions(reader) { + // for (const stage of this.#stages) { + // const stageWriters = this.#writers.get(stage); + // if (!stageWriters) { + // continue; + // } + // const condensedWriter = this._createWriter(); + + // for (let i = 1; i < stageWriters.length; i++) { + // if (stageWriters[i]) { + + // } + // } + + // // eslint-disable-next-line no-sparse-arrays + // const newWriters = [, condensedWriter]; + // if (stageWriters[0]) { + // newWriters[0] = stageWriters[0]; + // } + // this.#writers.set(stage, newWriters); + // } + // } + + importCachedStages(stages) { + if (!this.#workspaceSealed) { + throw new Error(`Unable to import cached stages: Workspace is not sealed`); + } + for (const {stage, reader} of stages) { + if (!this.#stages.includes(stage)) { + this.#stages.push(stage); + } + if (reader) { + this.#writers.set(stage, [reader]); + } else { + this.#writers.set(stage, []); + } + } + this.#currentVersion = 0; + this.useFinalStage(); + } + + getCurrentStage() { + return this.#currentStage; } - _addReadersFromWriter(style, readers, writer) { + /* Overwritten in ComponentProject subclass */ + _addWriterToReaders(style, readers, writer) { readers.push(writer); } diff --git a/packages/project/lib/specifications/types/Application.js b/packages/project/lib/specifications/types/Application.js index 1dc17b4bc1c..44f39b4ef6d 100644 --- a/packages/project/lib/specifications/types/Application.js +++ b/packages/project/lib/specifications/types/Application.js @@ -45,6 +45,21 @@ class Application extends ComponentProject { return fsPath.join(this.getRootPath(), this._webappPath); } + getSourcePaths() { + return [this.getSourcePath()]; + } + + getVirtualPath(sourceFilePath) { + const sourcePath = this.getSourcePath(); + if (sourceFilePath.startsWith(sourcePath)) { + const relSourceFilePath = fsPath.relative(sourcePath, sourceFilePath); + return `/resources/${this._namespace}/${relSourceFilePath}`; + } + + throw new Error( + `Unable to convert source path ${sourceFilePath} to virtual path for project ${this.getName()}`); + } + /* === Resource Access === */ /** * Get a resource reader for the sources of the project (excluding any test resources) @@ -107,13 +122,13 @@ class Application extends ComponentProject { /** * @private * @param {object} config Configuration object - * @param {object} buildDescription Cache metadata object + * @param {object} buildManifest Cache metadata object */ - async _parseConfiguration(config, buildDescription) { - await super._parseConfiguration(config, buildDescription); + async _parseConfiguration(config, buildManifest) { + await super._parseConfiguration(config, buildManifest); - if (buildDescription) { - this._namespace = buildDescription.namespace; + if (buildManifest) { + this._namespace = buildManifest.namespace; return; } this._namespace = await this._getNamespace(); diff --git a/packages/project/lib/specifications/types/Library.js b/packages/project/lib/specifications/types/Library.js index d3d2059a055..e118f39e6b6 100644 --- a/packages/project/lib/specifications/types/Library.js +++ b/packages/project/lib/specifications/types/Library.js @@ -56,6 +56,39 @@ class Library extends ComponentProject { return fsPath.join(this.getRootPath(), this._srcPath); } + getSourcePaths() { + const paths = [this.getSourcePath()]; + if (this._testPathExists) { + paths.push(fsPath.join(this.getRootPath(), this._testPath)); + } + return paths; + } + + getVirtualPath(sourceFilePath) { + const sourcePath = this.getSourcePath(); + if (sourceFilePath.startsWith(sourcePath)) { + const relSourceFilePath = fsPath.relative(sourcePath, sourceFilePath); + let virBasePath = "/resources/"; + if (!this._isSourceNamespaced) { + virBasePath += `${this._namespace}/`; + } + return posixPath.join(virBasePath, relSourceFilePath); + } + + const testPath = fsPath.join(this.getRootPath(), this._testPath); + if (sourceFilePath.startsWith(testPath)) { + const relSourceFilePath = fsPath.relative(testPath, sourceFilePath); + let virBasePath = "/test-resources/"; + if (!this._isSourceNamespaced) { + virBasePath += `${this._namespace}/`; + } + return posixPath.join(virBasePath, relSourceFilePath); + } + + throw new Error( + `Unable to convert source path ${sourceFilePath} to virtual path for project ${this.getName()}`); + } + /* === Resource Access === */ /** * Get a resource reader for the sources of the project (excluding any test resources) @@ -156,13 +189,13 @@ class Library extends ComponentProject { /** * @private * @param {object} config Configuration object - * @param {object} buildDescription Cache metadata object + * @param {object} buildManifest Cache metadata object */ - async _parseConfiguration(config, buildDescription) { - await super._parseConfiguration(config, buildDescription); + async _parseConfiguration(config, buildManifest) { + await super._parseConfiguration(config, buildManifest); - if (buildDescription) { - this._namespace = buildDescription.namespace; + if (buildManifest) { + this._namespace = buildManifest.namespace; return; } diff --git a/packages/project/lib/specifications/types/Module.js b/packages/project/lib/specifications/types/Module.js index a59c464f94a..dcd3a9a2176 100644 --- a/packages/project/lib/specifications/types/Module.js +++ b/packages/project/lib/specifications/types/Module.js @@ -31,6 +31,12 @@ class Module extends Project { throw new Error(`Projects of type module have more than one source path`); } + getSourcePaths() { + return this._paths.map(({fsBasePath}) => { + return fsBasePath; + }); + } + /* === Resource Access === */ _getStyledReader(style) { @@ -50,7 +56,7 @@ class Module extends Project { // const excludes = this.getBuilderResourcesExcludes(); // const reader = this._getReader(excludes); - // const writer = this._getWriter(); + // const writer = this._createWriter(); // return resourceFactory.createWorkspace({ // reader, // writer @@ -76,7 +82,7 @@ class Module extends Project { }); } - _getWriter() { + _createWriter() { if (!this._writer) { this._writer = resourceFactory.createAdapter({ virBasePath: "/" diff --git a/packages/project/lib/specifications/types/ThemeLibrary.js b/packages/project/lib/specifications/types/ThemeLibrary.js index d4644c78885..9412975721e 100644 --- a/packages/project/lib/specifications/types/ThemeLibrary.js +++ b/packages/project/lib/specifications/types/ThemeLibrary.js @@ -39,6 +39,25 @@ class ThemeLibrary extends Project { return fsPath.join(this.getRootPath(), this._srcPath); } + getSourcePaths() { + return [this.getSourcePath()]; + } + + getVirtualPath(sourceFilePath) { + const sourcePath = this.getSourcePath(); + if (sourceFilePath.startsWith(sourcePath)) { + const relSourceFilePath = fsPath.relative(sourcePath, sourceFilePath); + let virBasePath = "/resources/"; + if (!this._isSourceNamespaced) { + virBasePath += `${this._namespace}/`; + } + return `${virBasePath}${relSourceFilePath}`; + } + + throw new Error( + `Unable to convert source path ${sourceFilePath} to virtual path for project ${this.getName()}`); + } + /* === Resource Access === */ _getStyledReader(style) { @@ -62,7 +81,7 @@ class ThemeLibrary extends Project { // const excludes = this.getBuilderResourcesExcludes(); // const reader = this._getReader(excludes); - // const writer = this._getWriter(); + // const writer = this._createWriter(); // return resourceFactory.createWorkspace({ // reader, // writer @@ -93,7 +112,7 @@ class ThemeLibrary extends Project { return reader; } - _getWriter() { + _createWriter() { if (!this._writer) { this._writer = resourceFactory.createAdapter({ virBasePath: "/", diff --git a/packages/project/test/lib/build/ProjectBuilder.js b/packages/project/test/lib/build/ProjectBuilder.js index 64b36ab85e9..548703e5e32 100644 --- a/packages/project/test/lib/build/ProjectBuilder.js +++ b/packages/project/test/lib/build/ProjectBuilder.js @@ -16,6 +16,8 @@ function getMockProject(type, id = "b") { getVersion: noop, getReader: () => "reader", getWorkspace: () => "workspace", + sealWorkspace: noop, + createNewWorkspaceVersion: noop, }; } diff --git a/packages/project/test/lib/build/TaskRunner.js b/packages/project/test/lib/build/TaskRunner.js index 73c9e187648..97f59ce81d0 100644 --- a/packages/project/test/lib/build/TaskRunner.js +++ b/packages/project/test/lib/build/TaskRunner.js @@ -58,7 +58,9 @@ function getMockProject(type) { getCustomTasks: () => [], hasBuildManifest: () => false, getWorkspace: () => "workspace", - isFrameworkProject: () => false + isFrameworkProject: () => false, + sealWorkspace: noop, + createNewWorkspaceVersion: noop, }; } @@ -118,6 +120,10 @@ test.beforeEach(async (t) => { isLevelEnabled: sinon.stub().returns(true), }; + t.context.cache = { + setTasks: sinon.stub(), + }; + t.context.resourceFactory = { createReaderCollection: sinon.stub() .returns("reader collection") @@ -134,7 +140,7 @@ test.afterEach.always((t) => { }); test("Missing parameters", (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; t.throws(() => { new TaskRunner({ graph, @@ -152,6 +158,7 @@ test("Missing parameters", (t) => { taskUtil, taskRepository, log: projectBuildLogger, + cache, buildConfig }); }, { @@ -163,6 +170,7 @@ test("Missing parameters", (t) => { graph, taskRepository, log: projectBuildLogger, + cache, buildConfig }); }, { @@ -174,6 +182,7 @@ test("Missing parameters", (t) => { graph, taskUtil, log: projectBuildLogger, + cache, buildConfig }); }, { @@ -197,6 +206,7 @@ test("Missing parameters", (t) => { taskUtil, taskRepository, log: projectBuildLogger, + cache, }); }, { message: "TaskRunner: One or more mandatory parameters not provided" @@ -228,9 +238,9 @@ test("_initTasks: Project of type 'application'", async (t) => { }); test("_initTasks: Project of type 'library'", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskRunner = new TaskRunner({ - project: getMockProject("library"), graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project: getMockProject("library"), graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -254,13 +264,13 @@ test("_initTasks: Project of type 'library'", async (t) => { }); test("_initTasks: Project of type 'library' (framework project)", async (t) => { - const {graph, taskUtil, taskRepository, projectBuildLogger, TaskRunner} = t.context; + const {graph, taskUtil, taskRepository, projectBuildLogger, TaskRunner, cache} = t.context; const project = getMockProject("library"); project.isFrameworkProject = () => true; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -284,9 +294,9 @@ test("_initTasks: Project of type 'library' (framework project)", async (t) => { }); test("_initTasks: Project of type 'theme-library'", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskRunner = new TaskRunner({ - project: getMockProject("theme-library"), graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project: getMockProject("theme-library"), graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -300,13 +310,13 @@ test("_initTasks: Project of type 'theme-library'", async (t) => { }); test("_initTasks: Project of type 'theme-library' (framework project)", async (t) => { - const {graph, taskUtil, taskRepository, projectBuildLogger, TaskRunner} = t.context; + const {graph, taskUtil, taskRepository, projectBuildLogger, cache, TaskRunner} = t.context; const project = getMockProject("theme-library"); project.isFrameworkProject = () => true; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -320,9 +330,9 @@ test("_initTasks: Project of type 'theme-library' (framework project)", async (t }); test("_initTasks: Project of type 'module'", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskRunner = new TaskRunner({ - project: getMockProject("module"), graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project: getMockProject("module"), graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -330,9 +340,9 @@ test("_initTasks: Project of type 'module'", async (t) => { }); test("_initTasks: Unknown project type", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskRunner = new TaskRunner({ - project: getMockProject("pony"), graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project: getMockProject("pony"), graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(taskRunner._initTasks()); @@ -340,14 +350,14 @@ test("_initTasks: Unknown project type", async (t) => { }); test("_initTasks: Custom tasks", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "myTask", afterTask: "minify"}, {name: "myOtherTask", beforeTask: "replaceVersion"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); t.deepEqual(taskRunner._taskExecutionOrder, [ @@ -371,14 +381,14 @@ test("_initTasks: Custom tasks", async (t) => { }); test("_initTasks: Custom tasks with no standard tasks", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); project.getCustomTasks = () => [ {name: "myTask"}, {name: "myOtherTask", beforeTask: "myTask"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); t.deepEqual(taskRunner._taskExecutionOrder, [ @@ -388,14 +398,14 @@ test("_initTasks: Custom tasks with no standard tasks", async (t) => { }); test("_initTasks: Custom tasks with no standard tasks and second task defining no before-/afterTask", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); project.getCustomTasks = () => [ {name: "myTask"}, {name: "myOtherTask"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -407,13 +417,13 @@ test("_initTasks: Custom tasks with no standard tasks and second task defining n }); test("_initTasks: Custom tasks with both, before- and afterTask reference", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "myTask", beforeTask: "minify", afterTask: "replaceVersion"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -425,13 +435,13 @@ test("_initTasks: Custom tasks with both, before- and afterTask reference", asyn }); test("_initTasks: Custom tasks with no before-/afterTask reference", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "myTask"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -443,13 +453,13 @@ test("_initTasks: Custom tasks with no before-/afterTask reference", async (t) = }); test("_initTasks: Custom tasks without name", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: ""} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -460,13 +470,13 @@ test("_initTasks: Custom tasks without name", async (t) => { }); test("_initTasks: Custom task with name of standard tasks", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "replaceVersion", afterTask: "minify"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -478,7 +488,7 @@ test("_initTasks: Custom task with name of standard tasks", async (t) => { }); test("_initTasks: Multiple custom tasks with same name", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "myTask", afterTask: "minify"}, @@ -486,7 +496,7 @@ test("_initTasks: Multiple custom tasks with same name", async (t) => { {name: "myTask", afterTask: "minify"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); t.deepEqual(taskRunner._taskExecutionOrder, [ @@ -511,13 +521,13 @@ test("_initTasks: Multiple custom tasks with same name", async (t) => { }); test("_initTasks: Custom tasks with unknown beforeTask", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "myTask", beforeTask: "unknownTask"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -529,13 +539,13 @@ test("_initTasks: Custom tasks with unknown beforeTask", async (t) => { }); test("_initTasks: Custom tasks with unknown afterTask", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "myTask", afterTask: "unknownTask"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -547,14 +557,14 @@ test("_initTasks: Custom tasks with unknown afterTask", async (t) => { }); test("_initTasks: Custom tasks is unknown", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; graph.getExtension.returns(undefined); const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "myTask", afterTask: "minify"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -566,13 +576,13 @@ test("_initTasks: Custom tasks is unknown", async (t) => { }); test("_initTasks: Custom tasks with removed beforeTask", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getCustomTasks = () => [ {name: "myTask", beforeTask: "removedTask"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); const err = await t.throwsAsync(async () => { await taskRunner._initTasks(); @@ -585,10 +595,10 @@ test("_initTasks: Custom tasks with removed beforeTask", async (t) => { }); test("_initTasks: Create dependencies reader for all dependencies", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, resourceFactory, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, resourceFactory, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); t.is(graph.traverseBreadthFirst.callCount, 1, "ProjectGraph#traverseBreadthFirst called once"); @@ -631,7 +641,7 @@ test("_initTasks: Create dependencies reader for all dependencies", async (t) => }); test("Custom task is called correctly", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStub = sinon.stub(); const specVersionGteStub = sinon.stub().returns(false); const mockSpecVersion = { @@ -652,7 +662,7 @@ test("Custom task is called correctly", async (t) => { ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -692,7 +702,7 @@ test("Custom task is called correctly", async (t) => { }); test("Custom task with legacy spec version", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStub = sinon.stub(); const specVersionGteStub = sinon.stub().returns(false); const mockSpecVersion = { @@ -712,7 +722,7 @@ test("Custom task with legacy spec version", async (t) => { ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -752,7 +762,7 @@ test("Custom task with legacy spec version", async (t) => { }); test("Custom task with legacy spec version and requiredDependenciesCallback", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStub = sinon.stub(); const specVersionGteStub = sinon.stub().returns(false); const mockSpecVersion = { @@ -773,7 +783,7 @@ test("Custom task with legacy spec version and requiredDependenciesCallback", as ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -824,7 +834,7 @@ test("Custom task with legacy spec version and requiredDependenciesCallback", as }); test("Custom task with specVersion 3.0", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStub = sinon.stub(); const specVersionGteStub = sinon.stub().returns(true); const mockSpecVersion = { @@ -848,7 +858,7 @@ test("Custom task with specVersion 3.0", async (t) => { ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -921,7 +931,7 @@ test("Custom task with specVersion 3.0", async (t) => { }); test("Custom task with specVersion 3.0 and no requiredDependenciesCallback", async (t) => { - const {sinon, graph, taskUtil, taskRepository, projectBuildLogger, TaskRunner} = t.context; + const {sinon, graph, taskUtil, taskRepository, projectBuildLogger, cache, TaskRunner} = t.context; const taskStub = sinon.stub(); const specVersionGteStub = sinon.stub().returns(true); const mockSpecVersion = { @@ -944,7 +954,7 @@ test("Custom task with specVersion 3.0 and no requiredDependenciesCallback", asy ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -982,7 +992,7 @@ test("Custom task with specVersion 3.0 and no requiredDependenciesCallback", asy }); test("Multiple custom tasks with same name are called correctly", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStubA = sinon.stub(); const taskStubB = sinon.stub(); const taskStubC = sinon.stub(); @@ -1042,7 +1052,7 @@ test("Multiple custom tasks with same name are called correctly", async (t) => { {name: "myTask", afterTask: "myTask", configuration: "bird"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -1184,7 +1194,7 @@ test("Multiple custom tasks with same name are called correctly", async (t) => { }); test("Custom task: requiredDependenciesCallback returns unknown dependency", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStub = sinon.stub(); const specVersionGteStub = sinon.stub().returns(true); const mockSpecVersion = { @@ -1209,7 +1219,7 @@ test("Custom task: requiredDependenciesCallback returns unknown dependency", asy ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await t.throwsAsync(taskRunner._initTasks(), { message: @@ -1221,7 +1231,7 @@ test("Custom task: requiredDependenciesCallback returns unknown dependency", asy test("Custom task: requiredDependenciesCallback returns Array instead of Set", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStub = sinon.stub(); const specVersionGteStub = sinon.stub().returns(true); const mockSpecVersion = { @@ -1246,7 +1256,7 @@ test("Custom task: requiredDependenciesCallback returns Array instead of Set", a ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await t.throwsAsync(taskRunner._initTasks(), { message: @@ -1256,7 +1266,7 @@ test("Custom task: requiredDependenciesCallback returns Array instead of Set", a }); test("Custom task attached to a disabled task", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, sinon, customTask} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache, sinon, customTask} = t.context; const project = getMockProject("application"); const customTaskFnStub = sinon.stub(); @@ -1269,7 +1279,7 @@ test("Custom task attached to a disabled task", async (t) => { customTask.getTask = () => customTaskFnStub; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner.runTasks(); @@ -1296,7 +1306,7 @@ test("Custom task attached to a disabled task", async (t) => { }); test.serial("_addTask", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStub = sinon.stub(); taskRepository.getTask.withArgs("standardTask").resolves({ @@ -1305,7 +1315,7 @@ test.serial("_addTask", async (t) => { const project = getMockProject("module"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -1338,12 +1348,12 @@ test.serial("_addTask", async (t) => { }); test.serial("_addTask with options", async (t) => { - const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {sinon, graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const taskStub = sinon.stub(); const project = getMockProject("module"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -1384,10 +1394,10 @@ test.serial("_addTask with options", async (t) => { }); test("_addTask: Duplicate task", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -1405,10 +1415,10 @@ test("_addTask: Duplicate task", async (t) => { }); test("_addTask: Task already added to execution order", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); @@ -1424,13 +1434,13 @@ test("_addTask: Task already added to execution order", async (t) => { }); test("getRequiredDependencies: Custom Task", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); project.getCustomTasks = () => [ {name: "myTask"} ]; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); t.deepEqual(await taskRunner.getRequiredDependencies(), new Set([]), "Project with custom task >= specVersion 3.0 and no requiredDependenciesCallback " + @@ -1438,55 +1448,55 @@ test("getRequiredDependencies: Custom Task", async (t) => { }); test("getRequiredDependencies: Default application", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("application"); project.getBundles = () => []; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); t.deepEqual(await taskRunner.getRequiredDependencies(), new Set([]), "Default application project does not require dependencies"); }); test("getRequiredDependencies: Default library", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("library"); project.getBundles = () => []; const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); t.deepEqual(await taskRunner.getRequiredDependencies(), new Set(["dep.a", "dep.b"]), "Default library project requires dependencies"); }); test("getRequiredDependencies: Default theme-library", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("theme-library"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); t.deepEqual(await taskRunner.getRequiredDependencies(), new Set(["dep.a", "dep.b"]), "Default theme-library project requires dependencies"); }); test("getRequiredDependencies: Default module", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); t.deepEqual(await taskRunner.getRequiredDependencies(), new Set([]), "Default module project does not require dependencies"); }); test("_createDependenciesReader", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, resourceFactory, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, resourceFactory, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); graph.traverseBreadthFirst.reset(); // Ignore the call in initTask @@ -1543,11 +1553,11 @@ test("_createDependenciesReader", async (t) => { }); test("_createDependenciesReader: All dependencies required", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, resourceFactory, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, resourceFactory, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); graph.traverseBreadthFirst.reset(); // Ignore the call in initTask @@ -1560,11 +1570,11 @@ test("_createDependenciesReader: All dependencies required", async (t) => { }); test("_createDependenciesReader: No dependencies required", async (t) => { - const {graph, taskUtil, taskRepository, TaskRunner, resourceFactory, projectBuildLogger} = t.context; + const {graph, taskUtil, taskRepository, TaskRunner, resourceFactory, projectBuildLogger, cache} = t.context; const project = getMockProject("module"); const taskRunner = new TaskRunner({ - project, graph, taskUtil, taskRepository, log: projectBuildLogger, buildConfig + project, graph, taskUtil, taskRepository, log: projectBuildLogger, cache, buildConfig }); await taskRunner._initTasks(); graph.traverseBreadthFirst.reset(); // Ignore the call in initTask diff --git a/packages/project/test/lib/build/helpers/ProjectBuildContext.js b/packages/project/test/lib/build/helpers/ProjectBuildContext.js index 03f9a568325..74b06d49927 100644 --- a/packages/project/test/lib/build/helpers/ProjectBuildContext.js +++ b/packages/project/test/lib/build/helpers/ProjectBuildContext.js @@ -1,6 +1,7 @@ import test from "ava"; import sinon from "sinon"; import esmock from "esmock"; +import ProjectBuildCache from "../../../../lib/build/helpers/ProjectBuildCache.js"; import ResourceTagCollection from "@ui5/fs/internal/ResourceTagCollection"; test.beforeEach((t) => { @@ -315,7 +316,7 @@ test("getTaskUtil", (t) => { }); test.serial("getTaskRunner", async (t) => { - t.plan(3); + t.plan(4); const project = { getName: () => "project", getType: () => "type", @@ -325,10 +326,13 @@ test.serial("getTaskRunner", async (t) => { constructor(params) { t.true(params.log instanceof ProjectBuildLogger, "TaskRunner receives an instance of ProjectBuildLogger"); params.log = "log"; // replace log instance with string for deep comparison + t.true(params.cache instanceof ProjectBuildCache, "TaskRunner receives an instance of ProjectBuildCache"); + params.cache = "cache"; // replace cache instance with string for deep comparison t.deepEqual(params, { graph: "graph", project: project, log: "log", + cache: "cache", taskUtil: "taskUtil", taskRepository: "taskRepository", buildConfig: "buildConfig" From 2f216ee2817896ecc0b0961ba8a8c4111d017a36 Mon Sep 17 00:00:00 2001 From: Merlin Beutlberger Date: Tue, 18 Nov 2025 16:03:17 +0100 Subject: [PATCH 7/7] refactor(cli): Use cache in ui5 build Cherry-picked from: https://github.com/SAP/ui5-cli/commit/d29ead8326c43690c7c792bb15ff41402a3d9f25 JIRA: CPOUI5FOUNDATION-1174 --- packages/cli/lib/cli/commands/build.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cli/lib/cli/commands/build.js b/packages/cli/lib/cli/commands/build.js index df93ac5a12e..31e7b56062c 100644 --- a/packages/cli/lib/cli/commands/build.js +++ b/packages/cli/lib/cli/commands/build.js @@ -1,4 +1,5 @@ import baseMiddleware from "../middlewares/base.js"; +import path from "node:path"; const build = { command: "build", @@ -173,6 +174,7 @@ async function handleBuild(argv) { const buildSettings = graph.getRoot().getBuilderSettings() || {}; await graph.build({ graph, + cacheDir: path.join(graph.getRoot().getRootPath(), ".ui5-cache"), destPath: argv.dest, cleanDest: argv["clean-dest"], createBuildManifest: argv["create-build-manifest"],