Systemjs depcache output (#26)

# Thanks for opening a Pull Request

## Changes

Please outline the reason for the changes you are making:

This PR adds the ability to output the scripts in a `depcache` format
that's usable with single SPA / systemjs implementations

[Change description here]

Are there any breaking changes you are aware of in the PR?

Not directly. Updating arborist a major version, but its downstream
change removed node versions we already didn't support.

## General Checklist

* [x] Change is tested locally
* [x] Demo is updated to exercise change (if applicable)
* [x] [WIP] flag is removed from the title

[1]:https://docs.npmjs.com/about-semantic-versioning
[2]:https://docs.npmjs.com/cli/deprecate

[3]:https://github.com/meltwater/applet-orchard/blob/master/docs/README.md

.gitignore

@@ -1,5 +1,6 @@
 # Project specific files
 demo/index.html
+demo/depcache.json
 
 # Local developer environment configuration
 .screen_layout

README.md

@@ -26,6 +26,22 @@ We recommend using a separate source file from your output file to allow for sou
 orchard -i ./sourceFile.html -o ./outputFile.html
 ```
 
+## Outputting depcache json format
+
+If you're building an application using systemjs import maps, and utilizing the [depcache](https://github.com/guybedford/import-maps-extensions#depcache) format, you can output the json file to be used by the `orchard` cli tool with the following command:
+
+```bash
+orchard -o depcache.json --outputDepcache true
+```
+
+This outputs a json file that can be loaded by your solution, and that file will look something like:
+```json
+[
+  "https://unpkg.com/npm@1.0.0/index.js",
+  "https://cdn.jsdelivr.net/npm/uhtml@3.0.1/es.js"
+]
+```
+
 ### Usage Implications
 
 In order to selectively serve modern code to modern browsers the script tags generated from the orchard utilize the `type="module"` attribute for es6+ builds, and es5 compatible code uses script tags with the `nomodule` and `defer` attributes.

demo/package.json

@@ -5,6 +5,7 @@
   "main": "src/index.js",
   "types": "./types/index.d.ts",
   "scripts": {
+    "depcache": "node ../cjs/cli.entry.js -o depcache.json --outputDepcache true",
     "start": "node ../cjs/cli.entry.js -i index.source.html -o index.html && ws --spa index.html"
   },
   "devDependencies": {

package.json

@@ -16,6 +16,7 @@
     "pretest": "scripts/check-dependencies && eslint src/",
     "preversion": "scripts/preversion --commit-docs",
     "start": "npm run build && cd demo && npm ci && npm start",
+    "start:depcache": "npm run build && cd demo && npm ci && npm run depcache",
     "test": "scripts/test",
     "test:live": "nodemon -x 'scripts/test' -w js,json --watch src/",
     "update": "npx npm-check -uE",
@@ -33,23 +34,23 @@
   },
   "homepage": "https://github.com/meltwater/the-orchard#readme",
   "devDependencies": {
-    "@babel/cli": "7.22.10",
-    "@babel/core": "7.22.10",
-    "@babel/plugin-transform-modules-commonjs": "7.22.5",
-    "@babel/preset-env": "7.22.10",
-    "@babel/register": "7.22.5",
+    "@babel/cli": "7.24.7",
+    "@babel/core": "7.24.7",
+    "@babel/plugin-transform-modules-commonjs": "7.24.7",
+    "@babel/preset-env": "7.24.7",
+    "@babel/register": "7.24.6",
     "eslint": "8.46.0",
-    "figlet": "1.6.0",
+    "figlet": "1.7.0",
     "jasmine": "5.1.0",
-    "nodemon": "3.0.1"
+    "nodemon": "3.1.4"
   },
   "dependencies": {
     "@meltwater/coerce": "0.4.0",
-    "@npmcli/arborist": "6.3.0",
+    "@npmcli/arborist": "7.5.4",
     "argument-contracts": "1.2.3",
     "colors": "1.4.0",
     "js-yaml": "4.1.0",
-    "semver": "7.5.4",
+    "semver": "7.6.2",
     "sywac": "1.3.0"
   },
   "engines": {

src/build-depcache-output/index.js

@@ -0,0 +1,72 @@
+import ac from 'argument-contracts';
+import { checkForRequiredInitialization } from '../build-app-output/check-for-required-initialization';
+import { CliOptions, DO_NOT_INJECT } from '../cli-options';
+import fs from 'fs';
+import { getDependencyPackages } from '../build-app-output/get-dependency-packages';
+import { Logger } from '../logger';
+import path from 'path';
+import { readPackageDependencies } from '../build-app-output/read-package-dependencies';
+import { rollupLatestMajorVersions } from '../build-app-output/rollup-latest-major-versions';
+import { throwForConflictingMajorVersions } from '../build-app-output/throw-for-conflicting-major-versions';
+import { throwIfZerothLevelDepNotHighestMajorVersion } from '../build-app-output/throw-if-zeroth-level-dep-not-highest-major-version';
+import { buildDependencyArray } from '../build-app-output/build-dependency-array';
+import { pareDownToKnownPackages } from '../build-app-output/pare-down-to-known-packages';
+import { adjustOrderBasedOnChildDependencies } from '../build-app-output/adjust-order-based-on-child-dependencies';
+import { resolveRequiredDependencyScripts } from './resolve-required-dependency-scripts';
+
+const currentWorkingDirectory = process.cwd();
+
+export async function buildDepcacheOutput(cliOptions) {
+    ac.assertType(cliOptions, CliOptions, 'cliOptions');
+
+    Logger.setLoggingLevel(cliOptions.logging);
+    Logger.debug(`buildDepcacheOutput: cliOptions: ${JSON.stringify(cliOptions)}`);
+
+    const dependencies = readPackageDependencies(path.join(currentWorkingDirectory, cliOptions.pathToPackageJson));
+    Logger.debug('dependencies', JSON.stringify(dependencies, null, 2));
+
+    const dependencyMap = await getDependencyPackages(cliOptions.dependencyDirectory);
+    Logger.debug('dependencyMap', JSON.stringify(dependencyMap, null, 2));
+
+    const npmDependenciesWithChildDependencies = await buildDependencyArray({
+        currentWorkingDirectory,
+        pathToPackageJson: cliOptions.pathToPackageJson
+    });
+
+    const paredDownToKnownPackages = pareDownToKnownPackages({
+        dependencyMap,
+        npmDependenciesWithChildDependencies
+    });
+
+    Logger.debug('paredDownToKnownPackages', paredDownToKnownPackages);
+
+    const dependenciesReadyForRollup = paredDownToKnownPackages;
+
+    const rolledUpDeps = rollupLatestMajorVersions(dependenciesReadyForRollup);
+    throwForConflictingMajorVersions({ dependencies: rolledUpDeps, dependencyMap });
+    throwIfZerothLevelDepNotHighestMajorVersion(rolledUpDeps);
+    checkForRequiredInitialization({ dependencies: rolledUpDeps, dependencyMap });
+
+    const orderedDependencies = adjustOrderBasedOnChildDependencies({
+        npmDependenciesWithChildDependencies: rolledUpDeps
+    });
+
+    const dependencyScripts = resolveRequiredDependencyScripts({
+        dependencies: orderedDependencies,
+        dependencyMap: {
+            ...dependencyMap
+        }
+    });
+
+    const output = [
+        ...dependencyScripts
+    ];
+
+    if (cliOptions.injectFile && cliOptions.injectFile !== DO_NOT_INJECT) {
+        const fileContentToInjectInto = fs.readFileSync(cliOptions.injectFile, { encoding: 'utf8' });
+        const updatedFileContent = fileContentToInjectInto.replace(cliOptions.orchardInjectString, JSON.stringify(output, null, 2));
+        fs.writeFileSync(cliOptions.outputFile, updatedFileContent);
+    } else {
+        fs.writeFileSync(cliOptions.outputFile, JSON.stringify(output, null, 2));
+    }
+}

src/build-depcache-output/index.spec.js

@@ -0,0 +1,213 @@
+import ac from 'argument-contracts';
+import * as AdjustOrderBasedOnChildDependenciesModule from '../build-app-output/adjust-order-based-on-child-dependencies';
+import * as CheckForRequiredInitializationModule from '../build-app-output/check-for-required-initialization';
+import { CliOptions } from '../cli-options';
+import fs from 'fs';
+import * as BuildDependencyArrayModule from '../build-app-output/build-dependency-array';
+import * as GetDependencyPackagesModule from '../build-app-output/get-dependency-packages';
+import { Logger } from '../logger';
+import * as PareDownToKnownPackagesModule from '../build-app-output/pare-down-to-known-packages';
+import path from 'path';
+import * as ReadPackageDependenciesModule from '../build-app-output/read-package-dependencies';
+import * as ResolveRequiredDependencyScriptsModule from './resolve-required-dependency-scripts';
+import * as RollupLatestMajorVersionsModule from '../build-app-output/rollup-latest-major-versions';
+import * as ThrowForConflictingMajorVersionsModule from '../build-app-output/throw-for-conflicting-major-versions';
+import * as ThrowIfZerothLevelDepNotHighestMajorVersionModule from '../build-app-output/throw-if-zeroth-level-dep-not-highest-major-version';
+
+import { buildDepcacheOutput } from './index';
+
+describe('build depcache output', () => {
+    let cliOptions;
+
+    beforeEach(() => {
+
+        spyOn(ac, 'assertType');
+        spyOn(AdjustOrderBasedOnChildDependenciesModule, 'adjustOrderBasedOnChildDependencies').and.returnValue([]);
+        spyOn(CheckForRequiredInitializationModule, 'checkForRequiredInitialization');
+        spyOn(fs, 'writeFileSync');
+        spyOn(BuildDependencyArrayModule, 'buildDependencyArray').and.resolveTo([]);
+        spyOn(GetDependencyPackagesModule, 'getDependencyPackages').and.resolveTo([]);
+        spyOn(Logger, 'setLoggingLevel');
+        spyOn(PareDownToKnownPackagesModule, 'pareDownToKnownPackages').and.returnValue([]);
+        spyOn(ReadPackageDependenciesModule, 'readPackageDependencies');
+        spyOn(ResolveRequiredDependencyScriptsModule, 'resolveRequiredDependencyScripts').and.returnValue([]);
+        spyOn(RollupLatestMajorVersionsModule, 'rollupLatestMajorVersions').and.returnValue([]);
+        spyOn(ThrowForConflictingMajorVersionsModule, 'throwForConflictingMajorVersions');
+        spyOn(ThrowIfZerothLevelDepNotHighestMajorVersionModule, 'throwIfZerothLevelDepNotHighestMajorVersion');
+        spyOn(Logger, 'debug');
+
+        cliOptions = new CliOptions({
+            excludeDirectDependencies: false,
+            dependencyDirectory: 'her/there/everywhere',
+            openFileLimit: 4,
+            orchardInjectString: '<!-- wat -->',
+            outputFile: 'plumbus.html',
+            retryOpenFileSleepDuration: 10,
+            outputDepcache: true
+        });
+    });
+
+    it('should assert cliOptions is CliOptions', async () => {
+        await buildDepcacheOutput(cliOptions);
+
+        expect(ac.assertType).toHaveBeenCalledWith(cliOptions, CliOptions, 'cliOptions');
+    });
+
+    it('should set logging level from cliOptions', async () => {
+        const logging = 'complete deforestation';
+        await buildDepcacheOutput({
+            ...cliOptions,
+            logging
+        });
+
+        expect(Logger.setLoggingLevel).toHaveBeenCalledWith(logging);
+    });
+
+    it('should read package.json dependencies from cliOptions', async () => {
+        const pathToPackageJson = 'go/here/for/package.json';
+        await buildDepcacheOutput({
+            ...cliOptions,
+            pathToPackageJson
+        });
+
+        expect(ReadPackageDependenciesModule.readPackageDependencies).toHaveBeenCalledWith(path.join(process.cwd(), pathToPackageJson));
+    });
+
+    it('should get array of dependencies', async () => {
+        const openFileLimit = 'twenty';
+        const retryOpenFileSleepDuration = '1 siesta';
+
+        await buildDepcacheOutput({
+            ...cliOptions,
+            openFileLimit,
+            retryOpenFileSleepDuration
+        });
+
+        expect(BuildDependencyArrayModule.buildDependencyArray).toHaveBeenCalledWith(jasmine.objectContaining({
+            currentWorkingDirectory: jasmine.any(String),
+            pathToPackageJson: cliOptions.pathToPackageJson
+        }));
+    });
+
+    it('should rollup major versions', async () => {
+        const packageDependencies = [1, 2, 3];
+        PareDownToKnownPackagesModule.pareDownToKnownPackages.and.returnValue(packageDependencies);
+
+        await buildDepcacheOutput(cliOptions);
+
+        expect(RollupLatestMajorVersionsModule.rollupLatestMajorVersions).toHaveBeenCalledWith(packageDependencies);
+    });
+
+    it('should check for package major version conflicts', async () => {
+        const dependencyMap = { yes: 'maybe..... no' };
+        const packageDependencies = [1, 2, 3];
+        GetDependencyPackagesModule.getDependencyPackages.and.resolveTo(dependencyMap);
+        RollupLatestMajorVersionsModule.rollupLatestMajorVersions.and.returnValue(packageDependencies);
+
+        await buildDepcacheOutput(cliOptions);
+
+        expect(ThrowForConflictingMajorVersionsModule.throwForConflictingMajorVersions).toHaveBeenCalledWith({
+            dependencies: packageDependencies,
+            dependencyMap
+        });
+    });
+
+    it('should check for required initializations', async () => {
+        const dependencyMap = { yes: 'maybe..... no' };
+        const packageDependencies = [3, 2, 1];
+        GetDependencyPackagesModule.getDependencyPackages.and.resolveTo(dependencyMap);
+        RollupLatestMajorVersionsModule.rollupLatestMajorVersions.and.returnValue(packageDependencies);
+
+        await buildDepcacheOutput(cliOptions);
+
+        expect(CheckForRequiredInitializationModule.checkForRequiredInitialization).toHaveBeenCalledWith({ dependencies: packageDependencies, dependencyMap });
+    });
+
+    it('should checkout for package 0 level dependency not major version issues', async () => {
+        const dependencyMap = { yes: 'maybe..... no' };
+        const packageDependencies = [1, 2, 3];
+        GetDependencyPackagesModule.getDependencyPackages.and.resolveTo(dependencyMap);
+        RollupLatestMajorVersionsModule.rollupLatestMajorVersions.and.returnValue(packageDependencies);
+
+        await buildDepcacheOutput(cliOptions);
+
+        expect(ThrowIfZerothLevelDepNotHighestMajorVersionModule.throwIfZerothLevelDepNotHighestMajorVersion).toHaveBeenCalledWith(packageDependencies);
+    });
+
+    it('should resolve dependency scripts', async () => {
+        const dependencyMap = { all: 'The things!' };
+        GetDependencyPackagesModule.getDependencyPackages.and.resolveTo(dependencyMap);
+
+        await buildDepcacheOutput(cliOptions);
+
+        expect(ResolveRequiredDependencyScriptsModule.resolveRequiredDependencyScripts).toHaveBeenCalledWith(jasmine.objectContaining({
+            dependencyMap: jasmine.objectContaining(dependencyMap)
+        }));
+    });
+
+    it('should resolve dependency tree', async () => {
+        const packageDependencies = [1, 2, 3];
+        AdjustOrderBasedOnChildDependenciesModule.adjustOrderBasedOnChildDependencies.and.returnValue(packageDependencies);
+
+        await buildDepcacheOutput(cliOptions);
+
+        expect(ResolveRequiredDependencyScriptsModule.resolveRequiredDependencyScripts).toHaveBeenCalledWith(jasmine.objectContaining({
+            dependencies: packageDependencies
+        }));
+    });
+
+    describe('injection handling', () => {
+        beforeEach(() => {
+            spyOn(fs, 'readFileSync');
+        });
+
+        it('should read file to inject to', async () => {
+            const injectFile = 'inject/file/location.txt';
+            fs.readFileSync.and.returnValue('');
+
+            await buildDepcacheOutput({
+                ...cliOptions,
+                injectFile
+            });
+
+            expect(fs.readFileSync).toHaveBeenCalledWith(injectFile, { encoding: 'utf8' });
+        });
+
+        it('should inject output into the file', async () => {
+            const orchardInjectString = 'watwatwatwatwat';
+            const outputFile = 'The best output';
+            const beforeInjectionLocation = 'This file has been injected into!';
+            const afterInjectionLocation = 'YEAH BOIIIIIIIIIII!';
+            const fileContent = `${beforeInjectionLocation}${orchardInjectString}${afterInjectionLocation}`;
+            const output = 'The greatest output Evar';
+            fs.readFileSync.and.returnValue(fileContent);
+            ResolveRequiredDependencyScriptsModule.resolveRequiredDependencyScripts.and.returnValue([output])
+
+            await buildDepcacheOutput({
+                ...cliOptions,
+                injectFile: 'yarp.txt',
+                orchardInjectString,
+                outputFile
+            });
+
+
+            expect(fs.writeFileSync).toHaveBeenCalledWith(outputFile, jasmine.stringMatching(beforeInjectionLocation));
+            expect(fs.writeFileSync).toHaveBeenCalledWith(outputFile, jasmine.stringMatching(output));
+            expect(fs.writeFileSync).toHaveBeenCalledWith(outputFile, jasmine.stringMatching(afterInjectionLocation));
+        });
+    });
+
+    it('should output tags array', async () => {
+        const outputFile = 'The best output';
+
+        const dependencyOutput = 'AW YEAH!';
+        ResolveRequiredDependencyScriptsModule.resolveRequiredDependencyScripts.and.returnValue([dependencyOutput])
+
+        await buildDepcacheOutput({
+            ...cliOptions,
+            outputFile
+        });
+
+        expect(fs.writeFileSync).toHaveBeenCalledWith(outputFile, jasmine.stringMatching(dependencyOutput));
+    });
+});

src/build-depcache-output/resolve-required-dependency-scripts/index.js

@@ -0,0 +1,31 @@
+import ac from 'argument-contracts';
+import { ExternalPackageEntry } from '../../external-package-entry';
+import { Logger } from '../../logger';
+
+function getScripts({ externalPackageEntry, version }) {
+    ac.assertType(externalPackageEntry, ExternalPackageEntry, 'externalPackageEntry');
+    ac.assertString(version, 'version');
+
+    Logger.info(`Creating scripts for ${externalPackageEntry.packageName}`);
+
+    return externalPackageEntry.getEsmUrls(version);
+}
+
+export function resolveRequiredDependencyScripts({ dependencies, dependencyMap }) {
+    ac.assertType(dependencies, Object, 'dependencies');
+    ac.assertType(dependencyMap, Object, 'dependencyMap');
+
+    const dependencyScripts = dependencies.reduce((scriptsArray, dependency) => {
+        if (dependencyMap[dependency.packageName]) {
+            const scripts = getScripts({
+                externalPackageEntry: dependencyMap[dependency.packageName],
+                version: dependency.version
+            });
+
+            return scriptsArray.concat(scripts);
+        }
+        return scriptsArray;
+    }, []);
+
+    return dependencyScripts;
+}