handle require.resolve fallback when walking node_modules with no mai…

…n field in package.json (#733)

* handle require.resolve fallback

* test case for empty main field

* babel/runtime test case

* remove console logging

* use require.resolve.paths as require.resolve fallback

packages/cli/package.json

@@ -49,9 +49,11 @@
     "unified": "^9.2.0"
   },
   "devDependencies": {
+    "@babel/runtime": "^7.10.4",
     "@lion/button": "~0.12.0",
     "@lion/calendar": "~0.15.0",
     "@mapbox/rehype-prism": "^0.5.0",
+    "@types/trusted-types": "^2.0.2",
     "lit-element": "^2.4.0",
     "lit-redux-router": "^0.17.1",
     "lodash-es": "^4.17.20",

packages/cli/src/lib/node-modules-utils.js

@@ -1,6 +1,8 @@
+const fs = require('fs');
+
 // defer to NodeJS to find where on disk a package is located using require.resolve
 // and return the root absolute location
-function getNodeModulesResolveLocationForPackageName(packageName) {
+function getNodeModulesLocationForPackage(packageName) {
   let nodeModulesUrl;
 
   try {
@@ -20,9 +22,24 @@ function getNodeModulesResolveLocationForPackageName(packageName) {
 
       nodeModulesUrl = `${packageRootPath}${packageName}`;
     }
-
   } catch (e) {
-    console.debug(`Unable to look up package using NodeJS require.resolve for => ${packageName}`);
+    // require.resolve may fail in the event a package has no main in its package.json
+    // so as a fallback, ask for node_modules paths and find its location manually
+    // https://github.com/ProjectEvergreen/greenwood/issues/557#issuecomment-923332104
+    const locations = require.resolve.paths(packageName);
+
+    for (const location in locations) {
+      const nodeModulesPackageRoot = `${locations[location]}/${packageName}`;
+      const packageJsonLocation = `${nodeModulesPackageRoot}/package.json`;
+
+      if (fs.existsSync(packageJsonLocation)) {
+        nodeModulesUrl = nodeModulesPackageRoot;
+      }
+    }
+
+    if (!nodeModulesUrl) {
+      console.debug(`Unable to look up ${packageName} using NodeJS require.resolve.`);
+    }
   }
 
   return nodeModulesUrl;
@@ -42,6 +59,6 @@ function getPackageNameFromUrl(url) {
 }
 
 module.exports = {
-  getNodeModulesResolveLocationForPackageName,
+  getNodeModulesLocationForPackage,
   getPackageNameFromUrl
 };

packages/cli/src/plugins/resource/plugin-node-modules.js

@@ -8,7 +8,7 @@ const fs = require('fs');
 const path = require('path');
 const { nodeResolve } = require('@rollup/plugin-node-resolve');
 const replace = require('@rollup/plugin-replace');
-const { getNodeModulesResolveLocationForPackageName, getPackageNameFromUrl } = require('../../lib/node-modules-utils');
+const { getNodeModulesLocationForPackage, getPackageNameFromUrl } = require('../../lib/node-modules-utils');
 const { ResourceInterface } = require('../../lib/resource-interface');
 const walk = require('acorn-walk');
 
@@ -33,7 +33,7 @@ const getPackageEntryPath = (packageJson) => {
         : 'index.js'; // lastly, fallback to index.js
 
   // use .mjs version if it exists, for packages like redux
-  if (!Array.isArray(entry) && fs.existsSync(`${getNodeModulesResolveLocationForPackageName(packageJson.name)}/${entry.replace('.js', '.mjs')}`)) {
+  if (!Array.isArray(entry) && fs.existsSync(`${getNodeModulesLocationForPackage(packageJson.name)}/${entry.replace('.js', '.mjs')}`)) {
     entry = entry.replace('.js', '.mjs');
   }
 
@@ -47,7 +47,7 @@ const walkModule = (module, dependency) => {
   }), {
     ImportDeclaration(node) {
       let { value: sourceValue } = node.source;
-      const absoluteNodeModulesLocation = getNodeModulesResolveLocationForPackageName(dependency);
+      const absoluteNodeModulesLocation = getNodeModulesLocationForPackage(dependency);
 
       if (path.extname(sourceValue) === '' && sourceValue.indexOf('http') !== 0 && sourceValue.indexOf('./') < 0) {        
         if (!importMap[sourceValue]) {
@@ -103,7 +103,7 @@ const walkPackageJson = (packageJson = {}) => {
     const isJavascriptPackage = Array.isArray(entry) || typeof entry === 'string' && entry.endsWith('.js') || entry.endsWith('.mjs');
 
     if (isJavascriptPackage) {
-      const absoluteNodeModulesLocation = getNodeModulesResolveLocationForPackageName(dependency);
+      const absoluteNodeModulesLocation = getNodeModulesLocationForPackage(dependency);
 
       // https://nodejs.org/api/packages.html#packages_determining_module_system
       if (Array.isArray(entry)) {
@@ -214,7 +214,7 @@ class NodeModulesResource extends ResourceInterface {
     const bareUrl = this.getBareUrlPath(url);
     const { projectDirectory } = this.compilation.context;
     const packageName = getPackageNameFromUrl(bareUrl);
-    const absoluteNodeModulesLocation = getNodeModulesResolveLocationForPackageName(packageName);
+    const absoluteNodeModulesLocation = getNodeModulesLocationForPackage(packageName);
     const packagePathPieces = bareUrl.split('node_modules/')[1].split('/'); // double split to handle node_modules within nested paths
     let absoluteNodeModulesUrl;
 

packages/cli/test/cases/develop.default/develop.default.spec.js

@@ -29,13 +29,45 @@
  * package.json
  */
 const expect = require('chai').expect;
+const fs = require('fs');
 const { JSDOM } = require('jsdom');
 const path = require('path');
 const { getDependencyFiles, getSetupFiles } = require('../../../../../test/utils');
 const request = require('request');
 const Runner = require('gallinago').Runner;
 const runSmokeTest = require('../../../../../test/smoke-test');
 
+async function rreaddir (dir, allFiles = []) {
+  const files = (await fs.promises.readdir(dir)).map(f => path.join(dir, f));
+
+  allFiles.push(...files);
+
+  await Promise.all(files.map(async f => (
+    await fs.promises.stat(f)).isDirectory() && rreaddir(f, allFiles
+  )));
+
+  return allFiles;
+}
+
+// https://stackoverflow.com/a/30405105/417806
+async function copyFile(source, target) {
+  const rd = fs.createReadStream(source);
+  const wr = fs.createWriteStream(target);
+
+  try {
+    return await new Promise((resolve, reject) => {
+      rd.on('error', reject);
+      wr.on('error', reject);
+      wr.on('finish', resolve);
+      rd.pipe(wr);
+    });
+  } catch (error) {
+    console.error('ERROR', error);
+    rd.destroy();
+    wr.end();
+  }
+}
+
 describe('Develop Greenwood With: ', function() {
   const LABEL = 'Default Greenwood Configuration and Workspace';
   const cliPath = path.join(process.cwd(), 'packages/cli/src/index.js');
@@ -166,6 +198,39 @@ describe('Develop Greenwood With: ', function() {
         `${process.cwd()}/node_modules/singleton-manager/package.json`,
         `${outputPath}/node_modules/singleton-manager/`
       );
+      const trustedTypesPackageJson = await getDependencyFiles(
+        `${process.cwd()}/node_modules/@types/trusted-types/package.json`,
+        `${outputPath}/node_modules/@types/trusted-types/`
+      );
+
+      // manually copy all these @babel/runtime files recursively since there are too many of them to do it individually
+      const babelRuntimeLibs = await rreaddir(`${process.cwd()}/node_modules/@babel/runtime`);
+
+      await fs.promises.mkdir(`${outputPath}/node_modules/@babel/runtime`, { recursive: true });
+      await fs.promises.copyFile(`${process.cwd()}/node_modules/@babel/runtime/package.json`, `${outputPath}/node_modules/@babel/runtime/package.json`);
+      await Promise.all(babelRuntimeLibs.filter((asset) => {
+        const target = asset.replace(process.cwd(), __dirname);
+        const isDirectory = path.extname(target) === '';
+
+        if (isDirectory && !fs.existsSync(target)) {
+          fs.mkdirSync(target);
+        } else if (!isDirectory) {
+          return asset;
+        }
+      }).map((asset) => {
+        const target = asset.replace(process.cwd(), __dirname);
+
+        return copyFile(asset, target);
+      }));
+
+      const regeneratorRuntimeLibs = await getDependencyFiles(
+        `${process.cwd()}/node_modules/regenerator-runtime/*.js`,
+        `${outputPath}/node_modules/regenerator-runtime/`
+      );
+      const regeneratorRuntimeLibsPackageJson = await getDependencyFiles(
+        `${process.cwd()}/node_modules/regenerator-runtime/package.json`,
+        `${outputPath}/node_modules/regenerator-runtime/`
+      );
 
       await runner.setup(outputPath, [
         ...getSetupFiles(outputPath),
@@ -196,7 +261,10 @@ describe('Develop Greenwood With: ', function() {
         ...messageFormatLibs,
         ...messageFormatLibsPackageJson,
         ...singletonManagerLibsPackageJson,
-        ...singletonManagerLibs
+        ...singletonManagerLibs,
+        ...trustedTypesPackageJson,
+        ...regeneratorRuntimeLibs,
+        ...regeneratorRuntimeLibsPackageJson
       ]);
 
       return new Promise(async (resolve) => {

packages/cli/test/cases/develop.default/package.json

@@ -1,8 +1,10 @@
 {
   "name": "test-develop-command-default",
   "dependencies": {
+    "@babel/runtime": "^7.10.4",
     "@lion/button": "~0.12.0",
     "@lion/calendar": "~0.15.0",
+    "@types/trusted-types": "^2.0.2",
     "lit-element": "^2.4.0"
   }
 }

packages/plugin-polyfills/src/index.js

@@ -1,5 +1,5 @@
 const fs = require('fs');
-const { getNodeModulesResolveLocationForPackageName } = require('@greenwood/cli/src/lib/node-modules-utils');
+const { getNodeModulesLocationForPackage } = require('@greenwood/cli/src/lib/node-modules-utils');
 const path = require('path');
 const { ResourceInterface } = require('@greenwood/cli/src/lib/resource-interface');
 
@@ -15,7 +15,7 @@ class PolyfillsResource extends ResourceInterface {
   async optimize(url, body) {
     const polyfillPackageName = '@webcomponents/webcomponentsjs';
     const filename = 'webcomponents-loader.js';
-    const polyfillNodeModulesLocation = getNodeModulesResolveLocationForPackageName(polyfillPackageName);
+    const polyfillNodeModulesLocation = getNodeModulesLocationForPackage(polyfillPackageName);
 
     return new Promise(async (resolve, reject) => {
       try {

yarn.lock

@@ -2600,6 +2600,11 @@
     "@types/mime" "^1"
     "@types/node" "*"
 
+"@types/trusted-types@^2.0.2":
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.2.tgz#fc25ad9943bcac11cceb8168db4f275e0e72e756"
+  integrity sha512-F5DIZ36YVLE+PN+Zwws4kJogq47hNgX3Nx6WyDJ3kcplxyke3XIzB8uK5n/Lpm1HBsbGzd6nmGehL8cPekP+Tg==
+
 "@types/ungap__global-this@^0.3.1":
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/@types/ungap__global-this/-/ungap__global-this-0.3.1.tgz#18ce9f657da556037a29d50604335614ce703f4c"