diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParser.java index 458f7c676a..d1f1982aab 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParser.java @@ -10,6 +10,8 @@ import org.apache.commons.collections4.Predicate; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.blackduck.integration.detectable.detectable.codelocation.CodeLocation; import com.blackduck.integration.detectable.detectables.pnpm.lockfile.model.PnpmLockYaml; @@ -19,6 +21,8 @@ import com.blackduck.integration.util.NameVersion; public class PnpmLockYamlParser { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + private static final Predicate isNodeRoot = "."::equals; private PnpmYamlTransformer pnpmTransformer; @@ -41,6 +45,9 @@ public List parse(File parentFile, PnpmLockYaml pnpmLockYaml, private List createCodeLocationsFromRoot(File sourcePath, PnpmLockYaml pnpmLockYaml, @Nullable NameVersion projectNameVersion, PnpmLinkedPackageResolver linkedPackageResolver) throws IntegrationException { + if (pnpmLockYaml.packages == null) { + logger.warn("The pnpm-lock.yaml file has no 'packages' section. No resolved dependencies are present. The scan will continue with an empty dependency graph."); + } CodeLocation codeLocation = pnpmTransformer.generateCodeLocation(sourcePath, pnpmLockYaml, projectNameVersion, linkedPackageResolver); return Collections.singletonList(codeLocation); @@ -52,7 +59,14 @@ private List createCodeLocationsFromImports(File sourcePath, PnpmL if (MapUtils.isEmpty(pnpmLockYaml.importers)) { return Collections.emptyList(); } - + + if (pnpmLockYaml.packages == null) { + logger.warn("The pnpm-lock.yaml file contains {} importer(s) {} but has no 'packages' section. " + + "No resolved dependencies are available. All workspaces will have empty dependency graphs.", + pnpmLockYaml.importers.size(), + pnpmLockYaml.importers.keySet()); + } + ExcludedIncludedWildcardFilter workspacesFilter; if (excludedDirectories.isEmpty() && includedDirectories.isEmpty()) { workspacesFilter = null; // Include all diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParserInitial.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParserInitial.java index dfaa5cba88..7f534ad7e6 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParserInitial.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParserInitial.java @@ -4,9 +4,12 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.util.Collections; import java.util.List; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; @@ -29,6 +32,8 @@ */ public class PnpmLockYamlParserInitial { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + private final EnumListFilter dependencyFilter; private final List excludedDirectories; private final List includedDirectories; @@ -42,8 +47,19 @@ public PnpmLockYamlParserInitial(PnpmLockOptions pnpmLockOptions) { public List parse(File pnpmLockYamlFile, @Nullable NameVersion projectNameVersion, PnpmLinkedPackageResolver linkedPackageResolver) throws IOException, IntegrationException { PnpmLockYamlBase pnpmLockYaml = parseYamlFile(pnpmLockYamlFile); - + + if (pnpmLockYaml == null) { + logger.warn("The pnpm-lock.yaml file '{}' is empty and contains no parsable content. No dependencies will be extracted.", + pnpmLockYamlFile.getAbsolutePath()); + return Collections.emptyList(); + } + if (pnpmLockYaml instanceof PnpmLockYaml) { + if (pnpmLockYaml.lockfileVersion == null) { + throw new IntegrationException( + "The pnpm-lock.yaml file does not contain a 'lockfileVersion' field. " + + "This is required for parsing. Please regenerate the lock file by running 'pnpm install'."); + } PnpmYamlTransformer pnpmYamlTransformer = new PnpmYamlTransformer(dependencyFilter, pnpmLockYaml.lockfileVersion); PnpmLockYamlParser pnpmYamlParser = new PnpmLockYamlParser(pnpmYamlTransformer); return pnpmYamlParser.parse(pnpmLockYamlFile.getParentFile(), (PnpmLockYaml) pnpmLockYaml, linkedPackageResolver, projectNameVersion, excludedDirectories, includedDirectories); @@ -70,15 +86,41 @@ private PnpmLockYamlBase parseYamlFile(File pnpmLockYamlFile) throws FileNotFoun representer.getPropertyUtils().setSkipMissingProperties(true); LoaderOptions loaderOptions = new LoaderOptions(); + try { - // Try to read the lockfile into the current Yaml classes. It's more common and - // should hopefully work more of the time. + // Try to read the lockfile into the v6/v9 Yaml classes first (more common). + logger.debug("Parsing through v6/v9 format"); Yaml yaml = new Yaml(new Constructor(PnpmLockYaml.class, loaderOptions), representer); - return yaml.load(new FileReader(pnpmLockYamlFile)); + PnpmLockYamlBase result = yaml.load(new FileReader(pnpmLockYamlFile)); + + // If we got a valid result with a v6+ lockfileVersion, use it. + if (result != null && isV6OrNewer(result.lockfileVersion)) { + return result; + } } catch (ConstructorException e) { - // If the reading fails try to read a v5 Yaml. - Yaml yaml = new Yaml(new Constructor(PnpmLockYamlv5.class, loaderOptions), representer); - return yaml.load(new FileReader(pnpmLockYamlFile)); + // Fall through to try v5 parsing + } + + // Either: lockfileVersion was null, indicated v5, or a ConstructorException was thrown. + // Re-parse as v5. + logger.debug("Parsing through v5 format"); + Yaml yaml = new Yaml(new Constructor(PnpmLockYamlv5.class, loaderOptions), representer); + return yaml.load(new FileReader(pnpmLockYamlFile)); + } + + /** + * Returns true if the lockfileVersion string represents a v6 or newer pnpm lockfile. + * Returns false if the version is v5 or older, or null. + */ + private boolean isV6OrNewer(@Nullable String lockfileVersion) { + if (lockfileVersion == null) { + return false; + } + try { + return Double.parseDouble(lockfileVersion) >= 6.0; + } catch (NumberFormatException e) { + logger.debug("Unable to parse lockfileVersion '{}'; treating as v5/unknown.", lockfileVersion); + return false; } } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParserv5.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParserv5.java index 265c14439e..c364c6c07c 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParserv5.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmLockYamlParserv5.java @@ -47,6 +47,9 @@ private List createCodeLocationsFromRoot( @Nullable NameVersion projectNameVersion, PnpmLinkedPackageResolver linkedPackageResolver ) throws IntegrationException { + if (pnpmLockYaml.packages == null) { + logger.warn("The pnpm-lock.yaml file has no 'packages' section. No resolved dependencies are present. The scan will continue with an empty dependency graph."); + } CodeLocation codeLocation = pnpmTransformer.generateCodeLocation(sourcePath, pnpmLockYaml, projectNameVersion, linkedPackageResolver); return Collections.singletonList(codeLocation); } @@ -61,6 +64,13 @@ private List createCodeLocationsFromImports( return Collections.emptyList(); } + if (pnpmLockYaml.packages == null) { + logger.warn("The pnpm-lock.yaml file contains {} importer(s) {} but has no 'packages' section. " + + "No resolved dependencies are available. All workspaces will have empty dependency graphs.", + pnpmLockYaml.importers.size(), + pnpmLockYaml.importers.keySet()); + } + List codeLocations = new LinkedList<>(); for (Map.Entry projectPackageInfo : pnpmLockYaml.importers.entrySet()) { String projectKey = projectPackageInfo.getKey(); diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmYamlTransformer.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmYamlTransformer.java index a97993154d..efeccc0f1b 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmYamlTransformer.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmYamlTransformer.java @@ -19,7 +19,6 @@ import com.blackduck.integration.bdio.model.dependency.Dependency; import com.blackduck.integration.bdio.model.externalid.ExternalId; import com.blackduck.integration.detectable.detectable.codelocation.CodeLocation; -import com.blackduck.integration.detectable.detectable.exception.DetectableException; import com.blackduck.integration.detectable.detectable.util.EnumListFilter; import com.blackduck.integration.detectable.detectables.pnpm.lockfile.model.PnpmDependencyInfo; import com.blackduck.integration.detectable.detectables.pnpm.lockfile.model.PnpmDependencyType; @@ -80,7 +79,9 @@ private void buildGraph( @Nullable Map snapshots ) throws IntegrationException { if (packageMap == null) { - throw new DetectableException("Could not parse 'packages' section of the pnpm-lock.yaml file."); + logger.debug("No packages available to build dependency graph for workspace '{}'. Skipping graph construction.", + reportingProjectPackagePath != null ? reportingProjectPackagePath : "root"); + return; } for (Map.Entry packageEntry : packageMap.entrySet()) { String packageId = packageEntry.getKey(); diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmYamlTransformerv5.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmYamlTransformerv5.java index 488fd64e87..c8db5e64fd 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmYamlTransformerv5.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/pnpm/lockfile/process/PnpmYamlTransformerv5.java @@ -18,7 +18,6 @@ import com.blackduck.integration.bdio.model.dependency.Dependency; import com.blackduck.integration.bdio.model.externalid.ExternalId; import com.blackduck.integration.detectable.detectable.codelocation.CodeLocation; -import com.blackduck.integration.detectable.detectable.exception.DetectableException; import com.blackduck.integration.detectable.detectable.util.EnumListFilter; import com.blackduck.integration.detectable.detectables.pnpm.lockfile.model.PnpmDependencyType; import com.blackduck.integration.detectable.detectables.pnpm.lockfile.model.PnpmLockYamlv5; @@ -73,7 +72,9 @@ private void buildGraph( @Nullable String reportingProjectPackagePath ) throws IntegrationException { if (packageMap == null) { - throw new DetectableException("Could not parse 'packages' section of the pnpm-lock.yaml file."); + logger.debug("No packages available to build dependency graph for workspace '{}'. Skipping graph construction.", + reportingProjectPackagePath != null ? reportingProjectPackagePath : "root"); + return; } for (Map.Entry packageEntry : packageMap.entrySet()) { String packageId = packageEntry.getKey();