Fix npm purl lookup in info (#217)

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

src/fetchcode/package.py

@@ -71,6 +71,21 @@ def get_pypi_codeview_url(project_urls):
     return code_view_url
 
 
+def get_npm_field_url(value):
+    """
+    Return a URL from an npm ``repository`` or ``bugs`` field.
+
+    The registry stores these as either a URL string or an object with a ``url``.
+    """
+    if isinstance(value, str):
+        return value or None
+    if isinstance(value, dict):
+        url = value.get("url")
+        if isinstance(url, str):
+            return url or None
+    return None
+
+
 @router.route("pkg:cargo/.*")
 def get_cargo_data_from_purl(purl):
     """
@@ -121,28 +136,37 @@ def get_npm_data_from_purl(purl):
     purl = PackageURL.from_string(purl)
     base_path = "http://registry.npmjs.org"
     name = purl.name
+    namespace = purl.namespace
+    # Scoped packages are published as @scope/name. Keep namespace and name
+    # separate so the emitted Package PURL stays pkg:npm/%40scope/name.
+    registry_name = f"{namespace}/{name}" if namespace else name
     version = purl.version
-    api_url = f"{base_path}/{name}"
+    api_url = f"{base_path}/{registry_name}"
 
     response = get_response(api_url)
-    vcs_data = response.get("repository") or {}
-    bugs = response.get("bugs") or {}
-    download_url = f"{base_path}/{name}/-/{name}-{version}.tgz" if version else None
-    vcs_url = vcs_data.get("url")
-    bug_tracking_url = bugs.get("url")
+    download_url = f"{base_path}/{registry_name}/-/{name}-{version}.tgz" if version else None
+    vcs_url = get_npm_field_url(response.get("repository"))
+    bug_tracking_url = get_npm_field_url(response.get("bugs"))
     license = response.get("license")
     homepage_url = response.get("homepage")
 
-    versions = response.get("versions", [])
+    versions = response.get("versions") or {}
     for num in versions:
         version = versions[num]
-        version_purl = PackageURL(type=purl.type, name=name, version=version.get("version"))
-        repository = version.get("repository") or {}
-        bugs = response.get("bugs") or {}
+        if not isinstance(version, dict):
+            continue
+        version_purl = PackageURL(
+            type=purl.type,
+            namespace=namespace,
+            name=name,
+            version=version.get("version"),
+        )
         dist = version.get("dist") or {}
-        vcs_url = repository.get("url")
+        if not isinstance(dist, dict):
+            dist = {}
+        vcs_url = get_npm_field_url(version.get("repository"))
         download_url = dist.get("tarball")
-        bug_tracking_url = bugs.get("url")
+        bug_tracking_url = get_npm_field_url(version.get("bugs") or response.get("bugs"))
         declared_license = license
 
         if purl.version and version_purl.version != purl.version:

tests/test_package.py

@@ -59,15 +59,85 @@ def test_cargo_packages(mock_get):
     check_packages(packages, expected_data)
 
 
+SCOPED_NPM_REGISTRY = {
+    "name": "@angular/core",
+    "repository": {"url": "git+https://github.com/angular/angular.git"},
+    "bugs": {"url": "https://github.com/angular/angular/issues"},
+    "license": "MIT",
+    "homepage": "https://angular.io",
+    "versions": {
+        "1.0.0": {
+            "version": "1.0.0",
+            "repository": {"type": "git", "url": "git+https://github.com/angular/angular.git"},
+            "dist": {"tarball": "https://registry.npmjs.org/@angular/core/-/core-1.0.0.tgz"},
+        },
+        "1.0.1": {
+            "version": "1.0.1",
+            "repository": {"type": "git", "url": "git+https://github.com/angular/angular.git"},
+            "dist": {"tarball": "https://registry.npmjs.org/@angular/core/-/core-1.0.1.tgz"},
+        },
+    },
+}
+
+
 @mock.patch("fetchcode.package.get_response")
-def test_npm_packages(mock_get):
-    side_effect = [load_json("tests/data/npm_mock_data.json")]
-    purl = "pkg:npm/express"
-    expected_data = "tests/data/npm.json"
-    mock_get.side_effect = side_effect
-    packages = list(info(purl))
+def test_npm_scoped_packages(mock_get):
+    mock_get.return_value = SCOPED_NPM_REGISTRY
+    packages = list(info("pkg:npm/%40angular/core"))
+
+    mock_get.assert_called_once_with("http://registry.npmjs.org/@angular/core")
+    assert [p.version for p in packages] == ["1.0.0", "1.0.1"]
+    assert all(p.namespace == "@angular" and p.name == "core" for p in packages)
+    assert packages[0].purl == "pkg:npm/%40angular/core@1.0.0"
+    assert packages[0].api_url == "http://registry.npmjs.org/@angular/core"
+    assert packages[0].download_url == "https://registry.npmjs.org/@angular/core/-/core-1.0.0.tgz"
+    assert packages[0].homepage_url == "https://angular.io"
+    assert packages[0].bug_tracking_url == "https://github.com/angular/angular/issues"
+    assert packages[0].vcs_url == "git+https://github.com/angular/angular.git"
+    assert packages[0].declared_license == "MIT"
 
-    check_packages(packages, expected_data)
+
+@mock.patch("fetchcode.package.get_response")
+def test_npm_scoped_package_with_version(mock_get):
+    mock_get.return_value = SCOPED_NPM_REGISTRY
+    packages = list(info("pkg:npm/%40angular/core@1.0.1"))
+
+    mock_get.assert_called_once_with("http://registry.npmjs.org/@angular/core")
+    assert len(packages) == 1
+    package = packages[0]
+    assert package.namespace == "@angular"
+    assert package.name == "core"
+    assert package.version == "1.0.1"
+    assert package.purl == "pkg:npm/%40angular/core@1.0.1"
+    assert package.download_url == "https://registry.npmjs.org/@angular/core/-/core-1.0.1.tgz"
+
+
+@mock.patch("fetchcode.package.get_response")
+def test_npm_string_repository_and_bugs(mock_get):
+    mock_get.return_value = {
+        "name": "@babel/core",
+        "repository": "https://github.com/babel/babel.git",
+        "bugs": "https://github.com/babel/babel/issues",
+        "license": "MIT",
+        "homepage": "https://babeljs.io",
+        "versions": {
+            "7.0.0": {
+                "version": "7.0.0",
+                "repository": "https://github.com/babel/babel.git",
+                "bugs": "https://github.com/babel/babel/issues",
+                "dist": {"tarball": "https://registry.npmjs.org/@babel/core/-/core-7.0.0.tgz"},
+            }
+        },
+    }
+    packages = list(info("pkg:npm/%40babel/core@7.0.0"))
+
+    assert len(packages) == 1
+    package = packages[0]
+    assert package.namespace == "@babel"
+    assert package.name == "core"
+    assert package.vcs_url == "https://github.com/babel/babel.git"
+    assert package.bug_tracking_url == "https://github.com/babel/babel/issues"
+    assert package.download_url == "https://registry.npmjs.org/@babel/core/-/core-7.0.0.tgz"
 
 
 @mock.patch("fetchcode.package.get_response")