Correct .npmrc for newer versions of NPM

.github/workflows/integrationTests.yml

@@ -259,10 +259,10 @@ jobs:
         uses: gradle/gradle-build-action@v2
         with:
           arguments: clean build-info-extractor-maven3:test
-
+ 
   npm:
     needs: Pretest
-    name: npm (${{ matrix.os }})
+    name: npm (${{ matrix.os }}) < 8.19
     strategy:
       fail-fast: false
       matrix:
@@ -315,6 +315,61 @@ jobs:
           # We use localhost because npm does not support authenticating with registries started with 127.0.0.1
           BITESTS_PLATFORM_URL: http://localhost:8081
 
+  npm_9:
+    needs: Pretest
+    name: npm (${{ matrix.os }}) >= 8.19
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Install npm
+        uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+      - name: Config list
+        run: npm config ls -l
+      - name: Install Java
+        uses: actions/setup-java@v3
+        with:
+          java-version: "8"
+          distribution: "temurin"
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.x
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Setup Artifactory
+        run: |
+          go install github.com/jfrog/jfrog-testing-infra/local-rt-setup@latest
+          ~/go/bin/local-rt-setup
+        env:
+          RTLIC: ${{secrets.RTLIC}}
+          GOPROXY: direct
+
+      # Run tests
+      - name: Run Tests
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: clean build-info-extractor-npm:test
+        env:
+          # We use localhost because npm does not support authenticating with registries started with 127.0.0.1
+          BITESTS_PLATFORM_URL: http://localhost:8081
+
   NuGet:
     needs: Pretest
     name: NuGet (${{ matrix.os }})

.gitignore

@@ -18,3 +18,8 @@ atlassian-*.xml
 /itest/src/test/resources/org/jfrog/build/cache
 /build-info-extractor/src/test/resources/artifactory-bi.properties
 local.properties
+*.class
+/*/bin/test/org/jfrog/build
+/*/bin/test/*/snapshots/*.xml
+/*/bin/test/*/settings/build-info*.json
+/*/bin/test/*/pipLog.txt

build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/NpmDriver.java

@@ -131,4 +131,4 @@ private CommandResults runCommand(File workingDirectory, String[] args, List<Str
 
         return npmCommandRes;
     }
-}
+}

build-info-extractor-npm/src/main/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractor.java

@@ -5,6 +5,7 @@
 import org.jfrog.build.api.builder.ModuleType;
 import org.jfrog.build.api.util.Log;
 import org.jfrog.build.client.ProxyConfiguration;
+import org.jfrog.build.client.Version;
 import org.jfrog.build.extractor.BuildInfoExtractor;
 import org.jfrog.build.extractor.builder.ModuleBuilder;
 import org.jfrog.build.extractor.ci.BuildInfo;
@@ -189,9 +190,11 @@ private void createTempNpmrc(Path workingDir, List<String> commandArgs) throws I
             npmrcBuilder.append("proxy = ").append(this.npmProxy).append("\n");
         }
 
+        // Update Auth property for newer npm versions
+        handleNpmCompatibility(npmAuth, workingDir);
+
         // Save npm auth
         npmAuth.forEach((key, value) -> npmrcBuilder.append(key).append("=").append(value).append("\n"));
-
         // Write npmrc file
         try (FileWriter fileWriter = new FileWriter(npmrcPath.toFile());
              BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) {
@@ -200,6 +203,47 @@ private void createTempNpmrc(Path workingDir, List<String> commandArgs) throws I
         }
     }
 
+    /**
+     * Handles when Npm is at least version 8.19 the Auth related settings needing to be scoped to a specific registry.
+     * Results in transforming:
+     * 
+     * old NPMRC of :
+     * 
+     * registry=http://NO-NO-Repo/
+     * _auth={{AuthString}} 
+     * 
+     * 
+     * into the new NPMRC:
+     * 
+     * registry=http://NO-NO-Repo/
+     * //NO-NO-Repo/:_auth={{AuthString}}
+     * 
+     */
+    private void handleNpmCompatibility(Properties npmAuth, Path workingDir) throws IOException, InterruptedException{
+        Version npmVersion = new Version(this.npmDriver.version(workingDir.toFile()));
+        if(npmVersion.isAtLeast(new Version("8.19"))){
+            logger.debug("NPM version at least 8.19");
+            try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) {
+                String newAuthKey = artifactoryManager.getUrl();
+                    if (!StringUtils.endsWith(newAuthKey, "/")) {
+                        newAuthKey += "/";
+                    }
+                    newAuthKey += "api/npm/:";
+                    newAuthKey = newAuthKey.replaceAll("^http(s)?:","");
+
+                String[] checkList = { "_auth","_authToken","username","_password", "email", "certfile", "keyfile"};
+                for(String propKey: checkList){
+                    String prop = npmAuth.getProperty(propKey);
+                    if(prop != null){
+                        logger.debug("Found "+ propKey +", replacing with " + newAuthKey + propKey);
+                        npmAuth.setProperty(newAuthKey+propKey, prop);
+                        npmAuth.remove(propKey);
+                    }
+                }
+            }
+        }
+    }
+
     /**
      * Adds an array-value config to a StringBuilder of .npmrc file, in the following format:
      * key[] = value

build-info-extractor-npm/src/test/java/org/jfrog/build/extractor/npm/extractor/NpmBuildInfoExtractorTest.java

@@ -1,26 +1,64 @@
 package org.jfrog.build.extractor.npm.extractor;
 
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.jfrog.build.IntegrationTestsBase;
 import org.jfrog.build.extractor.builder.BuildInfoBuilder;
 import org.jfrog.build.extractor.builder.DependencyBuilder;
 import org.jfrog.build.extractor.builder.ModuleBuilder;
 import org.jfrog.build.extractor.ci.BuildInfo;
 import org.jfrog.build.extractor.ci.Dependency;
 import org.jfrog.build.extractor.ci.Module;
+import org.jfrog.build.extractor.npm.NpmDriver;
+import org.jfrog.build.extractor.npm.types.NpmProject;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.testng.Assert.*;
 
 import static org.jfrog.build.extractor.npm.extractor.NpmBuildInfoExtractor.getDependenciesMapFromBuild;
-import static org.testng.Assert.assertEquals;
 
 @Test
-public class NpmBuildInfoExtractorTest {
+public class NpmBuildInfoExtractorTest extends IntegrationTestsBase {
+
+    private static final String TEST_SPACE = "npm_test_space";
+    private static final File tempWorkspace = new File(System.getProperty("java.io.tmpdir"), TEST_SPACE);
+    private static final Path PROJECTS_ROOT = Paths.get(".").toAbsolutePath().normalize().resolve(Paths.get("src", "test", "resources", "org", "jfrog", "build", "extractor"));
+
+
+    private static final String NPM_LOCAL_REPO = "build-info-tests-npm-local";
+    private static final Set<String> DEV_SCOPE = Stream.of("dev").collect(Collectors.toSet());
+    private static final Set<String> PROD_SCOPE = Stream.of("prod").collect(Collectors.toSet());
+    private static final Set<String> DEV_PROD_SCOPE = Stream.of("prod", "dev").collect(Collectors.toSet());
+
+    @AfterMethod
+    protected void cleanup() throws IOException {
+        FileUtils.deleteDirectory(tempWorkspace);
+    }
 
+    @BeforeMethod
+    protected void init2() throws IOException {
+        FileUtils.forceMkdir(tempWorkspace);
+    }
     @DataProvider
     private Object[][] getDependenciesMapFromBuildProvider() {
         return new Object[][]{
@@ -122,4 +160,126 @@ private Module createTestModule(String id, List<Dependency> dependencies) {
             new DependencyBuilder().id("mod2dep1:2.1.0").sha1("sha1-mod2dep1").md5("md5-mod2dep1").build(),
             new DependencyBuilder().id("mod2dep2:2.2.0").sha1("sha1-mod2dep2").md5("md5-mod2dep2").build()
     };
+
+    private enum Project {
+        // Dependencies
+        ASGARD("asgard", "jfrog-asgard", "jfrog-asgard", "2.0.0", "a1fc28aa8733a161fa92d03379b71468d19292cd", "2fb7c420d2119831bc38559138d3444e"),
+        MIDGARD("midgard", "jfrog-midgard", "jfrog-midgard", "1.0.0", "547b8c7bb019863cc26438ef36e9b2d33668a626", "82f1558593727a7c89fb0b91859dab26"),
+        ALFHEIM("alfheim", "jfrog-alfheim", "jfrog-alfheim", "3.5.2", "f5592b523d2693649a94bbc2377cc653607a4053", "93e19985bb1c7c815abef052b67be244"),
+        SVARTALFHEIM("svartalfheim", "jfrog-svartalfheim", "jfrog-svartalfheim", "0.5.0", "473a5e001c67d716b8c9993245bd0ba2010c7374", "b1678118e32908b8e57f26fef1a23473"),
+
+        // Test projects
+        A("a", "NpmExtractorTest Project A", "package-name1", "v0.0.1", "", ""),
+        B("b", "NpmExtractorTest-Project-B", "package-name2", "0.0.2", "", ""),
+        C("c", "NpmExtractorTestProjectC", "package-name3", "=0.0.3", "", "");
+
+        private final File projectOrigin;
+        private final String targetDir;
+        private final String name;
+        private final String version;
+        private final String sha1;
+        private final String md5;
+
+        Project(String sourceDir, String targetDir, String name, String version, String sha1, String md5) {
+            this.projectOrigin = PROJECTS_ROOT.resolve(sourceDir).toFile();
+            this.targetDir = targetDir;
+            this.name = name;
+            this.version = version;
+            this.sha1 = sha1;
+            this.md5 = md5;
+        }
+
+        private String getModuleId() {
+            return String.format("%s:%s", name, version);
+        }
+
+        private String getPackedFileName() {
+            return String.format("%s-%s.tgz", name, version);
+        }
+
+        private String getDependencyId() {
+            return String.format("%s:%s", name, version);
+        }
+
+        private Dependency toDependency(String[][] requestedBy, Set<String> scope) {
+            return new DependencyBuilder().id(getDependencyId())
+                    .sha1(sha1)
+                    .md5(md5)
+                    .scopes(scope)
+                    .requestedBy(requestedBy)
+                    .build();
+        }
+
+        private String getRemotePath() {
+            return String.format("%s/-", name);
+        }
+
+        private String getTargetPath() {
+            return String.format("%s/%s", getRemotePath(), getPackedFileName());
+        }
+    }
+
+
+    @DataProvider
+    private Object[][] npmCiProvider() {
+        Dependency[] expectedDepsStep1 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"package-name1:v0.0.1"}}, PROD_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name1:v0.0.1"}}, PROD_SCOPE)};
+        Dependency[] expectedDepsStep2 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "@jscope/package-name2:0.0.2"}}, DEV_SCOPE), Project.MIDGARD.toDependency(new String[][]{{"@jscope/package-name2:0.0.2"}}, DEV_SCOPE), Project.ALFHEIM.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "@jscope/package-name2:0.0.2"}}, DEV_SCOPE)};
+        Dependency[] expectedDepsStep3 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "package-name3:=0.0.3"}, {"package-name3:=0.0.3"}}, DEV_PROD_SCOPE), Project.MIDGARD.toDependency(new String[][]{{"package-name3:=0.0.3"}}, DEV_SCOPE), Project.ALFHEIM.toDependency(new String[][]{{"jfrog-midgard:1.0.0", "package-name3:=0.0.3"}}, DEV_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE)};
+        Dependency[] expectedDepsStep4 = new Dependency[]{Project.ASGARD.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE), Project.SVARTALFHEIM.toDependency(new String[][]{{"package-name3:=0.0.3"}}, PROD_SCOPE)};
+        return new Object[][]{
+                {Project.A, expectedDepsStep1, "", true},
+                {Project.B, expectedDepsStep2, "", true},
+                {Project.B, new Dependency[]{}, "--production", false},
+                {Project.C, expectedDepsStep3, "", true},
+                {Project.C, expectedDepsStep4, "--only=production", true}
+        };
+    }
+
+    @SuppressWarnings("unused")
+    @Test(dataProvider = "npmCiProvider")
+    public void npmCiTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath) {
+        runNpmTest(project, expectedDependencies, args, packageJsonPath, true);
+    }
+
+    private void runNpmTest(Project project, Dependency[] expectedDependencies, String args, boolean packageJsonPath, boolean isNpmCi) {
+        args += " --verbose --no-audit";
+        Path projectDir = null;
+        try {
+            // Prepare.
+            projectDir = createProjectDir(project);
+            Path path = packageJsonPath ? projectDir.resolve("package.json") : projectDir;
+            if (isNpmCi) {
+                // Run npm install to generate package-lock.json file.
+                new NpmInstallCi(artifactoryManagerBuilder, localRepo1, args, log, path, null, null, null, false, null).execute();
+            }
+
+            NpmDriver driver = new NpmDriver(null);
+            List<String> commandArgs = StringUtils.isBlank(args) ? new ArrayList<>() : Arrays.asList(args.trim().split("\\s+"));
+            NpmProject proj = new NpmProject(commandArgs, localRepo1, path, isNpmCi);
+            // Execute command.
+
+            NpmBuildInfoExtractor buildExtractor = new NpmBuildInfoExtractor(artifactoryManagerBuilder, driver, log, null, null,null);
+            BuildInfo buildInfo = buildExtractor.extract(proj);
+
+            // Validate.
+            assertEquals(buildInfo.getModules().size(), 1);
+            Module module = buildInfo.getModules().get(0);
+            assertEquals(module.getType(), "npm");
+            assertEquals(module.getId(), project.getModuleId());
+            assertEqualsNoOrder(module.getDependencies().toArray(), expectedDependencies);
+        } catch (Exception e) {
+            fail(ExceptionUtils.getStackTrace(e));
+        } finally {
+            if (projectDir != null) {
+                FileUtils.deleteQuietly(projectDir.toFile());
+            }
+        }
+    }
+
+    private Path createProjectDir(Project project) throws IOException {
+        File projectDir = Files.createTempDirectory(project.targetDir).toFile();
+        FileUtils.copyDirectory(project.projectOrigin, projectDir);
+        return projectDir.toPath();
+    }
+
 }