Merge remote-tracking branch 'upstream/dev' into recursive_audit_scan

artifactory/commands/transferfiles/delayedartifactshandler.go

@@ -89,19 +89,25 @@ type DelayedArtifactsFile struct {
 }
 
 // Collect all the delayed artifact files that were created up to this point for the repository and transfer their artifacts using handleDelayedArtifactsFiles
-func consumeAllDelayFiles(base phaseBase) error {
+func consumeAllDelayFiles(base phaseBase) (err error) {
 	filesToConsume, err := getDelayFiles([]string{base.repoKey})
-	if err != nil {
-		return err
+	if err != nil || len(filesToConsume) == 0 {
+		return
 	}
 	delayFunctions := getDelayUploadComparisonFunctions(base.repoSummary.PackageType)
-	if len(filesToConsume) > 0 && len(delayFunctions) > 0 {
-		log.Info("Starting to handle delayed artifacts uploads...")
-		if err = handleDelayedArtifactsFiles(filesToConsume, base, delayFunctions[1:]); err == nil {
-			log.Info("Done handling delayed artifacts uploads.")
-		}
+	if len(delayFunctions) == 0 {
+		return
+	}
+
+	log.Info("Starting to handle delayed artifacts uploads...")
+	// Each delay function causes the transfer to skip a specific group of files.
+	// Within the handleDelayedArtifactsFiles function, we recursively remove the first delay function from the slice to transfer the first set of files every time.
+	if err = handleDelayedArtifactsFiles(filesToConsume, base, delayFunctions[1:]); err != nil {
+		return
 	}
-	return err
+
+	log.Info("Done handling delayed artifacts uploads.")
+	return deleteAllFiles(filesToConsume)
 }
 
 // Call consumeAllDelayFiles only if there are no failed transferred files for the repository up to this point.
@@ -182,13 +188,6 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon
 		if err = base.stateManager.ChangeDelayedFilesCountBy(uint64(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil {
 			log.Warn("Couldn't decrease the delayed files counter", err.Error())
 		}
-
-		// Remove the file, so it won't be consumed again.
-		if err = os.Remove(filePath); err != nil {
-			return errorutils.CheckError(err)
-		}
-
-		log.Debug("Done handling delayed artifacts file: '" + filePath + "'. Deleting it...")
 	}
 	return nil
 }

artifactory/commands/transferfiles/fileserror.go

@@ -3,13 +3,11 @@ package transferfiles
 import (
 	"errors"
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
 	"github.com/jfrog/gofrog/parallel"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles/api"
-	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 )
 
@@ -33,7 +31,8 @@ func (e *errorsRetryPhase) run() error {
 // Does so by creating and uploading by chunks, and polling on status.
 // Consumed errors files are deleted, new failures are written to new files.
 func (e *errorsRetryPhase) handlePreviousUploadFailures() error {
-	if len(e.errorsFilesToHandle) == 0 {
+	errorsFilesToHandle := e.errorsFilesToHandle
+	if len(errorsFilesToHandle) == 0 {
 		return nil
 	}
 	log.Info("Starting to handle previous upload failures...")
@@ -46,11 +45,13 @@ func (e *errorsRetryPhase) handlePreviousUploadFailures() error {
 	delayAction := func(phase phaseBase, addedDelayFiles []string) error {
 		return consumeAllDelayFiles(phase)
 	}
-	err := e.transferManager.doTransferWithProducerConsumer(action, delayAction)
-	if err == nil {
-		log.Info("Done handling previous upload failures.")
+
+	if err := e.transferManager.doTransferWithProducerConsumer(action, delayAction); err != nil {
+		return err
 	}
-	return err
+
+	log.Info("Done handling previous upload failures.")
+	return deleteAllFiles(errorsFilesToHandle)
 }
 
 func convertUploadStatusToFileRepresentation(statuses []ExtendedFileUploadStatusResponse) (files []api.FileRepresentation) {
@@ -83,24 +84,13 @@ func (e *errorsRetryPhase) handleErrorsFile(errFilePath string, pcWrapper *produ
 	}
 
 	// Upload
-	shouldStop, err := uploadByChunks(convertUploadStatusToFileRepresentation(failedFiles.Errors), uploadChunkChan, e.phaseBase, delayHelper, errorsChannelMng, pcWrapper)
-	if err != nil || shouldStop {
-		return err
-	}
-
-	// Remove the file, so it won't be consumed again.
-	err = os.Remove(errFilePath)
-	if err != nil {
-		return errorutils.CheckError(err)
-	}
-
-	log.Debug("Done handling errors file: '", errFilePath, "'. Deleting it...")
-	return nil
+	_, err = uploadByChunks(convertUploadStatusToFileRepresentation(failedFiles.Errors), uploadChunkChan, e.phaseBase, delayHelper, errorsChannelMng, pcWrapper)
+	return err
 }
 
 func (e *errorsRetryPhase) createErrorFilesHandleFunc(pcWrapper *producerConsumerWrapper, uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) errorFileHandlerFunc {
 	return func() parallel.TaskFunc {
-		return func(threadId int) error {
+		return func(int) error {
 			var errList []string
 			var err error
 			for _, errFile := range e.errorsFilesToHandle {

artifactory/commands/transferfiles/utils.go

@@ -735,3 +735,11 @@ func getUniqueErrorOrDelayFilePath(dirPath string, getFileNamePrefix func() stri
 	}
 	return
 }
+
+func deleteAllFiles(filesToDelete []string) (err error) {
+	for _, fileToDelete := range filesToDelete {
+		log.Debug("Deleting:", fileToDelete, "...")
+		err = errors.Join(err, errorutils.CheckError(os.Remove(fileToDelete)))
+	}
+	return
+}

artifactory/utils/projectconfig.go

@@ -2,16 +2,15 @@ package utils
 
 import (
 	"fmt"
-	"path/filepath"
-	"reflect"
-
-	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
-
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
+	xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 	"github.com/spf13/viper"
+	"path/filepath"
+	"reflect"
 )
 
 const (
@@ -38,6 +37,11 @@ const (
 	Terraform
 )
 
+// Associates a technology with another of a different type in the structure.
+// Docker is not present, as there is no docker-config command and, consequently, no docker.yaml file we need to operate on.
+var techType = map[coreutils.Technology]ProjectType{coreutils.Maven: Maven, coreutils.Gradle: Gradle, coreutils.Npm: Npm, coreutils.Yarn: Yarn, coreutils.Go: Go, coreutils.Pip: Pip,
+	coreutils.Pipenv: Pipenv, coreutils.Poetry: Poetry, coreutils.Nuget: Nuget, coreutils.Dotnet: Dotnet}
+
 var ProjectTypes = []string{
 	"go",
 	"pip",
@@ -187,3 +191,27 @@ func ReadResolutionOnlyConfiguration(confFilePath string) (*RepositoryConfig, er
 	}
 	return GetRepoConfigByPrefix(confFilePath, ProjectConfigResolverPrefix, vConfig)
 }
+
+// Verifies the existence of depsRepo. If it doesn't exist, it searches for a configuration file based on the technology type. If found, it assigns depsRepo in the AuditParams.
+func SetResolutionRepoIfExists(params xrayutils.AuditParams, tech coreutils.Technology) (err error) {
+	if params.DepsRepo() != "" {
+		return
+	}
+	configFilePath, exists, err := GetProjectConfFilePath(techType[tech])
+	if err != nil {
+		err = fmt.Errorf("failed while searching for %s.yaml config file: %s", tech.String(), err.Error())
+		return
+	}
+	if !exists {
+		log.Debug(fmt.Sprintf("No %s.yaml configuration file was found. Resolving dependencies from %s default registry", tech.String(), tech.String()))
+		return
+	}
+
+	repoConfig, err := ReadResolutionOnlyConfiguration(configFilePath)
+	if err != nil {
+		err = fmt.Errorf("failed while reading %s.yaml config file: %s", tech.String(), err.Error())
+		return
+	}
+	params.SetDepsRepo(repoConfig.targetRepo)
+	return
+}

go.mod

@@ -101,4 +101,4 @@ require (
 
 replace github.com/jfrog/jfrog-client-go => github.com/attiasas/jfrog-client-go v0.0.0-20231025121226-04e810f6f34a
 
-// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231019085746-e1b192457664
+replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231031143744-13f94ab07bbc

go.sum

@@ -198,8 +198,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jedib0t/go-pretty/v6 v6.4.8 h1:HiNzyMSEpsBaduKhmK+CwcpulEeBrTmxutz4oX/oWkg=
 github.com/jedib0t/go-pretty/v6 v6.4.8/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
-github.com/jfrog/build-info-go v1.9.14 h1:xVezJ16Vpm/boRBn3lI1THCQmkylm+6R4zYWxOQ0NSM=
-github.com/jfrog/build-info-go v1.9.14/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg=
+github.com/jfrog/build-info-go v1.8.9-0.20231031143744-13f94ab07bbc h1:MFejgCB90z7nA/KP48lF1t04tYuXAAQc53cBaFd9zcw=
+github.com/jfrog/build-info-go v1.8.9-0.20231031143744-13f94ab07bbc/go.mod h1:ujJ8XQZMdT2tMkLSMJNyDd1pCY+duwHdjV+9or9FLIg=
 github.com/jfrog/gofrog v1.3.1 h1:QqAwQXCVReT724uga1AYqG/ZyrNQ6f+iTxmzkb+YFQk=
 github.com/jfrog/gofrog v1.3.1/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0=
 github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=

xray/commands/audit/auditparams.go

@@ -92,3 +92,8 @@ func (params *AuditParams) SetThirdPartyApplicabilityScan(includeThirdPartyDeps
 	params.thirdPartyApplicabilityScan = includeThirdPartyDeps
 	return params
 }
+
+func (params *AuditParams) SetDepsRepo(depsRepo string) *AuditParams {
+	params.AuditBasicParams.SetDepsRepo(depsRepo)
+	return params
+}

xray/commands/audit/sca/npm/npm.go

@@ -1,8 +1,11 @@
 package npm
 
 import (
+	"errors"
+	"fmt"
 	biutils "github.com/jfrog/build-info-go/build/utils"
 	buildinfo "github.com/jfrog/build-info-go/entities"
+	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/npm"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca"
 	"github.com/jfrog/jfrog-cli-core/v2/xray/utils"
@@ -31,6 +34,17 @@ func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils
 
 	treeDepsParam := createTreeDepsParam(params)
 
+	restoreNpmrcFunc, err := configNpmResolutionServerIfNeeded(params)
+	if err != nil {
+		err = fmt.Errorf("failed while configuring a resolution server: %s", err.Error())
+		return
+	}
+	defer func() {
+		if restoreNpmrcFunc != nil {
+			err = errors.Join(err, restoreNpmrcFunc())
+		}
+	}()
+
 	// Calculate npm dependencies
 	dependenciesMap, err := biutils.CalculateDependenciesMap(npmExecutablePath, currentDir, packageInfo.BuildInfoModuleId(), treeDepsParam, log.Logger)
 	if err != nil {
@@ -47,14 +61,41 @@ func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils
 	return
 }
 
+// Generates a .npmrc file to configure an Artifactory server as the resolver server.
+func configNpmResolutionServerIfNeeded(params utils.AuditParams) (restoreNpmrcFunc func() error, err error) {
+	if params == nil {
+		err = fmt.Errorf("got empty params upon configuring resolution server")
+		return
+	}
+	serverDetails, err := params.ServerDetails()
+	if err != nil || serverDetails == nil {
+		return
+	}
+	depsRepo := params.DepsRepo()
+	if depsRepo == "" {
+		return
+	}
+
+	npmCmd := npm.NewNpmCommand("install", false).SetServerDetails(serverDetails)
+	if err = npmCmd.PreparePrerequisites(depsRepo); err != nil {
+		return
+	}
+	if err = npmCmd.CreateTempNpmrc(); err != nil {
+		return
+	}
+	restoreNpmrcFunc = npmCmd.RestoreNpmrcFunc()
+	return
+}
+
 func createTreeDepsParam(params utils.AuditParams) biutils.NpmTreeDepListParam {
 	if params == nil {
 		return biutils.NpmTreeDepListParam{
 			Args: addIgnoreScriptsFlag([]string{}),
 		}
 	}
 	npmTreeDepParam := biutils.NpmTreeDepListParam{
-		Args: addIgnoreScriptsFlag(params.Args()),
+		Args:               addIgnoreScriptsFlag(params.Args()),
+		InstallCommandArgs: params.InstallCommandArgs(),
 	}
 	if npmParams, ok := params.(utils.AuditNpmParams); ok {
 		npmTreeDepParam.IgnoreNodeModules = npmParams.NpmIgnoreNodeModules()

xray/commands/audit/sca/npm/npm_test.go

@@ -116,6 +116,7 @@ func TestIgnoreScripts(t *testing.T) {
 
 	// The package.json file contain a postinstall script running an "exit 1" command.
 	// Without the "--ignore-scripts" flag, the test will fail.
-	_, _, err := BuildDependencyTree(nil)
+	params := &utils.AuditBasicParams{}
+	_, _, err := BuildDependencyTree(params)
 	assert.NoError(t, err)
 }

xray/commands/audit/scarunner.go

@@ -10,6 +10,7 @@ import (
 	"github.com/jfrog/build-info-go/utils/pythonutils"
 	"github.com/jfrog/gofrog/datastructures"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
+	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca"
 	_go "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca/go"
@@ -203,6 +204,10 @@ func GetTechDependencyTree(params xrayutils.AuditParams, tech coreutils.Technolo
 	if err != nil {
 		return
 	}
+	err = utils.SetResolutionRepoIfExists(params, tech)
+	if err != nil {
+		return
+	}
 	var uniqueDeps []string
 	startTime := time.Now()
 	switch tech {

xray/utils/auditbasicparams.go

@@ -23,6 +23,7 @@ type AuditParams interface {
 	Progress() ioUtils.ProgressMgr
 	SetProgress(progress ioUtils.ProgressMgr)
 	Args() []string
+	InstallCommandArgs() []string
 	SetNpmScope(depType string) *AuditBasicParams
 	OutputFormat() OutputFormat
 	DepsRepo() string
@@ -44,6 +45,7 @@ type AuditBasicParams struct {
 	args                             []string
 	depsRepo                         string
 	ignoreConfigFile                 bool
+	installCommandArgs               []string
 }
 
 func (abp *AuditBasicParams) DirectDependencies() []string {
@@ -64,6 +66,10 @@ func (abp *AuditBasicParams) SetServerDetails(serverDetails *config.ServerDetail
 	return abp
 }
 
+func (abp *AuditBasicParams) SetInstallCommandArgs(installCommandArgs []string) *AuditBasicParams {
+	abp.installCommandArgs = installCommandArgs
+	return abp
+}
 func (abp *AuditBasicParams) PipRequirementsFile() string {
 	return abp.pipRequirementsFile
 }
@@ -121,6 +127,10 @@ func (abp *AuditBasicParams) Args() []string {
 	return abp.args
 }
 
+func (abp *AuditBasicParams) InstallCommandArgs() []string {
+	return abp.installCommandArgs
+}
+
 func (abp *AuditBasicParams) SetNpmScope(depType string) *AuditBasicParams {
 	switch depType {
 	case "devOnly":