diff --git a/buildscripts/download-jars.sh b/buildscripts/download-jars.sh index e5e6a5510..3e9f10a89 100755 --- a/buildscripts/download-jars.sh +++ b/buildscripts/download-jars.sh @@ -7,7 +7,7 @@ # https://github.com/jfrog/maven-dep-tree # Once you have updated the versions mentioned below, please execute this script from the root directory of the jfrog-cli-core to ensure the JAR files are updated. -GRADLE_DEP_TREE_VERSION="2.2.0" +GRADLE_DEP_TREE_VERSION="3.0.0" MAVEN_DEP_TREE_VERSION="1.0.0" curl -fL https://releases.jfrog.io/artifactory/oss-release-local/com/jfrog/gradle-dep-tree/${GRADLE_DEP_TREE_VERSION}/gradle-dep-tree-${GRADLE_DEP_TREE_VERSION}.jar -o xray/commands/audit/sca/java/gradle-dep-tree.jar diff --git a/xray/commands/audit/sca/java/gradle-dep-tree.jar b/xray/commands/audit/sca/java/gradle-dep-tree.jar index 2762a71ba..532445bc1 100644 Binary files a/xray/commands/audit/sca/java/gradle-dep-tree.jar and b/xray/commands/audit/sca/java/gradle-dep-tree.jar differ diff --git a/xray/commands/audit/sca/java/gradle.go b/xray/commands/audit/sca/java/gradle.go index d140efffd..933788192 100644 --- a/xray/commands/audit/sca/java/gradle.go +++ b/xray/commands/audit/sca/java/gradle.go @@ -2,11 +2,8 @@ package java import ( _ "embed" - "encoding/base64" - "encoding/json" "errors" "fmt" - "github.com/jfrog/gofrog/datastructures" "os" "os/exec" "path/filepath" @@ -58,60 +55,12 @@ allprojects { var gradleDepTreeJar []byte type depTreeManager struct { - dependenciesTree server *config.ServerDetails releasesRepo string depsRepo string useWrapper bool } -// dependenciesTree represents a map between dependencies to their children dependencies in multiple projects. -type dependenciesTree struct { - tree map[string][]dependenciesPaths -} - -// dependenciesPaths represents a map between dependencies to their children dependencies in a single project. -type dependenciesPaths struct { - Paths map[string]dependenciesPaths `json:"children"` -} - -// The gradle-dep-tree generates a JSON representation for the dependencies for each gradle build file in the project. -// parseDepTreeFiles iterates over those JSONs, and append them to the map of dependencies in dependenciesTree struct. -func (dtp *depTreeManager) parseDepTreeFiles(jsonFiles []byte) error { - outputFiles := strings.Split(strings.TrimSpace(string(jsonFiles)), "\n") - for _, path := range outputFiles { - tree, err := os.ReadFile(strings.TrimSpace(path)) - if err != nil { - return errorutils.CheckError(err) - } - - encodedFileName := path[strings.LastIndex(path, string(os.PathSeparator))+1:] - decodedFileName, err := base64.StdEncoding.DecodeString(encodedFileName) - if err != nil { - return errorutils.CheckError(err) - } - - if err = dtp.appendDependenciesPaths(tree, string(decodedFileName)); err != nil { - return errorutils.CheckError(err) - } - } - return nil -} - -func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName string) error { - var deps dependenciesPaths - if err := json.Unmarshal(jsonDepTree, &deps); err != nil { - return errorutils.CheckError(err) - } - if dtp.tree == nil { - dtp.tree = make(map[string][]dependenciesPaths) - } - if len(deps.Paths) > 0 { - dtp.tree[fileName] = append(dtp.tree[fileName], deps) - } - return nil -} - func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []*xrayUtils.GraphNode, uniqueDeps []string, err error) { manager := &depTreeManager{useWrapper: params.UseWrapper} if params.IgnoreConfigFile { @@ -130,7 +79,7 @@ func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []* if err != nil { return } - dependencyTree, uniqueDeps, err = manager.getGraphFromDepTree(outputFileContent) + dependencyTree, uniqueDeps, err = getGraphFromDepTree(outputFileContent) return } @@ -163,7 +112,7 @@ func (dtp *depTreeManager) createDepTreeScriptAndGetDir() (tmpDir string, err er if err != nil { return } - gradleDepTreeJarPath := filepath.Join(tmpDir, string(gradleDepTreeJarFile)) + gradleDepTreeJarPath := filepath.Join(tmpDir, gradleDepTreeJarFile) if err = errorutils.CheckError(os.WriteFile(gradleDepTreeJarPath, gradleDepTreeJar, 0666)); err != nil { return } @@ -237,42 +186,6 @@ func (dtp *depTreeManager) execGradleDepTree(depTreeDir string) (outputFileConte return } -// Assuming we ran gradle-dep-tree, getGraphFromDepTree receives the content of the depTreeOutputFile as input -func (dtp *depTreeManager) getGraphFromDepTree(outputFileContent []byte) ([]*xrayUtils.GraphNode, []string, error) { - if err := dtp.parseDepTreeFiles(outputFileContent); err != nil { - return nil, nil, err - } - var depsGraph []*xrayUtils.GraphNode - uniqueDepsSet := datastructures.MakeSet[string]() - for dependency, children := range dtp.tree { - directDependency := &xrayUtils.GraphNode{ - Id: GavPackageTypeIdentifier + dependency, - Nodes: []*xrayUtils.GraphNode{}, - } - for _, childPath := range children { - populateGradleDependencyTree(directDependency, childPath, uniqueDepsSet) - } - depsGraph = append(depsGraph, directDependency) - } - return depsGraph, uniqueDepsSet.ToSlice(), nil -} - -func populateGradleDependencyTree(currNode *xrayUtils.GraphNode, currNodeChildren dependenciesPaths, uniqueDepsSet *datastructures.Set[string]) { - uniqueDepsSet.Add(currNode.Id) - for gav, children := range currNodeChildren.Paths { - childNode := &xrayUtils.GraphNode{ - Id: GavPackageTypeIdentifier + gav, - Nodes: []*xrayUtils.GraphNode{}, - Parent: currNode, - } - if currNode.NodeHasLoop() { - return - } - populateGradleDependencyTree(childNode, children, uniqueDepsSet) - currNode.Nodes = append(currNode.Nodes, childNode) - } -} - func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDetails) (string, error) { if remoteRepo == "" || server.IsEmpty() { return "", nil diff --git a/xray/commands/audit/sca/java/gradle_test.go b/xray/commands/audit/sca/java/gradle_test.go index e9d09b742..27c4aaaa0 100644 --- a/xray/commands/audit/sca/java/gradle_test.go +++ b/xray/commands/audit/sca/java/gradle_test.go @@ -47,10 +47,10 @@ func TestGradleTreesWithoutConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, uniqueDeps, 11) - assert.Len(t, modulesDependencyTrees, 2) + assert.Len(t, uniqueDeps, 9) + assert.Len(t, modulesDependencyTrees, 5) // Check module - module := sca.GetAndAssertNode(t, modulesDependencyTrees, "webservice") + module := sca.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.example.gradle:webservice:1.0") assert.Len(t, module.Nodes, 7) // Check direct dependency @@ -71,10 +71,10 @@ func TestGradleTreesWithConfig(t *testing.T) { // Run getModulesDependencyTrees modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 3) - assert.Len(t, uniqueDeps, 11) + assert.Len(t, modulesDependencyTrees, 5) + assert.Len(t, uniqueDeps, 8) // Check module - module := sca.GetAndAssertNode(t, modulesDependencyTrees, "api") + module := sca.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test.gradle.publish:api:1.0-SNAPSHOT") assert.Len(t, module.Nodes, 4) // Check direct dependency @@ -86,22 +86,6 @@ func TestGradleTreesWithConfig(t *testing.T) { } } -func TestGradleTreesExcludeTestDeps(t *testing.T) { - // Create and change directory to test workspace - tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") - defer cleanUp() - assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) - - // Run getModulesDependencyTrees - modulesDependencyTrees, uniqueDeps, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true}) - if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) { - assert.Len(t, modulesDependencyTrees, 2) - assert.Len(t, uniqueDeps, 11) - // Check direct dependency - assert.Nil(t, sca.GetModule(modulesDependencyTrees, "services")) - } -} - func TestIsGradleWrapperExist(t *testing.T) { // Check Gradle wrapper doesn't exist isWrapperExist, err := isGradleWrapperExist() @@ -168,67 +152,6 @@ func TestGetDepTreeArtifactoryRepository(t *testing.T) { } } -func TestGetGraphFromDepTree(t *testing.T) { - // Create and change directory to test workspace - tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") - defer func() { - cleanUp() - }() - assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) - testCase := struct { - name string - expectedTree map[string]map[string]string - expectedUniqueDeps []string - }{ - name: "ValidOutputFileContent", - expectedTree: map[string]map[string]string{ - GavPackageTypeIdentifier + "shared": {}, - GavPackageTypeIdentifier + filepath.Base(tempDirPath): {}, - GavPackageTypeIdentifier + "services": {}, - GavPackageTypeIdentifier + "webservice": { - GavPackageTypeIdentifier + "junit:junit:4.11": "", - GavPackageTypeIdentifier + "commons-io:commons-io:1.2": "", - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": "", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", - GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": "", - }, - GavPackageTypeIdentifier + "api": { - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", - }, - }, - expectedUniqueDeps: []string{ - GavPackageTypeIdentifier + "webservice", - GavPackageTypeIdentifier + "junit:junit:4.11", - GavPackageTypeIdentifier + "commons-io:commons-io:1.2", - GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0", - GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0", - GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2", - GavPackageTypeIdentifier + "api", - GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4", - GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3", - GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2", - }, - } - - manager := &depTreeManager{} - outputFileContent, err := manager.runGradleDepTree() - assert.NoError(t, err) - depTree, uniqueDeps, err := (&depTreeManager{}).getGraphFromDepTree(outputFileContent) - assert.NoError(t, err) - assert.ElementsMatch(t, uniqueDeps, testCase.expectedUniqueDeps, "First is actual, Second is Expected") - - for _, dependency := range depTree { - depChild, exists := testCase.expectedTree[dependency.Id] - assert.True(t, exists) - assert.Equal(t, len(depChild), len(dependency.Nodes)) - } -} - func TestCreateDepTreeScript(t *testing.T) { manager := &depTreeManager{} tmpDir, err := manager.createDepTreeScriptAndGetDir() diff --git a/xray/commands/audit/sca/java/javautils.go b/xray/commands/audit/sca/java/javautils.go index 9653de43f..37009cb5b 100644 --- a/xray/commands/audit/sca/java/javautils.go +++ b/xray/commands/audit/sca/java/javautils.go @@ -1,12 +1,15 @@ package java import ( + "encoding/json" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" xrayutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" + "os" "strconv" + "strings" "time" buildinfo "github.com/jfrog/build-info-go/entities" @@ -58,7 +61,7 @@ func createGavDependencyTree(buildConfig *artifactoryUtils.BuildConfiguration) ( if len(generatedBuildsInfos) == 0 { return nil, nil, errorutils.CheckErrorf("Couldn't find build " + buildName + "/" + buildNumber) } - modules := []*xrayUtils.GraphNode{} + var modules []*xrayUtils.GraphNode uniqueDepsSet := datastructures.MakeSet[string]() for _, module := range generatedBuildsInfos[0].Modules { modules = append(modules, addModuleTree(module, uniqueDepsSet)) @@ -173,3 +176,73 @@ func (dm *dependencyMultimap) putChild(parent string, child *buildinfo.Dependenc func (dm *dependencyMultimap) getChildren(parent string) map[string]*buildinfo.Dependency { return dm.multimap[parent] } + +// The structure of a dependency tree of a module in a Gradle/Maven project, as created by the gradle-dep-tree and maven-dep-tree plugins. +type moduleDepTree struct { + Root string `json:"root"` + Nodes map[string]depTreeNode `json:"nodes"` +} + +type depTreeNode struct { + Children []string `json:"children"` +} + +// getGraphFromDepTree reads the output files of the gradle-dep-tree and maven-dep-tree plugins and returns them as a slice of GraphNodes. +// It takes the output of the plugin's run (which is a byte representation of a list of paths of the output files, separated by newlines) as input. +func getGraphFromDepTree(depTreeOutput []byte) (depsGraph []*xrayUtils.GraphNode, uniqueDeps []string, err error) { + modules, err := parseDepTreeFiles(depTreeOutput) + if err != nil { + return + } + uniqueDepsSet := datastructures.MakeSet[string]() + for _, moduleTree := range modules { + directDependency := &xrayUtils.GraphNode{ + Id: GavPackageTypeIdentifier + moduleTree.Root, + Nodes: []*xrayUtils.GraphNode{}, + } + populateDependencyTree(directDependency, moduleTree.Root, moduleTree, uniqueDepsSet) + depsGraph = append(depsGraph, directDependency) + } + uniqueDeps = uniqueDepsSet.ToSlice() + return +} + +func populateDependencyTree(currNode *xrayUtils.GraphNode, currNodeId string, moduleTree *moduleDepTree, uniqueDepsSet *datastructures.Set[string]) { + if currNode.NodeHasLoop() { + return + } + for _, childId := range moduleTree.Nodes[currNodeId].Children { + childGav := GavPackageTypeIdentifier + childId + childNode := &xrayUtils.GraphNode{ + Id: childGav, + Nodes: []*xrayUtils.GraphNode{}, + Parent: currNode, + } + uniqueDepsSet.Add(childGav) + populateDependencyTree(childNode, childId, moduleTree, uniqueDepsSet) + currNode.Nodes = append(currNode.Nodes, childNode) + } +} + +func parseDepTreeFiles(jsonFilePaths []byte) ([]*moduleDepTree, error) { + outputFilePaths := strings.Split(strings.TrimSpace(string(jsonFilePaths)), "\n") + var modules []*moduleDepTree + for _, path := range outputFilePaths { + results, err := parseDepTreeFile(path) + if err != nil { + return nil, err + } + modules = append(modules, results) + } + return modules, nil +} + +func parseDepTreeFile(path string) (results *moduleDepTree, err error) { + depTreeJson, err := os.ReadFile(strings.TrimSpace(path)) + if errorutils.CheckError(err) != nil { + return + } + results = &moduleDepTree{} + err = errorutils.CheckError(json.Unmarshal(depTreeJson, &results)) + return +} diff --git a/xray/commands/audit/sca/java/javautils_test.go b/xray/commands/audit/sca/java/javautils_test.go new file mode 100644 index 000000000..325a59408 --- /dev/null +++ b/xray/commands/audit/sca/java/javautils_test.go @@ -0,0 +1,68 @@ +package java + +import ( + "github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca" + "github.com/stretchr/testify/assert" + "os" + "path/filepath" + "testing" +) + +func TestGetGraphFromDepTree(t *testing.T) { + // Create and change directory to test workspace + tempDirPath, cleanUp := sca.CreateTestWorkspace(t, "gradle-example-ci-server") + defer func() { + cleanUp() + }() + assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700)) + testCase := struct { + name string + expectedTree map[string]map[string]string + expectedUniqueDeps []string + }{ + name: "ValidOutputFileContent", + expectedTree: map[string]map[string]string{ + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:" + filepath.Base(tempDirPath) + ":1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:services:1.0": {}, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:webservice:1.0": { + GavPackageTypeIdentifier + "junit:junit:4.11": "", + GavPackageTypeIdentifier + "commons-io:commons-io:1.2": "", + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": "", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2": "", + }, + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0": { + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7": "", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0": "", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4": "", + }, + }, + expectedUniqueDeps: []string{ + GavPackageTypeIdentifier + "junit:junit:4.11", + GavPackageTypeIdentifier + "commons-io:commons-io:1.2", + GavPackageTypeIdentifier + "org.apache.wicket:wicket:1.3.7", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:shared:1.0", + GavPackageTypeIdentifier + "org.jfrog.example.gradle:api:1.0", + GavPackageTypeIdentifier + "commons-collections:commons-collections:3.2", + GavPackageTypeIdentifier + "commons-lang:commons-lang:2.4", + GavPackageTypeIdentifier + "org.hamcrest:hamcrest-core:1.3", + GavPackageTypeIdentifier + "org.slf4j:slf4j-api:1.4.2", + }, + } + + manager := &depTreeManager{} + outputFileContent, err := manager.runGradleDepTree() + assert.NoError(t, err) + depTree, uniqueDeps, err := getGraphFromDepTree(outputFileContent) + assert.NoError(t, err) + assert.ElementsMatch(t, uniqueDeps, testCase.expectedUniqueDeps, "First is actual, Second is Expected") + + for _, dependency := range depTree { + depChild, exists := testCase.expectedTree[dependency.Id] + assert.True(t, exists) + assert.Equal(t, len(depChild), len(dependency.Nodes)) + } +}